From 9b6a9bd0a7ef31e3c6e4786d0c1a47d6b759941f Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:39:51 +0700 Subject: [PATCH 1/5] feat(benchmark): define phase 2c1 teaching image quality loop --- .../api/src/hcs_api/comfyui_teaching_image.py | 3 + .../src/hcs_api/teaching_image_benchmark.py | 954 +++++++++++++++ apps/api/tests/test_comfyui_teaching_image.py | 25 + .../tests/test_teaching_image_benchmark.py | 184 +++ benchmarks/phase2c1/cases.v1.json | 1032 +++++++++++++++++ docs/phase2c1-teaching-image-benchmark.md | 89 ++ 6 files changed, 2287 insertions(+) create mode 100644 apps/api/src/hcs_api/teaching_image_benchmark.py create mode 100644 apps/api/tests/test_teaching_image_benchmark.py create mode 100644 benchmarks/phase2c1/cases.v1.json create mode 100644 docs/phase2c1-teaching-image-benchmark.md diff --git a/apps/api/src/hcs_api/comfyui_teaching_image.py b/apps/api/src/hcs_api/comfyui_teaching_image.py index 2fdae37..bd990d8 100644 --- a/apps/api/src/hcs_api/comfyui_teaching_image.py +++ b/apps/api/src/hcs_api/comfyui_teaching_image.py @@ -605,6 +605,9 @@ def _execute_fixed_plan( {"filename": filename, "subfolder": "", "type": "output"} ) return _http_image(port, query, maximum_bytes), prompt_id + except KeyboardInterrupt: + _cancel_job_if_still_owned(plan, prompt_id) + raise except TeachingImageError: _cancel_job_if_still_owned(plan, prompt_id) raise diff --git a/apps/api/src/hcs_api/teaching_image_benchmark.py b/apps/api/src/hcs_api/teaching_image_benchmark.py new file mode 100644 index 0000000..3196298 --- /dev/null +++ b/apps/api/src/hcs_api/teaching_image_benchmark.py @@ -0,0 +1,954 @@ +"""Reproducible, teacher-reviewable benchmark for the fixed Phase 2C image path.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import shutil +import sys +import time +import uuid +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .comfyui_archive import load_runtime_manifest +from .comfyui_model import ( + WORKFLOW_PACK_SHA256, + ComfyUIModelError, + load_model_manifest, + load_workflow_pack, + model_installation_identity, + validate_model_installation, +) +from .comfyui_runtime import ( + ComfyUIRuntimeError, + runtime_installation_identity, + runtime_snapshot, +) +from .comfyui_teaching_image import ( + TeachingImageError, + TeachingImageRequest, + generate_teaching_image, + generation_capability_snapshot, + verify_png, +) +from .models import AssetManifest, TeachingImageProvenance, VerifiedImageArtifact + +BENCHMARK_SCHEMA = "hanclassstudio.teaching_image_benchmark.v1" +REVIEW_SCHEMA = "hanclassstudio.teacher_image_reviews.v1" +_MAX_JSON_BYTES = 32 * 1024 * 1024 +_REQUIRED_CATEGORIES = frozenset( + { + "single_object", + "person_action", + "person_count", + "spatial_relation", + "classroom_activity", + "daily_communication", + "emotion_expression", + "cultural_scene", + "event_sequence", + "hard_combination", + } +) +_FAILURE_LABELS = frozenset( + { + "wrong_count", + "wrong_action", + "wrong_scene", + "wrong_spatial_relation", + "missing_object", + "anatomy_defect", + "text_artifact", + "culturally_inappropriate", + "visually_confusing", + "not_teaching_usable", + } +) +_RATING_FIELDS = ( + "goal_relevance", + "instruction_following", + "person_object_count", + "action_accuracy", + "spatial_relation_accuracy", + "classroom_usability", + "visual_integrity", + "cultural_age_appropriateness", +) + + +class BenchmarkError(RuntimeError): + """A benchmark definition, state, or execution error.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +class BenchmarkBlockedError(BenchmarkError): + """A fail-closed blocker that requires an external/runtime change.""" + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +class BenchmarkFixedIdentity(_StrictModel): + model_package_id: str + model_version: str + model_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + workflow_pack_id: str + workflow_version: str + workflow_pack_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + runtime_id: Literal["comfyui"] + runtime_version: str + runtime_source_commit: str = Field(pattern=r"^[0-9a-f]{40}$") + prompt_profile_id: str + + +class BenchmarkRequest(_StrictModel): + purpose: Literal["classroom_scene", "vocabulary_image", "teaching_illustration"] + subject: str = Field(min_length=1, max_length=240) + action: str = Field(min_length=1, max_length=240) + environment: str = Field(min_length=1, max_length=240) + aspect_ratio: Literal["1:1", "4:3", "16:9"] + + +class BenchmarkExpected(_StrictModel): + people: int = Field(ge=0, le=20) + objects: list[str] = Field(max_length=20) + actions: list[str] = Field(max_length=12) + relations: list[str] = Field(max_length=12) + scene: str = Field(min_length=1, max_length=240) + + +class BenchmarkCase(_StrictModel): + case_id: str = Field(pattern=r"^[a-z][a-z0-9_]{2,63}$") + category: Literal[ + "single_object", + "person_action", + "person_count", + "spatial_relation", + "classroom_activity", + "daily_communication", + "emotion_expression", + "cultural_scene", + "event_sequence", + "hard_combination", + ] + title: str = Field(min_length=1, max_length=120) + teaching_goal: str = Field(min_length=1, max_length=500) + prompt: str = Field(min_length=1, max_length=800) + negative_prompt: str = Field(min_length=1, max_length=1200) + request: BenchmarkRequest + seeds: list[int] = Field(min_length=1, max_length=2) + expected: BenchmarkExpected + must_satisfy: list[str] = Field(min_length=1, max_length=12) + severe_failures: list[str] = Field(min_length=1, max_length=12) + + @model_validator(mode="after") + def _prompt_is_the_request_intent(self) -> BenchmarkCase: + expected = ( + f"subject: {self.request.subject}; action: {self.request.action}; " + f"environment: {self.request.environment}" + ) + if self.prompt != expected: + raise ValueError("case prompt must be the canonical controlled request intent") + if len(set(self.seeds)) != len(self.seeds): + raise ValueError("case seeds must be unique") + return self + + +class BenchmarkSpec(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_benchmark.v1"] = Field( + default=BENCHMARK_SCHEMA, alias="schema" + ) + benchmark_id: str = Field(pattern=r"^[a-z][a-z0-9-]{2,63}$") + version: str = Field(pattern=r"^\d+\.\d+\.\d+$") + fixed_identity: BenchmarkFixedIdentity + negative_prompt: str = Field(min_length=1, max_length=1200) + cases: list[BenchmarkCase] = Field(min_length=20, max_length=30) + + @model_validator(mode="after") + def _case_contract(self) -> BenchmarkSpec: + case_ids = [case.case_id for case in self.cases] + if len(set(case_ids)) != len(case_ids): + raise ValueError("benchmark case ids must be unique") + if {case.category for case in self.cases} != _REQUIRED_CATEGORIES: + raise ValueError("benchmark must cover every required teaching category") + if any(case.negative_prompt != self.negative_prompt for case in self.cases): + raise ValueError("every case must record the fixed negative prompt") + return self + + +class BenchmarkObservedIdentity(BenchmarkFixedIdentity): + runtime_installation_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + runtime_process_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + runtime_port: int = Field(ge=1024, le=65535) + model_installation_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + checked_at: str + + +class BenchmarkErrorRecord(_StrictModel): + code: str + message: str + attempt: int = Field(ge=1) + recoverable: bool = True + occurred_at: str + + +class BenchmarkTechnicalCheck(_StrictModel): + status: Literal["passed"] = "passed" + image_path: str + provenance_path: str + width: int + height: int + image_size_bytes: int + image_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + provenance_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + manifest_asset_id: str + request_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + execution_plan_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + checks: list[str] = Field(min_length=1) + + +class BenchmarkCaseResult(_StrictModel): + case_id: str + seed: int + status: Literal["pending", "running", "succeeded", "failed", "invalidated"] = "pending" + attempts: int = Field(default=0, ge=0) + started_at: str | None = None + completed_at: str | None = None + artifact: VerifiedImageArtifact | None = None + technical_checks: BenchmarkTechnicalCheck | None = None + error: BenchmarkErrorRecord | None = None + manual_review: None = None + + +class TeacherReviewRecord(_StrictModel): + schema_: Literal["hanclassstudio.teacher_image_review.v1"] = Field( + default="hanclassstudio.teacher_image_review.v1", alias="schema" + ) + case_key: str + case_id: str + seed: int + artifact_id: str + review_state: Literal["pending_review", "reviewed"] = "pending_review" + reviewer_id: str = "" + goal_relevance: int | None = Field(default=None, ge=1, le=5) + instruction_following: int | None = Field(default=None, ge=1, le=5) + person_object_count: int | None = Field(default=None, ge=1, le=5) + action_accuracy: int | None = Field(default=None, ge=1, le=5) + spatial_relation_accuracy: int | None = Field(default=None, ge=1, le=5) + classroom_usability: int | None = Field(default=None, ge=1, le=5) + visual_integrity: int | None = Field(default=None, ge=1, le=5) + cultural_age_appropriateness: int | None = Field(default=None, ge=1, le=5) + failure_labels: list[str] = Field(default_factory=list) + regeneration_required: bool | None = None + direct_courseware_use: bool | None = None + notes: str = Field(default="", max_length=3000) + + @model_validator(mode="after") + def _review_contract(self) -> TeacherReviewRecord: + if any(label not in _FAILURE_LABELS for label in self.failure_labels): + raise ValueError("unknown teacher failure label") + if len(set(self.failure_labels)) != len(self.failure_labels): + raise ValueError("teacher failure labels must be unique") + if self.review_state == "reviewed": + if any(getattr(self, field) is None for field in _RATING_FIELDS): + raise ValueError("reviewed records require all teacher ratings") + if self.regeneration_required is None or self.direct_courseware_use is None: + raise ValueError("reviewed records require use decisions") + return self + + +class BenchmarkRunState(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_benchmark_run.v1"] = Field( + default="hanclassstudio.teaching_image_benchmark_run.v1", alias="schema" + ) + run_id: str = Field(pattern=r"^[a-z0-9-]{8,80}$") + benchmark_id: str + benchmark_version: str + spec_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + output_dir: str + project_dir: str + selected_case_ids: list[str] + selected_case_seeds: dict[str, list[int]] + execution_identity: BenchmarkObservedIdentity | None = None + status: Literal["running", "paused", "completed", "blocked"] = "running" + results: dict[str, BenchmarkCaseResult] = Field(default_factory=dict) + started_at: str + updated_at: str + block: BenchmarkErrorRecord | None = None + + +def _iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _canonical(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + + +def _sha256(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _read_json(path: Path) -> Any: + try: + if path.stat().st_size > _MAX_JSON_BYTES: + raise BenchmarkError("json_too_large", f"JSON exceeds {_MAX_JSON_BYTES} bytes") + return json.loads(path.read_text(encoding="utf-8")) + except BenchmarkError: + raise + except (OSError, UnicodeDecodeError, ValueError) as exc: + raise BenchmarkError("json_invalid", f"Could not read JSON: {path}") from exc + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + temporary.replace(path) + + +def _expected_fixed_identity() -> BenchmarkFixedIdentity: + runtime = load_runtime_manifest() + model = load_model_manifest() + workflow = load_workflow_pack() + return BenchmarkFixedIdentity( + model_package_id=model.package_id, + model_version=model.version, + model_sha256=model.source.sha256, + workflow_pack_id=workflow.pack_id, + workflow_version=workflow.version, + workflow_pack_sha256=WORKFLOW_PACK_SHA256, + runtime_id=runtime.runtime_id, + runtime_version=runtime.version, + runtime_source_commit=runtime.source_commit, + prompt_profile_id=workflow.prompt_profile.id, + ) + + +def load_benchmark_spec(path: Path) -> BenchmarkSpec: + try: + spec = BenchmarkSpec.model_validate(_read_json(path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("benchmark_schema_invalid", str(exc)) from exc + fixed = _expected_fixed_identity() + if spec.fixed_identity != fixed: + raise BenchmarkError( + "benchmark_identity_invalid", + "Benchmark fixed identity does not match the repository Model/Workflow/Runtime contracts", + ) + workflow = load_workflow_pack() + if spec.negative_prompt != workflow.prompt_profile.negative: + raise BenchmarkError( + "benchmark_prompt_invalid", "Benchmark negative prompt differs from the fixed Workflow Pack" + ) + return spec + + +def _asset_id(case_id: str, seed: int) -> str: + return f"bmk-{case_id}-{seed}" + + +def _case_key(case_id: str, seed: int) -> str: + return f"{case_id}@{seed}" + + +def _request_for(case: BenchmarkCase, seed: int) -> TeachingImageRequest: + return TeachingImageRequest( + asset_id=_asset_id(case.case_id, seed), + purpose=case.request.purpose, + subject=case.request.subject, + action=case.request.action, + environment=case.request.environment, + aspect_ratio=case.request.aspect_ratio, + seed=seed, + source_trace=[ + "benchmark:phase2c1-teaching-image-quality", + f"case:{case.case_id}", + f"seed:{seed}", + ], + ) + + +def _capture_identity() -> BenchmarkObservedIdentity: + try: + capability = generation_capability_snapshot(deep=True) + if not capability.generation_ready: + error = capability.technical_error or { + "code": "generation_not_ready", + "message": "Runtime, model, and Workflow are not jointly ready", + } + raise BenchmarkBlockedError(error["code"], error["message"]) + runtime = runtime_snapshot(recover=False) + model = validate_model_installation(deep=False) + workflow = load_workflow_pack() + fixed = _expected_fixed_identity() + if ( + runtime.actual_port is None + or runtime.process_identity is None + or runtime.version != fixed.runtime_version + or runtime.source_commit != fixed.runtime_source_commit + or model.package_id != fixed.model_package_id + or model.version != fixed.model_version + or model.model_sha256 != fixed.model_sha256 + or workflow.pack_id != fixed.workflow_pack_id + or workflow.version != fixed.workflow_version + ): + raise BenchmarkBlockedError( + "benchmark_identity_invalid", + "Live Runtime, model, or Workflow identity does not match the fixed benchmark", + ) + return BenchmarkObservedIdentity( + **fixed.model_dump(), + runtime_installation_identity=runtime_installation_identity(), + runtime_process_identity=runtime.process_identity, + runtime_port=runtime.actual_port, + model_installation_identity=model_installation_identity(model), + checked_at=_iso(), + ) + except BenchmarkError: + raise + except (ComfyUIModelError, ComfyUIRuntimeError, OSError, ValueError) as exc: + code = getattr(exc, "code", "generation_not_ready") + message = getattr(exc, "message", str(exc)) + raise BenchmarkBlockedError(code, message) from exc + + +def _same_execution_identity( + expected: BenchmarkObservedIdentity, observed: BenchmarkObservedIdentity +) -> bool: + return expected.model_dump(exclude={"checked_at"}) == observed.model_dump( + exclude={"checked_at"} + ) + + +def _technical_check( + project_dir: Path, + case: BenchmarkCase, + seed: int, + artifact: VerifiedImageArtifact, +) -> BenchmarkTechnicalCheck: + workflow = load_workflow_pack() + image_path = project_dir / artifact.path + provenance_path = project_dir / artifact.provenance_ref + if not image_path.is_file() or not provenance_path.is_file(): + raise BenchmarkError("artifact_missing", "Generated image or provenance file is missing") + payload = image_path.read_bytes() + verified = verify_png( + payload, + expected_width=workflow.dimensions[case.request.aspect_ratio][0], + expected_height=workflow.dimensions[case.request.aspect_ratio][1], + maximum_bytes=workflow.output.maximum_bytes, + ) + if verified.sha256 != artifact.sha256 or verified.size_bytes != artifact.size_bytes: + raise BenchmarkError("artifact_hash_mismatch", "Image hash or size differs from artifact") + provenance_bytes = provenance_path.read_bytes() + provenance = TeachingImageProvenance.model_validate_json(provenance_bytes) + provenance_sha = hashlib.sha256(provenance_bytes).hexdigest() + if provenance_sha != artifact.provenance_sha256: + raise BenchmarkError("provenance_hash_mismatch", "Provenance hash differs from artifact") + request = _request_for(case, seed) + request_sha = _sha256(request.model_dump(mode="json", by_alias=True)) + if provenance.request_sha256 != request_sha: + raise BenchmarkError("request_provenance_mismatch", "Provenance does not identify this case request") + if provenance.negative_prompt != case.negative_prompt: + raise BenchmarkError("negative_prompt_mismatch", "Provenance negative prompt differs from benchmark") + manifest_path = project_dir / "assets/data/asset_manifest.json" + manifest = AssetManifest.model_validate_json(manifest_path.read_bytes()) + matches = [asset for asset in manifest.images if asset.id == request.asset_id] + if len(matches) != 1 or matches[0].review_state != "pending_review": + raise BenchmarkError("manifest_registration_invalid", "Asset Manifest entry is missing or not pending_review") + registered = matches[0].verified_image_artifact + if registered is None or registered.artifact_id != artifact.artifact_id: + raise BenchmarkError("manifest_registration_invalid", "Manifest artifact does not match execution artifact") + return BenchmarkTechnicalCheck( + image_path=artifact.path, + provenance_path=artifact.provenance_ref, + width=verified.width, + height=verified.height, + image_size_bytes=verified.size_bytes, + image_sha256=verified.sha256, + provenance_sha256=provenance_sha, + manifest_asset_id=request.asset_id, + request_sha256=provenance.request_sha256, + execution_plan_sha256=provenance.execution_plan_sha256, + checks=[ + "png_signature_crc_dimensions", + "image_sha256", + "provenance_sha256_and_identity", + "asset_manifest_single_pending_review_entry", + "fixed_negative_prompt", + ], + ) + + +def _result_is_reusable( + state: BenchmarkRunState, + result: BenchmarkCaseResult, + case: BenchmarkCase, + project_dir: Path, +) -> bool: + if result.status != "succeeded" or result.artifact is None: + return False + try: + _technical_check(project_dir, case, result.seed, result.artifact) + except (BenchmarkError, OSError, ValueError): + return False + return True + + +def _record_error(code: str, message: str, attempt: int, recoverable: bool = True) -> BenchmarkErrorRecord: + return BenchmarkErrorRecord( + code=code, + message=message, + attempt=attempt, + recoverable=recoverable, + occurred_at=_iso(), + ) + + +def run_benchmark( + spec_path: Path, + output_dir: Path, + *, + case_ids: list[str] | None = None, + case_limit: int | None = None, + max_attempts: int = 2, +) -> BenchmarkRunState: + """Run or resume a benchmark, saving state after every attempt.""" + if max_attempts < 1 or max_attempts > 3: + raise BenchmarkError("invalid_attempts", "max_attempts must be between 1 and 3") + spec = load_benchmark_spec(spec_path) + spec_sha = _sha256(spec.model_dump(mode="json", by_alias=True)) + output_dir = output_dir.resolve() + project_dir = output_dir / "project" + state_path = output_dir / "run-state.json" + case_by_id = {case.case_id: case for case in spec.cases} + selected = case_ids or [case.case_id for case in spec.cases] + if case_limit is not None: + selected = selected[:case_limit] + unknown = sorted(set(selected) - set(case_by_id)) + if unknown: + raise BenchmarkError("unknown_case", f"Unknown benchmark cases: {', '.join(unknown)}") + selected = list(dict.fromkeys(selected)) + selected_seeds = {case_id: case_by_id[case_id].seeds for case_id in selected} + + if state_path.exists(): + try: + state = BenchmarkRunState.model_validate(_read_json(state_path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("run_state_invalid", str(exc)) from exc + if state.spec_sha256 != spec_sha or state.selected_case_ids != selected: + raise BenchmarkError("run_state_mismatch", "Existing run state belongs to a different benchmark selection") + if state.selected_case_seeds != selected_seeds: + raise BenchmarkError("run_state_mismatch", "Existing run state has different case seeds") + if state.status == "blocked": + raise BenchmarkBlockedError( + state.block.code if state.block else "benchmark_blocked", + state.block.message if state.block else "Benchmark is blocked; start a new run after restoring identity", + ) + if state.execution_identity is None: + state.execution_identity = _capture_identity() + else: + observed = _capture_identity() + if not _same_execution_identity(state.execution_identity, observed): + state.status = "blocked" + state.block = _record_error( + "benchmark_identity_changed", + "Runtime process, installation, model installation, or fixed package identity changed; start a new run", + max((result.attempts for result in state.results.values()), default=0) + 1, + recoverable=False, + ) + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + raise BenchmarkBlockedError(state.block.code, state.block.message) + state.status = "running" + else: + observed = _capture_identity() + now = _iso() + state = BenchmarkRunState( + run_id=f"phase2c1-{int(time.time())}-{uuid.uuid4().hex[:8]}", + benchmark_id=spec.benchmark_id, + benchmark_version=spec.version, + spec_sha256=spec_sha, + output_dir=str(output_dir), + project_dir=str(project_dir), + selected_case_ids=selected, + selected_case_seeds=selected_seeds, + execution_identity=observed, + started_at=now, + updated_at=now, + ) + output_dir.mkdir(parents=True, exist_ok=True) + project_dir.mkdir(parents=True, exist_ok=True) + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + + try: + for case_id in selected: + case = case_by_id[case_id] + for seed in case.seeds: + key = _case_key(case_id, seed) + result = state.results.get(key) or BenchmarkCaseResult(case_id=case_id, seed=seed) + state.results[key] = result + if _result_is_reusable(state, result, case, project_dir): + continue + if result.status == "succeeded": + result.status = "invalidated" + result.artifact = None + result.technical_checks = None + result.status = "running" + result.started_at = result.started_at or _iso() + result.error = None + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + for attempt in range(result.attempts + 1, max_attempts + 1): + result.attempts = attempt + try: + observed = _capture_identity() + if state.execution_identity is None or not _same_execution_identity( + state.execution_identity, observed + ): + raise BenchmarkBlockedError( + "benchmark_identity_changed", + "Runtime or installation identity changed during the benchmark", + ) + artifact = generate_teaching_image(project_dir, _request_for(case, seed)) + checks = _technical_check(project_dir, case, seed, artifact) + result.status = "succeeded" + result.completed_at = _iso() + result.artifact = artifact + result.technical_checks = checks + result.error = None + break + except BenchmarkBlockedError as exc: + result.status = "invalidated" + result.error = _record_error(exc.code, exc.message, attempt, recoverable=False) + state.status = "blocked" + state.block = result.error + raise + except (BenchmarkError, TeachingImageError, ComfyUIModelError, ComfyUIRuntimeError, OSError, ValueError) as exc: + code = getattr(exc, "code", "benchmark_execution_failed") + message = getattr(exc, "message", str(exc)) + result.error = _record_error(code, message, attempt) + result.status = "failed" + result.artifact = None + result.technical_checks = None + if attempt < max_attempts: + continue + finally: + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + except KeyboardInterrupt: + state.status = "paused" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + return state + except BenchmarkBlockedError: + state.status = "blocked" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + raise + state.status = "completed" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + return state + + +def _load_state(output_dir: Path) -> BenchmarkRunState: + path = output_dir / "run-state.json" + try: + return BenchmarkRunState.model_validate(_read_json(path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("run_state_invalid", str(exc)) from exc + + +def _review_record_for(key: str, result: BenchmarkCaseResult) -> TeacherReviewRecord: + if result.artifact is None: + raise BenchmarkError("review_artifact_missing", f"No successful artifact for {key}") + return TeacherReviewRecord( + case_key=key, + case_id=result.case_id, + seed=result.seed, + artifact_id=result.artifact.artifact_id, + ) + + +def write_review_package(spec_path: Path, output_dir: Path) -> Path: + spec = load_benchmark_spec(spec_path) + state = _load_state(output_dir) + package_dir = output_dir / "review-package" + images_dir = package_dir / "images" + images_dir.mkdir(parents=True, exist_ok=True) + case_by_id = {case.case_id: case for case in spec.cases} + records: list[TeacherReviewRecord] = [] + cards: list[dict[str, Any]] = [] + project_dir = Path(state.project_dir) + for key, result in state.results.items(): + if result.status != "succeeded" or result.artifact is None: + continue + record = _review_record_for(key, result) + records.append(record) + case = case_by_id[result.case_id] + source = project_dir / result.artifact.path + target = images_dir / f"{key.replace('@', '-')}.png" + if not source.is_file(): + raise BenchmarkError("review_image_missing", f"Review image is missing: {source}") + shutil.copyfile(source, target) + cards.append( + { + "key": key, + "case": case.model_dump(mode="json", by_alias=True), + "seed": result.seed, + "artifact_id": result.artifact.artifact_id, + "image": f"images/{target.name}", + } + ) + _write_json( + package_dir / "teacher-reviews.pending.json", + { + "schema": REVIEW_SCHEMA, + "benchmark_id": spec.benchmark_id, + "benchmark_version": spec.version, + "run_id": state.run_id, + "records": [record.model_dump(mode="json", by_alias=True) for record in records], + }, + ) + readme = ( + "# Phase 2C.1 教师图片评审包\n\n" + "本包只包含真实生成图片、案例合同和空白评审记录。所有评分字段初始为空，" + "`pending_review` 不是通过结论。教师完成评分后，在界面点击 Download reviews.json，" + "再使用 `benchmark_phase2c1.py report --review-file ` 导入。\n\n" + "评审维度：教学目标相关性、指令遵循、人物/物体数量、动作、空间关系、课堂可用性、" + "视觉完整性、文化与年龄适切性、是否需要重新生成、是否可直接用于课件。\n" + ) + (package_dir / "README.md").write_text(readme, encoding="utf-8") + (package_dir / "index.html").write_text(_review_html(cards), encoding="utf-8") + return package_dir + + +def _review_html(cards: list[dict[str, Any]]) -> str: + encoded = json.dumps(cards, ensure_ascii=False).replace("<", "\\u003c") + labels = sorted(_FAILURE_LABELS) + score_fields = [ + ("goal_relevance", "教学目标相关性"), + ("instruction_following", "指令遵循"), + ("person_object_count", "人物和物体数量"), + ("action_accuracy", "动作准确性"), + ("spatial_relation_accuracy", "空间关系准确性"), + ("classroom_usability", "课堂可用性"), + ("visual_integrity", "视觉完整性"), + ("cultural_age_appropriateness", "文化与年龄适切性"), + ] + score_markup = "".join( + f'" + for field, label in score_fields + ) + failure_markup = "".join( + f'' + for label in labels + ) + return f""" + +Phase 2C.1 Teacher Image Review + +

Phase 2C.1 教师图片质量评审

+

所有评分初始为空。自动技术检查不等于教师结论；只有教师明确点击“标记已评审”后才会产生 reviewed 记录。

+
+""" + + +def _load_reviews(path: Path) -> list[TeacherReviewRecord]: + raw = _read_json(path) + if not isinstance(raw, dict) or raw.get("schema") != REVIEW_SCHEMA or not isinstance(raw.get("records"), list): + raise BenchmarkError("review_file_invalid", "Teacher review file has an unknown schema") + try: + return [TeacherReviewRecord.model_validate(item) for item in raw["records"]] + except (ValueError, TypeError) as exc: + raise BenchmarkError("review_file_invalid", str(exc)) from exc + + +def aggregate_benchmark( + spec_path: Path, + output_dir: Path, + *, + review_path: Path | None = None, +) -> dict[str, Any]: + spec = load_benchmark_spec(spec_path) + state = _load_state(output_dir) + case_by_id = {case.case_id: case for case in spec.cases} + category_counts: dict[str, dict[str, Any]] = defaultdict( + lambda: {"total": 0, "succeeded": 0, "failed": 0, "pending": 0, "technical_success_rate": None} + ) + technical_failures: Counter[str] = Counter() + for result in state.results.values(): + category = case_by_id[result.case_id].category + counts = category_counts[category] + counts["total"] += 1 + if result.status == "succeeded": + counts["succeeded"] += 1 + elif result.status == "failed": + counts["failed"] += 1 + if result.error: + technical_failures[result.error.code] += 1 + else: + counts["pending"] += 1 + for counts in category_counts.values(): + if counts["total"]: + counts["technical_success_rate"] = round(counts["succeeded"] / counts["total"], 4) + reviews: list[TeacherReviewRecord] = [] + if review_path is not None: + reviews = _load_reviews(review_path) + expected = { + key: result + for key, result in state.results.items() + if result.status == "succeeded" and result.artifact is not None + } + for review in reviews: + if review.case_key not in expected: + raise BenchmarkError("review_target_invalid", f"Review targets unknown or failed case: {review.case_key}") + if expected[review.case_key].artifact.artifact_id != review.artifact_id: + raise BenchmarkError("review_artifact_mismatch", f"Review artifact changed for {review.case_key}") + report = { + "schema": "hanclassstudio.teaching_image_benchmark_report.v1", + "benchmark_id": spec.benchmark_id, + "benchmark_version": spec.version, + "spec_sha256": _sha256(spec.model_dump(mode="json", by_alias=True)), + "run_id": state.run_id, + "run_status": state.status, + "generated_at": _iso(), + "selected_cases": len(state.results), + "succeeded": sum(result.status == "succeeded" for result in state.results.values()), + "failed": sum(result.status == "failed" for result in state.results.values()), + "pending_or_invalidated": sum(result.status not in {"succeeded", "failed"} for result in state.results.values()), + "technical_success_rate": round( + sum(result.status == "succeeded" for result in state.results.values()) / len(state.results), 4 + ) + if state.results + else None, + "category_results": dict(sorted(category_counts.items())), + "technical_failure_frequency": dict(sorted(technical_failures.items())), + "teacher_review": { + "conclusion": None if not reviews else "teacher_data_imported", + "records_imported": len(reviews), + "records_pending_without_teacher": max( + 0, + sum(result.status == "succeeded" for result in state.results.values()) + - sum(review.review_state == "reviewed" for review in reviews), + ), + "automatic_teacher_scores": False, + "failure_label_frequency": dict( + sorted(Counter(label for review in reviews for label in review.failure_labels).items()) + ), + }, + "review_package": str((output_dir / "review-package").resolve()), + "limitations": [ + "Technical success is not teaching quality success.", + "Teacher ratings and failure labels remain empty until a teacher submits the review package.", + "This benchmark does not change the fixed model, Workflow Pack, or production prompt profile.", + ], + } + _write_json(output_dir / "benchmark-report.json", report) + return report + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Phase 2C.1 controlled teaching-image benchmark") + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run or resume a benchmark") + run.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/cases.v1.json")) + run.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-benchmark")) + run.add_argument("--case-id", action="append", dest="case_ids") + run.add_argument("--case-limit", type=int) + run.add_argument("--max-attempts", type=int, default=2) + package = sub.add_parser("review-package", help="build portable teacher review package") + package.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/cases.v1.json")) + package.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-benchmark")) + report = sub.add_parser("report", help="aggregate technical results and optional teacher reviews") + report.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/cases.v1.json")) + report.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-benchmark")) + report.add_argument("--review-file", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "run": + state = run_benchmark( + args.spec, + args.output_dir, + case_ids=args.case_ids, + case_limit=args.case_limit, + max_attempts=args.max_attempts, + ) + print(json.dumps(state.model_dump(mode="json", by_alias=True), ensure_ascii=False, indent=2)) + elif args.command == "review-package": + print(write_review_package(args.spec, args.output_dir)) + elif args.command == "report": + print(json.dumps(aggregate_benchmark(args.spec, args.output_dir, review_path=args.review_file), ensure_ascii=False, indent=2)) + return 0 + except BenchmarkBlockedError as exc: + print(f"BLOCKED [{exc.code}]: {exc.message}", file=sys.stderr) + return 2 + except BenchmarkError as exc: + print(f"ERROR [{exc.code}]: {exc.message}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("PAUSED: interrupt received; run state was preserved", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/api/tests/test_comfyui_teaching_image.py b/apps/api/tests/test_comfyui_teaching_image.py index 00c195a..2a27c4b 100644 --- a/apps/api/tests/test_comfyui_teaching_image.py +++ b/apps/api/tests/test_comfyui_teaching_image.py @@ -256,6 +256,31 @@ def fake_json(_port, method, path, **kwargs): assert len(viewed) == 1 +def test_executor_cancels_current_job_when_batch_is_interrupted(monkeypatch) -> None: + _ready_runtime(monkeypatch) + plan = images.compile_teaching_image_request( + _request(), model_record=_record(), workflow=load_workflow_pack() + ) + submitted: dict[str, object] = {} + cancelled: list[str] = [] + + def fake_json(_port, method, path, **kwargs): + if method == "POST" and path == "/prompt": + submitted.update(kwargs["payload"]) + return {"prompt_id": submitted["prompt_id"], "node_errors": {}} + raise KeyboardInterrupt + + monkeypatch.setattr(images, "_http_json", fake_json) + monkeypatch.setattr( + images, + "_cancel_job_if_still_owned", + lambda _plan, prompt_id: cancelled.append(prompt_id), + ) + with pytest.raises(KeyboardInterrupt): + images._execute_fixed_plan(plan, 8188, 16 * 1024**2) + assert cancelled == [submitted["prompt_id"]] + + @pytest.mark.parametrize( "mismatch", ["client", "graph", "prior_output", "path_traversal"], diff --git a/apps/api/tests/test_teaching_image_benchmark.py b/apps/api/tests/test_teaching_image_benchmark.py new file mode 100644 index 0000000..1e76691 --- /dev/null +++ b/apps/api/tests/test_teaching_image_benchmark.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from hcs_api import teaching_image_benchmark as benchmark +from hcs_api.comfyui_teaching_image import TeachingImageError +from hcs_api.models import TeachingImageProvenance, VerifiedImageArtifact + +ROOT = Path(__file__).resolve().parents[3] +SPEC_PATH = ROOT / "benchmarks/phase2c1/cases.v1.json" + + +def _identity(checked_at: str = "2026-07-27T00:00:00+00:00") -> benchmark.BenchmarkObservedIdentity: + fixed = benchmark._expected_fixed_identity() + return benchmark.BenchmarkObservedIdentity( + **fixed.model_dump(), + runtime_installation_identity="a" * 64, + runtime_process_identity="b" * 64, + runtime_port=8188, + model_installation_identity="c" * 64, + checked_at=checked_at, + ) + + +def _artifact(asset_id: str, seed: int) -> VerifiedImageArtifact: + provenance = TeachingImageProvenance( + runtime_version="0.28.0", + runtime_source_commit="700821e1364eaab0e8f21c538a2131719fec57bf", + runtime_installation_identity="a" * 64, + runtime_process_identity="b" * 64, + runtime_port=8188, + model_package_id="hcs.sd15-teaching-illustration-fp16", + model_version="1.5-fp16-emaonly", + model_manifest_sha256="1" * 64, + model_sha256="e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916", + model_installation_identity="c" * 64, + workflow_pack_id="hcs.teaching-illustration-sd15-core", + workflow_version="1.0.0", + workflow_pack_sha256="e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e", + request_sha256="2" * 64, + execution_plan_sha256="3" * 64, + prompt_profile_id="soft-flat-educational-v1", + positive_prompt="fixed positive prompt", + negative_prompt="text, letters, words", + seed=seed, + steps=20, + cfg=7.0, + sampler_name="euler", + scheduler="normal", + denoise=1.0, + output_prefix="hcs_" + "4" * 20 + "_" + "5" * 12, + prompt_id="11111111-1111-4111-8111-111111111111", + source_trace=[f"case:{asset_id}"], + ) + return VerifiedImageArtifact( + artifact_id="img-" + "6" * 24, + asset_id=asset_id, + path=f"assets/images/{asset_id}.png", + width=512, + height=384, + size_bytes=100, + sha256="7" * 64, + provenance_ref=f"assets/data/image-provenance/{asset_id}.json", + provenance_sha256="8" * 64, + provenance=provenance, + ) + + +def _checks(artifact: VerifiedImageArtifact) -> benchmark.BenchmarkTechnicalCheck: + return benchmark.BenchmarkTechnicalCheck( + image_path=artifact.path, + provenance_path=artifact.provenance_ref, + width=artifact.width, + height=artifact.height, + image_size_bytes=artifact.size_bytes, + image_sha256=artifact.sha256, + provenance_sha256=artifact.provenance_sha256, + manifest_asset_id=artifact.asset_id, + request_sha256="2" * 64, + execution_plan_sha256="3" * 64, + checks=["fixture"], + ) + + +def test_benchmark_spec_has_fixed_identity_and_all_required_categories() -> None: + spec = benchmark.load_benchmark_spec(SPEC_PATH) + assert len(spec.cases) == 25 + assert {case.category for case in spec.cases} == benchmark._REQUIRED_CATEGORIES + assert all(case.prompt.startswith("subject: ") for case in spec.cases) + assert all(case.negative_prompt == spec.negative_prompt for case in spec.cases) + + +def test_execution_identity_ignores_check_timestamp_but_not_runtime_process() -> None: + assert benchmark._same_execution_identity(_identity(), _identity("later")) + changed = _identity() + changed.runtime_process_identity = "d" * 64 + assert not benchmark._same_execution_identity(_identity(), changed) + + +def test_run_continues_after_one_case_failure_and_resumes_idempotently( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + spec = benchmark.load_benchmark_spec(SPEC_PATH) + selected = [spec.cases[0].case_id, spec.cases[1].case_id] + observed = _identity() + monkeypatch.setattr(benchmark, "_capture_identity", lambda: observed) + calls: list[str] = [] + + def fake_generate(project_dir: Path, request): + calls.append(request.asset_id) + if request.asset_id.startswith("bmk-obj_apple"): + raise TeachingImageError("generation_failed", "fixture failure") + return _artifact(request.asset_id, request.seed) + + monkeypatch.setattr(benchmark, "generate_teaching_image", fake_generate) + monkeypatch.setattr(benchmark, "_technical_check", lambda *_args: _checks(_args[3])) + output = tmp_path / "run" + state = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) + assert state.status == "completed" + assert state.results[f"{selected[0]}@260101"].status == "failed" + assert state.results[f"{selected[1]}@260102"].status == "succeeded" + assert len(calls) == 2 + + monkeypatch.setattr(benchmark, "_result_is_reusable", lambda *_args: True) + resumed = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) + assert resumed.status == "completed" + assert len(calls) == 2 + + +def test_identity_change_blocks_unfinished_cases(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + observed = _identity() + changed = _identity() + changed.runtime_process_identity = "d" * 64 + captures = iter([observed, changed]) + monkeypatch.setattr(benchmark, "_capture_identity", lambda: next(captures)) + with pytest.raises(benchmark.BenchmarkBlockedError, match="identity"): + benchmark.run_benchmark( + SPEC_PATH, + tmp_path / "run", + case_ids=["obj_apple_01"], + max_attempts=1, + ) + state = benchmark.BenchmarkRunState.model_validate( + json.loads((tmp_path / "run/run-state.json").read_text(encoding="utf-8")) + ) + assert state.status == "blocked" + assert state.block and state.block.code == "benchmark_identity_changed" + + +def test_teacher_review_defaults_are_empty_and_reviewed_requires_all_fields() -> None: + pending = benchmark.TeacherReviewRecord( + case_key="obj_apple_01@260101", + case_id="obj_apple_01", + seed=260101, + artifact_id="img-" + "6" * 24, + ) + assert pending.review_state == "pending_review" + assert pending.goal_relevance is None + with pytest.raises(ValueError, match="all teacher ratings"): + reviewed_payload = pending.model_dump() + reviewed_payload["review_state"] = "reviewed" + benchmark.TeacherReviewRecord.model_validate(reviewed_payload) + + +def test_report_does_not_infer_teacher_conclusions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + observed = _identity() + monkeypatch.setattr(benchmark, "_capture_identity", lambda: observed) + artifact = _artifact("bmk-obj_apple_01-260101", 260101) + monkeypatch.setattr(benchmark, "generate_teaching_image", lambda *_args: artifact) + monkeypatch.setattr(benchmark, "_technical_check", lambda *_args: _checks(artifact)) + state = benchmark.run_benchmark( + SPEC_PATH, + tmp_path / "run", + case_ids=["obj_apple_01"], + max_attempts=1, + ) + assert state.status == "completed" + report = benchmark.aggregate_benchmark(SPEC_PATH, tmp_path / "run") + assert report["technical_success_rate"] == 1.0 + assert report["teacher_review"]["conclusion"] is None + assert report["teacher_review"]["records_pending_without_teacher"] == 1 + assert report["teacher_review"]["failure_label_frequency"] == {} diff --git a/benchmarks/phase2c1/cases.v1.json b/benchmarks/phase2c1/cases.v1.json new file mode 100644 index 0000000..7b233ff --- /dev/null +++ b/benchmarks/phase2c1/cases.v1.json @@ -0,0 +1,1032 @@ +{ + "schema": "hanclassstudio.teaching_image_benchmark.v1", + "benchmark_id": "phase2c1-teaching-image-quality", + "version": "1.0.0", + "fixed_identity": { + "model_package_id": "hcs.sd15-teaching-illustration-fp16", + "model_version": "1.5-fp16-emaonly", + "model_sha256": "e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916", + "workflow_pack_id": "hcs.teaching-illustration-sd15-core", + "workflow_version": "1.0.0", + "workflow_pack_sha256": "e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e", + "runtime_id": "comfyui", + "runtime_version": "0.28.0", + "runtime_source_commit": "700821e1364eaab0e8f21c538a2131719fec57bf", + "prompt_profile_id": "soft-flat-educational-v1" + }, + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity", + "cases": [ + { + "case_id": "obj_apple_01", + "category": "single_object", + "title": "苹果", + "teaching_goal": "Learner can recognize 苹果 as a single everyday object.", + "prompt": "subject: one red apple; action: resting on a small table; environment: a plain bright teaching surface", + "request": { + "purpose": "vocabulary_image", + "subject": "one red apple", + "action": "resting on a small table", + "environment": "a plain bright teaching surface", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260101 + ], + "expected": { + "people": 0, + "objects": [ + "one red apple", + "small table" + ], + "actions": [], + "relations": [], + "scene": "plain teaching surface" + }, + "must_satisfy": [ + "one clearly recognizable red apple is the focal object", + "no extra people are present" + ], + "severe_failures": [ + "missing apple", + "multiple apples when singularity is unclear", + "text artifact" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "obj_book_01", + "category": "single_object", + "title": "书", + "teaching_goal": "Learner can recognize 书 as a single classroom object.", + "prompt": "subject: one blue book; action: open on a desk; environment: a simple classroom table", + "request": { + "purpose": "vocabulary_image", + "subject": "one blue book", + "action": "open on a desk", + "environment": "a simple classroom table", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260102 + ], + "expected": { + "people": 0, + "objects": [ + "one blue book", + "desk" + ], + "actions": [ + "open" + ], + "relations": [ + "book on desk" + ], + "scene": "simple classroom table" + }, + "must_satisfy": [ + "book is the clear focal object", + "book is visibly open on a desk" + ], + "severe_failures": [ + "book absent", + "book replaced by a phone or laptop", + "unreadable fake writing dominating the image" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "obj_umbrella_01", + "category": "single_object", + "title": "雨伞", + "teaching_goal": "Learner can recognize 雨伞 as an everyday object.", + "prompt": "subject: one yellow umbrella; action: standing closed beside a doorway; environment: a clean home entryway", + "request": { + "purpose": "vocabulary_image", + "subject": "one yellow umbrella", + "action": "standing closed beside a doorway", + "environment": "a clean home entryway", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260103 + ], + "expected": { + "people": 0, + "objects": [ + "one yellow umbrella", + "doorway" + ], + "actions": [], + "relations": [ + "umbrella beside doorway" + ], + "scene": "clean home entryway" + }, + "must_satisfy": [ + "umbrella silhouette is unmistakable", + "one umbrella is shown" + ], + "severe_failures": [ + "umbrella missing", + "several umbrellas with unclear count", + "umbrella becomes an unrelated object" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "action_wave_01", + "category": "person_action", + "title": "挥手", + "teaching_goal": "Learner can connect 挥手 with a person visibly waving.", + "prompt": "subject: one child; action: waving one raised hand hello; environment: a bright uncluttered classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one child", + "action": "waving one raised hand hello", + "environment": "a bright uncluttered classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260104 + ], + "expected": { + "people": 1, + "objects": [], + "actions": [ + "wave with one raised hand" + ], + "relations": [], + "scene": "bright uncluttered classroom" + }, + "must_satisfy": [ + "one child is visible", + "raised hand clearly communicates waving" + ], + "severe_failures": [ + "wrong action such as sitting or sleeping", + "extra people", + "severe hand anatomy defect" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "action_drink_01", + "category": "person_action", + "title": "喝水", + "teaching_goal": "Learner can connect 喝水 with a person drinking from a cup.", + "prompt": "subject: one adult student; action: drinking water from a cup; environment: a simple classroom desk", + "request": { + "purpose": "classroom_scene", + "subject": "one adult student", + "action": "drinking water from a cup", + "environment": "a simple classroom desk", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260105 + ], + "expected": { + "people": 1, + "objects": [ + "cup", + "desk" + ], + "actions": [ + "drink water" + ], + "relations": [ + "cup near mouth" + ], + "scene": "simple classroom desk" + }, + "must_satisfy": [ + "cup is visibly held near the mouth", + "one student is shown" + ], + "severe_failures": [ + "person not drinking", + "cup absent", + "dangerous or culturally inappropriate context" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "action_open_01", + "category": "person_action", + "title": "开门", + "teaching_goal": "Learner can connect 开门 with a person opening a door.", + "prompt": "subject: one person; action: opening a blue door with one hand; environment: a tidy apartment entrance", + "request": { + "purpose": "classroom_scene", + "subject": "one person", + "action": "opening a blue door with one hand", + "environment": "a tidy apartment entrance", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260106 + ], + "expected": { + "people": 1, + "objects": [ + "blue door" + ], + "actions": [ + "open door" + ], + "relations": [ + "hand touching door" + ], + "scene": "tidy apartment entrance" + }, + "must_satisfy": [ + "door and hand contact are clear", + "one person is shown" + ], + "severe_failures": [ + "door missing", + "person merely standing", + "extra limbs or severe anatomy defect" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "count_one_01", + "category": "person_count", + "title": "一个学生", + "teaching_goal": "Learner can distinguish one person from a group.", + "prompt": "subject: exactly one student; action: standing and smiling; environment: a plain classroom wall", + "request": { + "purpose": "classroom_scene", + "subject": "exactly one student", + "action": "standing and smiling", + "environment": "a plain classroom wall", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260107 + ], + "expected": { + "people": 1, + "objects": [], + "actions": [ + "stand", + "smile" + ], + "relations": [], + "scene": "plain classroom wall" + }, + "must_satisfy": [ + "exactly one human figure is visible", + "figure is standing" + ], + "severe_failures": [ + "wrong_count", + "crowd or partial extra person", + "face or body not visually complete" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "count_two_01", + "category": "person_count", + "title": "两个朋友", + "teaching_goal": "Learner can identify two people in a simple scene.", + "prompt": "subject: exactly two friends; action: standing side by side and smiling; environment: a simple park path", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two friends", + "action": "standing side by side and smiling", + "environment": "a simple park path", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260108 + ], + "expected": { + "people": 2, + "objects": [], + "actions": [ + "stand side by side", + "smile" + ], + "relations": [ + "two people side by side" + ], + "scene": "simple park path" + }, + "must_satisfy": [ + "exactly two complete people are visible", + "people stand side by side" + ], + "severe_failures": [ + "wrong_count", + "one person or crowd", + "people merge into an unreadable figure" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "count_three_01", + "category": "person_count", + "title": "三个学生", + "teaching_goal": "Learner can identify three people in a classroom.", + "prompt": "subject: exactly three students; action: sitting together at one table; environment: a bright language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "exactly three students", + "action": "sitting together at one table", + "environment": "a bright language classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260109 + ], + "expected": { + "people": 3, + "objects": [ + "one table" + ], + "actions": [ + "sit together" + ], + "relations": [ + "three people around one table" + ], + "scene": "bright language classroom" + }, + "must_satisfy": [ + "three complete people are discernible", + "one shared table is visible" + ], + "severe_failures": [ + "wrong_count", + "four or more people", + "table or people are missing" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "space_left_01", + "category": "spatial_relation", + "title": "左边", + "teaching_goal": "Learner can interpret 左边 using a clear left-right relation.", + "prompt": "subject: a red ball and a blue box; action: the red ball is to the left of the blue box; environment: a plain tabletop", + "request": { + "purpose": "vocabulary_image", + "subject": "a red ball and a blue box", + "action": "the red ball is to the left of the blue box", + "environment": "a plain tabletop", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260110 + ], + "expected": { + "people": 0, + "objects": [ + "red ball", + "blue box" + ], + "actions": [], + "relations": [ + "red ball left of blue box" + ], + "scene": "plain tabletop" + }, + "must_satisfy": [ + "both objects are separate and visible", + "red ball is clearly on the left" + ], + "severe_failures": [ + "wrong_spatial_relation", + "one object missing", + "objects overlap so relation is unclear" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "space_under_01", + "category": "spatial_relation", + "title": "下面", + "teaching_goal": "Learner can interpret 下面 using an above-below relation.", + "prompt": "subject: a cat and a chair; action: the cat is under the chair; environment: a clean living room", + "request": { + "purpose": "vocabulary_image", + "subject": "a cat and a chair", + "action": "the cat is under the chair", + "environment": "a clean living room", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260111 + ], + "expected": { + "people": 0, + "objects": [ + "cat", + "chair" + ], + "actions": [], + "relations": [ + "cat under chair" + ], + "scene": "clean living room" + }, + "must_satisfy": [ + "chair is above the cat", + "cat and chair are both recognizable" + ], + "severe_failures": [ + "wrong_spatial_relation", + "cat missing", + "chair missing" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "space_between_01", + "category": "spatial_relation", + "title": "中间", + "teaching_goal": "Learner can interpret 中间 in a three-object arrangement.", + "prompt": "subject: a small green plant between two books; action: the plant is in the middle of the books; environment: a neat desk", + "request": { + "purpose": "vocabulary_image", + "subject": "a small green plant between two books", + "action": "the plant is in the middle of the books", + "environment": "a neat desk", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260112 + ], + "expected": { + "people": 0, + "objects": [ + "small green plant", + "two books" + ], + "actions": [], + "relations": [ + "plant between two books" + ], + "scene": "neat desk" + }, + "must_satisfy": [ + "two books flank the plant", + "plant is visually central" + ], + "severe_failures": [ + "wrong_spatial_relation", + "fewer than two books", + "plant absent" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "class_read_01", + "category": "classroom_activity", + "title": "读书", + "teaching_goal": "Learner can recognize a classroom reading activity.", + "prompt": "subject: one teacher and one student; action: the student reads a book while the teacher listens; environment: a calm Chinese language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one teacher and one student", + "action": "the student reads a book while the teacher listens", + "environment": "a calm Chinese language classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260113 + ], + "expected": { + "people": 2, + "objects": [ + "book", + "classroom desk" + ], + "actions": [ + "student reads", + "teacher listens" + ], + "relations": [ + "teacher and student at desk" + ], + "scene": "calm Chinese language classroom" + }, + "must_satisfy": [ + "book is visible", + "student and teacher roles are visually plausible", + "scene reads as a classroom" + ], + "severe_failures": [ + "wrong_scene", + "reading activity absent", + "crowd or no classroom cues" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "class_write_01", + "category": "classroom_activity", + "title": "写汉字", + "teaching_goal": "Learner can recognize a guided writing activity without relying on generated text.", + "prompt": "subject: one teacher and one student; action: the teacher points while the student writes in a notebook; environment: an uncluttered language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one teacher and one student", + "action": "the teacher points while the student writes in a notebook", + "environment": "an uncluttered language classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260114 + ], + "expected": { + "people": 2, + "objects": [ + "notebook", + "pencil", + "desk" + ], + "actions": [ + "teacher points", + "student writes" + ], + "relations": [ + "student writes at desk" + ], + "scene": "uncluttered language classroom" + }, + "must_satisfy": [ + "writing posture and notebook are clear", + "no readable generated text is required", + "teacher-student arrangement is plausible" + ], + "severe_failures": [ + "text artifact", + "wrong_scene", + "writing action absent" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "class_pair_01", + "category": "classroom_activity", + "title": "两人对话", + "teaching_goal": "Learner can recognize a pair speaking activity.", + "prompt": "subject: exactly two language students; action: facing each other and practicing a short conversation; environment: a friendly classroom pair-work table", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two language students", + "action": "facing each other and practicing a short conversation", + "environment": "a friendly classroom pair-work table", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260115 + ], + "expected": { + "people": 2, + "objects": [ + "pair-work table" + ], + "actions": [ + "speak to each other" + ], + "relations": [ + "two students face each other" + ], + "scene": "friendly classroom pair-work table" + }, + "must_satisfy": [ + "exactly two students are visible", + "face-to-face orientation is clear", + "no speech text is needed" + ], + "severe_failures": [ + "wrong_count", + "text artifact", + "students face away from each other" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "daily_greet_01", + "category": "daily_communication", + "title": "问候", + "teaching_goal": "Learner can recognize a polite everyday greeting.", + "prompt": "subject: one adult and one older adult; action: smiling and greeting each other respectfully at a doorway; environment: a welcoming home entrance", + "request": { + "purpose": "classroom_scene", + "subject": "one adult and one older adult", + "action": "smiling and greeting each other respectfully at a doorway", + "environment": "a welcoming home entrance", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260116 + ], + "expected": { + "people": 2, + "objects": [ + "doorway" + ], + "actions": [ + "greet respectfully", + "smile" + ], + "relations": [ + "two people face each other" + ], + "scene": "welcoming home entrance" + }, + "must_satisfy": [ + "greeting posture is readable", + "age relationship is respectful", + "no text is needed" + ], + "severe_failures": [ + "wrong_scene", + "people ignore each other", + "culturally inappropriate interaction" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "daily_shop_01", + "category": "daily_communication", + "title": "买东西", + "teaching_goal": "Learner can recognize a simple shopping exchange.", + "prompt": "subject: one shopper and one shopkeeper; action: handing a small bag across a counter; environment: a clean neighborhood shop", + "request": { + "purpose": "classroom_scene", + "subject": "one shopper and one shopkeeper", + "action": "handing a small bag across a counter", + "environment": "a clean neighborhood shop", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260117 + ], + "expected": { + "people": 2, + "objects": [ + "small bag", + "shop counter" + ], + "actions": [ + "hand a bag" + ], + "relations": [ + "bag passes across counter" + ], + "scene": "clean neighborhood shop" + }, + "must_satisfy": [ + "counter and bag are visible", + "two roles are plausible", + "exchange is clear" + ], + "severe_failures": [ + "wrong_scene", + "bag absent", + "crowded unreadable shop" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "daily_bus_01", + "category": "daily_communication", + "title": "问路", + "teaching_goal": "Learner can recognize asking for directions in daily life.", + "prompt": "subject: one visitor and one local person; action: the visitor points at a simple map while asking for directions; environment: a quiet city street corner", + "request": { + "purpose": "classroom_scene", + "subject": "one visitor and one local person", + "action": "the visitor points at a simple map while asking for directions", + "environment": "a quiet city street corner", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260118 + ], + "expected": { + "people": 2, + "objects": [ + "simple map" + ], + "actions": [ + "point at map", + "ask for directions" + ], + "relations": [ + "two people look at map" + ], + "scene": "quiet city street corner" + }, + "must_satisfy": [ + "map is visible without readable text", + "visitor and local attend to each other" + ], + "severe_failures": [ + "text artifact", + "wrong_scene", + "map absent" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "emotion_happy_01", + "category": "emotion_expression", + "title": "高兴", + "teaching_goal": "Learner can associate 高兴 with a clearly happy expression.", + "prompt": "subject: one child; action: smiling broadly and holding a small gift; environment: a warm family room", + "request": { + "purpose": "vocabulary_image", + "subject": "one child", + "action": "smiling broadly and holding a small gift", + "environment": "a warm family room", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260119 + ], + "expected": { + "people": 1, + "objects": [ + "small gift" + ], + "actions": [ + "smile broadly", + "hold gift" + ], + "relations": [], + "scene": "warm family room" + }, + "must_satisfy": [ + "facial expression is visibly happy", + "gift is secondary and recognizable" + ], + "severe_failures": [ + "emotion unreadable", + "sad or frightened expression", + "severe face defect" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "emotion_tired_01", + "category": "emotion_expression", + "title": "累", + "teaching_goal": "Learner can associate 累 with a tired but safe everyday expression.", + "prompt": "subject: one office worker; action: sitting with tired shoulders and a cup of water; environment: a quiet desk after work", + "request": { + "purpose": "vocabulary_image", + "subject": "one office worker", + "action": "sitting with tired shoulders and a cup of water", + "environment": "a quiet desk after work", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260120 + ], + "expected": { + "people": 1, + "objects": [ + "cup", + "desk" + ], + "actions": [ + "sit tiredly" + ], + "relations": [ + "person at desk" + ], + "scene": "quiet desk after work" + }, + "must_satisfy": [ + "tired posture is clear", + "scene remains age-appropriate and safe" + ], + "severe_failures": [ + "emotion unreadable", + "person appears injured", + "visually confusing posture" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "culture_tea_01", + "category": "cultural_scene", + "title": "喝茶", + "teaching_goal": "Learner can recognize a respectful Chinese tea-sharing scene.", + "prompt": "subject: two adults; action: respectfully sharing tea at a small round table; environment: a calm Chinese tea room with simple ceramic cups", + "request": { + "purpose": "classroom_scene", + "subject": "two adults", + "action": "respectfully sharing tea at a small round table", + "environment": "a calm Chinese tea room with simple ceramic cups", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260121 + ], + "expected": { + "people": 2, + "objects": [ + "round table", + "ceramic cups", + "tea pot" + ], + "actions": [ + "share tea" + ], + "relations": [ + "two people around table" + ], + "scene": "calm Chinese tea room" + }, + "must_satisfy": [ + "tea objects are recognizable", + "interaction is respectful", + "no stereotypes or decorative text dominate" + ], + "severe_failures": [ + "culturally_inappropriate", + "wrong_scene", + "text artifact" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "culture_festival_01", + "category": "cultural_scene", + "title": "春节", + "teaching_goal": "Learner can recognize a family celebration connected to Spring Festival.", + "prompt": "subject: three family members; action: sitting together for a respectful Spring Festival meal; environment: a warm Chinese family dining room with red decorations but no writing", + "request": { + "purpose": "classroom_scene", + "subject": "three family members", + "action": "sitting together for a respectful Spring Festival meal", + "environment": "a warm Chinese family dining room with red decorations but no writing", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260122 + ], + "expected": { + "people": 3, + "objects": [ + "dining table", + "shared meal", + "simple red decorations" + ], + "actions": [ + "sit together for meal" + ], + "relations": [ + "family around table" + ], + "scene": "warm Chinese family dining room" + }, + "must_satisfy": [ + "family meal is clear", + "decorations are simple and text-free", + "age and interaction are respectful" + ], + "severe_failures": [ + "culturally_inappropriate", + "wrong_count", + "text artifact or stereotyped costume" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "sequence_arrive_01", + "category": "event_sequence", + "title": "先后顺序", + "teaching_goal": "Learner can infer a simple before-and-after event from one visual scene.", + "prompt": "subject: one student arriving at school; action: holding a backpack at the school entrance just before entering; environment: a clear morning school doorway", + "request": { + "purpose": "classroom_scene", + "subject": "one student arriving at school", + "action": "holding a backpack at the school entrance just before entering", + "environment": "a clear morning school doorway", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260123 + ], + "expected": { + "people": 1, + "objects": [ + "backpack", + "school doorway" + ], + "actions": [ + "arrive before entering" + ], + "relations": [ + "student outside doorway" + ], + "scene": "clear morning school doorway" + }, + "must_satisfy": [ + "arrival-before-entry state is visually plausible", + "backpack and doorway are clear" + ], + "severe_failures": [ + "wrong_scene", + "student already inside with no doorway context", + "visually confusing composition" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "hard_count_action_space_01", + "category": "hard_combination", + "title": "数量+动作+方位", + "teaching_goal": "Learner can attempt a combined count, action, and spatial-relation prompt.", + "prompt": "subject: exactly two children and one red ball; action: one child stands left and passes the ball to the other child on the right; environment: a simple school playground", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two children and one red ball", + "action": "one child stands left and passes the ball to the other child on the right", + "environment": "a simple school playground", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260124 + ], + "expected": { + "people": 2, + "objects": [ + "one red ball" + ], + "actions": [ + "pass ball" + ], + "relations": [ + "one child left of the other", + "ball between children" + ], + "scene": "simple school playground" + }, + "must_satisfy": [ + "exactly two children are visible", + "one ball is visible", + "left-right relation and pass action are at least interpretable" + ], + "severe_failures": [ + "wrong_count", + "wrong_action", + "wrong_spatial_relation", + "ball missing" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "hard_role_culture_01", + "category": "hard_combination", + "title": "角色+文化+课堂", + "teaching_goal": "Learner can attempt a culturally respectful classroom role-play scene.", + "prompt": "subject: one teacher and two students; action: students politely greet the teacher before a Chinese lesson; environment: a calm classroom with a simple tea table and no written signs", + "request": { + "purpose": "classroom_scene", + "subject": "one teacher and two students", + "action": "students politely greet the teacher before a Chinese lesson", + "environment": "a calm classroom with a simple tea table and no written signs", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260125 + ], + "expected": { + "people": 3, + "objects": [ + "simple tea table" + ], + "actions": [ + "students greet teacher politely" + ], + "relations": [ + "students face teacher" + ], + "scene": "calm Chinese lesson classroom" + }, + "must_satisfy": [ + "three people are discernible", + "teacher-student role relationship is plausible", + "scene is respectful and text-free" + ], + "severe_failures": [ + "wrong_count", + "culturally_inappropriate", + "wrong_scene", + "text artifact" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + } + ] +} diff --git a/docs/phase2c1-teaching-image-benchmark.md b/docs/phase2c1-teaching-image-benchmark.md new file mode 100644 index 0000000..231a12e --- /dev/null +++ b/docs/phase2c1-teaching-image-benchmark.md @@ -0,0 +1,89 @@ +# Phase 2C.1 教师图片质量基准 + +Phase 2C.1 是评测切片，不改变 Phase 2C 的固定模型、Workflow Pack 或生产 prompt profile。 +它把真实生成结果与教师教学判断分开记录：技术成功只表示受控入口生成并登记了一个可验证 PNG， +不表示图片教学可用。 + +## 固定基准合同 + +案例定义位于 [`benchmarks/phase2c1/cases.v1.json`](../benchmarks/phase2c1/cases.v1.json)，schema 为 +`hanclassstudio.teaching_image_benchmark.v1`，版本 `1.0.0`，首轮包含 25 个案例，覆盖： + +- single object、person action、person count、spatial relation； +- classroom activity、daily communication、emotion/expression、cultural scene； +- event sequence、hard combination。 + +每个案例固定记录教学目标、受控 request intent、canonical prompt、固定 negative prompt、seed、尺寸、 +预期人物/物体/动作/关系/场景、必须满足条件和严重失败条件。`prompt` 是可审计的 request intent； +执行器不会把案例 JSON 当作任意 ComfyUI graph 或 raw prompt API。 + +模型、Workflow Pack 和 Runtime identity 必须与仓库中的固定 package 合同一致： + +- Model `hcs.sd15-teaching-illustration-fp16`, version `1.5-fp16-emaonly`； +- Workflow `hcs.teaching-illustration-sd15-core`, version `1.0.0`，digest `e25c17976054…1751e`； +- ComfyUI Runtime `0.28.0`, source commit `700821e1364eaab0e8f21c538a2131719fec57bf`； +- prompt profile `soft-flat-educational-v1`。 + +运行开始时会做 deep `generation_ready` 检查并记录 Runtime installation/process、model installation、 +port 和 package identities。恢复时这些 identity 必须完全一致（只忽略检查时间）；Runtime 重启、模型 +替换或 Workflow/package identity 变化会将未完成任务置为 `blocked`，要求新 run，不沿用旧 readiness。 + +## 执行器与恢复 + +实现位于 [`apps/api/src/hcs_api/teaching_image_benchmark.py`](../apps/api/src/hcs_api/teaching_image_benchmark.py)。 +它只调用 Phase 2C 的 `generate_teaching_image()` 受控入口，逐 case/seed 保存 +`runtime/.../run-state.json`： + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_benchmark run \ + --spec benchmarks/phase2c1/cases.v1.json \ + --output-dir runtime/phase2c1-pilot \ + --case-id obj_apple_01 \ + --case-id action_wave_01 \ + --case-id class_read_01 +``` + +再次运行相同命令会读取已有 state：已经通过技术检查的 artifact 幂等跳过，缺失或不一致的 artifact +会重新生成；单个 case 的失败会记录 code/message/attempt 并继续后续 case。`Ctrl-C` 会保存 +`paused` 状态，并定向取消当前受控 ComfyUI job；再次运行即可恢复。生成结果永远保持 +`pending_review`。 + +完整基准只需去掉 `--case-id` 参数并使用新的 output directory。pilot 与完整基准使用不同目录，避免 +把 pilot 结果误当作完整结果。 + +## 技术检查与报告 + +每个成功结果必须同时通过：PNG signature/CRC/尺寸/大小检查、图片 SHA-256、provenance SHA-256、 +request SHA、固定 negative prompt、Asset Manifest 单一条目和 `pending_review` 状态检查。 + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_benchmark report \ + --spec benchmarks/phase2c1/cases.v1.json \ + --output-dir runtime/phase2c1-pilot +``` + +报告中的 `technical_success_rate` 不是教学成功率；没有教师文件时 +`teacher_review.conclusion` 必须为 `null`，失败标签频率也保持为空。技术错误 code 与教师 failure +labels 不混用。 + +## 教师评审包 + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_benchmark review-package \ + --spec benchmarks/phase2c1/cases.v1.json \ + --output-dir runtime/phase2c1-pilot +``` + +命令生成 Git 忽略的 `runtime/phase2c1-pilot/review-package/`，包含本地图片、案例说明、空白 +`pending_review` JSON 和离线 `index.html`。界面要求教师填写：教学目标相关性、指令遵循、数量、动作、 +空间关系、课堂可用性、视觉完整性、文化与年龄适切性、是否需要重新生成、是否可直接用于课件， +并可选择结构化失败标签。界面不会预填评分；只有教师明确标记已评审后才会导出 `reviewed` 记录。 + +## 范围边界 + +该基准不做 prompt 调优、不改模型、不改 Workflow Pack、不加入 LoRA/ControlNet/custom nodes， +不自动推断教师结论，也不把图片或 Runtime/cache/report 复制进 Git。真实运行若受 macOS arm64、 +磁盘、下载或 Runtime 状态阻断，state 会保留精确 blocker 和恢复命令，不通过 mock 宣称真实基准完成。 From 180554d3293a757179d4b283dc0e13f3b1be3927 Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:51:50 +0700 Subject: [PATCH 2/5] fix(benchmark): retry completed failures and guard review completion --- apps/api/src/hcs_api/teaching_image_benchmark.py | 14 +++++++++++++- apps/api/tests/test_teaching_image_benchmark.py | 13 ++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/apps/api/src/hcs_api/teaching_image_benchmark.py b/apps/api/src/hcs_api/teaching_image_benchmark.py index 3196298..ddbaa79 100644 --- a/apps/api/src/hcs_api/teaching_image_benchmark.py +++ b/apps/api/src/hcs_api/teaching_image_benchmark.py @@ -585,8 +585,10 @@ def run_benchmark( state.updated_at = _iso() _write_json(state_path, state.model_dump(mode="json", by_alias=True)) raise BenchmarkBlockedError(state.block.code, state.block.message) + was_completed = state.status == "completed" state.status = "running" else: + was_completed = False observed = _capture_identity() now = _iso() state = BenchmarkRunState( @@ -615,6 +617,11 @@ def run_benchmark( state.results[key] = result if _result_is_reusable(state, result, case, project_dir): continue + if was_completed and result.status in {"failed", "invalidated"}: + # A completed run is a terminal snapshot. A later invocation + # is an explicit retry of its failed/invalidated cases, while + # a paused run keeps its cumulative attempt count for resume. + result.attempts = 0 if result.status == "succeeded": result.status = "invalidated" result.artifact = None @@ -800,7 +807,12 @@ def _review_html(cards: list[dict[str, Any]]) -> str: for(const checkbox of article.querySelectorAll('.failure input')) checkbox.addEventListener('change',e=>{{r.failure_labels=[...article.querySelectorAll('.failure input:checked')].map(x=>x.value);}}); for(const select of article.querySelectorAll('[data-decision]')) select.addEventListener('change',e=>r[e.target.dataset.decision]=e.target.value===''?null:e.target.value==='true'); article.querySelector('[data-notes]').addEventListener('input',e=>r.notes=e.target.value); - article.querySelector('[data-review]').addEventListener('click',()=>{{r.review_state='reviewed'; article.querySelector('[data-status]').textContent=' reviewed'; article.querySelector('[data-status]').className='reviewed';}}); + article.querySelector('[data-review]').addEventListener('click',()=>{{ + const complete=scoreFields.every(field=>Number.isInteger(r[field])&&r[field]>=1&&r[field]<=5) + &&typeof r.regeneration_required==='boolean'&&typeof r.direct_courseware_use==='boolean'; + if(!complete){{window.alert('请先完成全部评分和两个使用决策，再标记已评审。');return;}} + r.review_state='reviewed'; article.querySelector('[data-status]').textContent=' reviewed'; article.querySelector('[data-status]').className='reviewed'; + }}); root.appendChild(article); }} document.querySelector('#count').textContent=` ${{cards.length}} 张待评审图片`; }} diff --git a/apps/api/tests/test_teaching_image_benchmark.py b/apps/api/tests/test_teaching_image_benchmark.py index 1e76691..b1085e4 100644 --- a/apps/api/tests/test_teaching_image_benchmark.py +++ b/apps/api/tests/test_teaching_image_benchmark.py @@ -107,10 +107,12 @@ def test_run_continues_after_one_case_failure_and_resumes_idempotently( observed = _identity() monkeypatch.setattr(benchmark, "_capture_identity", lambda: observed) calls: list[str] = [] + fail_apple = True def fake_generate(project_dir: Path, request): + nonlocal fail_apple calls.append(request.asset_id) - if request.asset_id.startswith("bmk-obj_apple"): + if request.asset_id.startswith("bmk-obj_apple") and fail_apple: raise TeachingImageError("generation_failed", "fixture failure") return _artifact(request.asset_id, request.seed) @@ -123,10 +125,15 @@ def fake_generate(project_dir: Path, request): assert state.results[f"{selected[1]}@260102"].status == "succeeded" assert len(calls) == 2 - monkeypatch.setattr(benchmark, "_result_is_reusable", lambda *_args: True) + fail_apple = False resumed = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) assert resumed.status == "completed" - assert len(calls) == 2 + assert resumed.results[f"{selected[0]}@260101"].status == "succeeded" + assert len(calls) == 3 + + idempotent = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) + assert idempotent.status == "completed" + assert len(calls) == 3 def test_identity_change_blocks_unfinished_cases(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: From 22d09fa55b3825684143c5a960d700ade9a279ef Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:55:57 +0700 Subject: [PATCH 3/5] feat(ablation): define controlled sd15 parameter experiment --- .../src/hcs_api/teaching_image_ablation.py | 1016 +++++++++++++++++ .../api/tests/test_teaching_image_ablation.py | 198 ++++ benchmarks/phase2c1/sd15-ablation.v1.json | 195 ++++ 3 files changed, 1409 insertions(+) create mode 100644 apps/api/src/hcs_api/teaching_image_ablation.py create mode 100644 apps/api/tests/test_teaching_image_ablation.py create mode 100644 benchmarks/phase2c1/sd15-ablation.v1.json diff --git a/apps/api/src/hcs_api/teaching_image_ablation.py b/apps/api/src/hcs_api/teaching_image_ablation.py new file mode 100644 index 0000000..821db0f --- /dev/null +++ b/apps/api/src/hcs_api/teaching_image_ablation.py @@ -0,0 +1,1016 @@ +"""Controlled SD 1.5 parameter/prompt ablation benchmark. + +This module is evaluation-only. It reuses the fixed TeachingImageRequest entry +point and temporarily changes only the compiled plan for three named variants; +the production Workflow Pack and its defaults remain unchanged. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import html +import io +import json +import shutil +import threading +import time +import uuid +from collections import Counter, defaultdict +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +from PIL import Image +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from . import comfyui_teaching_image as image_runtime +from .comfyui_model import load_workflow_pack +from .comfyui_teaching_image import ( + TeachingImageError, + TeachingImageRequest, + generate_teaching_image, + verify_png, +) +from .models import AssetManifest, TeachingImageProvenance, VerifiedImageArtifact +from .teaching_image_benchmark import ( + BenchmarkBlockedError, + BenchmarkError, + BenchmarkExpected, + BenchmarkFixedIdentity, + BenchmarkObservedIdentity, + BenchmarkRequest, + _capture_identity, + _expected_fixed_identity, + _read_json, + _same_execution_identity, + _sha256, + _write_json, +) + +ABLATION_SCHEMA = "hanclassstudio.teaching_image_ablation.v1" +REVIEW_SCHEMA = "hanclassstudio.teaching_image_ablation_review.v1" +_CONFIG_IDS = ("A", "B", "C") +_CASE_CATEGORIES = frozenset( + { + "single_object", + "spatial_relation", + "two_person_communication", + "person_action", + "person_count", + "classroom_activity", + } +) +_FAILURE_LABELS = frozenset( + { + "wrong_count", + "wrong_action", + "wrong_scene", + "wrong_spatial_relation", + "missing_object", + "anatomy_defect", + "text_artifact", + "merged_people", + "visually_confusing", + "not_teaching_usable", + } +) +_RATING_FIELDS = ( + "visual_quality", + "prompt_adherence", + "object_count_accuracy", + "action_accuracy", + "spatial_relation_accuracy", + "teaching_usability", + "anatomy_quality", +) +_STATE_LOCK = threading.RLock() + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +class AblationConfig(_StrictModel): + config_id: Literal["A", "B", "C"] + name: str = Field(min_length=1, max_length=100) + sampler_name: Literal["euler", "dpmpp_2m"] + scheduler: Literal["normal", "karras"] + steps: int = Field(ge=1, le=100) + cfg: float = Field(gt=0, le=30) + prompt_profile_id: str = Field(min_length=1, max_length=100) + positive_prefix: str | None = Field(default=None, max_length=800) + positive_suffix: str | None = Field(default=None, max_length=400) + negative_prompt: str | None = Field(default=None, max_length=1600) + + @model_validator(mode="after") + def _fixed_variant_contract(self) -> AblationConfig: + expected = { + "A": ("euler", "normal", 20, 7.0, "soft-flat-educational-v1"), + "B": ("dpmpp_2m", "karras", 30, 6.0, "soft-flat-educational-v1"), + "C": ("dpmpp_2m", "karras", 30, 6.0, "ablation-clean-modern-v1"), + }[self.config_id] + if (self.sampler_name, self.scheduler, self.steps, self.cfg, self.prompt_profile_id) != expected: + raise ValueError(f"configuration {self.config_id} has mutable parameters") + if self.config_id == "C": + if not self.positive_prefix or not self.positive_suffix or not self.negative_prompt: + raise ValueError("configuration C requires its fixed prompt profile") + elif any(value is not None for value in (self.positive_prefix, self.positive_suffix, self.negative_prompt)): + raise ValueError("configurations A and B inherit the fixed Workflow Pack prompt profile") + return self + + +class AblationCase(_StrictModel): + case_id: str = Field(pattern=r"^[a-z][a-z0-9_]{2,63}$") + category: Literal[ + "single_object", + "spatial_relation", + "two_person_communication", + "person_action", + "person_count", + "classroom_activity", + ] + teaching_goal: str = Field(min_length=1, max_length=500) + prompt: str = Field(min_length=1, max_length=800) + request: BenchmarkRequest + expected: BenchmarkExpected + must_satisfy: list[str] = Field(min_length=1, max_length=12) + severe_failures: list[str] = Field(min_length=1, max_length=12) + seeds: list[int] = Field(min_length=3, max_length=3) + + @model_validator(mode="after") + def _case_contract(self) -> AblationCase: + expected_prompt = ( + f"subject: {self.request.subject}; action: {self.request.action}; " + f"environment: {self.request.environment}" + ) + if self.prompt != expected_prompt: + raise ValueError("case prompt must be the canonical controlled request intent") + if len(set(self.seeds)) != 3: + raise ValueError("each ablation case must have exactly three unique seeds") + if self.request.aspect_ratio != "4:3": + raise ValueError("the ablation uses the fixed 4:3 dimensions for every case") + return self + + +class AblationSpec(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_ablation.v1"] = Field( + default=ABLATION_SCHEMA, alias="schema" + ) + experiment_id: Literal["phase2c1-sd15-ablation"] + version: str = Field(pattern=r"^\d+\.\d+\.\d+$") + fixed_identity: BenchmarkFixedIdentity + configurations: list[AblationConfig] = Field(min_length=3, max_length=3) + cases: list[AblationCase] = Field(min_length=6, max_length=6) + + @model_validator(mode="after") + def _experiment_contract(self) -> AblationSpec: + if [config.config_id for config in self.configurations] != list(_CONFIG_IDS): + raise ValueError("ablation configurations must be ordered A, B, C") + if len({case.case_id for case in self.cases}) != 6: + raise ValueError("ablation case ids must be unique") + if {case.category for case in self.cases} != _CASE_CATEGORIES: + raise ValueError("ablation must cover the six required representative categories") + return self + + +class AblationErrorRecord(_StrictModel): + code: str + message: str + attempt: int = Field(ge=1) + recoverable: bool = True + occurred_at: str + + +class AblationVisualPrecheck(_StrictModel): + status: Literal["passed", "warning"] + blank_like: bool + near_solid: bool + distinct_colors: int = Field(ge=0) + sampled_pixels: int = Field(ge=0) + + +class AblationTechnicalCheck(_StrictModel): + status: Literal["passed"] = "passed" + image_path: str + provenance_path: str + width: int + height: int + image_size_bytes: int + image_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + provenance_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + manifest_asset_id: str + request_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + execution_plan_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + duration_seconds: float = Field(ge=0) + visual_precheck: AblationVisualPrecheck + checks: list[str] = Field(min_length=1) + + +class AblationCaseResult(_StrictModel): + case_id: str + config_id: Literal["A", "B", "C"] + seed: int + status: Literal["pending", "running", "succeeded", "failed", "invalidated"] = "pending" + attempts: int = Field(default=0, ge=0) + started_at: str | None = None + completed_at: str | None = None + duration_seconds: float = Field(default=0, ge=0) + artifact: VerifiedImageArtifact | None = None + technical_checks: AblationTechnicalCheck | None = None + error: AblationErrorRecord | None = None + manual_review: None = None + + +class AblationReviewRecord(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_ablation_review.v1"] = Field( + default=REVIEW_SCHEMA, alias="schema" + ) + review_key: str + case_id: str + config_id: Literal["A", "B", "C"] + seed: int + artifact_id: str + review_state: Literal["pending_review", "reviewed"] = "pending_review" + reviewer_id: str = "" + visual_quality: int | None = Field(default=None, ge=1, le=5) + prompt_adherence: int | None = Field(default=None, ge=1, le=5) + object_count_accuracy: int | None = Field(default=None, ge=1, le=5) + action_accuracy: int | None = Field(default=None, ge=1, le=5) + spatial_relation_accuracy: int | None = Field(default=None, ge=1, le=5) + teaching_usability: int | None = Field(default=None, ge=1, le=5) + anatomy_quality: int | None = Field(default=None, ge=1, le=5) + needs_regeneration: bool | None = None + failure_tags: list[str] = Field(default_factory=list) + reviewer_notes: str = Field(default="", max_length=3000) + + @model_validator(mode="after") + def _review_contract(self) -> AblationReviewRecord: + if any(tag not in _FAILURE_LABELS for tag in self.failure_tags): + raise ValueError("unknown ablation failure tag") + if len(set(self.failure_tags)) != len(self.failure_tags): + raise ValueError("ablation failure tags must be unique") + if self.review_state == "reviewed": + if any(getattr(self, field) is None for field in _RATING_FIELDS): + raise ValueError("reviewed ablation records require every rating") + if self.needs_regeneration is None: + raise ValueError("reviewed ablation records require needs_regeneration") + return self + + +class AblationRunState(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_ablation_run.v1"] = Field( + default="hanclassstudio.teaching_image_ablation_run.v1", alias="schema" + ) + run_id: str = Field(pattern=r"^[a-z0-9-]{8,100}$") + experiment_id: str + experiment_version: str + spec_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + output_dir: str + project_dir: str + selected_case_ids: list[str] + selected_config_ids: list[str] + selected_case_seeds: dict[str, list[int]] + execution_identity: BenchmarkObservedIdentity | None = None + status: Literal["running", "paused", "completed", "blocked"] = "running" + results: dict[str, AblationCaseResult] = Field(default_factory=dict) + started_at: str + updated_at: str + block: AblationErrorRecord | None = None + + +def _iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _error(code: str, message: str, attempt: int, recoverable: bool = True) -> AblationErrorRecord: + return AblationErrorRecord( + code=code, + message=message, + attempt=attempt, + recoverable=recoverable, + occurred_at=_iso(), + ) + + +def load_ablation_spec(path: Path) -> AblationSpec: + try: + spec = AblationSpec.model_validate(_read_json(path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("ablation_schema_invalid", str(exc)) from exc + fixed = _expected_fixed_identity() + if spec.fixed_identity != fixed: + raise BenchmarkError( + "ablation_identity_invalid", + "Ablation fixed identity does not match the repository Model/Workflow/Runtime contracts", + ) + workflow = load_workflow_pack() + baseline = spec.configurations[0] + if ( + workflow.sampling.sampler_name != baseline.sampler_name + or workflow.sampling.scheduler != baseline.scheduler + or workflow.sampling.steps != baseline.steps + or workflow.sampling.cfg != baseline.cfg + or workflow.prompt_profile.id != baseline.prompt_profile_id + ): + raise BenchmarkError("ablation_baseline_invalid", "Configuration A no longer matches the fixed Workflow Pack") + return spec + + +def _case_key(case_id: str, config_id: str, seed: int) -> str: + return f"{case_id}@{config_id}@{seed}" + + +def _request_for(case: AblationCase, config: AblationConfig, seed: int) -> TeachingImageRequest: + return TeachingImageRequest( + asset_id=f"abl-{case.case_id}-{config.config_id}-{seed}", + purpose=case.request.purpose, + subject=case.request.subject, + action=case.request.action, + environment=case.request.environment, + aspect_ratio=case.request.aspect_ratio, + seed=seed, + source_trace=[ + "ablation:phase2c1-sd15-ablation", + f"case:{case.case_id}", + f"config:{config.config_id}", + f"seed:{seed}", + ], + ) + + +def _purpose_label(request: TeachingImageRequest) -> str: + return { + "classroom_scene": "classroom situation scene", + "vocabulary_image": "clear vocabulary concept image", + "teaching_illustration": "teaching courseware illustration", + }[request.purpose] + + +def _fixed_positive(request: TeachingImageRequest, workflow: Any) -> str: + return ( + f"{workflow.prompt_profile.positive_prefix}, {_purpose_label(request)}, " + f"subject: {request.subject}, action: {request.action}, environment: {request.environment}, " + f"{workflow.prompt_profile.positive_suffix}" + ) + + +def _variant_prompts( + request: TeachingImageRequest, + config: AblationConfig, + workflow: Any, + original_positive: str, + original_negative: str, +) -> tuple[str, str]: + if config.config_id != "C": + return original_positive, original_negative + assert config.positive_prefix and config.positive_suffix and config.negative_prompt + return ( + ( + f"{config.positive_prefix}, {_purpose_label(request)}, subject: {request.subject}, " + f"action: {request.action}, environment: {request.environment}, {config.positive_suffix}" + ), + config.negative_prompt, + ) + + +@contextmanager +def _compiled_variant(config: AblationConfig): + """Temporarily bind one named experiment variant to the fixed compiler. + + The graph still comes exclusively from ``_fixed_graph``. A process-local + lock prevents two variants from changing the compiler concurrently. + """ + + original = image_runtime.compile_teaching_image_request + + def compile_variant(request: TeachingImageRequest, *, model_record: Any, workflow: Any): + plan = original(request, model_record=model_record, workflow=workflow) + if config.config_id == "A": + return plan + positive, negative = _variant_prompts( + request, config, workflow, plan.positive_prompt, plan.negative_prompt + ) + unsigned = plan.model_dump(mode="json") + unsigned.pop("execution_plan_sha256", None) + unsigned.update( + positive_prompt=positive, + negative_prompt=negative, + steps=config.steps, + cfg=config.cfg, + sampler_name=config.sampler_name, + scheduler=config.scheduler, + ) + return image_runtime.CompiledTeachingImagePlan( + **unsigned, + execution_plan_sha256=_sha256(unsigned), + ) + + with _STATE_LOCK: + image_runtime.compile_teaching_image_request = compile_variant + try: + yield + finally: + image_runtime.compile_teaching_image_request = original + + +def _visual_precheck(payload: bytes) -> AblationVisualPrecheck: + try: + with Image.open(io.BytesIO(payload)) as image: + rgb = image.convert("RGB") + rgb.thumbnail((64, 48)) + pixels = list(rgb.getdata()) + except Exception as exc: + raise BenchmarkError("image_visual_precheck_failed", "Generated PNG could not be sampled safely") from exc + if not pixels: + raise BenchmarkError("blank_image", "Generated PNG has no pixels") + colors = set(pixels) + channel_ranges = [max(pixel[index] for pixel in pixels) - min(pixel[index] for pixel in pixels) for index in range(3)] + near_solid = len(colors) <= 8 or max(channel_ranges) <= 5 + blank_like = len(colors) <= 2 + return AblationVisualPrecheck( + status="warning" if near_solid else "passed", + blank_like=blank_like, + near_solid=near_solid, + distinct_colors=len(colors), + sampled_pixels=len(pixels), + ) + + +def _technical_check( + project_dir: Path, + case: AblationCase, + config: AblationConfig, + seed: int, + artifact: VerifiedImageArtifact, + identity: BenchmarkObservedIdentity, + duration_seconds: float, +) -> AblationTechnicalCheck: + image_path = project_dir / artifact.path + provenance_path = project_dir / artifact.provenance_ref + if not image_path.is_file() or not provenance_path.is_file(): + raise BenchmarkError("artifact_missing", "Ablation image or provenance file is missing") + payload = image_path.read_bytes() + verified = verify_png(payload, expected_width=512, expected_height=384, maximum_bytes=32 * 1024**2) + if verified.sha256 != artifact.sha256 or verified.size_bytes != artifact.size_bytes: + raise BenchmarkError("artifact_hash_mismatch", "Ablation image hash or size differs from artifact") + precheck = _visual_precheck(payload) + if precheck.blank_like: + raise BenchmarkError("blank_image", "Ablation image is blank-like") + provenance_bytes = provenance_path.read_bytes() + provenance = TeachingImageProvenance.model_validate_json(provenance_bytes) + provenance_sha = hashlib.sha256(provenance_bytes).hexdigest() + if provenance_sha != artifact.provenance_sha256: + raise BenchmarkError("provenance_hash_mismatch", "Ablation provenance hash differs from artifact") + request = _request_for(case, config, seed) + request_sha = _sha256(request.model_dump(mode="json", by_alias=True)) + workflow = load_workflow_pack() + expected_positive = _fixed_positive(request, workflow) + if config.config_id == "C": + expected_positive, expected_negative = _variant_prompts( + request, config, workflow, expected_positive, workflow.prompt_profile.negative + ) + else: + expected_negative = workflow.prompt_profile.negative + if provenance.request_sha256 != request_sha: + raise BenchmarkError("request_provenance_mismatch", "Provenance does not identify this ablation request") + if provenance.seed != seed or provenance.source_trace != request.source_trace: + raise BenchmarkError("provenance_case_mismatch", "Provenance case/config/seed trace differs") + if ( + provenance.runtime_version != identity.runtime_version + or provenance.runtime_source_commit != identity.runtime_source_commit + or provenance.runtime_installation_identity != identity.runtime_installation_identity + or provenance.runtime_process_identity != identity.runtime_process_identity + or provenance.runtime_port != identity.runtime_port + or provenance.model_package_id != identity.model_package_id + or provenance.model_version != identity.model_version + or provenance.model_sha256 != identity.model_sha256 + or provenance.model_installation_identity != identity.model_installation_identity + or provenance.workflow_pack_id != identity.workflow_pack_id + or provenance.workflow_version != identity.workflow_version + or provenance.workflow_pack_sha256 != identity.workflow_pack_sha256 + ): + raise BenchmarkError("provenance_identity_mismatch", "Provenance identity differs from the run identity") + if ( + provenance.steps != config.steps + or provenance.cfg != config.cfg + or provenance.sampler_name != config.sampler_name + or provenance.scheduler != config.scheduler + or provenance.positive_prompt != expected_positive + or provenance.negative_prompt != expected_negative + ): + raise BenchmarkError("provenance_config_mismatch", "Provenance sampling or prompt does not match config") + manifest_path = project_dir / "assets/data/asset_manifest.json" + manifest = AssetManifest.model_validate_json(manifest_path.read_bytes()) + matches = [asset for asset in manifest.images if asset.id == request.asset_id] + if len(matches) != 1 or matches[0].review_state != "pending_review": + raise BenchmarkError("manifest_registration_invalid", "Ablation Asset Manifest entry is missing or not pending") + registered = matches[0].verified_image_artifact + if registered is None or registered.artifact_id != artifact.artifact_id: + raise BenchmarkError("manifest_registration_invalid", "Manifest artifact does not match ablation output") + return AblationTechnicalCheck( + image_path=artifact.path, + provenance_path=artifact.provenance_ref, + width=verified.width, + height=verified.height, + image_size_bytes=verified.size_bytes, + image_sha256=verified.sha256, + provenance_sha256=provenance_sha, + manifest_asset_id=request.asset_id, + request_sha256=request_sha, + execution_plan_sha256=provenance.execution_plan_sha256, + duration_seconds=duration_seconds, + visual_precheck=precheck, + checks=[ + "png_signature_crc_dimensions", + "image_sha256", + "visual_nonblank_precheck", + "provenance_sha256_and_identity", + "sampling_and_prompt_config", + "asset_manifest_single_pending_review_entry", + ], + ) + + +def _result_is_reusable( + project_dir: Path, + case: AblationCase, + config: AblationConfig, + result: AblationCaseResult, + identity: BenchmarkObservedIdentity, +) -> bool: + if result.status != "succeeded" or result.artifact is None or result.technical_checks is None: + return False + try: + _technical_check(project_dir, case, config, result.seed, result.artifact, identity, result.duration_seconds) + except (BenchmarkError, TeachingImageError, OSError, ValueError): + return False + return True + + +def run_ablation( + spec_path: Path, + output_dir: Path, + *, + case_ids: list[str] | None = None, + config_ids: list[str] | None = None, + max_attempts: int = 2, +) -> AblationRunState: + if max_attempts < 1 or max_attempts > 3: + raise BenchmarkError("invalid_attempts", "max_attempts must be between 1 and 3") + spec = load_ablation_spec(spec_path) + spec_sha = _sha256(spec.model_dump(mode="json", by_alias=True)) + output_dir = output_dir.resolve() + project_dir = output_dir / "project" + state_path = output_dir / "run-state.json" + cases = {case.case_id: case for case in spec.cases} + configs = {config.config_id: config for config in spec.configurations} + selected_cases = list(dict.fromkeys(case_ids or [case.case_id for case in spec.cases])) + selected_configs = list(dict.fromkeys(config_ids or list(_CONFIG_IDS))) + if set(selected_cases) - set(cases): + raise BenchmarkError("unknown_case", "Ablation selection contains an unknown case") + if set(selected_configs) - set(configs) or not selected_configs: + raise BenchmarkError("unknown_config", "Ablation selection contains an unknown configuration") + selected_seeds = {case_id: cases[case_id].seeds for case_id in selected_cases} + if state_path.exists(): + try: + state = AblationRunState.model_validate(_read_json(state_path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("ablation_state_invalid", str(exc)) from exc + if ( + state.spec_sha256 != spec_sha + or state.selected_case_ids != selected_cases + or state.selected_config_ids != selected_configs + or state.selected_case_seeds != selected_seeds + ): + raise BenchmarkError("ablation_state_mismatch", "Existing state belongs to a different ablation selection") + if state.status == "blocked": + raise BenchmarkBlockedError( + state.block.code if state.block else "ablation_blocked", + state.block.message if state.block else "Ablation run is blocked; restore identity and start a new run", + ) + observed = _capture_identity() + if state.execution_identity is None: + state.execution_identity = observed + elif not _same_execution_identity(state.execution_identity, observed): + state.status = "blocked" + state.block = _error( + "ablation_identity_changed", + "Runtime process, installation, model, or fixed package identity changed", + max((result.attempts for result in state.results.values()), default=0) + 1, + recoverable=False, + ) + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + raise BenchmarkBlockedError(state.block.code, state.block.message) + was_completed = state.status == "completed" + state.status = "running" + else: + observed = _capture_identity() + now = _iso() + was_completed = False + state = AblationRunState( + run_id=f"phase2c1-ablation-{int(time.time())}-{uuid.uuid4().hex[:8]}", + experiment_id=spec.experiment_id, + experiment_version=spec.version, + spec_sha256=spec_sha, + output_dir=str(output_dir), + project_dir=str(project_dir), + selected_case_ids=selected_cases, + selected_config_ids=selected_configs, + selected_case_seeds=selected_seeds, + execution_identity=observed, + started_at=now, + updated_at=now, + ) + output_dir.mkdir(parents=True, exist_ok=True) + project_dir.mkdir(parents=True, exist_ok=True) + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + try: + for case_id in selected_cases: + case = cases[case_id] + for config_id in selected_configs: + config = configs[config_id] + for seed in case.seeds: + key = _case_key(case_id, config_id, seed) + result = state.results.get(key) or AblationCaseResult( + case_id=case_id, config_id=config_id, seed=seed + ) + state.results[key] = result + if _result_is_reusable(project_dir, case, config, result, state.execution_identity): + continue + if was_completed and result.status in {"failed", "invalidated"}: + result.attempts = 0 + if result.status == "succeeded": + result.status = "invalidated" + result.artifact = None + result.technical_checks = None + result.status = "running" + result.started_at = result.started_at or _iso() + result.error = None + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + for attempt in range(result.attempts + 1, max_attempts + 1): + result.attempts = attempt + started = time.monotonic() + try: + observed = _capture_identity() + if state.execution_identity is None or not _same_execution_identity( + state.execution_identity, observed + ): + raise BenchmarkBlockedError( + "ablation_identity_changed", + "Runtime identity changed during the ablation", + ) + request = _request_for(case, config, seed) + with _compiled_variant(config): + artifact = generate_teaching_image(project_dir, request) + duration = time.monotonic() - started + checks = _technical_check( + project_dir, + case, + config, + seed, + artifact, + state.execution_identity, + duration, + ) + result.status = "succeeded" + result.completed_at = _iso() + result.duration_seconds += duration + result.artifact = artifact + result.technical_checks = checks + result.error = None + break + except BenchmarkBlockedError as exc: + result.status = "invalidated" + result.error = _error(exc.code, exc.message, attempt, recoverable=False) + state.status = "blocked" + state.block = result.error + raise + except (BenchmarkError, TeachingImageError, OSError, ValueError) as exc: + duration = time.monotonic() - started + result.duration_seconds += duration + code = getattr(exc, "code", "ablation_execution_failed") + message = getattr(exc, "message", str(exc)) + result.error = _error(code, message, attempt) + result.status = "failed" + result.artifact = None + result.technical_checks = None + if attempt < max_attempts: + continue + finally: + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + except KeyboardInterrupt: + state.status = "paused" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + return state + except BenchmarkBlockedError: + state.status = "blocked" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + raise + state.status = "completed" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + return state + + +def _review_record_for(key: str, result: AblationCaseResult) -> AblationReviewRecord: + if result.artifact is None: + raise BenchmarkError("review_artifact_missing", f"No successful artifact for {key}") + return AblationReviewRecord( + review_key=key, + case_id=result.case_id, + config_id=result.config_id, + seed=result.seed, + artifact_id=result.artifact.artifact_id, + ) + + +def write_review_package(spec_path: Path, output_dir: Path) -> Path: + spec = load_ablation_spec(spec_path) + state = AblationRunState.model_validate(_read_json(output_dir / "run-state.json")) + package_dir = output_dir / "review-package" + images_dir = package_dir / "images" + images_dir.mkdir(parents=True, exist_ok=True) + case_by_id = {case.case_id: case for case in spec.cases} + config_by_id = {config.config_id: config for config in spec.configurations} + records: list[AblationReviewRecord] = [] + cards: list[dict[str, Any]] = [] + for key, result in state.results.items(): + if result.status != "succeeded" or result.artifact is None: + continue + source = Path(state.project_dir) / result.artifact.path + if not source.is_file(): + raise BenchmarkError("review_image_missing", f"Review image is missing: {source}") + target = images_dir / f"{key.replace('@', '-')}.png" + shutil.copyfile(source, target) + records.append(_review_record_for(key, result)) + cards.append( + { + "key": key, + "case": case_by_id[result.case_id].model_dump(mode="json", by_alias=True), + "config": config_by_id[result.config_id].model_dump(mode="json", by_alias=True), + "seed": result.seed, + "artifact_id": result.artifact.artifact_id, + "image": f"images/{target.name}", + } + ) + cards.sort(key=lambda card: (card["case"]["case_id"], card["seed"], card["config"]["config_id"])) + _write_json( + package_dir / "teacher-reviews.pending.json", + { + "schema": REVIEW_SCHEMA, + "experiment_id": spec.experiment_id, + "experiment_version": spec.version, + "run_id": state.run_id, + "records": [record.model_dump(mode="json", by_alias=True) for record in records], + }, + ) + with (package_dir / "teacher-reviews.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "review_key", "case_id", "config_id", "seed", "artifact_id", "review_state", "reviewer_id", + *_RATING_FIELDS, "needs_regeneration", "failure_tags", "reviewer_notes", + ], + ) + writer.writeheader() + for record in records: + values = record.model_dump(mode="json", by_alias=True) + writer.writerow( + { + field: "|".join(values[field]) if field == "failure_tags" else values.get(field) + for field in writer.fieldnames + } + ) + _write_json( + package_dir / "experiment-manifest.json", + { + "schema": ABLATION_SCHEMA, + "experiment_id": spec.experiment_id, + "experiment_version": spec.version, + "run_id": state.run_id, + "fixed_identity": spec.fixed_identity.model_dump(mode="json"), + "configurations": [config.model_dump(mode="json") for config in spec.configurations], + "cases": [case.model_dump(mode="json") for case in spec.cases], + "results": [result.model_dump(mode="json", by_alias=True) for result in state.results.values()], + }, + ) + (package_dir / "README.md").write_text( + "# Phase 2C.1 SD 1.5 参数消融评审包\n\n" + "本包只报告固定 A/B/C 配置的真实图片和空白人工评审表。自动技术检查不等于教师评价。\n\n" + "评分字段：visual_quality、prompt_adherence、object_count_accuracy、action_accuracy、" + "spatial_relation_accuracy、teaching_usability、anatomy_quality、needs_regeneration、" + "failure_tags、reviewer_notes。请在真实教师评审后提交 JSON 或 CSV，不要由模型代填。\n", + encoding="utf-8", + ) + (package_dir / "comparison.html").write_text(_comparison_html(cards), encoding="utf-8") + return package_dir + + +def _comparison_html(cards: list[dict[str, Any]]) -> str: + groups: dict[tuple[str, int], list[dict[str, Any]]] = defaultdict(list) + for card in cards: + groups[(card["case"]["case_id"], card["seed"])].append(card) + parts = [ + "", + "Phase 2C.1 SD 1.5 Ablation Comparison", + ( + "" + ), + "

Phase 2C.1 SD 1.5 参数消融对比

", + "

A/B/C 配置使用相同案例、尺寸、模型、Runtime 和 seed。技术通过不等于教师教学结论。

", + "

JSON 评审表 · CSV 评审表

", + ] + for (case_id, seed), group in sorted(groups.items()): + case = group[0]["case"] + parts.append( + f"

{html.escape(case_id)} · seed {seed}

" + f"

{html.escape(case['teaching_goal'])}

" + f"

必须满足：{html.escape('；'.join(case['must_satisfy']))}

" + ) + for card in sorted(group, key=lambda item: item["config"]["config_id"]): + config = card["config"] + parts.append( + f"

配置 {html.escape(config['config_id'])} · {html.escape(config['name'])}

" + f"

{html.escape(config['sampler_name'])} / {html.escape(config['scheduler'])} / " + f"{config['steps']} steps / CFG {config['cfg']}

" + f"{html.escape(case_id)} {html.escape(config[" + f"

artifact {html.escape(card['artifact_id'])}

" + ) + parts.append("
") + return "\n".join(parts) + "\n" + + +def _load_reviews(path: Path) -> list[AblationReviewRecord]: + raw = _read_json(path) + if not isinstance(raw, dict) or raw.get("schema") != REVIEW_SCHEMA or not isinstance(raw.get("records"), list): + raise BenchmarkError("review_file_invalid", "Ablation review file has an unknown schema") + try: + return [AblationReviewRecord.model_validate(item) for item in raw["records"]] + except (ValueError, TypeError) as exc: + raise BenchmarkError("review_file_invalid", str(exc)) from exc + + +def aggregate_ablation( + spec_path: Path, + output_dir: Path, + *, + review_path: Path | None = None, +) -> dict[str, Any]: + spec = load_ablation_spec(spec_path) + state = AblationRunState.model_validate(_read_json(output_dir / "run-state.json")) + case_by_id = {case.case_id: case for case in spec.cases} + config_by_id = {config.config_id: config for config in spec.configurations} + config_results: dict[str, dict[str, Any]] = {} + case_results: dict[str, dict[str, Any]] = {} + technical_failures: Counter[str] = Counter() + warnings: Counter[str] = Counter() + for config_id in _CONFIG_IDS: + config_results[config_id] = { + "name": config_by_id[config_id].name, + "total": 0, + "succeeded": 0, + "failed": 0, + "pending": 0, + "technical_success_rate": None, + "mean_duration_seconds": None, + "near_solid_warnings": 0, + } + durations: dict[str, list[float]] = defaultdict(list) + for result in state.results.values(): + config_counts = config_results[result.config_id] + config_counts["total"] += 1 + case_counts = case_results.setdefault( + result.case_id, + {"category": case_by_id[result.case_id].category, "total": 0, "succeeded": 0, "failed": 0, "pending": 0}, + ) + case_counts["total"] += 1 + if result.status == "succeeded": + config_counts["succeeded"] += 1 + case_counts["succeeded"] += 1 + if result.technical_checks: + durations[result.config_id].append(result.duration_seconds) + if result.technical_checks.visual_precheck.near_solid: + config_counts["near_solid_warnings"] += 1 + warnings["near_solid"] += 1 + elif result.status == "failed": + config_counts["failed"] += 1 + case_counts["failed"] += 1 + if result.error: + technical_failures[result.error.code] += 1 + else: + config_counts["pending"] += 1 + case_counts["pending"] += 1 + for config_id, counts in config_results.items(): + if counts["total"]: + counts["technical_success_rate"] = round(counts["succeeded"] / counts["total"], 4) + if durations[config_id]: + counts["mean_duration_seconds"] = round(sum(durations[config_id]) / len(durations[config_id]), 3) + reviews: list[AblationReviewRecord] = [] + if review_path is not None: + reviews = _load_reviews(review_path) + expected = { + key: result for key, result in state.results.items() if result.status == "succeeded" and result.artifact + } + for review in reviews: + if review.review_key not in expected: + raise BenchmarkError("review_target_invalid", f"Review targets unknown result: {review.review_key}") + if expected[review.review_key].artifact.artifact_id != review.artifact_id: + raise BenchmarkError("review_artifact_mismatch", f"Review artifact changed: {review.review_key}") + succeeded = sum(result.status == "succeeded" for result in state.results.values()) + report = { + "schema": "hanclassstudio.teaching_image_ablation_report.v1", + "experiment_id": spec.experiment_id, + "experiment_version": spec.version, + "spec_sha256": _sha256(spec.model_dump(mode="json", by_alias=True)), + "run_id": state.run_id, + "run_status": state.status, + "generated_at": _iso(), + "total": len(state.results), + "succeeded": succeeded, + "failed": sum(result.status == "failed" for result in state.results.values()), + "pending_or_invalidated": sum(result.status not in {"succeeded", "failed"} for result in state.results.values()), + "technical_success_rate": round(succeeded / len(state.results), 4) if state.results else None, + "config_results": config_results, + "case_results": dict(sorted(case_results.items())), + "technical_failure_frequency": dict(sorted(technical_failures.items())), + "automatic_visual_precheck_warnings": dict(sorted(warnings.items())), + "teacher_review": { + "conclusion": None if not reviews else "teacher_data_imported", + "records_imported": len(reviews), + "records_pending_without_teacher": max(0, succeeded - sum(review.review_state == "reviewed" for review in reviews)), + "automatic_teacher_scores": False, + "failure_tag_frequency": dict(sorted(Counter(tag for review in reviews for tag in review.failure_tags).items())), + }, + "decision": { + "status": "pending_teacher_review", + "message": "Technical output is complete, but parameter/prompt/model quality ranking requires real teacher scores.", + "production_configuration_changed": False, + }, + "review_package": str((output_dir / "review-package").resolve()), + "limitations": [ + "Technical success and automatic visual prechecks are not teacher quality judgments.", + "The fixed production Workflow Pack and default prompt profile were not changed.", + "No conclusion about SD 1.5 capability ceiling is made before teacher review.", + ], + } + _write_json(output_dir / "ablation-report.json", report) + return report + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Phase 2C.1 SD 1.5 controlled ablation benchmark") + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run or resume the ablation") + run.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/sd15-ablation.v1.json")) + run.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-sd15-ablation")) + run.add_argument("--case-id", action="append", dest="case_ids") + run.add_argument("--config-id", action="append", dest="config_ids") + run.add_argument("--max-attempts", type=int, default=2) + package = sub.add_parser("review-package", help="write comparison and blank teacher review package") + package.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/sd15-ablation.v1.json")) + package.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-sd15-ablation")) + report = sub.add_parser("report", help="aggregate technical results and optional reviews") + report.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/sd15-ablation.v1.json")) + report.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-sd15-ablation")) + report.add_argument("--review-file", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "run": + state = run_ablation( + args.spec, + args.output_dir, + case_ids=args.case_ids, + config_ids=args.config_ids, + max_attempts=args.max_attempts, + ) + print(json.dumps(state.model_dump(mode="json", by_alias=True), ensure_ascii=False, indent=2)) + elif args.command == "review-package": + print(write_review_package(args.spec, args.output_dir)) + else: + print(json.dumps(aggregate_ablation(args.spec, args.output_dir, review_path=args.review_file), ensure_ascii=False, indent=2)) + return 0 + except BenchmarkBlockedError as exc: + print(json.dumps({"error": exc.code, "message": exc.message}, ensure_ascii=False)) + return 2 + except (BenchmarkError, TeachingImageError) as exc: + print(json.dumps({"error": getattr(exc, "code", "ablation_failed"), "message": str(exc)}, ensure_ascii=False)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/api/tests/test_teaching_image_ablation.py b/apps/api/tests/test_teaching_image_ablation.py new file mode 100644 index 0000000..d2ea7c3 --- /dev/null +++ b/apps/api/tests/test_teaching_image_ablation.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from hcs_api import teaching_image_ablation as ablation +from hcs_api.comfyui_teaching_image import TeachingImageError +from hcs_api.models import TeachingImageProvenance, VerifiedImageArtifact + +ROOT = Path(__file__).resolve().parents[3] +SPEC_PATH = ROOT / "benchmarks/phase2c1/sd15-ablation.v1.json" + + +def _identity(checked_at: str = "2026-07-27T00:00:00+00:00") -> ablation.BenchmarkObservedIdentity: + fixed = ablation._expected_fixed_identity() + return ablation.BenchmarkObservedIdentity( + **fixed.model_dump(), + runtime_installation_identity="a" * 64, + runtime_process_identity="b" * 64, + runtime_port=8188, + model_installation_identity="c" * 64, + checked_at=checked_at, + ) + + +def _artifact(asset_id: str, seed: int) -> VerifiedImageArtifact: + provenance = TeachingImageProvenance( + runtime_version="0.28.0", + runtime_source_commit="700821e1364eaab0e8f21c538a2131719fec57bf", + runtime_installation_identity="a" * 64, + runtime_process_identity="b" * 64, + runtime_port=8188, + model_package_id="hcs.sd15-teaching-illustration-fp16", + model_version="1.5-fp16-emaonly", + model_manifest_sha256="1" * 64, + model_sha256="e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916", + model_installation_identity="c" * 64, + workflow_pack_id="hcs.teaching-illustration-sd15-core", + workflow_version="1.0.0", + workflow_pack_sha256="e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e", + request_sha256="2" * 64, + execution_plan_sha256="3" * 64, + prompt_profile_id="soft-flat-educational-v1", + positive_prompt="fixed positive prompt", + negative_prompt="text, letters, words", + seed=seed, + steps=20, + cfg=7.0, + sampler_name="euler", + scheduler="normal", + denoise=1.0, + output_prefix="hcs_" + "4" * 20 + "_" + "5" * 12, + prompt_id="11111111-1111-4111-8111-111111111111", + source_trace=["ablation:phase2c1-sd15-ablation"], + ) + return VerifiedImageArtifact( + artifact_id="img-" + "6" * 24, + asset_id=asset_id, + path=f"assets/images/{asset_id}.png", + width=512, + height=384, + size_bytes=100, + sha256="7" * 64, + provenance_ref=f"assets/data/image-provenance/{asset_id}.json", + provenance_sha256="8" * 64, + provenance=provenance, + ) + + +def _checks(artifact: VerifiedImageArtifact) -> ablation.AblationTechnicalCheck: + return ablation.AblationTechnicalCheck( + image_path=artifact.path, + provenance_path=artifact.provenance_ref, + width=artifact.width, + height=artifact.height, + image_size_bytes=artifact.size_bytes, + image_sha256=artifact.sha256, + provenance_sha256=artifact.provenance_sha256, + manifest_asset_id=artifact.asset_id, + request_sha256="2" * 64, + execution_plan_sha256="3" * 64, + duration_seconds=0.1, + visual_precheck=ablation.AblationVisualPrecheck( + status="passed", blank_like=False, near_solid=False, distinct_colors=10, sampled_pixels=10 + ), + checks=["fixture"], + ) + + +def test_spec_has_exactly_54_fixed_jobs_and_three_named_configs() -> None: + spec = ablation.load_ablation_spec(SPEC_PATH) + assert len(spec.cases) == 6 + assert [config.config_id for config in spec.configurations] == ["A", "B", "C"] + assert sum(len(case.seeds) for case in spec.cases) * len(spec.configurations) == 54 + assert spec.configurations[1].sampler_name == "dpmpp_2m" + assert spec.configurations[2].prompt_profile_id == "ablation-clean-modern-v1" + + +def test_visual_precheck_flags_uniform_png(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeImage: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def convert(self, _mode): + return self + + def thumbnail(self, _size): + return None + + def getdata(self): + return [(1, 1, 1)] * 4 + + monkeypatch.setattr(ablation.Image, "open", lambda *_args: FakeImage()) + result = ablation._visual_precheck(b"fixture") + assert result.blank_like is True + assert result.near_solid is True + assert result.status == "warning" + + +def test_runner_continues_and_retries_completed_failures( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + observed = _identity() + monkeypatch.setattr(ablation, "_capture_identity", lambda: observed) + calls: list[str] = [] + fail_once = True + + def fake_generate(_project_dir: Path, request): + nonlocal fail_once + calls.append(request.asset_id) + if request.asset_id.startswith("abl-obj_apple_01-A") and fail_once: + raise TeachingImageError("generation_failed", "fixture failure") + return _artifact(request.asset_id, request.seed) + + monkeypatch.setattr(ablation, "generate_teaching_image", fake_generate) + monkeypatch.setattr(ablation, "_technical_check", lambda *_args: _checks(_args[4])) + output = tmp_path / "run" + state = ablation.run_ablation( + SPEC_PATH, + output, + case_ids=["obj_apple_01"], + config_ids=["A", "B", "C"], + max_attempts=1, + ) + assert len(state.results) == 9 + assert state.results["obj_apple_01@A@260101"].status == "failed" + assert state.results["obj_apple_01@B@260101"].status == "succeeded" + fail_once = False + resumed = ablation.run_ablation( + SPEC_PATH, + output, + case_ids=["obj_apple_01"], + config_ids=["A", "B", "C"], + max_attempts=1, + ) + assert all(result.status == "succeeded" for result in resumed.results.values()) + assert len(calls) == 12 + again = ablation.run_ablation( + SPEC_PATH, + output, + case_ids=["obj_apple_01"], + config_ids=["A", "B", "C"], + max_attempts=1, + ) + assert again.status == "completed" + assert len(calls) == 12 + + +def test_identity_change_blocks_existing_run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + first = _identity() + second = _identity() + second.runtime_process_identity = "d" * 64 + captures = iter([first, second]) + monkeypatch.setattr(ablation, "_capture_identity", lambda: next(captures)) + with pytest.raises(ablation.BenchmarkBlockedError, match="identity"): + ablation.run_ablation(SPEC_PATH, tmp_path / "run", case_ids=["obj_apple_01"], config_ids=["A"], max_attempts=1) + state = json.loads((tmp_path / "run/run-state.json").read_text(encoding="utf-8")) + assert state["status"] == "blocked" + assert state["block"]["code"] == "ablation_identity_changed" + + +def test_review_record_stays_pending_until_all_scores_and_decision_exist() -> None: + pending = ablation.AblationReviewRecord( + review_key="obj_apple_01@A@260101", + case_id="obj_apple_01", + config_id="A", + seed=260101, + artifact_id="img-" + "6" * 24, + ) + assert pending.review_state == "pending_review" + payload = pending.model_dump() + payload["review_state"] = "reviewed" + with pytest.raises(ValueError, match="every rating"): + ablation.AblationReviewRecord.model_validate(payload) diff --git a/benchmarks/phase2c1/sd15-ablation.v1.json b/benchmarks/phase2c1/sd15-ablation.v1.json new file mode 100644 index 0000000..0ef38ee --- /dev/null +++ b/benchmarks/phase2c1/sd15-ablation.v1.json @@ -0,0 +1,195 @@ +{ + "schema": "hanclassstudio.teaching_image_ablation.v1", + "experiment_id": "phase2c1-sd15-ablation", + "version": "1.0.0", + "fixed_identity": { + "model_package_id": "hcs.sd15-teaching-illustration-fp16", + "model_version": "1.5-fp16-emaonly", + "model_sha256": "e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916", + "workflow_pack_id": "hcs.teaching-illustration-sd15-core", + "workflow_version": "1.0.0", + "workflow_pack_sha256": "e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e", + "runtime_id": "comfyui", + "runtime_version": "0.28.0", + "runtime_source_commit": "700821e1364eaab0e8f21c538a2131719fec57bf", + "prompt_profile_id": "soft-flat-educational-v1" + }, + "configurations": [ + { + "config_id": "A", + "name": "current-baseline", + "sampler_name": "euler", + "scheduler": "normal", + "steps": 20, + "cfg": 7.0, + "prompt_profile_id": "soft-flat-educational-v1", + "positive_prefix": null, + "positive_suffix": null, + "negative_prompt": null + }, + { + "config_id": "B", + "name": "sampling-only", + "sampler_name": "dpmpp_2m", + "scheduler": "karras", + "steps": 30, + "cfg": 6.0, + "prompt_profile_id": "soft-flat-educational-v1", + "positive_prefix": null, + "positive_suffix": null, + "negative_prompt": null + }, + { + "config_id": "C", + "name": "sampling-plus-clean-modern-prompt", + "sampler_name": "dpmpp_2m", + "scheduler": "karras", + "steps": 30, + "cfg": 6.0, + "prompt_profile_id": "ablation-clean-modern-v1", + "positive_prefix": "clean modern educational illustration, clear visual hierarchy, simple natural shapes, balanced composition, soft controlled colors, clear separation between people and objects, plain background, no text", + "positive_suffix": "clear focal subject, readable action, uncluttered teaching image, age-appropriate, culturally respectful", + "negative_prompt": "text, letters, caption, watermark, logo, extra limbs, fused hands, malformed hands, duplicate people, merged bodies, cropped subjects, distorted furniture, blurry, low resolution, oversaturated" + } + ], + "cases": [ + { + "case_id": "obj_apple_01", + "category": "single_object", + "teaching_goal": "Learner can recognize 苹果 as a single everyday object.", + "prompt": "subject: one red apple; action: resting on a small table; environment: a plain bright teaching surface", + "request": { + "purpose": "vocabulary_image", + "subject": "one red apple", + "action": "resting on a small table", + "environment": "a plain bright teaching surface", + "aspect_ratio": "4:3" + }, + "expected": { + "people": 0, + "objects": ["one red apple", "small table"], + "actions": [], + "relations": [], + "scene": "plain teaching surface" + }, + "must_satisfy": ["one clearly recognizable red apple is the focal object", "no extra people are present"], + "severe_failures": ["missing apple", "multiple competing objects dominate", "unreadable image"], + "seeds": [260101, 260201, 260301] + }, + { + "case_id": "space_between_01", + "category": "spatial_relation", + "teaching_goal": "Learner can interpret 中间 in a three-object arrangement.", + "prompt": "subject: a small green plant between two books; action: the plant is in the middle of the books; environment: a neat desk", + "request": { + "purpose": "classroom_scene", + "subject": "a small green plant between two books", + "action": "the plant is in the middle of the books", + "environment": "a neat desk", + "aspect_ratio": "4:3" + }, + "expected": { + "people": 0, + "objects": ["small green plant", "two books"], + "actions": [], + "relations": ["plant between two books"], + "scene": "neat desk" + }, + "must_satisfy": ["two books flank the plant", "plant is visually central"], + "severe_failures": ["plant not between books", "missing book", "ambiguous arrangement"], + "seeds": [260112, 260212, 260312] + }, + { + "case_id": "daily_greet_01", + "category": "two_person_communication", + "teaching_goal": "Learner can recognize a polite everyday greeting.", + "prompt": "subject: one adult and one older adult; action: smiling and greeting each other respectfully at a doorway; environment: a welcoming home entrance", + "request": { + "purpose": "classroom_scene", + "subject": "one adult and one older adult", + "action": "smiling and greeting each other respectfully at a doorway", + "environment": "a welcoming home entrance", + "aspect_ratio": "4:3" + }, + "expected": { + "people": 2, + "objects": ["doorway"], + "actions": ["greet respectfully", "smile"], + "relations": ["two people face each other"], + "scene": "welcoming home entrance" + }, + "must_satisfy": ["greeting posture is readable", "age relationship is respectful", "no text is needed"], + "severe_failures": ["wrong number of people", "no interaction", "culturally inappropriate depiction"], + "seeds": [260116, 260216, 260316] + }, + { + "case_id": "action_wave_01", + "category": "person_action", + "teaching_goal": "Learner can connect 挥手 with a person visibly waving.", + "prompt": "subject: one child; action: waving one raised hand hello; environment: a bright uncluttered classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one child", + "action": "waving one raised hand hello", + "environment": "a bright uncluttered classroom", + "aspect_ratio": "4:3" + }, + "expected": { + "people": 1, + "objects": [], + "actions": ["wave with one raised hand"], + "relations": [], + "scene": "bright uncluttered classroom" + }, + "must_satisfy": ["one child is visible", "raised hand clearly communicates waving"], + "severe_failures": ["wrong number of people", "wrong action", "malformed hands dominate"], + "seeds": [260104, 260204, 260304] + }, + { + "case_id": "count_three_01", + "category": "person_count", + "teaching_goal": "Learner can identify three people in a classroom.", + "prompt": "subject: exactly three students; action: sitting together at one table; environment: a bright language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "exactly three students", + "action": "sitting together at one table", + "environment": "a bright language classroom", + "aspect_ratio": "4:3" + }, + "expected": { + "people": 3, + "objects": ["one table"], + "actions": ["sit together"], + "relations": ["three people around one table"], + "scene": "bright language classroom" + }, + "must_satisfy": ["three complete people are discernible", "one shared table is visible"], + "severe_failures": ["wrong count", "merged bodies", "missing table"], + "seeds": [260109, 260209, 260309] + }, + { + "case_id": "class_pair_01", + "category": "classroom_activity", + "teaching_goal": "Learner can recognize a pair speaking activity.", + "prompt": "subject: exactly two language students; action: facing each other and practicing a short conversation; environment: a friendly classroom pair-work table", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two language students", + "action": "facing each other and practicing a short conversation", + "environment": "a friendly classroom pair-work table", + "aspect_ratio": "4:3" + }, + "expected": { + "people": 2, + "objects": ["pair-work table"], + "actions": ["speak to each other"], + "relations": ["two students face each other"], + "scene": "friendly classroom pair-work table" + }, + "must_satisfy": ["exactly two students are visible", "face-to-face orientation is clear", "no speech text is needed"], + "severe_failures": ["wrong count", "no face-to-face interaction", "speech text artifact"], + "seeds": [260115, 260215, 260315] + } + ] +} From de9aa1afb9991cec7fb60ed61ce3a29cab5765cf Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:02:06 +0700 Subject: [PATCH 4/5] fix(ablation): bind prompt profile in provenance trace --- apps/api/src/hcs_api/teaching_image_ablation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/hcs_api/teaching_image_ablation.py b/apps/api/src/hcs_api/teaching_image_ablation.py index 821db0f..fb31fd6 100644 --- a/apps/api/src/hcs_api/teaching_image_ablation.py +++ b/apps/api/src/hcs_api/teaching_image_ablation.py @@ -336,6 +336,7 @@ def _request_for(case: AblationCase, config: AblationConfig, seed: int) -> Teach "ablation:phase2c1-sd15-ablation", f"case:{case.case_id}", f"config:{config.config_id}", + f"prompt-profile:{config.prompt_profile_id}", f"seed:{seed}", ], ) From 35c0bd9fb11bb9715a84c0a9fc4a8743e6f8a8bf Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:45:00 +0700 Subject: [PATCH 5/5] docs(ablation): record full sd15 experiment results --- docs/phase2c1-sd15-ablation-results.md | 71 ++++++++++++++++++++++++++ docs/phase2c1-sd15-ablation.md | 66 ++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 docs/phase2c1-sd15-ablation-results.md create mode 100644 docs/phase2c1-sd15-ablation.md diff --git a/docs/phase2c1-sd15-ablation-results.md b/docs/phase2c1-sd15-ablation-results.md new file mode 100644 index 0000000..b14c0b3 --- /dev/null +++ b/docs/phase2c1-sd15-ablation-results.md @@ -0,0 +1,71 @@ +# Phase 2C.1 SD 1.5 消融首轮结果 + +本报告区分自动技术结果和人工教学评审。教师评审尚未导入时，不把工程抽样观察称为教师结论。 + +## 固定身份 + +- Model：`hcs.sd15-teaching-illustration-fp16` `1.5-fp16-emaonly` +- Model SHA-256：`e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916` +- Workflow：`hcs.teaching-illustration-sd15-core` `1.0.0` +- Workflow SHA-256：`e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e` +- Runtime：ComfyUI `0.28.0`, source commit `700821e1364eaab0e8f21c538a2131719fec57bf` +- Platform：macOS arm64 opt-in lifecycle + +## Pilot + +Pilot `obj_apple_01 × A/B/C × 3 seeds` 生成 9/9 成功，A/B/C 的 provenance 参数与 prompt trace 均通过。 +技术失败和近纯色预警均为 0。Pilot review package 有 9 条空白评审记录。 + +## 完整运行 + +Run id：`phase2c1-ablation-1785135763-338802ab`。 + +| 配置 | 总数 | 技术成功 | 技术失败 | 成功率 | 平均生成秒数 | 近纯色预警 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| A 当前基线 | 18 | 18 | 0 | 1.0 | 35.011 | 0 | +| B 仅采样参数 | 18 | 18 | 0 | 1.0 | 49.012 | 0 | +| C 参数 + prompt | 18 | 18 | 0 | 1.0 | 45.585 | 0 | +| **合计** | **54** | **54** | **0** | **1.0** | — | **0** | + +六个案例类别均为 9/9 技术成功：单物体、空间关系、两人交际、人物动作、精确人数、课堂活动。 +没有缺失输出、重试、pending/invalidated 结果或技术 failure code。 + +## 评审包 + +完整评审包位于 Git 忽略目录： +`runtime/phase2c1-sd15-ablation-full/review-package/`。 + +- `comparison.html`：按 case/seed 并排显示 A/B/C； +- `experiment-manifest.json`：案例、配置、身份、结果和 provenance 索引； +- `teacher-reviews.pending.json`：54 条 `pending_review`； +- `teacher-reviews.csv`：54 条空白人工评审表； +- `images/`：54 张评审副本。 + +当前人工评审为 54 pending、0 reviewed、0 imported；failure tag frequency 为空，不能据此声称教师 +认为任何配置更好或更差。 + +## 工程抽样观察（待教师确认） + +以下是对少量对照图的工程检查，不是自动评分，也不是教师结论： + +- `action_wave_01` 的抽样中，A/B 出现多人场景，C 出现手部特写；这提示 `wrong_count` / `wrong_action` + 需要教师确认； +- `space_between_01` 的抽样中，三组都出现了植物、书本/盆栽关系偏离目标的迹象；这提示 + `missing_object` / `wrong_spatial_relation` 需要教师确认； +- `daily_greet_01` 的抽样中，C 有明显疑似文字伪影；这提示 `text_artifact` 和交际动作准确性需要教师 + 确认； +- `class_pair_01` 的抽样中，配置间构图和人物朝向差异较大，不能仅凭技术通过判定课堂可用性。 + +这些观察没有写入评审记录，必须由真实教师逐图评分后才能聚合。 + +## 当前决策 + +当前证据证明三组配置都能稳定通过受控生成和技术合同，但尚不足以判断“参数改善是否显著”或“瓶颈 +更偏参数还是模型”。因此本轮不更新固定 prompt/sampler，不修改生产配置，也不宣称 SD 1.5 已达到教学 +质量门槛。 + +下一步门槛：完成 54 张的真实教师评审；若 B/C 在视觉完整性上提升但人数/动作/空间关系仍失败，限制 +SD 1.5 使用范围并另开更强模型评估；若 C 在多数简单案例的教师评分稳定提升且无回归，另开小型生产 +prompt/sampler PR；若三组评分差异小，停止继续调参并评估新的固定 Model Package。 + +图片、模型、缓存、Runtime、state、report 和评审包均未提交 Git。 diff --git a/docs/phase2c1-sd15-ablation.md b/docs/phase2c1-sd15-ablation.md new file mode 100644 index 0000000..6498197 --- /dev/null +++ b/docs/phase2c1-sd15-ablation.md @@ -0,0 +1,66 @@ +# Phase 2C.1 SD 1.5 教学图片参数消融 + +这是一个评测与决策切片，不修改生产 Model Package、Workflow Pack、Provider Hub 默认配置或生产 +prompt profile。实现位于 +[`apps/api/src/hcs_api/teaching_image_ablation.py`](../apps/api/src/hcs_api/teaching_image_ablation.py)， +实验合同位于 [`benchmarks/phase2c1/sd15-ablation.v1.json`](../benchmarks/phase2c1/sd15-ablation.v1.json)。 + +## 固定设计 + +实验固定同一个 SD 1.5 模型、Runtime、Workflow、4:3 尺寸和 6 个案例，每个案例 3 个 seed： + +- `obj_apple_01`：单个词汇物体；`260101`, `260201`, `260301` +- `space_between_01`：室内/桌面空间关系；`260112`, `260212`, `260312` +- `daily_greet_01`：两个人静态交际；`260116`, `260216`, `260316` +- `action_wave_01`：明确人物动作；`260104`, `260204`, `260304` +- `count_three_01`：精确人数；`260109`, `260209`, `260309` +- `class_pair_01`：多人课堂活动；`260115`, `260215`, `260315` + +三组配置只在评测 runner 内绑定到受控 `TeachingImageRequest` 编译计划： + +| 配置 | sampler | scheduler | steps | CFG | prompt | +| --- | --- | --- | ---: | ---: | --- | +| A | Euler | normal | 20 | 7 | 固定 `soft-flat-educational-v1` | +| B | DPM++ 2M | Karras | 30 | 6 | 固定 `soft-flat-educational-v1` | +| C | DPM++ 2M | Karras | 30 | 6 | 评测专用 `ablation-clean-modern-v1` | + +C 的正向描述固定包含 clean modern educational illustration、clear visual hierarchy、simple natural +shapes、balanced composition、soft controlled colors、clear separation between people and objects、 +plain background、no text；负面提示固定包含 text/letters/caption、watermark/logo、extra limbs、 +fused/malformed hands、duplicate/merged people、cropped subjects、distorted furniture、blurry、 +low resolution、oversaturated。 + +每张图片的 request、case、configuration、prompt profile、seed、artifact 和 provenance 通过 source trace、 +request SHA、execution plan SHA 和 Asset Manifest 绑定。实验 runner 不接受任意 ComfyUI graph。 + +## 执行与评审 + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_ablation run \ + --spec benchmarks/phase2c1/sd15-ablation.v1.json \ + --output-dir runtime/phase2c1-sd15-ablation-full +``` + +runner 支持每次尝试保存 state、Ctrl-C 后 `paused` 恢复、幂等 artifact 技术复核、失败重试和 Runtime/model/ +Workflow identity 变化后的 fail-closed。真实 macOS Runtime 启动所属进程必须保持存活；本轮复用了已验证 +的 Phase 2C Runtime/model 安装，输出项目仍在本 worktree 的 ignored `runtime/` 目录。 + +评审包由 `review-package` 命令生成，包含 `comparison.html`、原图、`experiment-manifest.json`、空白 +`teacher-reviews.pending.json` 和 `teacher-reviews.csv`。人工字段为 visual_quality、prompt_adherence、 +object_count_accuracy、action_accuracy、spatial_relation_accuracy、teaching_usability、anatomy_quality、 +needs_regeneration、failure_tags、reviewer_notes。任何自动预检都不填教师字段。 + +## 自动检查边界 + +自动检查 PNG signature/CRC/尺寸/大小、图片 SHA-256、非空/近纯色预检、provenance SHA-256、固定 Runtime/ +model/Workflow identity、采样参数、prompt profile、request/plan identity、生成耗时和 Asset Manifest 单一 +`pending_review` 条目。自动检查结果不是教师教学质量结论。 + +## 决策门槛 + +只有在真实教师评审导入后，才比较三组的教学评分和 failure tags。若 B/C 只改善清晰度而关系/人数/动作 +仍失败，应限制 SD 1.5 为低复杂度图并另开更强模型评估；若 C 在简单案例上稳定提升且无回归，另开 +生产 prompt/sampler 变更 PR；若三组差异小，停止继续榨取 SD 1.5 并评估新的固定 Model Package。 + +本 PR 不执行上述生产变更。