diff --git a/core/intelligence/curation_holdout_guard_contract.py b/core/intelligence/curation_holdout_guard_contract.py new file mode 100644 index 00000000..5c8497b7 --- /dev/null +++ b/core/intelligence/curation_holdout_guard_contract.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from core.intelligence.curation_review_v2_contract import SelectionScope + +HOLDOUT_GUARD_VERSION = "curation-holdout-guard-r1" + + +def _non_empty(value: str, field_name: str) -> str: + normalized = str(value).strip() + if not normalized: + raise ValueError(f"{field_name} must not be empty") + return normalized + + +class HoldoutSelectionInput(StrEnum): + SOURCE_LIBRARY_ELIGIBILITY = "source_library_eligibility" + SET_ROLE = "set_role" + DETERMINISTIC_SEED = "deterministic_seed" + BPM_STRATUM = "bpm_stratum" + STYLE_STRATUM = "style_stratum" + ENERGY_STRATUM = "energy_stratum" + CHALLENGER_SCORE = "challenger_score" + CHALLENGER_PREFERENCE = "challenger_preference" + SOURCE_CHALLENGER_DISAGREEMENT = "source_challenger_disagreement" + FAILURE_CLASS = "failure_class" + + +_REPRESENTATIVE_ALLOWED_INPUTS = frozenset( + { + HoldoutSelectionInput.SOURCE_LIBRARY_ELIGIBILITY, + HoldoutSelectionInput.SET_ROLE, + HoldoutSelectionInput.DETERMINISTIC_SEED, + HoldoutSelectionInput.BPM_STRATUM, + HoldoutSelectionInput.STYLE_STRATUM, + HoldoutSelectionInput.ENERGY_STRATUM, + } +) + + +@dataclass(frozen=True, slots=True) +class DevelopmentEvidenceExclusionRegistry: + registry_id: str + registry_version: str + case_ids: tuple[str, ...] + scenario_fingerprints: tuple[str, ...] + evidence_refs: tuple[str, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "registry_id", _non_empty(self.registry_id, "registry_id")) + object.__setattr__( + self, + "registry_version", + _non_empty(self.registry_version, "registry_version"), + ) + case_ids = tuple(_non_empty(item, "development_case_id") for item in self.case_ids) + scenarios = tuple( + _non_empty(item, "development_scenario_fingerprint") + for item in self.scenario_fingerprints + ) + if not case_ids or not scenarios: + raise ValueError("development exclusion registry must be non-empty") + if len(set(case_ids)) != len(case_ids): + raise ValueError("development exclusion case_ids must be unique") + if len(set(scenarios)) != len(scenarios): + raise ValueError("development exclusion scenario_fingerprints must be unique") + refs = tuple(_non_empty(item, "development_evidence_ref") for item in self.evidence_refs) + if not refs: + raise ValueError("development exclusion registry requires evidence refs") + object.__setattr__(self, "case_ids", case_ids) + object.__setattr__(self, "scenario_fingerprints", scenarios) + object.__setattr__(self, "evidence_refs", refs) + + +@dataclass(frozen=True, slots=True) +class HoldoutSelectionBasis: + basis_id: str + basis_version: str + selection_scope: SelectionScope + selection_inputs: tuple[HoldoutSelectionInput, ...] + evidence_refs: tuple[str, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "basis_id", _non_empty(self.basis_id, "basis_id")) + object.__setattr__(self, "basis_version", _non_empty(self.basis_version, "basis_version")) + inputs = tuple(self.selection_inputs) + if not inputs or len(set(inputs)) != len(inputs): + raise ValueError("selection_inputs must be non-empty and unique") + if self.selection_scope is SelectionScope.REPRESENTATIVE_HOLDOUT: + disallowed = tuple(item for item in inputs if item not in _REPRESENTATIVE_ALLOWED_INPUTS) + if disallowed: + names = ",".join(item.value for item in disallowed) + raise ValueError( + f"representative holdout selection cannot depend on model outcomes: {names}" + ) + if HoldoutSelectionInput.DETERMINISTIC_SEED not in inputs: + raise ValueError("representative holdout selection must bind deterministic_seed") + refs = tuple(_non_empty(item, "selection_basis_evidence_ref") for item in self.evidence_refs) + if not refs: + raise ValueError("holdout selection basis requires evidence refs") + object.__setattr__(self, "selection_inputs", inputs) + object.__setattr__(self, "evidence_refs", refs) + + +@dataclass(frozen=True, slots=True) +class CurationAssignmentBatchManifest: + batch_id: str + batch_version: str + assignment_seed_commitment: str + reviewer_refs: tuple[str, ...] + case_ids: tuple[str, ...] + assignment_fingerprints: tuple[str, ...] + generated_at: str + algorithm_identity_hidden: bool = True + activation_authorized: bool = False + personal_dj_model_training_authorized: bool = False + + def __post_init__(self) -> None: + for field_name in ( + "batch_id", + "batch_version", + "assignment_seed_commitment", + "generated_at", + ): + object.__setattr__(self, field_name, _non_empty(getattr(self, field_name), field_name)) + if self.batch_version != HOLDOUT_GUARD_VERSION: + raise ValueError("unsupported assignment batch version") + reviewers = tuple(_non_empty(item, "reviewer_ref") for item in self.reviewer_refs) + cases = tuple(_non_empty(item, "case_id") for item in self.case_ids) + fingerprints = tuple( + _non_empty(item, "assignment_fingerprint") for item in self.assignment_fingerprints + ) + if not reviewers or len(set(reviewers)) != len(reviewers): + raise ValueError("reviewer_refs must be non-empty and unique") + if not cases or len(set(cases)) != len(cases): + raise ValueError("case_ids must be non-empty and unique") + expected = len(reviewers) * len(cases) + if len(fingerprints) != expected or len(set(fingerprints)) != expected: + raise ValueError("assignment_fingerprints must cover reviewer x case exactly once") + object.__setattr__(self, "reviewer_refs", reviewers) + object.__setattr__(self, "case_ids", cases) + object.__setattr__(self, "assignment_fingerprints", fingerprints) + if not self.algorithm_identity_hidden: + raise ValueError("assignment batch requires algorithm identity hiding") + if self.activation_authorized: + raise ValueError("assignment batch cannot authorize optimizer activation") + if self.personal_dj_model_training_authorized: + raise ValueError("assignment batch cannot authorize Personal DJ Model training") + + +__all__ = [ + "HOLDOUT_GUARD_VERSION", + "CurationAssignmentBatchManifest", + "DevelopmentEvidenceExclusionRegistry", + "HoldoutSelectionBasis", + "HoldoutSelectionInput", +] diff --git a/core/intelligence/curation_review_v2_contract.py b/core/intelligence/curation_review_v2_contract.py new file mode 100644 index 00000000..30ad6092 --- /dev/null +++ b/core/intelligence/curation_review_v2_contract.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from enum import StrEnum + +from core.intelligence.curated_real_library_review_contract import CuratedSetRole +from core.intelligence.human_preference_calibration_contract import CalibrationVerdict, ResolvedPreference + +CURATION_REVIEW_PROTOCOL_VERSION = "curation-review-v2" +CURATION_CALIBRATION_VERSION = "curation-preference-calibration-v3" +HOLDOUT_VALIDATION_MANIFEST_VERSION = "holdout-validation-manifest-r1" +CURATION_AUDITION_MODE = "sequence_curation_only" + + +def _non_empty(value: str, field_name: str) -> str: + normalized = str(value).strip() + if not normalized: + raise ValueError(f"{field_name} must not be empty") + return normalized + + +def _unit(value: float | None, field_name: str) -> float | None: + if value is None: + return None + numeric = float(value) + if not math.isfinite(numeric) or not 0.0 <= numeric <= 1.0: + raise ValueError(f"{field_name} must be between 0 and 1") + return numeric + + +def _score(value: float, field_name: str) -> float: + numeric = float(value) + if not math.isfinite(numeric) or not 1.0 <= numeric <= 5.0: + raise ValueError(f"{field_name} must be between 1 and 5") + return numeric + + +def _ref(value: tuple[str, str], field_name: str) -> tuple[str, str]: + if len(value) != 2: + raise ValueError(f"{field_name} must contain exactly two values") + return ( + _non_empty(value[0], f"{field_name}[0]"), + _non_empty(value[1], f"{field_name}[1]"), + ) + + +class EvidenceRole(StrEnum): + DEVELOPMENT_CALIBRATION = "development_calibration" + HOLDOUT_VALIDATION = "holdout_validation" + + +class SelectionScope(StrEnum): + REPRESENTATIVE_HOLDOUT = "representative_holdout" + DIAGNOSTIC_CHALLENGE_SET = "diagnostic_challenge_set" + + +class EvaluationScope(StrEnum): + PERSONAL_DJ_CALIBRATION = "personal_dj_calibration" + MULTI_DJ_PRODUCT_EVALUATION = "multi_dj_product_evaluation" + + +class CurationReviewDimension(StrEnum): + ENERGY_FLOW = "energy_flow" + DRAMATURGICAL_FIT = "dramaturgical_fit" + SET_COHERENCE = "set_coherence" + ALTERNATIVE_USEFULNESS = "alternative_usefulness" + TRACK_SELECTION_FIT = "track_selection_fit" + + +REQUIRED_CURATION_REVIEW_DIMENSIONS_V2 = tuple(CurationReviewDimension) + + +class CurationPreference(StrEnum): + PLAN_A = "plan_a" + PLAN_B = "plan_b" + TIE = "tie" + ABSTAIN = "abstain" + + +class HoldoutSystemOutcome(StrEnum): + REVIEWABLE_PAIR = "reviewable_pair" + TECHNICALLY_IDENTICAL_PAIR = "technically_identical_pair" + NO_MEANINGFUL_ALTERNATIVE = "no_meaningful_alternative" + MISSING_REQUIRED_EVIDENCE = "missing_required_evidence" + SOURCE_GENERATION_FAILED = "source_generation_failed" + CHALLENGER_NOT_PROVEN = "challenger_not_proven" + + +@dataclass(frozen=True, slots=True) +class CurationBlindAssignmentV2: + assignment_id: str + case_id: str + reviewer_ref: str + slot_a_plan_id: str + slot_b_plan_id: str + assignment_fingerprint: str + algorithm_identity_hidden: bool = True + + def __post_init__(self) -> None: + for field_name in ( + "assignment_id", + "case_id", + "reviewer_ref", + "slot_a_plan_id", + "slot_b_plan_id", + "assignment_fingerprint", + ): + object.__setattr__(self, field_name, _non_empty(getattr(self, field_name), field_name)) + if self.slot_a_plan_id == self.slot_b_plan_id: + raise ValueError("curation blind assignment slots must reference distinct plans") + if not self.algorithm_identity_hidden: + raise ValueError("curation review requires algorithm identity to remain hidden") + + +@dataclass(frozen=True, slots=True) +class CurationDimensionPairRating: + dimension: CurationReviewDimension + plan_a_score: float + plan_b_score: float + + def __post_init__(self) -> None: + object.__setattr__(self, "plan_a_score", _score(self.plan_a_score, "plan_a_score")) + object.__setattr__(self, "plan_b_score", _score(self.plan_b_score, "plan_b_score")) + + +@dataclass(frozen=True, slots=True) +class CurationDJReviewV2: + review_id: str + assignment_id: str + reviewer_ref: str + evidence_role: EvidenceRole + curation_packet_fingerprint: str + source_blinded_packet_fingerprint: str + preference: CurationPreference + ratings: tuple[CurationDimensionPairRating, ...] + confidence: float + observed_at: str + audition_mode: str = CURATION_AUDITION_MODE + algorithm_identity_was_hidden: bool = True + execution_quality_excluded_from_curation_judgment: bool = True + reason_codes: tuple[str, ...] = () + notes: str = "" + activation_authorized: bool = False + personal_dj_model_training_authorized: bool = False + + def __post_init__(self) -> None: + for field_name in ( + "review_id", + "assignment_id", + "reviewer_ref", + "curation_packet_fingerprint", + "source_blinded_packet_fingerprint", + "observed_at", + ): + object.__setattr__(self, field_name, _non_empty(getattr(self, field_name), field_name)) + if self.audition_mode != CURATION_AUDITION_MODE: + raise ValueError("curation review must use sequence_curation_only audition mode") + if not self.algorithm_identity_was_hidden: + raise ValueError("curation review is invalid when algorithm identity was visible") + if not self.execution_quality_excluded_from_curation_judgment: + raise ValueError("curation review must exclude live transition execution quality") + ratings = tuple(self.ratings) + dimensions = tuple(item.dimension for item in ratings) + if len(dimensions) != len(REQUIRED_CURATION_REVIEW_DIMENSIONS_V2) or set(dimensions) != set( + REQUIRED_CURATION_REVIEW_DIMENSIONS_V2 + ): + raise ValueError("curation review requires all V2 dimensions exactly once") + object.__setattr__(self, "ratings", ratings) + object.__setattr__(self, "confidence", _unit(self.confidence, "confidence")) + object.__setattr__(self, "reason_codes", tuple(dict.fromkeys(self.reason_codes))) + object.__setattr__(self, "notes", str(self.notes)) + if self.activation_authorized: + raise ValueError("curation review cannot authorize optimizer activation") + if self.personal_dj_model_training_authorized: + raise ValueError("curation review cannot authorize Personal DJ Model training") + + +@dataclass(frozen=True, slots=True) +class HoldoutValidationManifest: + holdout_id: str + holdout_version: str + evidence_role: EvidenceRole + selection_scope: SelectionScope + source_snapshot_ref: tuple[str, str] + case_selection_policy_ref: tuple[str, str] + selection_seed_commitment: str + selected_case_ids: tuple[str, ...] + scenario_fingerprints: tuple[str, ...] + required_set_roles: tuple[CuratedSetRole, ...] + source_optimizer_sha: str + challenger_sha: str + challenger_policy_ref: tuple[str, str] + challenger_config_digest: str + calibration_policy_digest: str + source_evidence_revisions: tuple[str, ...] + generated_at: str + human_labels_available_at_freeze: bool = False + algorithm_identity_hidden: bool = True + activation_authorized: bool = False + personal_dj_model_training_authorized: bool = False + + def __post_init__(self) -> None: + for field_name in ( + "holdout_id", + "holdout_version", + "selection_seed_commitment", + "source_optimizer_sha", + "challenger_sha", + "challenger_config_digest", + "calibration_policy_digest", + "generated_at", + ): + object.__setattr__(self, field_name, _non_empty(getattr(self, field_name), field_name)) + if self.holdout_version != HOLDOUT_VALIDATION_MANIFEST_VERSION: + raise ValueError("unsupported holdout manifest version") + if self.evidence_role is not EvidenceRole.HOLDOUT_VALIDATION: + raise ValueError("holdout manifest must use holdout_validation evidence role") + object.__setattr__(self, "source_snapshot_ref", _ref(self.source_snapshot_ref, "source_snapshot_ref")) + object.__setattr__( + self, + "case_selection_policy_ref", + _ref(self.case_selection_policy_ref, "case_selection_policy_ref"), + ) + object.__setattr__( + self, + "challenger_policy_ref", + _ref(self.challenger_policy_ref, "challenger_policy_ref"), + ) + case_ids = tuple(_non_empty(item, "selected_case_id") for item in self.selected_case_ids) + scenarios = tuple(_non_empty(item, "scenario_fingerprint") for item in self.scenario_fingerprints) + if not case_ids or len(set(case_ids)) != len(case_ids): + raise ValueError("selected_case_ids must be non-empty and unique") + if len(scenarios) != len(case_ids) or len(set(scenarios)) != len(scenarios): + raise ValueError("scenario_fingerprints must be unique and align with selected cases") + object.__setattr__(self, "selected_case_ids", case_ids) + object.__setattr__(self, "scenario_fingerprints", scenarios) + roles = tuple(self.required_set_roles) + if not roles or len(set(roles)) != len(roles): + raise ValueError("required_set_roles must be non-empty and unique") + object.__setattr__(self, "required_set_roles", roles) + revisions = tuple(_non_empty(item, "source_evidence_revision") for item in self.source_evidence_revisions) + if not revisions: + raise ValueError("source_evidence_revisions must not be empty") + object.__setattr__(self, "source_evidence_revisions", revisions) + if self.human_labels_available_at_freeze: + raise ValueError("holdout must be frozen before human labels are available") + if not self.algorithm_identity_hidden: + raise ValueError("holdout validation requires algorithm identity to remain hidden") + if self.activation_authorized: + raise ValueError("holdout manifest cannot authorize optimizer activation") + if self.personal_dj_model_training_authorized: + raise ValueError("holdout manifest cannot authorize Personal DJ Model training") + + +@dataclass(frozen=True, slots=True) +class HoldoutCaseExecutionEvidence: + case_id: str + set_role: CuratedSetRole + outcome: HoldoutSystemOutcome + scenario_fingerprint: str + evidence_refs: tuple[str, ...] + reason_codes: tuple[str, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "case_id", _non_empty(self.case_id, "case_id")) + object.__setattr__( + self, + "scenario_fingerprint", + _non_empty(self.scenario_fingerprint, "scenario_fingerprint"), + ) + refs = tuple(_non_empty(item, "evidence_ref") for item in self.evidence_refs) + if not refs: + raise ValueError("holdout case execution evidence must contain evidence refs") + object.__setattr__(self, "evidence_refs", refs) + object.__setattr__(self, "reason_codes", tuple(dict.fromkeys(self.reason_codes))) + + @property + def human_review_eligible(self) -> bool: + return self.outcome is HoldoutSystemOutcome.REVIEWABLE_PAIR + + +@dataclass(frozen=True, slots=True) +class CurationCalibrationPolicyV3: + policy_id: str = "curation-preference-calibration-v3" + policy_version: str = CURATION_CALIBRATION_VERSION + minimum_cases: int = 12 + minimum_reviewed_reviewable_case_fraction: float = 1.0 + minimum_exact_agreement_rate: float = 0.65 + minimum_decisive_agreement_rate: float = 0.70 + minimum_confidence_weighted_decisive_agreement: float = 0.70 + maximum_false_winner_on_human_tie_rate: float = 0.25 + minimum_meaningful_alternative_availability_rate: float = 0.70 + minimum_independent_reviewers_for_multi_dj: int = 3 + maximum_assignment_imbalance: int = 1 + required_set_roles: tuple[CuratedSetRole, ...] = tuple(CuratedSetRole) + activation_authorized: bool = False + personal_dj_model_training_authorized: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "policy_id", _non_empty(self.policy_id, "policy_id")) + object.__setattr__(self, "policy_version", _non_empty(self.policy_version, "policy_version")) + if self.minimum_cases <= 0 or self.minimum_independent_reviewers_for_multi_dj <= 0: + raise ValueError("minimum case/reviewer counts must be positive") + if self.maximum_assignment_imbalance < 0: + raise ValueError("maximum_assignment_imbalance must be non-negative") + for field_name in ( + "minimum_reviewed_reviewable_case_fraction", + "minimum_exact_agreement_rate", + "minimum_decisive_agreement_rate", + "minimum_confidence_weighted_decisive_agreement", + "maximum_false_winner_on_human_tie_rate", + "minimum_meaningful_alternative_availability_rate", + ): + object.__setattr__(self, field_name, _unit(getattr(self, field_name), field_name)) + roles = tuple(self.required_set_roles) + if not roles or len(set(roles)) != len(roles): + raise ValueError("required_set_roles must be non-empty and unique") + object.__setattr__(self, "required_set_roles", roles) + if self.activation_authorized: + raise ValueError("V3 calibration cannot authorize optimizer activation") + if self.personal_dj_model_training_authorized: + raise ValueError("V3 calibration cannot authorize Personal DJ Model training") + + +@dataclass(frozen=True, slots=True) +class CurationCasePreferenceCalibration: + case_id: str + set_role: CuratedSetRole + review_id: str + reviewer_ref: str + assignment_id: str + human_preference: ResolvedPreference + challenger_preference: ResolvedPreference + confidence: float + exact_agreement: bool | None + decisive_agreement: bool | None + reason_codes: tuple[str, ...] = () + activation_authorized: bool = False + personal_dj_model_training_authorized: bool = False + + def __post_init__(self) -> None: + for field_name in ("case_id", "review_id", "reviewer_ref", "assignment_id"): + object.__setattr__(self, field_name, _non_empty(getattr(self, field_name), field_name)) + object.__setattr__(self, "confidence", _unit(self.confidence, "confidence")) + object.__setattr__(self, "reason_codes", tuple(dict.fromkeys(self.reason_codes))) + if self.activation_authorized: + raise ValueError("case calibration cannot authorize optimizer activation") + if self.personal_dj_model_training_authorized: + raise ValueError("case calibration cannot authorize Personal DJ Model training") + + +@dataclass(frozen=True, slots=True) +class CurationCalibrationReportV3: + report_id: str + policy_ref: tuple[str, str] + evidence_role: EvidenceRole + evaluation_scope: EvaluationScope + selection_scope: SelectionScope | None + independent_validation: bool + representative_performance_claim_allowed: bool + selected_case_count: int + reviewable_pair_count: int + non_reviewable_system_outcome_count: int + human_reviewed_case_count: int + reviewer_count: int + reviewable_pair_fraction: float + meaningful_alternative_availability_rate: float + exact_agreement_rate: float | None + decisive_agreement_rate: float | None + confidence_weighted_decisive_agreement: float | None + false_winner_on_human_tie_rate: float | None + reviewer_disagreement_rate: float | None + macro_reviewer_agreement_rate: float | None + pooled_agreement_rate: float | None + plan_a_greedy_count: int + plan_b_greedy_count: int + plan_a_beam_count: int + plan_b_beam_count: int + covered_set_roles: tuple[CuratedSetRole, ...] + missing_set_roles: tuple[CuratedSetRole, ...] + outcome_counts: tuple[tuple[HoldoutSystemOutcome, int], ...] + case_evidence: tuple[CurationCasePreferenceCalibration, ...] + verdict: CalibrationVerdict + explanation_codes: tuple[str, ...] = () + activation_authorized: bool = False + personal_dj_model_training_authorized: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "report_id", _non_empty(self.report_id, "report_id")) + object.__setattr__(self, "policy_ref", _ref(self.policy_ref, "policy_ref")) + for field_name in ( + "selected_case_count", + "reviewable_pair_count", + "non_reviewable_system_outcome_count", + "human_reviewed_case_count", + "reviewer_count", + "plan_a_greedy_count", + "plan_b_greedy_count", + "plan_a_beam_count", + "plan_b_beam_count", + ): + value = int(getattr(self, field_name)) + if value < 0: + raise ValueError(f"{field_name} must be non-negative") + object.__setattr__(self, field_name, value) + if self.reviewable_pair_count + self.non_reviewable_system_outcome_count != self.selected_case_count: + raise ValueError("system outcome counts must cover the selected-case denominator exactly") + for field_name in ( + "reviewable_pair_fraction", + "meaningful_alternative_availability_rate", + "exact_agreement_rate", + "decisive_agreement_rate", + "confidence_weighted_decisive_agreement", + "false_winner_on_human_tie_rate", + "reviewer_disagreement_rate", + "macro_reviewer_agreement_rate", + "pooled_agreement_rate", + ): + object.__setattr__(self, field_name, _unit(getattr(self, field_name), field_name)) + if self.evidence_role is EvidenceRole.DEVELOPMENT_CALIBRATION and self.independent_validation: + raise ValueError("development calibration cannot claim independent validation") + if self.selection_scope is SelectionScope.DIAGNOSTIC_CHALLENGE_SET and self.representative_performance_claim_allowed: + raise ValueError("diagnostic challenge sets cannot make representative performance claims") + object.__setattr__(self, "covered_set_roles", tuple(self.covered_set_roles)) + object.__setattr__(self, "missing_set_roles", tuple(self.missing_set_roles)) + object.__setattr__(self, "outcome_counts", tuple(self.outcome_counts)) + object.__setattr__(self, "case_evidence", tuple(self.case_evidence)) + object.__setattr__(self, "explanation_codes", tuple(dict.fromkeys(self.explanation_codes))) + if self.activation_authorized: + raise ValueError("calibration report cannot authorize optimizer activation") + if self.personal_dj_model_training_authorized: + raise ValueError("calibration report cannot authorize Personal DJ Model training") + + +__all__ = [ + "CURATION_AUDITION_MODE", + "CURATION_CALIBRATION_VERSION", + "CURATION_REVIEW_PROTOCOL_VERSION", + "HOLDOUT_VALIDATION_MANIFEST_VERSION", + "REQUIRED_CURATION_REVIEW_DIMENSIONS_V2", + "CurationBlindAssignmentV2", + "CurationCalibrationPolicyV3", + "CurationCalibrationReportV3", + "CurationCasePreferenceCalibration", + "CurationDJReviewV2", + "CurationDimensionPairRating", + "CurationPreference", + "CurationReviewDimension", + "EvaluationScope", + "EvidenceRole", + "HoldoutCaseExecutionEvidence", + "HoldoutSystemOutcome", + "HoldoutValidationManifest", + "SelectionScope", +] diff --git a/docs/bundles/BUNDLE_69_CURATION_REVIEW_PROTOCOL_V2.md b/docs/bundles/BUNDLE_69_CURATION_REVIEW_PROTOCOL_V2.md new file mode 100644 index 00000000..695faaac --- /dev/null +++ b/docs/bundles/BUNDLE_69_CURATION_REVIEW_PROTOCOL_V2.md @@ -0,0 +1,197 @@ +# Bundle 69 — Curation Review Protocol V2 + Holdout-Safe Calibration + +## Why this bundle exists + +Skill Tester found that the legacy six-dimension Human DJ Review mixed two different experimental questions: + +1. is this a good **set sequence / curation plan**? +2. did the DJ happen to perform a **smooth transition** this time? + +The legacy reviewer workspace showed Plan A/B track ordering while requiring `transition_smoothness` and `phrase_alignment`, but did not bind a standardized mix recipe or rendered transition. Bundle 68 then used the single overall human preference as calibration truth for the Bundle 67 curation challenger. + +That creates execution contamination. + +The same original 12 cases also influenced Bundle 65/67 design, so they are development evidence rather than independent validation. + +## Core rule + +`CURATION_REVIEW != TRANSITION_FEASIBILITY_REVIEW != HUMAN_EXECUTION_REVIEW` + +Bundle 69 implements the curation side only. + +## Curation Review V2 + +New protocol: + +- `protocol_version = curation-review-v2` +- `audition_mode = sequence_curation_only` +- explicit execution-quality exclusion acknowledgement +- separate V2 packet/submission schema and fingerprint +- legacy Bundle 63 submission schema rejected + +Required dimensions, Plan A and B scored 1..5: + +1. `energy_flow` +2. `dramaturgical_fit` +3. `set_coherence` +4. `alternative_usefulness` +5. `track_selection_fit` + +Case outcome: + +- `plan_a` +- `plan_b` +- `tie` +- `abstain` + +Human preference is explicit and is never derived from dimension sums. + +`transition_smoothness` and `phrase_alignment` are not Curation Review V2 dimensions. + +## Transition evidence + +Transition feasibility remains a separate future evidence stream. + +Until APPLAYLIST supplies a deterministic transition proposal or immutable standardized preview, transition review should be represented as not assessable rather than a forced numeric score. + +No transition or execution data enter the V3 curation calibration API. + +## Evidence roles + +### development_calibration + +The existing R2 12-case set is permanently treated as development/calibration evidence because its qualitative failures directly influenced Bundle 65/67. + +It may support personal-DJ calibration and debugging, but not independent validation or a general superiority claim. + +### holdout_validation + +A fresh holdout requires a frozen manifest before human labels exist. + +The manifest binds: + +- selected case IDs and scenario fingerprints +- source snapshot +- case-selection policy and seed commitment +- source optimizer SHA +- challenger SHA/policy/config digest +- calibration policy digest +- source evidence revisions +- set-role coverage +- label-unavailable-at-freeze assertion + +Changing candidate/config/threshold identity after labels invalidates the binding for the changed candidate. + +## Selection scope + +### representative_holdout + +Selection cannot depend on human labels, challenger score, challenger preference, or favorable source/challenger disagreement. + +This is the only scope that may set `representative_performance_claim_allowed=true`. + +### diagnostic_challenge_set + +May intentionally select hard/disagreement/failure cases. + +It is diagnostic only and cannot be represented as representative performance. + +## Survivorship protection + +Once a representative holdout is frozen, every selected case stays in the denominator. + +System outcomes: + +- `reviewable_pair` +- `technically_identical_pair` +- `no_meaningful_alternative` +- `missing_required_evidence` +- `source_generation_failed` +- `challenger_not_proven` + +Only `reviewable_pair` may receive a human A/B review. + +A non-reviewable system outcome must not be rewritten as human `abstain`. + +The report exposes: + +- selected case count +- reviewable pair count +- non-reviewable system outcome count +- human reviewed case count +- reviewable pair fraction +- meaningful alternative availability rate +- per-outcome counts + +Poor complete product evidence yields negative evidence, not hidden case deletion. + +## Personal vs multi-DJ scope + +### personal_dj_calibration + +One genuine reviewer may calibrate the challenger against that DJ's preferences. + +This is not a market-wide claim. + +### multi_dj_product_evaluation + +Requires policy-defined independent reviewer coverage; R1 defaults to three reviewer identities before this scope can be complete. + +A/B placement must be counterbalanced and reviewer-specific. Reviewer disagreement remains visible through pooled, macro/per-reviewer and disagreement metrics. + +## Bounded verdicts + +Bundle 69 reuses the bounded calibration vocabulary: + +- `INCOMPLETE` +- `DOES_NOT_SUPPORT_ACTIVATION` +- `SUPPORTS_FURTHER_EVALUATION` + +For development evidence, `SUPPORTS_FURTHER_EVALUATION` means only that a fresh holdout is justified. + +It does not mean validated superiority and never authorizes optimizer activation. + +## Skill Tester attack history + +The design was attacked before implementation for: + +1. transition execution contamination; +2. old/new packet schema ambiguity; +3. personal vs general reviewer scope confusion; +4. underspecified alternative usefulness; +5. missing structural contamination invariant; +6. development/holdout circular validation; +7. challenger-dependent holdout selection bias; +8. A/B slot position bias; +9. survivorship bias from dropping weak/unreviewable system outcomes; +10. hidden multi-DJ disagreement. + +Protocol V2.4 received `PASS_FOR_IMPLEMENTATION_DESIGN` before this implementation branch was opened. + +## Non-claims + +- `independent_validation=true` means independent holdout-case validation under this protocol, not independent laboratory replication. +- three reviewers is a bounded minimum for product-evaluation scope, not universal statistical proof. +- no p-value, significance, or market-wide accuracy claim is emitted. +- personal-DJ calibration remains personal. + +## Privacy / security + +The new protocol/calibration services are pure over supplied evidence: + +- no audio reads +- no MIR/provider execution +- no filesystem persistence +- no network/cloud upload +- no hidden telemetry +- no TransitionAssessment mutation +- no optimizer/ranking mutation + +## Authority + +- `MERGE_AUTHORIZATION=NO` +- `OPTIMIZER_RANKING_ACTIVATION=NO` +- `RELEASE_AUTHORIZATION=NO` +- `DEPLOY_AUTHORIZATION=NO` +- `PRODUCTION_ACTIVATION=NO` +- `PDM_TRAINING=NO` diff --git a/services/intelligence/curation_holdout_guard_v1.py b/services/intelligence/curation_holdout_guard_v1.py new file mode 100644 index 00000000..42ec5cb0 --- /dev/null +++ b/services/intelligence/curation_holdout_guard_v1.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Sequence + +from core.intelligence.curated_real_library_review_contract import CuratedReviewCase +from core.intelligence.curation_holdout_guard_contract import ( + HOLDOUT_GUARD_VERSION, + CurationAssignmentBatchManifest, + DevelopmentEvidenceExclusionRegistry, + HoldoutSelectionBasis, +) +from core.intelligence.curation_review_v2_contract import ( + CurationBlindAssignmentV2, + HoldoutValidationManifest, +) + + +class CurationHoldoutGuardError(ValueError): + """Fail-closed holdout lineage / assignment verification error.""" + + +def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def assignment_seed_commitment(private_seed: str) -> str: + normalized = str(private_seed).strip() + if not normalized: + raise CurationHoldoutGuardError("assignment private seed must not be empty") + return _sha256_text(f"applaylist-assignment-seed-r1|{normalized}") + + +def _assignment_fingerprint( + *, + private_seed: str, + reviewer_ref: str, + case_id: str, + slot_a_plan_id: str, + slot_b_plan_id: str, +) -> str: + material = "|".join( + ( + "applaylist-curation-assignment-r1", + private_seed, + reviewer_ref, + case_id, + slot_a_plan_id, + slot_b_plan_id, + ) + ) + return _sha256_text(material) + + +def build_counterbalanced_assignment_batch( + *, + cases: Sequence[CuratedReviewCase], + reviewer_refs: Sequence[str], + private_seed: str, + generated_at: str, +) -> tuple[tuple[CurationBlindAssignmentV2, ...], CurationAssignmentBatchManifest]: + source_cases = tuple(cases) + reviewers = tuple(sorted({str(item).strip() for item in reviewer_refs if str(item).strip()})) + if not source_cases: + raise CurationHoldoutGuardError("assignment batch requires source cases") + if len({item.case_id for item in source_cases}) != len(source_cases): + raise CurationHoldoutGuardError("assignment source case identities must be unique") + if not reviewers: + raise CurationHoldoutGuardError("assignment batch requires reviewer identities") + normalized_seed = str(private_seed).strip() + if not normalized_seed: + raise CurationHoldoutGuardError("assignment private seed must not be empty") + + # Freeze one deterministic case order from the private seed. The alternating + # placement plus reviewer-specific parity offset guarantees as-even-as-possible + # A/B allocation for each reviewer and across the batch while keeping mappings + # unpredictable without the private seed. + ordered_cases = tuple( + sorted( + source_cases, + key=lambda case: _sha256_text( + f"applaylist-case-order-r1|{normalized_seed}|{case.case_id}" + ), + ) + ) + + assignments: list[CurationBlindAssignmentV2] = [] + fingerprints: list[str] = [] + for reviewer_index, reviewer_ref in enumerate(reviewers): + reviewer_offset = int( + _sha256_text( + f"applaylist-reviewer-offset-r1|{normalized_seed}|{reviewer_ref}" + ), + 16, + ) % 2 + # Alternate reviewer parity as a second counterbalance axis. + reviewer_offset ^= reviewer_index % 2 + for case_index, case in enumerate(ordered_cases): + greedy_in_a = (case_index + reviewer_offset) % 2 == 0 + slot_a = case.greedy_plan.plan_id if greedy_in_a else case.beam_plan.plan_id + slot_b = case.beam_plan.plan_id if greedy_in_a else case.greedy_plan.plan_id + fingerprint = _assignment_fingerprint( + private_seed=normalized_seed, + reviewer_ref=reviewer_ref, + case_id=case.case_id, + slot_a_plan_id=slot_a, + slot_b_plan_id=slot_b, + ) + assignment_id = f"curation-assignment:{fingerprint[:32]}" + assignments.append( + CurationBlindAssignmentV2( + assignment_id=assignment_id, + case_id=case.case_id, + reviewer_ref=reviewer_ref, + slot_a_plan_id=slot_a, + slot_b_plan_id=slot_b, + assignment_fingerprint=fingerprint, + algorithm_identity_hidden=True, + ) + ) + fingerprints.append(fingerprint) + + commitment = assignment_seed_commitment(normalized_seed) + manifest_material = "|".join( + ( + commitment, + *reviewers, + *(case.case_id for case in ordered_cases), + *sorted(fingerprints), + ) + ) + manifest = CurationAssignmentBatchManifest( + batch_id=f"curation-assignment-batch:{_sha256_text(manifest_material)}", + batch_version=HOLDOUT_GUARD_VERSION, + assignment_seed_commitment=commitment, + reviewer_refs=reviewers, + case_ids=tuple(case.case_id for case in ordered_cases), + assignment_fingerprints=tuple(sorted(fingerprints)), + generated_at=str(generated_at).strip(), + algorithm_identity_hidden=True, + activation_authorized=False, + personal_dj_model_training_authorized=False, + ) + return tuple(assignments), manifest + + +def validate_assignment_batch( + *, + cases: Sequence[CuratedReviewCase], + assignments: Sequence[CurationBlindAssignmentV2], + manifest: CurationAssignmentBatchManifest, + private_seed: str, +) -> None: + source_cases = tuple(cases) + actual_assignments = tuple(assignments) + expected_assignments, expected_manifest = build_counterbalanced_assignment_batch( + cases=source_cases, + reviewer_refs=manifest.reviewer_refs, + private_seed=private_seed, + generated_at=manifest.generated_at, + ) + if manifest.assignment_seed_commitment != expected_manifest.assignment_seed_commitment: + raise CurationHoldoutGuardError("assignment seed commitment mismatch") + if manifest.case_ids != expected_manifest.case_ids: + raise CurationHoldoutGuardError("assignment batch case identities mismatch") + if manifest.assignment_fingerprints != expected_manifest.assignment_fingerprints: + raise CurationHoldoutGuardError("assignment batch fingerprints mismatch") + + expected_by_key = { + (item.reviewer_ref, item.case_id): item for item in expected_assignments + } + actual_by_key: dict[tuple[str, str], CurationBlindAssignmentV2] = {} + for item in actual_assignments: + key = (item.reviewer_ref, item.case_id) + if key in actual_by_key: + raise CurationHoldoutGuardError("duplicate reviewer/case assignment") + actual_by_key[key] = item + if set(actual_by_key) != set(expected_by_key): + raise CurationHoldoutGuardError("assignment batch does not cover reviewer x case exactly") + for key, expected in expected_by_key.items(): + actual = actual_by_key[key] + if ( + actual.assignment_id != expected.assignment_id + or actual.slot_a_plan_id != expected.slot_a_plan_id + or actual.slot_b_plan_id != expected.slot_b_plan_id + or actual.assignment_fingerprint != expected.assignment_fingerprint + ): + raise CurationHoldoutGuardError("reviewer-specific assignment derivation mismatch") + + +def validate_holdout_lineage( + *, + cases: Sequence[CuratedReviewCase], + holdout_manifest: HoldoutValidationManifest, + development_registry: DevelopmentEvidenceExclusionRegistry, + selection_basis: HoldoutSelectionBasis, +) -> None: + source_cases = tuple(cases) + if selection_basis.selection_scope is not holdout_manifest.selection_scope: + raise CurationHoldoutGuardError("holdout selection scope/basis mismatch") + + case_ids = {item.case_id for item in source_cases} + scenario_fingerprints = {item.scenario_fingerprint for item in source_cases} + development_case_ids = set(development_registry.case_ids) + development_scenarios = set(development_registry.scenario_fingerprints) + overlap_cases = case_ids & development_case_ids + overlap_scenarios = scenario_fingerprints & development_scenarios + if overlap_cases: + raise CurationHoldoutGuardError( + "holdout contains prior development case identity" + ) + if overlap_scenarios: + raise CurationHoldoutGuardError( + "holdout contains prior development scenario fingerprint" + ) + + +__all__ = [ + "CurationHoldoutGuardError", + "assignment_seed_commitment", + "build_counterbalanced_assignment_batch", + "validate_assignment_batch", + "validate_holdout_lineage", +] diff --git a/services/intelligence/curation_preference_calibration_v3.py b/services/intelligence/curation_preference_calibration_v3.py new file mode 100644 index 00000000..63d7d2bf --- /dev/null +++ b/services/intelligence/curation_preference_calibration_v3.py @@ -0,0 +1,589 @@ +from __future__ import annotations + +import hashlib +import json +from collections import Counter, defaultdict +from collections.abc import Mapping + +from core.intelligence.competitive_curation_contract import ShadowPathComparison, ShadowPathPreference +from core.intelligence.curated_real_library_review_contract import ( + CuratedReviewCase, + ReviewPlanStrategy, +) +from core.intelligence.curation_holdout_guard_contract import ( + CurationAssignmentBatchManifest, + DevelopmentEvidenceExclusionRegistry, + HoldoutSelectionBasis, +) +from core.intelligence.curation_review_v2_contract import ( + CurationBlindAssignmentV2, + CurationCalibrationPolicyV3, + CurationCalibrationReportV3, + CurationCasePreferenceCalibration, + CurationDJReviewV2, + CurationPreference, + EvaluationScope, + EvidenceRole, + HoldoutCaseExecutionEvidence, + HoldoutSystemOutcome, + HoldoutValidationManifest, + SelectionScope, +) +from core.intelligence.human_preference_calibration_contract import CalibrationVerdict, ResolvedPreference +from services.intelligence.curation_holdout_guard_v1 import ( + CurationHoldoutGuardError, + validate_assignment_batch, + validate_holdout_lineage, +) + + +class CurationPreferenceCalibrationV3Error(ValueError): + """Fail-closed error for curation-only calibration evidence.""" + + +def _stable_id(prefix: str, payload: object) -> str: + encoded = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + return f"{prefix}:{hashlib.sha256(encoded).hexdigest()}" + + +def _rate(numerator: int, denominator: int) -> float | None: + if denominator <= 0: + return None + return numerator / denominator + + +def _path_strategy_map(case: CuratedReviewCase) -> dict[str, ResolvedPreference]: + return { + case.greedy_plan.path_id: ResolvedPreference.GREEDY, + case.beam_plan.path_id: ResolvedPreference.BEAM, + } + + +def _plan_strategy_map(case: CuratedReviewCase) -> dict[str, ResolvedPreference]: + return { + case.greedy_plan.plan_id: ResolvedPreference.GREEDY, + case.beam_plan.plan_id: ResolvedPreference.BEAM, + } + + +def _resolve_human_preference( + *, + case: CuratedReviewCase, + assignment: CurationBlindAssignmentV2, + review: CurationDJReviewV2, +) -> ResolvedPreference: + if assignment.case_id != case.case_id: + raise CurationPreferenceCalibrationV3Error("assignment case_id mismatch") + if assignment.reviewer_ref != review.reviewer_ref: + raise CurationPreferenceCalibrationV3Error("assignment reviewer mismatch") + if assignment.assignment_id != review.assignment_id: + raise CurationPreferenceCalibrationV3Error("review assignment mismatch") + if not assignment.algorithm_identity_hidden or not review.algorithm_identity_was_hidden: + raise CurationPreferenceCalibrationV3Error("curation calibration requires blinded evidence") + if not review.execution_quality_excluded_from_curation_judgment: + raise CurationPreferenceCalibrationV3Error("execution-contaminated review is invalid") + + by_plan = _plan_strategy_map(case) + if {assignment.slot_a_plan_id, assignment.slot_b_plan_id} != set(by_plan): + raise CurationPreferenceCalibrationV3Error("assignment does not bind exactly to source plans") + + if review.preference is CurationPreference.PLAN_A: + return by_plan[assignment.slot_a_plan_id] + if review.preference is CurationPreference.PLAN_B: + return by_plan[assignment.slot_b_plan_id] + if review.preference is CurationPreference.TIE: + return ResolvedPreference.TIE + if review.preference is CurationPreference.ABSTAIN: + return ResolvedPreference.ABSTAIN + raise CurationPreferenceCalibrationV3Error("unsupported curation preference") + + +def _resolve_challenger_preference( + *, + case: CuratedReviewCase, + comparison: ShadowPathComparison, +) -> ResolvedPreference: + by_path = _path_strategy_map(case) + if {comparison.left_path_id, comparison.right_path_id} != set(by_path): + raise CurationPreferenceCalibrationV3Error("challenger comparison path binding mismatch") + if comparison.activation_authorized: + raise CurationPreferenceCalibrationV3Error("challenger comparison exceeds shadow authority") + if comparison.preference is ShadowPathPreference.LEFT: + return by_path[comparison.left_path_id] + if comparison.preference is ShadowPathPreference.RIGHT: + return by_path[comparison.right_path_id] + if comparison.preference is ShadowPathPreference.TIE: + return ResolvedPreference.TIE + if comparison.preference is ShadowPathPreference.NOT_PROVEN: + return ResolvedPreference.NOT_PROVEN + raise CurationPreferenceCalibrationV3Error("unsupported challenger preference") + + +def calibrate_curation_case_v3( + *, + case: CuratedReviewCase, + assignment: CurationBlindAssignmentV2, + review: CurationDJReviewV2, + comparison: ShadowPathComparison, +) -> CurationCasePreferenceCalibration: + human = _resolve_human_preference(case=case, assignment=assignment, review=review) + challenger = _resolve_challenger_preference(case=case, comparison=comparison) + + reasons: list[str] = [] + exact: bool | None + decisive: bool | None + if human is ResolvedPreference.ABSTAIN: + exact = None + decisive = None + reasons.append("curation_human_abstain_excluded_from_accuracy") + else: + exact = human is challenger + reasons.append( + "curation_challenger_exactly_agrees_with_human" + if exact + else "curation_challenger_disagrees_with_human" + ) + if human in (ResolvedPreference.GREEDY, ResolvedPreference.BEAM): + decisive = human is challenger + if challenger is ResolvedPreference.TIE: + reasons.append("curation_challenger_tie_on_human_decisive") + elif challenger is ResolvedPreference.NOT_PROVEN: + reasons.append("curation_challenger_not_proven_on_human_decisive") + else: + decisive = None + if human is ResolvedPreference.TIE and challenger in ( + ResolvedPreference.GREEDY, + ResolvedPreference.BEAM, + ): + reasons.append("curation_challenger_false_winner_on_human_tie") + if challenger is ResolvedPreference.NOT_PROVEN: + reasons.append("curation_challenger_preference_not_proven") + + return CurationCasePreferenceCalibration( + case_id=case.case_id, + set_role=case.set_role, + review_id=review.review_id, + reviewer_ref=review.reviewer_ref, + assignment_id=assignment.assignment_id, + human_preference=human, + challenger_preference=challenger, + confidence=review.confidence, + exact_agreement=exact, + decisive_agreement=decisive, + reason_codes=tuple(reasons), + activation_authorized=False, + personal_dj_model_training_authorized=False, + ) + + +def _validate_holdout_binding( + *, + cases: tuple[CuratedReviewCase, ...], + manifest: HoldoutValidationManifest, + expected_source_optimizer_sha: str, + expected_challenger_sha: str, + expected_challenger_config_digest: str, + expected_calibration_policy_digest: str, +) -> None: + case_ids = tuple(item.case_id for item in cases) + scenario_fingerprints = tuple(item.scenario_fingerprint for item in cases) + if case_ids != manifest.selected_case_ids: + raise CurationPreferenceCalibrationV3Error("holdout selected case identities changed after freeze") + if scenario_fingerprints != manifest.scenario_fingerprints: + raise CurationPreferenceCalibrationV3Error("holdout scenario fingerprints changed after freeze") + if expected_source_optimizer_sha != manifest.source_optimizer_sha: + raise CurationPreferenceCalibrationV3Error("source optimizer revision does not match holdout freeze") + if expected_challenger_sha != manifest.challenger_sha: + raise CurationPreferenceCalibrationV3Error("challenger revision does not match holdout freeze") + if expected_challenger_config_digest != manifest.challenger_config_digest: + raise CurationPreferenceCalibrationV3Error("challenger configuration changed after holdout freeze") + if expected_calibration_policy_digest != manifest.calibration_policy_digest: + raise CurationPreferenceCalibrationV3Error("calibration policy changed after holdout freeze") + + +def _assignment_position_counts( + *, + cases_by_id: Mapping[str, CuratedReviewCase], + assignments: tuple[CurationBlindAssignmentV2, ...], +) -> tuple[int, int, int, int]: + a_greedy = 0 + b_greedy = 0 + a_beam = 0 + b_beam = 0 + for assignment in assignments: + case = cases_by_id.get(assignment.case_id) + if case is None: + raise CurationPreferenceCalibrationV3Error("assignment references unknown case") + by_plan = { + case.greedy_plan.plan_id: ReviewPlanStrategy.GREEDY_RECOMMEND_NEXT, + case.beam_plan.plan_id: ReviewPlanStrategy.BOUNDED_BEAM, + } + if {assignment.slot_a_plan_id, assignment.slot_b_plan_id} != set(by_plan): + raise CurationPreferenceCalibrationV3Error("assignment plan identity mismatch") + if by_plan[assignment.slot_a_plan_id] is ReviewPlanStrategy.GREEDY_RECOMMEND_NEXT: + a_greedy += 1 + b_beam += 1 + else: + a_beam += 1 + b_greedy += 1 + return a_greedy, b_greedy, a_beam, b_beam + + +def build_curation_calibration_report_v3( + *, + all_cases: tuple[CuratedReviewCase, ...], + assignments: tuple[CurationBlindAssignmentV2, ...], + reviews: tuple[CurationDJReviewV2, ...], + comparisons_by_case: Mapping[str, ShadowPathComparison], + system_outcomes: tuple[HoldoutCaseExecutionEvidence, ...], + evidence_role: EvidenceRole, + evaluation_scope: EvaluationScope, + policy: CurationCalibrationPolicyV3 = CurationCalibrationPolicyV3(), + holdout_manifest: HoldoutValidationManifest | None = None, + development_exclusion_registry: DevelopmentEvidenceExclusionRegistry | None = None, + selection_basis: HoldoutSelectionBasis | None = None, + assignment_batch_manifest: CurationAssignmentBatchManifest | None = None, + assignment_private_seed: str | None = None, + expected_source_optimizer_sha: str = "development", + expected_challenger_sha: str = "development", + expected_challenger_config_digest: str = "development", + expected_calibration_policy_digest: str = "development", +) -> CurationCalibrationReportV3: + cases = tuple(all_cases) + if not cases: + raise CurationPreferenceCalibrationV3Error("curation calibration requires source cases") + case_ids = tuple(item.case_id for item in cases) + if len(set(case_ids)) != len(case_ids): + raise CurationPreferenceCalibrationV3Error("source case identities must be unique") + cases_by_id = {item.case_id: item for item in cases} + + holdout_guard_values = ( + development_exclusion_registry, + selection_basis, + assignment_batch_manifest, + assignment_private_seed, + ) + if evidence_role is EvidenceRole.HOLDOUT_VALIDATION: + if holdout_manifest is None: + raise CurationPreferenceCalibrationV3Error("holdout validation requires frozen manifest") + if any(item is None for item in holdout_guard_values): + raise CurationPreferenceCalibrationV3Error( + "holdout validation requires development exclusion, selection basis, and assignment proof" + ) + assert development_exclusion_registry is not None + assert selection_basis is not None + assert assignment_batch_manifest is not None + assert assignment_private_seed is not None + if holdout_manifest.case_selection_policy_ref != ( + selection_basis.basis_id, + selection_basis.basis_version, + ): + raise CurationPreferenceCalibrationV3Error( + "holdout selection basis does not match frozen selection policy ref" + ) + _validate_holdout_binding( + cases=cases, + manifest=holdout_manifest, + expected_source_optimizer_sha=expected_source_optimizer_sha, + expected_challenger_sha=expected_challenger_sha, + expected_challenger_config_digest=expected_challenger_config_digest, + expected_calibration_policy_digest=expected_calibration_policy_digest, + ) + try: + validate_holdout_lineage( + cases=cases, + holdout_manifest=holdout_manifest, + development_registry=development_exclusion_registry, + selection_basis=selection_basis, + ) + validate_assignment_batch( + cases=cases, + assignments=assignments, + manifest=assignment_batch_manifest, + private_seed=assignment_private_seed, + ) + except CurationHoldoutGuardError as exc: + raise CurationPreferenceCalibrationV3Error(str(exc)) from exc + selection_scope: SelectionScope | None = holdout_manifest.selection_scope + independent_validation = True + representative_allowed = selection_scope is SelectionScope.REPRESENTATIVE_HOLDOUT + else: + if holdout_manifest is not None: + raise CurationPreferenceCalibrationV3Error("development evidence cannot bind a holdout manifest") + if any(item is not None for item in holdout_guard_values): + raise CurationPreferenceCalibrationV3Error( + "development evidence cannot bind holdout-only guard artifacts" + ) + selection_scope = None + independent_validation = False + representative_allowed = False + + outcomes_by_case = {item.case_id: item for item in system_outcomes} + if len(outcomes_by_case) != len(system_outcomes) or set(outcomes_by_case) != set(case_ids): + raise CurationPreferenceCalibrationV3Error("system outcomes must cover selected cases exactly once") + for case in cases: + outcome = outcomes_by_case[case.case_id] + if outcome.set_role is not case.set_role or outcome.scenario_fingerprint != case.scenario_fingerprint: + raise CurationPreferenceCalibrationV3Error("system outcome source binding mismatch") + + assignments_by_id: dict[str, CurationBlindAssignmentV2] = {} + for assignment in assignments: + if assignment.assignment_id in assignments_by_id: + raise CurationPreferenceCalibrationV3Error("duplicate curation assignment identity") + assignments_by_id[assignment.assignment_id] = assignment + + review_ids: set[str] = set() + calibration_evidence: list[CurationCasePreferenceCalibration] = [] + reviewed_case_ids: set[str] = set() + reviewers: set[str] = set() + reviews_by_case: dict[str, list[CurationDJReviewV2]] = defaultdict(list) + for review in reviews: + if review.review_id in review_ids: + raise CurationPreferenceCalibrationV3Error("duplicate curation review identity") + review_ids.add(review.review_id) + if review.evidence_role is not evidence_role: + raise CurationPreferenceCalibrationV3Error("review evidence role mismatch") + assignment = assignments_by_id.get(review.assignment_id) + if assignment is None: + raise CurationPreferenceCalibrationV3Error("review references unknown assignment") + case = cases_by_id.get(assignment.case_id) + if case is None: + raise CurationPreferenceCalibrationV3Error("assignment references unknown case") + outcome = outcomes_by_case[case.case_id] + if not outcome.human_review_eligible: + raise CurationPreferenceCalibrationV3Error( + "human review cannot replace a non-reviewable machine/system outcome" + ) + comparison = comparisons_by_case.get(case.case_id) + if comparison is None: + raise CurationPreferenceCalibrationV3Error("reviewable case requires challenger comparison") + calibration_evidence.append( + calibrate_curation_case_v3( + case=case, + assignment=assignment, + review=review, + comparison=comparison, + ) + ) + reviewed_case_ids.add(case.case_id) + reviewers.add(review.reviewer_ref) + reviews_by_case[case.case_id].append(review) + + reviewable_case_ids = { + case_id + for case_id, outcome in outcomes_by_case.items() + if outcome.outcome is HoldoutSystemOutcome.REVIEWABLE_PAIR + } + if not reviewed_case_ids <= reviewable_case_ids: + raise CurationPreferenceCalibrationV3Error("reviewed case set exceeds reviewable case set") + + reviewable_pair_count = len(reviewable_case_ids) + selected_count = len(cases) + non_reviewable_count = selected_count - reviewable_pair_count + reviewable_fraction = reviewable_pair_count / selected_count + meaningful_availability = reviewable_fraction + reviewed_reviewable_fraction = ( + 1.0 if reviewable_pair_count == 0 else len(reviewed_case_ids) / reviewable_pair_count + ) + + non_abstain = [ + item for item in calibration_evidence if item.human_preference is not ResolvedPreference.ABSTAIN + ] + decisive = [ + item + for item in calibration_evidence + if item.human_preference in (ResolvedPreference.GREEDY, ResolvedPreference.BEAM) + ] + human_ties = [item for item in calibration_evidence if item.human_preference is ResolvedPreference.TIE] + exact_count = sum(item.exact_agreement is True for item in non_abstain) + decisive_count = sum(item.decisive_agreement is True for item in decisive) + false_winner_count = sum( + item.human_preference is ResolvedPreference.TIE + and item.challenger_preference in (ResolvedPreference.GREEDY, ResolvedPreference.BEAM) + for item in calibration_evidence + ) + exact_rate = _rate(exact_count, len(non_abstain)) + decisive_rate = _rate(decisive_count, len(decisive)) + false_winner_rate = _rate(false_winner_count, len(human_ties)) + confidence_denominator = sum(item.confidence for item in decisive) + confidence_weighted = ( + None + if confidence_denominator <= 0.0 + else sum(item.confidence for item in decisive if item.decisive_agreement is True) + / confidence_denominator + ) + + reviewer_rates: list[float] = [] + for reviewer in sorted(reviewers): + items = [ + item + for item in calibration_evidence + if item.reviewer_ref == reviewer and item.human_preference is not ResolvedPreference.ABSTAIN + ] + if items: + reviewer_rates.append(sum(item.exact_agreement is True for item in items) / len(items)) + macro_agreement = None if not reviewer_rates else sum(reviewer_rates) / len(reviewer_rates) + + disagreement_cases = 0 + multi_review_cases = 0 + for case_reviews in reviews_by_case.values(): + preferences = [ + review.preference + for review in case_reviews + if review.preference is not CurationPreference.ABSTAIN + ] + if len(preferences) >= 2: + multi_review_cases += 1 + if len(set(preferences)) > 1: + disagreement_cases += 1 + disagreement_rate = _rate(disagreement_cases, multi_review_cases) + + a_greedy, b_greedy, a_beam, b_beam = _assignment_position_counts( + cases_by_id=cases_by_id, + assignments=assignments, + ) + assignment_imbalance = abs(a_greedy - b_greedy) + + covered_roles = tuple(sorted({item.set_role for item in cases}, key=lambda item: item.value)) + missing_roles = tuple(role for role in policy.required_set_roles if role not in covered_roles) + outcome_counter = Counter(item.outcome for item in system_outcomes) + outcome_counts = tuple( + (outcome, outcome_counter.get(outcome, 0)) for outcome in HoldoutSystemOutcome + ) + + explanations: list[str] = [] + if evidence_role is EvidenceRole.DEVELOPMENT_CALIBRATION: + explanations.append("development_evidence_not_independent_validation") + + incomplete = False + if selected_count < policy.minimum_cases: + incomplete = True + explanations.append("curation_calibration_case_count_below_policy") + if missing_roles: + incomplete = True + explanations.append("curation_calibration_required_set_roles_missing") + if reviewed_reviewable_fraction < policy.minimum_reviewed_reviewable_case_fraction: + incomplete = True + explanations.append("curation_calibration_reviewable_case_coverage_below_policy") + if assignment_imbalance > policy.maximum_assignment_imbalance: + incomplete = True + explanations.append("curation_calibration_assignment_position_imbalance") + if evaluation_scope is EvaluationScope.MULTI_DJ_PRODUCT_EVALUATION: + if len(reviewers) < policy.minimum_independent_reviewers_for_multi_dj: + incomplete = True + explanations.append("curation_calibration_independent_reviewer_count_below_policy") + + if incomplete: + verdict = CalibrationVerdict.INCOMPLETE + else: + failed = False + if meaningful_availability < policy.minimum_meaningful_alternative_availability_rate: + failed = True + explanations.append("curation_meaningful_alternative_availability_below_policy") + if reviewable_pair_count > 0: + if exact_rate is None or exact_rate < policy.minimum_exact_agreement_rate: + failed = True + explanations.append("curation_exact_agreement_below_policy") + if decisive_rate is None or decisive_rate < policy.minimum_decisive_agreement_rate: + failed = True + explanations.append("curation_decisive_agreement_below_policy") + if ( + confidence_weighted is None + or confidence_weighted < policy.minimum_confidence_weighted_decisive_agreement + ): + failed = True + explanations.append("curation_confidence_weighted_agreement_below_policy") + if ( + false_winner_rate is not None + and false_winner_rate > policy.maximum_false_winner_on_human_tie_rate + ): + failed = True + explanations.append("curation_false_winner_on_human_tie_above_policy") + verdict = ( + CalibrationVerdict.DOES_NOT_SUPPORT_ACTIVATION + if failed + else CalibrationVerdict.SUPPORTS_FURTHER_EVALUATION + ) + if not failed: + explanations.append("curation_calibration_supports_further_evaluation_only") + if evidence_role is EvidenceRole.DEVELOPMENT_CALIBRATION: + explanations.append("development_result_requires_fresh_holdout_validation") + elif selection_scope is SelectionScope.DIAGNOSTIC_CHALLENGE_SET: + explanations.append("diagnostic_result_not_representative_performance") + + payload = { + "policy": [policy.policy_id, policy.policy_version], + "evidence_role": evidence_role.value, + "evaluation_scope": evaluation_scope.value, + "selection_scope": None if selection_scope is None else selection_scope.value, + "cases": case_ids, + "outcomes": [(item.case_id, item.outcome.value) for item in system_outcomes], + "reviews": [ + ( + item.case_id, + item.review_id, + item.reviewer_ref, + item.human_preference.value, + item.challenger_preference.value, + item.confidence, + ) + for item in sorted( + calibration_evidence, + key=lambda value: (value.case_id, value.review_id), + ) + ], + } + + return CurationCalibrationReportV3( + report_id=_stable_id("curation-preference-calibration-v3", payload), + policy_ref=(policy.policy_id, policy.policy_version), + evidence_role=evidence_role, + evaluation_scope=evaluation_scope, + selection_scope=selection_scope, + independent_validation=independent_validation, + representative_performance_claim_allowed=representative_allowed, + selected_case_count=selected_count, + reviewable_pair_count=reviewable_pair_count, + non_reviewable_system_outcome_count=non_reviewable_count, + human_reviewed_case_count=len(reviewed_case_ids), + reviewer_count=len(reviewers), + reviewable_pair_fraction=reviewable_fraction, + meaningful_alternative_availability_rate=meaningful_availability, + exact_agreement_rate=exact_rate, + decisive_agreement_rate=decisive_rate, + confidence_weighted_decisive_agreement=confidence_weighted, + false_winner_on_human_tie_rate=false_winner_rate, + reviewer_disagreement_rate=disagreement_rate, + macro_reviewer_agreement_rate=macro_agreement, + pooled_agreement_rate=exact_rate, + plan_a_greedy_count=a_greedy, + plan_b_greedy_count=b_greedy, + plan_a_beam_count=a_beam, + plan_b_beam_count=b_beam, + covered_set_roles=covered_roles, + missing_set_roles=missing_roles, + outcome_counts=outcome_counts, + case_evidence=tuple( + sorted( + calibration_evidence, + key=lambda value: (value.case_id, value.review_id), + ) + ), + verdict=verdict, + explanation_codes=tuple(explanations), + activation_authorized=False, + personal_dj_model_training_authorized=False, + ) + + +__all__ = [ + "CurationPreferenceCalibrationV3Error", + "build_curation_calibration_report_v3", + "calibrate_curation_case_v3", +] diff --git a/services/intelligence/curation_review_execution_v2.py b/services/intelligence/curation_review_execution_v2.py new file mode 100644 index 00000000..9f47432e --- /dev/null +++ b/services/intelligence/curation_review_execution_v2.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from typing import Any + +from core.intelligence.curated_real_library_review_contract import CuratedSetRole +from core.intelligence.curation_review_v2_contract import ( + CURATION_AUDITION_MODE, + CURATION_REVIEW_PROTOCOL_VERSION, + REQUIRED_CURATION_REVIEW_DIMENSIONS_V2, + CurationDJReviewV2, + CurationDimensionPairRating, + CurationPreference, + CurationReviewDimension, + EvidenceRole, +) + +CURATION_REVIEW_PACKET_SCHEMA_V2 = "applaylist-curation-review-packet-v2" +CURATION_REVIEW_SUBMISSION_SCHEMA_V2 = "applaylist-curation-review-submission-v2" + +_ALLOWED_PREFERENCES = tuple(item.value for item in CurationPreference) +_REQUIRED_DIMENSIONS = tuple(item.value for item in REQUIRED_CURATION_REVIEW_DIMENSIONS_V2) + + +class CurationReviewExecutionV2Error(ValueError): + """Fail-closed error for curation-only human review evidence.""" + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _token(value: object, field: str, *, maximum: int = 512) -> str: + if not isinstance(value, str): + raise CurationReviewExecutionV2Error(f"{field} must be text") + normalized = value.strip() + if ( + not normalized + or normalized != value + or len(normalized) > maximum + or any(ord(character) < 32 for character in normalized) + ): + raise CurationReviewExecutionV2Error(f"{field} is invalid") + return normalized + + +def curation_packet_fingerprint(packet: Mapping[str, Any]) -> str: + material = dict(packet) + supplied = material.pop("curation_packet_fingerprint", None) + if supplied is not None and not isinstance(supplied, str): + raise CurationReviewExecutionV2Error("curation_packet_fingerprint must be text") + encoded = json.dumps( + material, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + return _sha256_bytes(encoded) + + +def validate_curation_review_packet_v2(packet: Mapping[str, Any]) -> dict[str, Any]: + if packet.get("schema") != CURATION_REVIEW_PACKET_SCHEMA_V2: + raise CurationReviewExecutionV2Error("unsupported curation review packet schema") + if packet.get("protocol_version") != CURATION_REVIEW_PROTOCOL_VERSION: + raise CurationReviewExecutionV2Error("unsupported curation review protocol version") + if packet.get("audition_mode") != CURATION_AUDITION_MODE: + raise CurationReviewExecutionV2Error("curation packet must use sequence_curation_only audition mode") + if packet.get("algorithm_identity_hidden") is not True: + raise CurationReviewExecutionV2Error("algorithm identity must remain hidden") + if packet.get("activation_authorized") is not False: + raise CurationReviewExecutionV2Error("curation packet cannot authorize activation") + if packet.get("personal_dj_model_training_authorized") is not False: + raise CurationReviewExecutionV2Error("curation packet cannot authorize Personal DJ Model training") + + try: + evidence_role = EvidenceRole(str(packet.get("evidence_role"))) + except ValueError as exc: + raise CurationReviewExecutionV2Error("unknown curation evidence role") from exc + + source_blinded_packet_fingerprint = _token( + packet.get("source_blinded_packet_fingerprint"), + "source_blinded_packet_fingerprint", + ) + reviewer_ref = _token(packet.get("reviewer_ref"), "reviewer_ref") + generated_at = _token(packet.get("generated_at"), "generated_at") + + snapshot_ref = packet.get("source_snapshot_ref") + if not isinstance(snapshot_ref, list) or len(snapshot_ref) != 2: + raise CurationReviewExecutionV2Error("source_snapshot_ref must contain exactly two values") + normalized_snapshot_ref = [ + _token(snapshot_ref[0], "source_snapshot_ref[0]"), + _token(snapshot_ref[1], "source_snapshot_ref[1]"), + ] + + cases = packet.get("cases") + if not isinstance(cases, list) or not cases: + raise CurationReviewExecutionV2Error("curation packet cases must be a non-empty array") + + case_ids: set[str] = set() + assignment_ids: set[str] = set() + normalized_cases: list[dict[str, Any]] = [] + for raw in cases: + if not isinstance(raw, dict): + raise CurationReviewExecutionV2Error("curation case must be an object") + allowed_keys = { + "case_id", + "set_role", + "assignment_id", + "plan_a", + "plan_b", + "required_review_dimensions", + "allowed_preference", + "execution_quality_exclusion_required", + } + if set(raw) != allowed_keys: + raise CurationReviewExecutionV2Error("curation case has unexpected fields") + case_id = _token(raw.get("case_id"), "case_id") + assignment_id = _token(raw.get("assignment_id"), "assignment_id") + if case_id in case_ids or assignment_id in assignment_ids: + raise CurationReviewExecutionV2Error("case/assignment identities must be unique") + case_ids.add(case_id) + assignment_ids.add(assignment_id) + try: + role = CuratedSetRole(str(raw.get("set_role"))).value + except ValueError as exc: + raise CurationReviewExecutionV2Error("unknown curated set role") from exc + + plans: dict[str, list[str]] = {} + for key in ("plan_a", "plan_b"): + value = raw.get(key) + if not isinstance(value, list) or not value: + raise CurationReviewExecutionV2Error(f"{key} must be a non-empty array") + plans[key] = [_token(item, f"{key} display name") for item in value] + + dimensions = raw.get("required_review_dimensions") + if not isinstance(dimensions, list) or tuple(dimensions) != _REQUIRED_DIMENSIONS: + raise CurationReviewExecutionV2Error("curation dimensions do not match V2 protocol") + preferences = raw.get("allowed_preference") + if not isinstance(preferences, list) or tuple(preferences) != _ALLOWED_PREFERENCES: + raise CurationReviewExecutionV2Error("curation preference choices do not match V2 protocol") + if raw.get("execution_quality_exclusion_required") is not True: + raise CurationReviewExecutionV2Error("curation case must require execution-quality exclusion") + + normalized_cases.append( + { + "case_id": case_id, + "set_role": role, + "assignment_id": assignment_id, + "plan_a": plans["plan_a"], + "plan_b": plans["plan_b"], + "required_review_dimensions": list(_REQUIRED_DIMENSIONS), + "allowed_preference": list(_ALLOWED_PREFERENCES), + "execution_quality_exclusion_required": True, + } + ) + + expected_fingerprint = curation_packet_fingerprint(packet) + if packet.get("curation_packet_fingerprint") != expected_fingerprint: + raise CurationReviewExecutionV2Error("curation packet fingerprint mismatch") + + return { + "schema": CURATION_REVIEW_PACKET_SCHEMA_V2, + "protocol_version": CURATION_REVIEW_PROTOCOL_VERSION, + "generated_at": generated_at, + "evidence_role": evidence_role.value, + "source_blinded_packet_fingerprint": source_blinded_packet_fingerprint, + "source_snapshot_ref": normalized_snapshot_ref, + "reviewer_ref": reviewer_ref, + "audition_mode": CURATION_AUDITION_MODE, + "algorithm_identity_hidden": True, + "cases": normalized_cases, + "activation_authorized": False, + "personal_dj_model_training_authorized": False, + "curation_packet_fingerprint": expected_fingerprint, + } + + +def _number(value: object, field: str, minimum: float, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CurationReviewExecutionV2Error(f"{field} must be numeric") + result = float(value) + if not minimum <= result <= maximum: + raise CurationReviewExecutionV2Error(f"{field} must be between {minimum} and {maximum}") + return result + + +def parse_curation_review_submission_v2( + *, + packet: Mapping[str, Any], + submission: Mapping[str, Any], +) -> tuple[CurationDJReviewV2, ...]: + validated_packet = validate_curation_review_packet_v2(packet) + if submission.get("schema") != CURATION_REVIEW_SUBMISSION_SCHEMA_V2: + raise CurationReviewExecutionV2Error("unsupported curation review submission schema") + if submission.get("protocol_version") != CURATION_REVIEW_PROTOCOL_VERSION: + raise CurationReviewExecutionV2Error("submission protocol version mismatch") + if submission.get("curation_packet_fingerprint") != validated_packet["curation_packet_fingerprint"]: + raise CurationReviewExecutionV2Error("submission does not bind to curation packet") + if submission.get("reviewer_ref") != validated_packet["reviewer_ref"]: + raise CurationReviewExecutionV2Error("submission reviewer does not bind to curation packet") + if submission.get("activation_authorized") is not False: + raise CurationReviewExecutionV2Error("submission cannot authorize activation") + if submission.get("personal_dj_model_training_authorized") is not False: + raise CurationReviewExecutionV2Error("submission cannot authorize Personal DJ Model training") + + reviews = submission.get("reviews") + if not isinstance(reviews, list): + raise CurationReviewExecutionV2Error("submission reviews must be an array") + + by_assignment = {item["assignment_id"]: item for item in validated_packet["cases"]} + seen_assignments: set[str] = set() + parsed: list[CurationDJReviewV2] = [] + for raw in reviews: + if not isinstance(raw, dict): + raise CurationReviewExecutionV2Error("review must be an object") + assignment_id = _token(raw.get("assignment_id"), "assignment_id") + case_id = _token(raw.get("case_id"), "case_id") + case = by_assignment.get(assignment_id) + if case is None or case["case_id"] != case_id: + raise CurationReviewExecutionV2Error("review assignment/case binding mismatch") + if assignment_id in seen_assignments: + raise CurationReviewExecutionV2Error("duplicate review assignment in submission") + seen_assignments.add(assignment_id) + + if raw.get("algorithm_identity_was_hidden") is not True: + raise CurationReviewExecutionV2Error("algorithm identity exposure invalidates curation review") + if raw.get("execution_quality_excluded_from_curation_judgment") is not True: + raise CurationReviewExecutionV2Error("execution-quality exclusion acknowledgement is required") + if raw.get("audition_mode") != CURATION_AUDITION_MODE: + raise CurationReviewExecutionV2Error("review audition mode mismatch") + if raw.get("activation_authorized") is not False: + raise CurationReviewExecutionV2Error("review cannot authorize activation") + if raw.get("personal_dj_model_training_authorized") is not False: + raise CurationReviewExecutionV2Error("review cannot authorize Personal DJ Model training") + + try: + preference = CurationPreference(str(raw.get("preference"))) + except ValueError as exc: + raise CurationReviewExecutionV2Error("unknown curation preference") from exc + + ratings_raw = raw.get("ratings") + if not isinstance(ratings_raw, list): + raise CurationReviewExecutionV2Error("ratings must be an array") + ratings: list[CurationDimensionPairRating] = [] + for rating in ratings_raw: + if not isinstance(rating, dict) or set(rating) != { + "dimension", + "plan_a_score", + "plan_b_score", + }: + raise CurationReviewExecutionV2Error("invalid curation rating shape") + try: + dimension = CurationReviewDimension(str(rating["dimension"])) + except ValueError as exc: + raise CurationReviewExecutionV2Error("unknown curation review dimension") from exc + ratings.append( + CurationDimensionPairRating( + dimension=dimension, + plan_a_score=_number(rating["plan_a_score"], "plan_a_score", 1.0, 5.0), + plan_b_score=_number(rating["plan_b_score"], "plan_b_score", 1.0, 5.0), + ) + ) + + reason_codes_raw = raw.get("reason_codes", []) + if not isinstance(reason_codes_raw, list): + raise CurationReviewExecutionV2Error("reason_codes must be an array") + reason_codes = tuple(_token(item, "reason_code") for item in reason_codes_raw) + notes = raw.get("notes", "") + if not isinstance(notes, str) or len(notes) > 4000: + raise CurationReviewExecutionV2Error("notes must be bounded text") + + parsed.append( + CurationDJReviewV2( + review_id=_token(raw.get("review_id"), "review_id"), + assignment_id=assignment_id, + reviewer_ref=validated_packet["reviewer_ref"], + evidence_role=EvidenceRole(validated_packet["evidence_role"]), + curation_packet_fingerprint=validated_packet["curation_packet_fingerprint"], + source_blinded_packet_fingerprint=validated_packet["source_blinded_packet_fingerprint"], + preference=preference, + ratings=tuple(ratings), + confidence=_number(raw.get("confidence"), "confidence", 0.0, 1.0), + observed_at=_token(raw.get("observed_at"), "observed_at"), + audition_mode=CURATION_AUDITION_MODE, + algorithm_identity_was_hidden=True, + execution_quality_excluded_from_curation_judgment=True, + reason_codes=reason_codes, + notes=notes, + activation_authorized=False, + personal_dj_model_training_authorized=False, + ) + ) + + return tuple(parsed) + + +__all__ = [ + "CURATION_REVIEW_PACKET_SCHEMA_V2", + "CURATION_REVIEW_SUBMISSION_SCHEMA_V2", + "CurationReviewExecutionV2Error", + "curation_packet_fingerprint", + "parse_curation_review_submission_v2", + "validate_curation_review_packet_v2", +] diff --git a/tests/test_curation_review_protocol_v2.py b/tests/test_curation_review_protocol_v2.py new file mode 100644 index 00000000..dcdce209 --- /dev/null +++ b/tests/test_curation_review_protocol_v2.py @@ -0,0 +1,717 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from core.intelligence.competitive_curation_contract import ( + ShadowPathComparison, + ShadowPathPreference, +) +from core.intelligence.curated_real_library_review_contract import ( + CuratedReviewCase, + CuratedSetRole, + HumanDJReview, + HumanDimensionPairRating, + HumanPlanPreference, + HumanReviewDimension, + ReviewPlanStrategy, + ReviewableSetPlan, +) +from core.intelligence.curation_holdout_guard_contract import ( + DevelopmentEvidenceExclusionRegistry, + HoldoutSelectionBasis, + HoldoutSelectionInput, +) +from core.intelligence.curation_review_v2_contract import ( + CURATION_AUDITION_MODE, + HOLDOUT_VALIDATION_MANIFEST_VERSION, + CurationBlindAssignmentV2, + CurationCalibrationPolicyV3, + CurationDJReviewV2, + CurationDimensionPairRating, + CurationPreference, + CurationReviewDimension, + EvaluationScope, + EvidenceRole, + HoldoutCaseExecutionEvidence, + HoldoutSystemOutcome, + HoldoutValidationManifest, + SelectionScope, +) +from core.intelligence.human_preference_calibration_contract import CalibrationVerdict +from services.intelligence.curation_holdout_guard_v1 import ( + build_counterbalanced_assignment_batch, +) +from services.intelligence.curation_preference_calibration_v3 import ( + CurationPreferenceCalibrationV3Error, + build_curation_calibration_report_v3, +) +from services.intelligence.curation_review_execution_v2 import ( + CURATION_REVIEW_PACKET_SCHEMA_V2, + CURATION_REVIEW_SUBMISSION_SCHEMA_V2, + CurationReviewExecutionV2Error, + curation_packet_fingerprint, + parse_curation_review_submission_v2, + validate_curation_review_packet_v2, +) + +ROLES = tuple(CuratedSetRole) +HOLDOUT_PRIVATE_SEED = "synthetic-holdout-assignment-private-seed" +HOLDOUT_GENERATED_AT = "2026-08-23T06:00:00Z" + + +def _plan(case_index: int, strategy: ReviewPlanStrategy) -> ReviewableSetPlan: + suffix = "g" if strategy is ReviewPlanStrategy.GREEDY_RECOMMEND_NEXT else "b" + return ReviewableSetPlan( + plan_id=f"plan-{case_index}-{suffix}", + strategy=strategy, + result_id=f"result-{case_index}-{suffix}", + path_id=f"path-{case_index}-{suffix}", + ordered_track_ids=(f"track-{case_index}-0", f"track-{case_index}-{suffix}"), + transition_ids=(f"transition-{case_index}-{suffix}",), + evidence_refs=(f"evidence-{case_index}-{suffix}",), + ) + + +def _case(index: int) -> CuratedReviewCase: + role = ROLES[(index - 1) % len(ROLES)] + return CuratedReviewCase( + case_id=f"case-{index:02d}", + snapshot_ref=("snapshot", "1"), + scenario_fingerprint=f"scenario-{index:02d}", + set_role=role, + benchmark_ref=("benchmark", "1"), + greedy_plan=_plan(index, ReviewPlanStrategy.GREEDY_RECOMMEND_NEXT), + beam_plan=_plan(index, ReviewPlanStrategy.BOUNDED_BEAM), + engineering_acceptance_passed=True, + evidence_refs=(f"case-evidence-{index}",), + ) + + +def _cases() -> tuple[CuratedReviewCase, ...]: + return tuple(_case(index) for index in range(1, 13)) + + +def _assignment(case: CuratedReviewCase, reviewer: str, index: int) -> CurationBlindAssignmentV2: + greedy_in_a = index % 2 == 0 + return CurationBlindAssignmentV2( + assignment_id=f"assignment-{reviewer}-{index}", + case_id=case.case_id, + reviewer_ref=reviewer, + slot_a_plan_id=case.greedy_plan.plan_id if greedy_in_a else case.beam_plan.plan_id, + slot_b_plan_id=case.beam_plan.plan_id if greedy_in_a else case.greedy_plan.plan_id, + assignment_fingerprint=f"assignment-fingerprint-{reviewer}-{index}", + algorithm_identity_hidden=True, + ) + + +def _ratings(a: float = 4.0, b: float = 3.0) -> tuple[CurationDimensionPairRating, ...]: + return tuple( + CurationDimensionPairRating(dimension=dimension, plan_a_score=a, plan_b_score=b) + for dimension in CurationReviewDimension + ) + + +def _review( + assignment: CurationBlindAssignmentV2, + *, + role: EvidenceRole = EvidenceRole.DEVELOPMENT_CALIBRATION, + preference: CurationPreference = CurationPreference.PLAN_A, + confidence: float = 0.9, +) -> CurationDJReviewV2: + return CurationDJReviewV2( + review_id=f"review-{assignment.assignment_id}", + assignment_id=assignment.assignment_id, + reviewer_ref=assignment.reviewer_ref, + evidence_role=role, + curation_packet_fingerprint="curation-packet-fingerprint", + source_blinded_packet_fingerprint="legacy-source-fingerprint", + preference=preference, + ratings=_ratings(), + confidence=confidence, + observed_at="2026-08-23T06:00:00Z", + audition_mode=CURATION_AUDITION_MODE, + algorithm_identity_was_hidden=True, + execution_quality_excluded_from_curation_judgment=True, + ) + + +def _comparison(case: CuratedReviewCase, preference: ShadowPathPreference = ShadowPathPreference.LEFT) -> ShadowPathComparison: + return ShadowPathComparison( + left_path_id=case.greedy_plan.path_id, + right_path_id=case.beam_plan.path_id, + left_score=0.8, + right_score=0.7, + right_minus_left=-0.1, + preference=preference, + reason_codes=("shadow-test",), + activation_authorized=False, + ) + + +def _outcome( + case: CuratedReviewCase, + outcome: HoldoutSystemOutcome = HoldoutSystemOutcome.REVIEWABLE_PAIR, +) -> HoldoutCaseExecutionEvidence: + return HoldoutCaseExecutionEvidence( + case_id=case.case_id, + set_role=case.set_role, + outcome=outcome, + scenario_fingerprint=case.scenario_fingerprint, + evidence_refs=(f"outcome-evidence-{case.case_id}",), + ) + + +def _packet() -> dict[str, object]: + packet: dict[str, object] = { + "schema": CURATION_REVIEW_PACKET_SCHEMA_V2, + "protocol_version": "curation-review-v2", + "generated_at": "2026-08-23T06:00:00Z", + "evidence_role": "development_calibration", + "source_blinded_packet_fingerprint": "legacy-source-fingerprint", + "source_snapshot_ref": ["snapshot", "1"], + "reviewer_ref": "dj-01", + "audition_mode": "sequence_curation_only", + "algorithm_identity_hidden": True, + "cases": [ + { + "case_id": "case-01", + "set_role": "opening", + "assignment_id": "assignment-dj-01-1", + "plan_a": ["Track 1", "Track 2"], + "plan_b": ["Track 1", "Track 3"], + "required_review_dimensions": [item.value for item in CurationReviewDimension], + "allowed_preference": [item.value for item in CurationPreference], + "execution_quality_exclusion_required": True, + } + ], + "activation_authorized": False, + "personal_dj_model_training_authorized": False, + } + packet["curation_packet_fingerprint"] = curation_packet_fingerprint(packet) + return packet + + +def _submission(packet: dict[str, object]) -> dict[str, object]: + return { + "schema": CURATION_REVIEW_SUBMISSION_SCHEMA_V2, + "protocol_version": "curation-review-v2", + "curation_packet_fingerprint": packet["curation_packet_fingerprint"], + "reviewer_ref": "dj-01", + "reviews": [ + { + "review_id": "review-01", + "case_id": "case-01", + "assignment_id": "assignment-dj-01-1", + "preference": "plan_a", + "ratings": [ + {"dimension": item.value, "plan_a_score": 4, "plan_b_score": 3} + for item in CurationReviewDimension + ], + "confidence": 0.9, + "observed_at": "2026-08-23T06:10:00Z", + "audition_mode": "sequence_curation_only", + "algorithm_identity_was_hidden": True, + "execution_quality_excluded_from_curation_judgment": True, + "reason_codes": [], + "notes": "sequence only", + "activation_authorized": False, + "personal_dj_model_training_authorized": False, + } + ], + "activation_authorized": False, + "personal_dj_model_training_authorized": False, + } + + +def _selection_basis( + scope: SelectionScope = SelectionScope.REPRESENTATIVE_HOLDOUT, +) -> HoldoutSelectionBasis: + return HoldoutSelectionBasis( + basis_id="selection", + basis_version="1", + selection_scope=scope, + selection_inputs=( + HoldoutSelectionInput.SOURCE_LIBRARY_ELIGIBILITY, + HoldoutSelectionInput.SET_ROLE, + HoldoutSelectionInput.DETERMINISTIC_SEED, + HoldoutSelectionInput.BPM_STRATUM, + HoldoutSelectionInput.STYLE_STRATUM, + HoldoutSelectionInput.ENERGY_STRATUM, + ), + evidence_refs=("selection-policy-evidence",), + ) + + +def _development_registry() -> DevelopmentEvidenceExclusionRegistry: + return DevelopmentEvidenceExclusionRegistry( + registry_id="prior-development-r2", + registry_version="1", + case_ids=tuple(f"legacy-development-case-{index:02d}" for index in range(1, 13)), + scenario_fingerprints=tuple( + f"legacy-development-scenario-{index:02d}" for index in range(1, 13) + ), + evidence_refs=("legacy-development-registry-evidence",), + ) + + +def _holdout_manifest(cases: tuple[CuratedReviewCase, ...], *, scope: SelectionScope = SelectionScope.REPRESENTATIVE_HOLDOUT) -> HoldoutValidationManifest: + return HoldoutValidationManifest( + holdout_id="holdout-01", + holdout_version=HOLDOUT_VALIDATION_MANIFEST_VERSION, + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + selection_scope=scope, + source_snapshot_ref=("snapshot", "1"), + case_selection_policy_ref=("selection", "1"), + selection_seed_commitment="seed-commitment", + selected_case_ids=tuple(case.case_id for case in cases), + scenario_fingerprints=tuple(case.scenario_fingerprint for case in cases), + required_set_roles=tuple(CuratedSetRole), + source_optimizer_sha="source-sha", + challenger_sha="challenger-sha", + challenger_policy_ref=("competitive-curation-shadow", "competitive-curation-r1"), + challenger_config_digest="challenger-config", + calibration_policy_digest="calibration-policy", + source_evidence_revisions=("evidence-r1",), + generated_at="2026-08-23T06:00:00Z", + human_labels_available_at_freeze=False, + algorithm_identity_hidden=True, + ) + + +def _holdout_assignments(cases: tuple[CuratedReviewCase, ...]): + return build_counterbalanced_assignment_batch( + cases=cases, + reviewer_refs=("dj-01",), + private_seed=HOLDOUT_PRIVATE_SEED, + generated_at=HOLDOUT_GENERATED_AT, + ) + + +def _holdout_guard_kwargs(cases: tuple[CuratedReviewCase, ...], *, scope: SelectionScope = SelectionScope.REPRESENTATIVE_HOLDOUT) -> dict[str, object]: + assignments, assignment_manifest = _holdout_assignments(cases) + return { + "assignments": assignments, + "development_exclusion_registry": _development_registry(), + "selection_basis": _selection_basis(scope), + "assignment_batch_manifest": assignment_manifest, + "assignment_private_seed": HOLDOUT_PRIVATE_SEED, + } + + +def test_v2_packet_rejects_legacy_bundle63_submission_schema() -> None: + packet = _packet() + submission = _submission(packet) + submission["schema"] = "applaylist-human-dj-review-submission-r1" + + with pytest.raises(CurationReviewExecutionV2Error, match="unsupported curation review submission schema"): + parse_curation_review_submission_v2(packet=packet, submission=submission) + + +def test_v2_packet_requires_execution_quality_exclusion() -> None: + packet = _packet() + submission = _submission(packet) + submission["reviews"][0]["execution_quality_excluded_from_curation_judgment"] = False # type: ignore[index] + + with pytest.raises(CurationReviewExecutionV2Error, match="execution-quality exclusion"): + parse_curation_review_submission_v2(packet=packet, submission=submission) + + +def test_v2_packet_rejects_transition_fields_in_curation_case() -> None: + packet = _packet() + packet["cases"][0]["transition_smoothness"] = 5 # type: ignore[index] + packet["curation_packet_fingerprint"] = curation_packet_fingerprint(packet) + + with pytest.raises(CurationReviewExecutionV2Error, match="unexpected fields"): + validate_curation_review_packet_v2(packet) + + +def test_curation_review_requires_all_five_dimensions() -> None: + assignment = _assignment(_case(1), "dj-01", 1) + with pytest.raises(ValueError, match="all V2 dimensions"): + CurationDJReviewV2( + review_id="review", + assignment_id=assignment.assignment_id, + reviewer_ref="dj-01", + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + curation_packet_fingerprint="packet", + source_blinded_packet_fingerprint="source", + preference=CurationPreference.PLAN_A, + ratings=_ratings()[:-1], + confidence=0.8, + observed_at="2026-08-23T06:00:00Z", + ) + + +def test_legacy_human_review_is_a_distinct_contract_not_curation_v2() -> None: + legacy = HumanDJReview( + review_id="legacy-review", + assignment_id="legacy-assignment", + reviewer_ref="dj-01", + preference=HumanPlanPreference.PLAN_A, + ratings=tuple( + HumanDimensionPairRating(dimension=dimension, plan_a_score=4, plan_b_score=3) + for dimension in HumanReviewDimension + ), + confidence=0.8, + observed_at="2026-08-23T06:00:00Z", + ) + assert not isinstance(legacy, CurationDJReviewV2) + + +def test_development_calibration_cannot_claim_independent_validation() -> None: + cases = _cases() + assignments = tuple(_assignment(case, "dj-01", index) for index, case in enumerate(cases, 1)) + reviews = tuple(_review(assignment) for assignment in assignments) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple(_outcome(case) for case in cases) + + report = build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + ) + + assert report.independent_validation is False + assert report.representative_performance_claim_allowed is False + assert "development_evidence_not_independent_validation" in report.explanation_codes + if report.verdict is CalibrationVerdict.SUPPORTS_FURTHER_EVALUATION: + assert "development_result_requires_fresh_holdout_validation" in report.explanation_codes + else: + assert "development_result_requires_fresh_holdout_validation" not in report.explanation_codes + + +def test_non_reviewable_system_failures_stay_in_denominator_and_are_negative_evidence() -> None: + cases = _cases() + outcomes = tuple( + _outcome( + case, + HoldoutSystemOutcome.NO_MEANINGFUL_ALTERNATIVE + if index <= 8 + else HoldoutSystemOutcome.REVIEWABLE_PAIR, + ) + for index, case in enumerate(cases, 1) + ) + reviewable = [case for index, case in enumerate(cases, 1) if index > 8] + assignments = tuple(_assignment(case, "dj-01", index) for index, case in enumerate(reviewable, 9)) + reviews = tuple(_review(assignment) for assignment in assignments) + comparisons = {case.case_id: _comparison(case) for case in reviewable} + + report = build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + policy=CurationCalibrationPolicyV3(maximum_assignment_imbalance=4), + ) + + assert report.selected_case_count == 12 + assert report.reviewable_pair_count == 4 + assert report.non_reviewable_system_outcome_count == 8 + assert report.reviewable_pair_fraction == pytest.approx(4 / 12) + assert report.verdict is CalibrationVerdict.DOES_NOT_SUPPORT_ACTIVATION + assert "curation_meaningful_alternative_availability_below_policy" in report.explanation_codes + + +def test_human_review_cannot_be_attached_to_non_reviewable_machine_outcome() -> None: + cases = _cases() + assignments = tuple(_assignment(case, "dj-01", index) for index, case in enumerate(cases, 1)) + reviews = tuple(_review(assignment) for assignment in assignments) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple( + _outcome(case, HoldoutSystemOutcome.TECHNICALLY_IDENTICAL_PAIR if index == 1 else HoldoutSystemOutcome.REVIEWABLE_PAIR) + for index, case in enumerate(cases, 1) + ) + + with pytest.raises(CurationPreferenceCalibrationV3Error, match="cannot replace a non-reviewable"): + build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + ) + + +def test_holdout_requires_all_guard_artifacts() -> None: + cases = _cases() + manifest = _holdout_manifest(cases) + with pytest.raises(CurationPreferenceCalibrationV3Error, match="requires development exclusion"): + build_curation_calibration_report_v3( + all_cases=cases, + assignments=(), + reviews=(), + comparisons_by_case={}, + system_outcomes=tuple(_outcome(case) for case in cases), + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + holdout_manifest=manifest, + expected_source_optimizer_sha="source-sha", + expected_challenger_sha="challenger-sha", + expected_challenger_config_digest="challenger-config", + expected_calibration_policy_digest="calibration-policy", + ) + + +def test_holdout_rejects_prior_development_case_overlap() -> None: + cases = _cases() + manifest = _holdout_manifest(cases) + assignments, assignment_manifest = _holdout_assignments(cases) + reviews = tuple( + _review(assignment, role=EvidenceRole.HOLDOUT_VALIDATION) for assignment in assignments + ) + registry = DevelopmentEvidenceExclusionRegistry( + registry_id="development-overlap", + registry_version="1", + case_ids=tuple(case.case_id for case in cases), + scenario_fingerprints=tuple(f"other-{index}" for index in range(12)), + evidence_refs=("development-overlap-evidence",), + ) + + with pytest.raises(CurationPreferenceCalibrationV3Error, match="prior development case identity"): + build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case={case.case_id: _comparison(case) for case in cases}, + system_outcomes=tuple(_outcome(case) for case in cases), + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + holdout_manifest=manifest, + development_exclusion_registry=registry, + selection_basis=_selection_basis(), + assignment_batch_manifest=assignment_manifest, + assignment_private_seed=HOLDOUT_PRIVATE_SEED, + expected_source_optimizer_sha="source-sha", + expected_challenger_sha="challenger-sha", + expected_challenger_config_digest="challenger-config", + expected_calibration_policy_digest="calibration-policy", + ) + + +def test_representative_selection_basis_rejects_challenger_dependent_inputs() -> None: + with pytest.raises(ValueError, match="cannot depend on model outcomes"): + HoldoutSelectionBasis( + basis_id="selection", + basis_version="1", + selection_scope=SelectionScope.REPRESENTATIVE_HOLDOUT, + selection_inputs=( + HoldoutSelectionInput.DETERMINISTIC_SEED, + HoldoutSelectionInput.SET_ROLE, + HoldoutSelectionInput.CHALLENGER_SCORE, + ), + evidence_refs=("selection-evidence",), + ) + + +def test_diagnostic_selection_basis_may_use_challenger_dependent_inputs() -> None: + basis = HoldoutSelectionBasis( + basis_id="selection-diagnostic", + basis_version="1", + selection_scope=SelectionScope.DIAGNOSTIC_CHALLENGE_SET, + selection_inputs=( + HoldoutSelectionInput.DETERMINISTIC_SEED, + HoldoutSelectionInput.CHALLENGER_PREFERENCE, + HoldoutSelectionInput.SOURCE_CHALLENGER_DISAGREEMENT, + HoldoutSelectionInput.FAILURE_CLASS, + ), + evidence_refs=("selection-evidence",), + ) + assert basis.selection_scope is SelectionScope.DIAGNOSTIC_CHALLENGE_SET + + +def test_counterbalanced_assignment_builder_balances_personal_slots() -> None: + cases = _cases() + assignments, _ = _holdout_assignments(cases) + greedy_in_a = sum( + assignment.slot_a_plan_id + == next(case.greedy_plan.plan_id for case in cases if case.case_id == assignment.case_id) + for assignment in assignments + ) + assert len(assignments) == 12 + assert greedy_in_a == 6 + + +def test_holdout_freeze_rejects_post_label_challenger_mutation() -> None: + cases = _cases() + manifest = _holdout_manifest(cases) + guard = _holdout_guard_kwargs(cases) + assignments = guard.pop("assignments") + assert isinstance(assignments, tuple) + reviews = tuple( + _review(assignment, role=EvidenceRole.HOLDOUT_VALIDATION) for assignment in assignments + ) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple(_outcome(case) for case in cases) + + with pytest.raises(CurationPreferenceCalibrationV3Error, match="challenger revision"): + build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + holdout_manifest=manifest, + **guard, + expected_source_optimizer_sha="source-sha", + expected_challenger_sha="changed-after-labels", + expected_challenger_config_digest="challenger-config", + expected_calibration_policy_digest="calibration-policy", + ) + + +def test_holdout_rejects_assignment_private_seed_mismatch() -> None: + cases = _cases() + manifest = _holdout_manifest(cases) + guard = _holdout_guard_kwargs(cases) + assignments = guard.pop("assignments") + assert isinstance(assignments, tuple) + reviews = tuple( + _review(assignment, role=EvidenceRole.HOLDOUT_VALIDATION) for assignment in assignments + ) + guard["assignment_private_seed"] = "wrong-private-seed" + + with pytest.raises(CurationPreferenceCalibrationV3Error, match="assignment seed commitment mismatch"): + build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case={case.case_id: _comparison(case) for case in cases}, + system_outcomes=tuple(_outcome(case) for case in cases), + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + holdout_manifest=manifest, + **guard, + expected_source_optimizer_sha="source-sha", + expected_challenger_sha="challenger-sha", + expected_challenger_config_digest="challenger-config", + expected_calibration_policy_digest="calibration-policy", + ) + + +def test_representative_holdout_can_claim_only_protocol_bounded_independent_validation() -> None: + cases = _cases() + manifest = _holdout_manifest(cases) + guard = _holdout_guard_kwargs(cases) + assignments = guard.pop("assignments") + assert isinstance(assignments, tuple) + reviews = tuple( + _review(assignment, role=EvidenceRole.HOLDOUT_VALIDATION) for assignment in assignments + ) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple(_outcome(case) for case in cases) + + report = build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + holdout_manifest=manifest, + **guard, + expected_source_optimizer_sha="source-sha", + expected_challenger_sha="challenger-sha", + expected_challenger_config_digest="challenger-config", + expected_calibration_policy_digest="calibration-policy", + ) + + assert report.independent_validation is True + assert report.representative_performance_claim_allowed is True + + +def test_diagnostic_holdout_never_allows_representative_performance_claim() -> None: + cases = _cases() + manifest = _holdout_manifest(cases, scope=SelectionScope.DIAGNOSTIC_CHALLENGE_SET) + guard = _holdout_guard_kwargs(cases, scope=SelectionScope.DIAGNOSTIC_CHALLENGE_SET) + assignments = guard.pop("assignments") + assert isinstance(assignments, tuple) + reviews = tuple( + _review(assignment, role=EvidenceRole.HOLDOUT_VALIDATION) for assignment in assignments + ) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple(_outcome(case) for case in cases) + + report = build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + holdout_manifest=manifest, + **guard, + expected_source_optimizer_sha="source-sha", + expected_challenger_sha="challenger-sha", + expected_challenger_config_digest="challenger-config", + expected_calibration_policy_digest="calibration-policy", + ) + + assert report.independent_validation is True + assert report.representative_performance_claim_allowed is False + + +def test_personal_dj_assignment_position_imbalance_fails_completeness() -> None: + cases = _cases() + assignments = tuple( + CurationBlindAssignmentV2( + assignment_id=f"assignment-dj-01-{index}", + case_id=case.case_id, + reviewer_ref="dj-01", + slot_a_plan_id=case.greedy_plan.plan_id, + slot_b_plan_id=case.beam_plan.plan_id, + assignment_fingerprint=f"fingerprint-{index}", + ) + for index, case in enumerate(cases, 1) + ) + reviews = tuple(_review(assignment) for assignment in assignments) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple(_outcome(case) for case in cases) + + report = build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + evaluation_scope=EvaluationScope.PERSONAL_DJ_CALIBRATION, + ) + + assert report.verdict is CalibrationVerdict.INCOMPLETE + assert "curation_calibration_assignment_position_imbalance" in report.explanation_codes + + +def test_multi_dj_scope_requires_minimum_independent_reviewers() -> None: + cases = _cases() + assignments = tuple(_assignment(case, "dj-01", index) for index, case in enumerate(cases, 1)) + reviews = tuple(_review(assignment) for assignment in assignments) + comparisons = {case.case_id: _comparison(case) for case in cases} + outcomes = tuple(_outcome(case) for case in cases) + + report = build_curation_calibration_report_v3( + all_cases=cases, + assignments=assignments, + reviews=reviews, + comparisons_by_case=comparisons, + system_outcomes=outcomes, + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + evaluation_scope=EvaluationScope.MULTI_DJ_PRODUCT_EVALUATION, + ) + + assert report.verdict is CalibrationVerdict.INCOMPLETE + assert "curation_calibration_independent_reviewer_count_below_policy" in report.explanation_codes diff --git a/tests/test_curation_review_protocol_v2_security.py b/tests/test_curation_review_protocol_v2_security.py new file mode 100644 index 00000000..083cb556 --- /dev/null +++ b/tests/test_curation_review_protocol_v2_security.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import inspect + +import pytest + +import services.intelligence.curation_holdout_guard_v1 as holdout_guard_module +import services.intelligence.curation_preference_calibration_v3 as calibration_module +import services.intelligence.curation_review_execution_v2 as review_module +from core.intelligence.curated_real_library_review_contract import CuratedSetRole +from core.intelligence.curation_review_v2_contract import ( + HOLDOUT_VALIDATION_MANIFEST_VERSION, + CurationCalibrationPolicyV3, + CurationDJReviewV2, + CurationDimensionPairRating, + CurationPreference, + CurationReviewDimension, + EvidenceRole, + HoldoutValidationManifest, + SelectionScope, +) +from services.intelligence.curation_review_execution_v2 import ( + CurationReviewExecutionV2Error, + curation_packet_fingerprint, + validate_curation_review_packet_v2, +) + + +def _ratings() -> tuple[CurationDimensionPairRating, ...]: + return tuple( + CurationDimensionPairRating(dimension=dimension, plan_a_score=4, plan_b_score=3) + for dimension in CurationReviewDimension + ) + + +def test_curation_review_structurally_rejects_activation_and_pdm_training() -> None: + kwargs = dict( + review_id="review", + assignment_id="assignment", + reviewer_ref="dj-01", + evidence_role=EvidenceRole.DEVELOPMENT_CALIBRATION, + curation_packet_fingerprint="packet", + source_blinded_packet_fingerprint="source", + preference=CurationPreference.PLAN_A, + ratings=_ratings(), + confidence=0.9, + observed_at="2026-08-23T06:00:00Z", + ) + with pytest.raises(ValueError, match="optimizer activation"): + CurationDJReviewV2(**kwargs, activation_authorized=True) + with pytest.raises(ValueError, match="Personal DJ Model training"): + CurationDJReviewV2(**kwargs, personal_dj_model_training_authorized=True) + + +def test_holdout_manifest_rejects_labels_available_at_freeze() -> None: + with pytest.raises(ValueError, match="before human labels"): + HoldoutValidationManifest( + holdout_id="holdout", + holdout_version=HOLDOUT_VALIDATION_MANIFEST_VERSION, + evidence_role=EvidenceRole.HOLDOUT_VALIDATION, + selection_scope=SelectionScope.REPRESENTATIVE_HOLDOUT, + source_snapshot_ref=("snapshot", "1"), + case_selection_policy_ref=("selection", "1"), + selection_seed_commitment="seed", + selected_case_ids=tuple(f"case-{index}" for index in range(12)), + scenario_fingerprints=tuple(f"scenario-{index}" for index in range(12)), + required_set_roles=tuple(CuratedSetRole), + source_optimizer_sha="source", + challenger_sha="challenger", + challenger_policy_ref=("challenger", "1"), + challenger_config_digest="config", + calibration_policy_digest="policy", + source_evidence_revisions=("evidence",), + generated_at="2026-08-23T06:00:00Z", + human_labels_available_at_freeze=True, + ) + + +def test_curation_calibration_policy_cannot_authorize_activation_or_training() -> None: + with pytest.raises(ValueError, match="optimizer activation"): + CurationCalibrationPolicyV3(activation_authorized=True) + with pytest.raises(ValueError, match="Personal DJ Model training"): + CurationCalibrationPolicyV3(personal_dj_model_training_authorized=True) + + +def test_packet_authority_flags_fail_closed() -> None: + packet: dict[str, object] = { + "schema": "applaylist-curation-review-packet-v2", + "protocol_version": "curation-review-v2", + "generated_at": "2026-08-23T06:00:00Z", + "evidence_role": "development_calibration", + "source_blinded_packet_fingerprint": "source", + "source_snapshot_ref": ["snapshot", "1"], + "reviewer_ref": "dj-01", + "audition_mode": "sequence_curation_only", + "algorithm_identity_hidden": True, + "cases": [ + { + "case_id": "case-01", + "set_role": "opening", + "assignment_id": "assignment-01", + "plan_a": ["A"], + "plan_b": ["B"], + "required_review_dimensions": [item.value for item in CurationReviewDimension], + "allowed_preference": [item.value for item in CurationPreference], + "execution_quality_exclusion_required": True, + } + ], + "activation_authorized": True, + "personal_dj_model_training_authorized": False, + } + packet["curation_packet_fingerprint"] = curation_packet_fingerprint(packet) + with pytest.raises(CurationReviewExecutionV2Error, match="cannot authorize activation"): + validate_curation_review_packet_v2(packet) + + +def test_curation_protocol_services_are_pure_no_io_network_or_provider_execution() -> None: + source = ( + inspect.getsource(review_module) + + inspect.getsource(calibration_module) + + inspect.getsource(holdout_guard_module) + ) + forbidden = ( + "pathlib", + "Path(", + "open(", + "sqlite3", + "subprocess", + "socket", + "requests", + "httpx", + "urllib", + "analyze_real_tracks", + "provider.execute", + ) + for token in forbidden: + assert token not in source