Improve Jev synthetic data auditing and selection provenance

src/tasksource/jev/synthetic/config.py

@@ -70,6 +70,33 @@ class AnnotatorConfig:
     model: str = "jev-mock"
 
 
+@dataclass
+class TeacherAuditConfig:
+    # Independent post-Jev sanity check. The auditor sees state/questions but
+    # never Jev's probabilities; comparison happens locally afterwards.
+    enabled: bool = False
+    # When absent, inherit the critic provider. For a genuinely independent
+    # check, explicitly choose a different provider/model from the generator.
+    provider: ProviderConfig | None = None
+    model: str = "deepseek-v4-flash-0731"
+    prompt_version: str = "teacher_audit_v1"
+    temperature: float = 0.0
+    requests_per_minute: int = 40
+    min_teacher_confidence: float = 0.85
+    min_auditor_confidence: float = 0.80
+    # False keeps flagged examples in the released artifact and records the
+    # disagreement. True makes the stage also emit/use audit_passed.jsonl.
+    drop_confident_disagreements: bool = False
+
+    def __post_init__(self):
+        for name, value in (
+            ("min_teacher_confidence", self.min_teacher_confidence),
+            ("min_auditor_confidence", self.min_auditor_confidence),
+        ):
+            if not 0 <= value <= 1:
+                raise ValueError(f"{name} must be in [0, 1], got {value}")
+
+
 @dataclass
 class SelectionConfig:
     # Buckets over observed Jev ambiguity (max_prob based, see select.py).
@@ -81,6 +108,20 @@ class SelectionConfig:
         "near_uniform": 0.05,
     })
     max_per_family: int = 0  # 0 = no cap
+    # When downsampling, this share is selected by a teacher-independent hash
+    # before confidence-bucket balancing. This leaves an unbiased reservoir for
+    # diagnosing selection effects.
+    unfiltered_fraction: float = 0.20
+    seed_salt: str = "jev-synthetic-selection-v2"
+
+    def __post_init__(self):
+        if not 0 <= self.unfiltered_fraction <= 1:
+            raise ValueError(
+                f"unfiltered_fraction must be in [0, 1], got {self.unfiltered_fraction}")
+        if not self.buckets or any(float(weight) < 0 for weight in self.buckets.values()):
+            raise ValueError("selection buckets must be a non-empty map of non-negative weights")
+        if sum(float(weight) for weight in self.buckets.values()) <= 0:
+            raise ValueError("selection bucket weights must have positive total mass")
 
 
 @dataclass
@@ -105,17 +146,26 @@ class AppConfig:
     sampler: SamplerConfig = field(default_factory=SamplerConfig)
     critic: CriticConfig = field(default_factory=CriticConfig)
     annotator: AnnotatorConfig = field(default_factory=AnnotatorConfig)
+    teacher_audit: TeacherAuditConfig = field(default_factory=TeacherAuditConfig)
     selection: SelectionConfig = field(default_factory=SelectionConfig)
     split: SplitConfig = field(default_factory=SplitConfig)
 
     def critic_provider(self) -> ProviderConfig:
         """Resolved critic provider (explicit, else the generator provider)."""
         return self.critic.provider if self.critic.provider is not None else self.provider
 
+    def teacher_audit_provider(self) -> ProviderConfig:
+        """Resolved teacher-audit provider (explicit, else the critic provider)."""
+        return (self.teacher_audit.provider
+                if self.teacher_audit.provider is not None else self.critic_provider())
+
     def to_dict(self) -> dict:
         critic = dict(self.critic.__dict__)
         critic["provider"] = (self.critic.provider.__dict__
                               if self.critic.provider is not None else None)
+        teacher_audit = dict(self.teacher_audit.__dict__)
+        teacher_audit["provider"] = (self.teacher_audit.provider.__dict__
+                                     if self.teacher_audit.provider is not None else None)
         return {
             "run_name": self.run_name,
             "output_dir": self.output_dir,
@@ -124,6 +174,7 @@ def to_dict(self) -> dict:
             "sampler": self.sampler.__dict__,
             "critic": critic,
             "annotator": self.annotator.__dict__,
+            "teacher_audit": teacher_audit,
             "selection": self.selection.__dict__,
             "split": self.split.__dict__,
         }
@@ -139,32 +190,46 @@ def _merge(base: dict, override: dict) -> dict:
     return out
 
 
+def _nested_provider(raw, fallback: ProviderConfig, model: str) -> ProviderConfig | None:
+    if isinstance(raw, dict):
+        return ProviderConfig(**raw)
+    if isinstance(raw, str):
+        # Legacy: bare name inherits the parent connection details.
+        return ProviderConfig(
+            name=raw, api_key_env=fallback.api_key_env,
+            base_url=fallback.base_url, model=model)
+    return None
+
+
 def load_config(path: str) -> AppConfig:
     """Load a config.yaml into a typed AppConfig."""
     with open(path, encoding="utf-8") as handle:
         raw = yaml.safe_load(handle) or {}
     defaults = AppConfig().to_dict()
     merged = _merge(defaults, raw)
     provider = ProviderConfig(**merged["provider"])
-    critic_raw = merged["critic"]
+
+    critic_raw = dict(merged["critic"])
     critic_provider_raw = critic_raw.pop("provider", None)
-    if isinstance(critic_provider_raw, dict):
-        critic_provider = ProviderConfig(**critic_provider_raw)
-    elif isinstance(critic_provider_raw, str):
-        # Legacy: bare name inherits the generator's connection details.
-        critic_provider = ProviderConfig(
-            name=critic_provider_raw, api_key_env=provider.api_key_env,
-            base_url=provider.base_url, model=critic_raw.get("model", provider.model))
-    else:
-        critic_provider = None  # inherit the generator provider at use site
+    critic_provider = _nested_provider(
+        critic_provider_raw, provider, critic_raw.get("model", provider.model))
+    critic = CriticConfig(provider=critic_provider, **critic_raw)
+
+    audit_raw = dict(merged["teacher_audit"])
+    audit_provider_raw = audit_raw.pop("provider", None)
+    audit_provider = _nested_provider(
+        audit_provider_raw, critic_provider or provider,
+        audit_raw.get("model", (critic_provider or provider).model))
+
     return AppConfig(
         run_name=merged["run_name"],
         output_dir=merged["output_dir"],
         provider=provider,
         generation=GenerationConfig(**merged["generation"]),
         sampler=SamplerConfig(**merged["sampler"]),
-        critic=CriticConfig(provider=critic_provider, **critic_raw),
+        critic=critic,
         annotator=AnnotatorConfig(**merged["annotator"]),
+        teacher_audit=TeacherAuditConfig(provider=audit_provider, **audit_raw),
         selection=SelectionConfig(**merged["selection"]),
         split=SplitConfig(**merged["split"]),
     )

src/tasksource/jev/synthetic/configs/albert_deepseek_v4_flash_jev_audited.yaml

@@ -0,0 +1,84 @@
+# DeepSeek generation + independent OpenAI-model teacher audit + pinned Jev targets.
+# This is intentionally more expensive than the overnight baseline. The auditor
+# never sees Jev's probabilities; it independently answers each question and only
+# flags high-confidence disagreements.
+run_name: deepseek_v4_flash_jev_audited
+output_dir: .synthetic_runs
+
+provider:
+  name: albert
+  api_key_env: ALBERT_API_KEY
+  base_url: https://albert.api.etalab.gouv.fr/v1
+  model: deepseek-v4-flash-0731
+
+generation:
+  temperature: 0.8
+  concurrency: 10
+  max_output_tokens: 4000
+  seed: 42
+  prompt_version: generate_v2
+  n_states: 4000
+
+sampler:
+  seed: 42
+  n_states: 4000
+  questions_per_state: {'1': 0.6, '2': 0.25, '3': 0.1, '4': 0.05}
+  question_formats: {choice: 0.4, noul: 0.3, score: 0.3}
+  probability_mixed_formats: 0.85
+  probability_all_formats_if_n_ge_3: 0.7
+
+critic:
+  enabled: true
+  provider:
+    name: albert
+    api_key_env: ALBERT_API_KEY
+    base_url: https://albert.api.etalab.gouv.fr/v1
+    model: deepseek-v4-flash-0731
+  model: deepseek-v4-flash-0731
+  prompt_version: critic_v1
+  temperature: 0.0
+  requests_per_minute: 40
+
+annotator:
+  name: jev
+  version: jev-1.13
+  base_url: https://openrouter.ai
+  api_path: /api/alpha/decisions
+  api_key_env: OPENROUTER_API_KEY
+  model: typesafe/jev-1.13
+
+teacher_audit:
+  enabled: true
+  provider:
+    name: openrouter
+    api_key_env: OPENROUTER_API_KEY
+    base_url: https://openrouter.ai/api/v1
+    model: openai/gpt-4.1-mini
+  model: openai/gpt-4.1-mini
+  prompt_version: teacher_audit_v1
+  temperature: 0.0
+  requests_per_minute: 40
+  min_teacher_confidence: 0.85
+  min_auditor_confidence: 0.80
+  # Keep flagged rows in the public artifact by default so filtering choices are
+  # auditable. Set true for a clean training-only subset.
+  drop_confident_disagreements: false
+
+selection:
+  buckets:
+    very_confident: 0.2
+    confident: 0.3
+    moderately_ambiguous: 0.3
+    high_ambiguity: 0.15
+    near_uniform: 0.05
+  # When --n-target downsamples, reserve this fraction without consulting Jev
+  # confidence, then fill the rest with the requested ambiguity mixture.
+  unfiltered_fraction: 0.20
+  seed_salt: jev-synthetic-selection-v2
+
+split:
+  train: 0.85
+  validation: 0.075
+  test: 0.075
+  ood_fraction: 0.05
+  seed_salt: jev-synthetic-split-v1

src/tasksource/jev/synthetic/prompts/teacher_audit_v1.txt

@@ -0,0 +1,19 @@
+You are an independent auditor for generated decision examples.
+
+You are NOT given another model's answer. Answer each question from the state
+alone. Do not invent missing evidence. If the state is ambiguous or incomplete,
+lower your confidence instead of forcing certainty.
+
+BUNDLE:
+{{BUNDLE_JSON}}
+
+For every question:
+- choice: answer with the exact option string from `options`
+- score: answer with the zero-based integer index into the ordered `options`
+- noul: answer with JSON true or false
+- confidence: number in [0, 1] representing confidence in that answer
+- issues: short list of any ambiguity, missing evidence, malformed rubric, or
+  other reason the answer should not be trusted
+
+Output STRICT JSON only:
+{"answers": [{"question_id": "...", "answer": ..., "confidence": 0.0, "issues": []}]}