Merge pull request #35 from interscript/fix/byt5-decode

fix(distill): byt5 decode_joined mojibake — all Arabic labels poisoned

Author: ronaldtse

docs/RESULTS.md

@@ -121,11 +121,13 @@ reproducing its documented tier on this replication.
 | Teacher (r6, run-006-morph) | 1.32% |
 | **Student (33M from-scratch)** | **83.08%** — REJECTED |
 
-Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude
-with the same collapse signature as the Thai tiny tier (train loss
-converges, test generalization absent). Verdict: sub-100M from-scratch
-byte students do not generalize for diacritization any more than for
-G2P — a pretrained backbone is non-negotiable. The Arabic client tier
-therefore ships at the ByT5-small rung (ara-diac-small) or parks until
-byte-level pretraining exists. The 30MB tier is closed as a negative
-result across both task families.
+Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude.
+
+**RETRACTION (2026-08-24):** this verdict is CONFOUNDED — every Arabic
+label generated before the byt5 `decode_joined` fix was mojibake
+(double-encoded targets); both Arabic students trained on corrupted
+labels, and their identical DER scores are the bare-text constant, not
+a capacity result. The numbers stand as measured but the capacity
+conclusion for Arabic is UNPROVEN pending a clean-label re-run. The
+Thai tiny verdict is unaffected (umt5/sentencepiece labels were
+byte-exact); the pretrained-backbone law rests on Thai evidence.

src/api/inference.py

@@ -0,0 +1,145 @@
+"""IMF v1 inference endpoint for api.interscript.org (Modal, CPU).
+
+Serves the shipped models from the secryst-models volume using the
+exact ONNX kv decode the WO03 parity gate verified. Cold start loads
+the fp32 zip (~30-60s); sessions are cached per container.
+
+    modal deploy src/api/inference.py
+
+Auth: X-API-Key header must match the `api-inference-key` secret.
+"""
+
+import hmac
+from pathlib import Path
+
+import modal
+
+models_volume = modal.Volume.from_name("secryst-models")
+
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .pip_install("onnxruntime==1.23.2", "pyyaml>=6.0", "fastapi>=0.115")
+    .add_local_dir(str(Path(__file__).resolve().parent.parent), "/root/interscript-ml", copy=True)
+    .workdir("/root/interscript-ml")
+    .env({"IMAGE_REV": "5"})
+)
+
+app = modal.App("interscript-inference", image=image)
+
+MAX_INPUT_BYTES = 4000
+MAX_OUTPUT_TOKENS = 8192
+ALLOWED_TASKS = ("diacritization", "g2p")
+
+_sessions: dict[str, tuple] = {}
+
+
+def _zip_path(model_id: str) -> Path:
+    # model_id arrives from the request body — paths are built ONLY from
+    # the server's own volume listing; the user value is used solely in
+    # an equality comparison, never in path construction (CWE-22)
+    import glob
+
+    wanted = f"{model_id}-fp32.zip"
+    for z in glob.glob("/v/imf/*/*-fp32.zip"):
+        if z.rsplit("/", 1)[1] == wanted:
+            return Path(z)
+    raise KeyError(model_id)
+
+
+def _get_sessions(model_id: str) -> tuple:
+    import sys
+
+    sys.path.insert(0, "/root/interscript-ml/src")
+    from imf.parity import _sessions_from_zip  # the parity-verified loader
+
+    if model_id not in _sessions:
+        _sessions[model_id] = _sessions_from_zip(_zip_path(model_id))
+    return _sessions[model_id]
+
+
+def _metadata(model_id: str) -> dict:
+    import zipfile
+
+    import yaml
+
+    with zipfile.ZipFile(_zip_path(model_id)) as zf:
+        return yaml.safe_load(zf.read("metadata.yaml"))
+
+
+def _decode(tokens: list) -> str:
+    # ByT5 token ids are byte+3, with trailing EOS (id 1)
+    return bytes(t - 3 for t in tokens if t >= 3).decode("utf-8", "replace")
+
+
+def make_api():
+    import os
+
+    from fastapi import FastAPI, HTTPException, Request
+
+    api = FastAPI(title="Interscript inference", version="1.0.0")
+
+    @api.post("/infer")
+    async def infer(request: Request) -> dict:
+
+        key = request.headers.get("x-api-key", "")
+        if not key or not hmac.compare_digest(key, os.environ.get("API_INFERENCE_KEY") or ""):
+            raise HTTPException(401, "invalid or missing X-API-Key")
+        try:
+            body = await request.json()
+        except Exception:
+            raise HTTPException(400, "body must be JSON {model, input}") from None
+        if not isinstance(body, dict) or not body.get("model") or not body.get("input"):
+            raise HTTPException(400, "body must be {model, input}")
+
+        return await _run_infer(body)
+
+    async def _run_infer(body):
+        import sys
+
+        models_volume.reload()
+        try:
+            meta = _metadata(body["model"])
+        except KeyError:
+            raise HTTPException(404, f"unknown model {body['model']}") from None
+        if meta.get("task") not in ALLOWED_TASKS:
+            raise HTTPException(400, f"model {body['model']} task {meta.get('task')} is not served")
+
+        if len(body["input"].encode("utf-8")) > MAX_INPUT_BYTES:
+            raise HTTPException(413, f"input exceeds {MAX_INPUT_BYTES} bytes")
+
+        sys.path.insert(0, "/root/interscript-ml/src")
+        from imf.export import onnx_greedy_kv
+
+        enc, kv = _get_sessions(body["model"])
+        max_len = min(MAX_OUTPUT_TOKENS, 3 * len(body["input"].encode("utf-8")) + 256)
+        output = _decode(onnx_greedy_kv(enc, kv, body["input"], max_len))
+        return {
+            "model": body["model"],
+            "task": meta["task"],
+            "source_script": meta.get("source_script"),
+            "input": body["input"],
+            "output": output,
+        }
+
+    @api.get("/health")
+    def health() -> dict:
+        import glob
+
+        models_volume.reload()
+        return {"ok": True, "models": len(glob.glob("/v/imf/*/*-fp32.zip"))}
+
+    return api
+
+
+@app.function(
+    cpu=4,
+    memory=8 * 1024,
+    timeout=10 * 60,
+    volumes={"/v": models_volume},
+    secrets=[modal.Secret.from_name("api-inference-key")],
+    # cold starts (~30-60s model load) instead of a 24/7 warm container
+    # — money discipline; bump once traffic justifies it
+)
+@modal.asgi_app()
+def web():
+    return make_api()

src/gpu/modal_distill.py

@@ -116,6 +116,9 @@
         "max_len": 1450,
         "label_beams": "1",
         "out": "rababa_arabic_distill_small/run-002",
+        # v2: every label generated before the byt5 decode_joined fix is
+        # mojibake (double-encoded); relabel from scratch on the new file
+        "labels_file": "teacher_labels_v2.jsonl",
         "mode": "sequence",
         "note": "r6 canonical (2.5793 DER); gate <= 3.07 windowed DER-CE",
     },
@@ -233,11 +236,24 @@
 
 
 def decode_joined(tok, ids) -> str:
-    """Correct decode for umt5 teachers: 5.x batch_decode inserts spurious
-    spaces between sentencepiece pieces; pieces must join directly (the
-    targets are unspaced IPA strings)."""
+    """Decode teacher generations correctly per tokenizer family.
+
+    umt5/sentencepiece: 5.x batch_decode inserts spurious spaces between
+    pieces; joining convert_ids_to_tokens is correct.
+
+    byt5/byte-level: convert_ids_to_tokens returns each byte token as a
+    RAW CHARACTER (latin-1 view) — joining them produces mojibake. This
+    poisoned every Arabic label generated under 5.14 (both students
+    trained on double-encoded targets and scored the identical bare-text
+    DER). batch_decode is byte-exact here, so branch on it: if its
+    output round-trips the same token ids it is authoritative.
+    """
     skip = {tok.pad_token, tok.eos_token, tok.bos_token}
-    return "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip)
+    joined = "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip)
+    if joined and all(ord(c) < 256 for c in joined):
+        # byte-level vocab decoded to raw chars — use the byte-exact path
+        return tok.batch_decode([ids], skip_special_tokens=True)[0]
+    return joined
 
 
 @app.function(