feat(device-eval): on-device open-ended + RAG eval, host-proxy fidelity validated

configs/config-v0.2.0/manifests/healthbench_device_sample.json

@@ -0,0 +1,155 @@
+{
+ "name": "healthbench-device-fidelity-sample-50ps",
+ "ids": [
+  "mamabench_v0.2_healthbench_oss-eval_1c5c847f-acae-42e8-a5b1-49b78ca766ba",
+  "mamabench_v0.2_healthbench_oss-eval_23809964-caaa-4fac-9210-40d64d1dea58",
+  "mamabench_v0.2_healthbench_oss-eval_c843acd8-1aec-4eb9-86d6-2d79f21b46a0",
+  "mamabench_v0.2_healthbench_oss-eval_7f41c0f3-04c9-46e4-8f5f-7de76a793399",
+  "mamabench_v0.2_healthbench_oss-eval_90030f57-9b7c-454e-bb06-e0b96644b53d",
+  "mamabench_v0.2_healthbench_oss-eval_6e9da1e2-48f9-4b4c-9226-112abb012dcf",
+  "mamabench_v0.2_healthbench_oss-eval_477f7950-360b-4462-815a-20fa14e9667a",
+  "mamabench_v0.2_healthbench_oss-eval_c3f19a38-f4c6-4a8a-b4ba-1b6eb44e463e",
+  "mamabench_v0.2_healthbench_oss-eval_abe78331-8e84-45d2-b4b3-c5251db6b722",
+  "mamabench_v0.2_healthbench_oss-eval_d35e660f-cb4c-4fe1-b503-3107bb13a57b",
+  "mamabench_v0.2_healthbench_oss-eval_cb5286fd-8dc3-4188-9031-406826f78961",
+  "mamabench_v0.2_healthbench_oss-eval_87ff241a-9fa9-4362-bd87-91d4229869f2",
+  "mamabench_v0.2_healthbench_oss-eval_e9b7f69b-69ee-486b-805c-c98439ad969a",
+  "mamabench_v0.2_healthbench_oss-eval_79e197ab-8409-43c1-b476-d2bc632deca0",
+  "mamabench_v0.2_healthbench_oss-eval_01787f37-5adc-4cfe-bf79-62b9a43ac482",
+  "mamabench_v0.2_healthbench_oss-eval_76b40740-ab88-4aec-aa77-3de4a20af938",
+  "mamabench_v0.2_healthbench_oss-eval_8cba5bc7-d127-473b-884d-39024b3609f6",
+  "mamabench_v0.2_healthbench_oss-eval_21302167-c258-4b49-8ef4-be907774c8c9",
+  "mamabench_v0.2_healthbench_oss-eval_3af15d2d-7eb5-4bd8-902f-5986ac790c4e",
+  "mamabench_v0.2_healthbench_oss-eval_aa02e0ed-a35a-4c31-85ae-3ac2b72aec53",
+  "mamabench_v0.2_healthbench_oss-eval_599deef4-e03a-446c-b2b5-44fe294f512f",
+  "mamabench_v0.2_healthbench_oss-eval_e392b996-4400-45cd-a3a2-b0907a7dfadb",
+  "mamabench_v0.2_healthbench_oss-eval_2b77fd41-55f9-4a4b-869f-79c2a6b042cf",
+  "mamabench_v0.2_healthbench_oss-eval_eefd92ef-79df-4b4b-bfdc-fa62bda236b0",
+  "mamabench_v0.2_healthbench_oss-eval_2d878243-5d98-489a-b8f2-8809f813b6cf",
+  "mamabench_v0.2_healthbench_oss-eval_d9b1540b-e7df-42af-94e7-08defb1a34ae",
+  "mamabench_v0.2_healthbench_oss-eval_9dea1433-b4b8-44d5-8841-3f208c29df8a",
+  "mamabench_v0.2_healthbench_oss-eval_08691500-9c5b-4b29-8c0c-f818cb0c0728",
+  "mamabench_v0.2_healthbench_oss-eval_89a3c017-0a9b-4481-b9f2-350d0545377c",
+  "mamabench_v0.2_healthbench_oss-eval_e72beb48-8541-41f7-9934-5650b0749832",
+  "mamabench_v0.2_healthbench_oss-eval_8b47384a-ba74-4f50-aa1e-7d195eadf7ab",
+  "mamabench_v0.2_healthbench_oss-eval_f5a77a0d-cc92-41e5-910d-3e774c09ced2",
+  "mamabench_v0.2_healthbench_oss-eval_a87d2dc9-ca9c-49fc-84a4-818049e5f1ab",
+  "mamabench_v0.2_healthbench_oss-eval_ac3caa21-8cfc-49d9-acbc-d4a9822df88b",
+  "mamabench_v0.2_healthbench_oss-eval_89169855-80e8-4234-8865-f3e89559f028",
+  "mamabench_v0.2_healthbench_oss-eval_15811008-38d6-4ae2-90d5-004f0bcca7e7",
+  "mamabench_v0.2_healthbench_oss-eval_19d9f810-e8e6-45b8-98ee-bca17ec347af",
+  "mamabench_v0.2_healthbench_oss-eval_af6ff14b-1cf0-481c-9b6e-19be1dbc4595",
+  "mamabench_v0.2_healthbench_oss-eval_5ed6eebd-3a37-4738-8356-16c9e760d62b",
+  "mamabench_v0.2_healthbench_oss-eval_dd5be674-118a-4002-8a16-66baa884f390",
+  "mamabench_v0.2_healthbench_oss-eval_5ef0055c-f276-41bb-b42c-0d5c34b254c1",
+  "mamabench_v0.2_healthbench_oss-eval_964e73e0-ffad-4722-a33e-4f61c37f4efb",
+  "mamabench_v0.2_healthbench_oss-eval_019f591a-8320-40ec-83d0-75cbd9258f56",
+  "mamabench_v0.2_healthbench_oss-eval_50a8890e-b72d-47b7-9354-b81501ca81a3",
+  "mamabench_v0.2_healthbench_oss-eval_9a455446-d2cc-4f2b-b82c-9243cf9a6dd2",
+  "mamabench_v0.2_healthbench_oss-eval_70c10085-1386-441f-81f5-4d21cf60ca94",
+  "mamabench_v0.2_healthbench_oss-eval_711c4c23-3946-496d-aa83-7d3c2c33526c",
+  "mamabench_v0.2_healthbench_oss-eval_7558237f-34c5-475e-b6a2-ea9bbccf90d4",
+  "mamabench_v0.2_healthbench_oss-eval_249ddc30-defb-4b9c-957e-cfa522060b22",
+  "mamabench_v0.2_healthbench_oss-eval_b00e7c9f-d069-4736-9789-21fe4316a2e2",
+  "mamabench_v0.2_healthbench_consensus_efa19e50-86f0-435a-9851-5f07105f91be",
+  "mamabench_v0.2_healthbench_consensus_b9dbca6d-54f4-4d45-92c6-daa0a7802332",
+  "mamabench_v0.2_healthbench_consensus_ac470100-0201-4bff-8b1f-10b83f038f70",
+  "mamabench_v0.2_healthbench_consensus_b0d7a2da-c34f-4458-b9ae-1b6f0a2641ed",
+  "mamabench_v0.2_healthbench_consensus_fd2998f6-f1e1-4115-b9ed-f93062c69a51",
+  "mamabench_v0.2_healthbench_consensus_9f7075b5-ed05-4b71-9db9-42fdfd03860f",
+  "mamabench_v0.2_healthbench_consensus_c968b43d-c6c0-420a-9139-8da36dfaf170",
+  "mamabench_v0.2_healthbench_consensus_e6fe887a-4a2c-4416-a007-46826d06abe3",
+  "mamabench_v0.2_healthbench_consensus_c0b746b0-8209-4504-b8a3-59251db480e2",
+  "mamabench_v0.2_healthbench_consensus_19aba1c6-8cd3-49bd-bfc1-5af65d85da8c",
+  "mamabench_v0.2_healthbench_consensus_91dad885-2b6d-4be8-b2fb-a8228c92f38c",
+  "mamabench_v0.2_healthbench_consensus_711c4c23-3946-496d-aa83-7d3c2c33526c",
+  "mamabench_v0.2_healthbench_consensus_c3115c69-cc42-44f5-a7e3-704cd0e94e3e",
+  "mamabench_v0.2_healthbench_consensus_23f803c7-e7ef-4e01-a221-1b644c712910",
+  "mamabench_v0.2_healthbench_consensus_ab7c72ec-6e79-416f-8a93-8f3413bf57be",
+  "mamabench_v0.2_healthbench_consensus_37165947-d0f6-4677-a0e5-cc70ec977d5a",
+  "mamabench_v0.2_healthbench_consensus_e126273b-9d59-4a79-9bb0-391db5f8eb0a",
+  "mamabench_v0.2_healthbench_consensus_ef3f7972-e436-4093-b06f-6039f747ae07",
+  "mamabench_v0.2_healthbench_consensus_952c0b6e-d3b1-4db9-833e-97d3e95bb748",
+  "mamabench_v0.2_healthbench_consensus_8bbdffcf-a460-4d88-a466-a58f8235a1a0",
+  "mamabench_v0.2_healthbench_consensus_3c244c68-7374-47ff-a1a0-dc1bdecaec61",
+  "mamabench_v0.2_healthbench_consensus_ece7367d-6c83-4057-a60f-62f9e1b1d8b5",
+  "mamabench_v0.2_healthbench_consensus_4a2ee46a-97f3-4259-8733-2f95d7603074",
+  "mamabench_v0.2_healthbench_consensus_7379f677-902d-4871-8c72-7dff214ce6e4",
+  "mamabench_v0.2_healthbench_consensus_9114a635-aae2-464a-bf3f-6d628cf2b98f",
+  "mamabench_v0.2_healthbench_consensus_aedeea33-3bea-45fe-8214-90ba6f40752c",
+  "mamabench_v0.2_healthbench_consensus_037cdb42-109a-44f5-90f1-09b731842fb2",
+  "mamabench_v0.2_healthbench_consensus_6ef3e0fe-319f-4992-9a45-5ef853a273b8",
+  "mamabench_v0.2_healthbench_consensus_cc1d44d0-769e-4d84-aa33-e17d9fd4fb9f",
+  "mamabench_v0.2_healthbench_consensus_459b5cc4-6a16-403c-8b17-1670fa8cdac0",
+  "mamabench_v0.2_healthbench_consensus_86fbe585-01b5-46d5-bdb0-c28ab2ff6f9c",
+  "mamabench_v0.2_healthbench_consensus_1c75f6cf-5ee7-49f7-9b4e-ccc4f902fead",
+  "mamabench_v0.2_healthbench_consensus_3691ff82-35f9-4bd3-9d6f-a62fce1d8704",
+  "mamabench_v0.2_healthbench_consensus_397d5f29-4ac2-41b2-ad2e-1ae775aff8e5",
+  "mamabench_v0.2_healthbench_consensus_baa76281-a534-423c-bec1-1100023466a0",
+  "mamabench_v0.2_healthbench_consensus_61bf651c-a509-4f40-bd27-ee9e276f5939",
+  "mamabench_v0.2_healthbench_consensus_6a131b5f-278c-415d-b28d-46aad3b18066",
+  "mamabench_v0.2_healthbench_consensus_9ada42f8-e939-4584-88a5-75d9035f7f49",
+  "mamabench_v0.2_healthbench_consensus_5b1cdedb-d1ec-4bfb-8aa1-de54dd2bd17e",
+  "mamabench_v0.2_healthbench_consensus_6888187f-5088-4579-881d-7f3f73586c28",
+  "mamabench_v0.2_healthbench_consensus_fe502ce3-aa6a-49c3-bec1-2890262011ea",
+  "mamabench_v0.2_healthbench_consensus_1d786666-4dbb-4dec-ad2c-1b7302fe6a02",
+  "mamabench_v0.2_healthbench_consensus_415a3bca-609e-4c16-a690-6ed2af93fc57",
+  "mamabench_v0.2_healthbench_consensus_5256737f-0b8b-4ba3-9abd-74ab00bd117e",
+  "mamabench_v0.2_healthbench_consensus_cf2fc93c-d09f-4c27-b219-4fbe8ac46a70",
+  "mamabench_v0.2_healthbench_consensus_478b5044-cdbc-4cc7-a126-dc78ac3305f9",
+  "mamabench_v0.2_healthbench_consensus_cc2902e3-38d3-40cc-83cc-639313d0e897",
+  "mamabench_v0.2_healthbench_consensus_b9c1bbea-e333-40c4-afc6-f5eaf927aa0e",
+  "mamabench_v0.2_healthbench_consensus_ca9307c4-6a12-4f71-8630-4d6284bab914",
+  "mamabench_v0.2_healthbench_consensus_1956b7b9-0b75-43a4-8ef6-e4f726bf5f98",
+  "mamabench_v0.2_healthbench_hard_d817d21a-6f01-4a17-8772-cbd433049d6a",
+  "mamabench_v0.2_healthbench_hard_d0a7dc08-94bd-4d27-991a-2af7c919ce7a",
+  "mamabench_v0.2_healthbench_hard_68b20b72-f599-4cd0-a724-097d50ecd1b8",
+  "mamabench_v0.2_healthbench_hard_3534668a-1e1d-421b-82b3-ecef3f114e3a",
+  "mamabench_v0.2_healthbench_hard_4bd394a3-e8e2-4346-9ccc-5e055cc2a94e",
+  "mamabench_v0.2_healthbench_hard_555650e8-47aa-4fde-a90b-cd88baee20e6",
+  "mamabench_v0.2_healthbench_hard_037cdb42-109a-44f5-90f1-09b731842fb2",
+  "mamabench_v0.2_healthbench_hard_1ee4752e-13c7-405c-93d7-8930f9160ea1",
+  "mamabench_v0.2_healthbench_hard_459c3554-6a0a-46e7-a2da-3c86507327ed",
+  "mamabench_v0.2_healthbench_hard_1cd1c06f-d27d-49c1-83a0-b3d427a2976b",
+  "mamabench_v0.2_healthbench_hard_a0cc9508-1d8f-40ee-bf6e-f8997cb750cc",
+  "mamabench_v0.2_healthbench_hard_9ea199de-bc30-408e-a0f9-7740f910c6e2",
+  "mamabench_v0.2_healthbench_hard_51e55472-4d53-4a31-b40e-61feb9ff6310",
+  "mamabench_v0.2_healthbench_hard_59c18c75-2774-4939-abab-0da869419e24",
+  "mamabench_v0.2_healthbench_hard_6ed3b046-c538-4d21-8d4b-39cd1504e44b",
+  "mamabench_v0.2_healthbench_hard_8aadd85d-1aa2-4046-a854-884c6ec4ff34",
+  "mamabench_v0.2_healthbench_hard_4c5225e6-5217-47d1-b7f0-9a15d4f4317f",
+  "mamabench_v0.2_healthbench_hard_78c02ee0-52f2-47d2-b5dc-d7c312727448",
+  "mamabench_v0.2_healthbench_hard_9b98fb94-8a98-4c31-8bba-db9385a6b8a0",
+  "mamabench_v0.2_healthbench_hard_18cdc57b-d52e-406b-a16c-6e2f5538d08b",
+  "mamabench_v0.2_healthbench_hard_06d12149-94dc-4797-b3ef-e9efa9759c09",
+  "mamabench_v0.2_healthbench_hard_db3d5b2d-1b83-4737-824c-a9be94167b37",
+  "mamabench_v0.2_healthbench_hard_c3f19a38-f4c6-4a8a-b4ba-1b6eb44e463e",
+  "mamabench_v0.2_healthbench_hard_8404a07b-4ba1-476c-a22c-a20e084b4153",
+  "mamabench_v0.2_healthbench_hard_d608127c-ac7f-4fd6-90de-8b2103259468",
+  "mamabench_v0.2_healthbench_hard_c45130e5-d0a9-4eb0-90ac-d4d8b56ac94f",
+  "mamabench_v0.2_healthbench_hard_bfb9a07b-7088-4832-bbf1-32b72c308b75",
+  "mamabench_v0.2_healthbench_hard_0ffac3f1-44e4-4934-a25c-e6a31fe5a838",
+  "mamabench_v0.2_healthbench_hard_3e2d1d7a-66c8-4cf0-a94f-dbf269887497",
+  "mamabench_v0.2_healthbench_hard_b81c059a-1137-47cf-bab8-ab5eb8ff34d5",
+  "mamabench_v0.2_healthbench_hard_ff1e5dd0-33a2-419e-b31c-4e9cd286c9f2",
+  "mamabench_v0.2_healthbench_hard_b7ddde65-702e-4255-b1d7-09d01248e76d",
+  "mamabench_v0.2_healthbench_hard_63507bb7-dc6a-4a69-8fc4-4b4c9c9c2901",
+  "mamabench_v0.2_healthbench_hard_ebff7dc3-4423-4d51-9d56-873a67a75e96",
+  "mamabench_v0.2_healthbench_hard_e9b7f69b-69ee-486b-805c-c98439ad969a",
+  "mamabench_v0.2_healthbench_hard_eac8fe1e-ef50-4342-a239-fcebfff6001f",
+  "mamabench_v0.2_healthbench_hard_7b821797-070d-4fa4-80f1-cd2e9e1d61a1",
+  "mamabench_v0.2_healthbench_hard_5a45b11c-6c3c-4a3e-8cae-5b7c4abe6e56",
+  "mamabench_v0.2_healthbench_hard_d92a738c-4e7c-4637-9362-61a7c2bea804",
+  "mamabench_v0.2_healthbench_hard_c28ac978-31e6-463d-b97b-3dadcf059db4",
+  "mamabench_v0.2_healthbench_hard_00656524-cc51-47a3-bfb5-85e7096ee1c8",
+  "mamabench_v0.2_healthbench_hard_1dc0b953-ef66-4a78-9989-7cf1053b5b41",
+  "mamabench_v0.2_healthbench_hard_ca976526-9bac-476e-8860-9e0914d5fc39",
+  "mamabench_v0.2_healthbench_hard_352832df-973e-4468-956b-13f2df962bb6",
+  "mamabench_v0.2_healthbench_hard_6b403cd8-97c6-4849-ab22-81e9753156ff",
+  "mamabench_v0.2_healthbench_hard_eb89fb31-0d41-4928-8ca0-d34e2c5a78ea",
+  "mamabench_v0.2_healthbench_hard_65449e9d-ac3f-4cb0-8805-58fcb4ffbdce",
+  "mamabench_v0.2_healthbench_hard_4d3be4fb-4168-49f9-836d-ff9871e14c29",
+  "mamabench_v0.2_healthbench_hard_1e93a526-e2b4-42a1-901a-d6697bb0ead9",
+  "mamabench_v0.2_healthbench_hard_eb97bae4-430e-45cd-a065-2df3ab5c600e"
+ ]
+}

configs/config-v0.2.0/reports/device-vs-host-fidelity-20260611.html

@@ -0,0 +1,98 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Device (LiteRT) vs Host (cluster GGUF) fidelity — v0.2 open-ended</title>
+<style>
+  :root {
+    --ink:#1a1a1a; --muted:#555; --soft:#666; --rule:#d9d9d9; --bg:#fafafa;
+    --good:#117a3b; --good-bg:#e6f4ec; --mid:#9a5a00; --mid-bg:#fcf3e1;
+    --bad:#9c2727; --bad-bg:#fbe7e7; --baseline:#1a4e8a; --accent:#444;
+  }
+  body { font:15px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif; color:var(--ink); max-width:920px; margin:32px auto; padding:0 24px; background:#fff; }
+  h1 { font-size:22px; margin:0 0 4px; }
+  h2 { font-size:17px; margin:28px 0 10px; border-bottom:1px solid var(--rule); padding-bottom:4px; }
+  p { margin:8px 0; }
+  small.meta { color:var(--muted); }
+  .tldr { background:var(--bg); border-left:3px solid var(--accent); padding:12px 16px; margin:16px 0; }
+  .tldr p { margin:6px 0; }
+  table { width:100%; border-collapse:collapse; margin:8px 0 16px; font-size:14px; }
+  th,td { padding:8px 10px; text-align:right; border-bottom:1px solid var(--rule); }
+  th:first-child, td:first-child { text-align:left; }
+  thead th { background:#f4f4f4; font-weight:600; border-bottom:2px solid #bbb; }
+  .cell-good { background:var(--good-bg); color:var(--good); font-weight:600; }
+  .cell-mid  { background:var(--mid-bg);  color:var(--mid);  font-weight:600; }
+  .cell-bad  { background:var(--bad-bg);  color:var(--bad);  font-weight:600; }
+  .legend { font-size:12.5px; color:var(--muted); margin:-8px 0 16px; }
+  .callout { background:var(--bg); border-left:3px solid var(--accent); padding:10px 14px; font-size:13px; color:var(--muted); margin:10px 0; }
+  .footnote { color:var(--muted); font-size:12.5px; margin-top:18px; padding-top:12px; border-top:1px solid var(--rule); }
+  code { background:#f4f4f4; padding:1px 4px; border-radius:3px; font-size:13px; }
+</style>
+</head>
+<body>
+
+<h1>Device (LiteRT) vs Host (cluster GGUF) fidelity — v0.2 open-ended</h1>
+<small class="meta">MAM-AI v0.2 · is the host proxy a good stand-in for the deployed device? · gpt-oss-120b judge both sides · 2026-06-11</small>
+
+<div class="tldr">
+<p><strong>The host proxy is validated.</strong> Phase B was generated on a <em>host</em> proxy (cluster llama.cpp + Q4_0 GGUF); the deployed app runs <strong>LiteRT-LM on-device</strong>. Re-running the same questions on the real device and scoring with the same judge:</p>
+<ul style="margin:6px 0 4px 18px;">
+  <li><strong>Device tracks host closely on both tracks (SAQ + Healthbench) and both arms.</strong></li>
+  <li><strong>No-RAG (the clean, identical-input comparison): device is consistently slightly <em>better</em></strong> — so the host numbers are a mildly <em>pessimistic</em> lower bound, not an overstatement.</li>
+  <li><strong>+RAG: device ≈ host</strong> (device slightly higher recall; these arms also exercise device-side retrieval, so differences mix retrieval + generation).</li>
+  <li><strong>Every Phase B conclusion holds on-device</strong> — low recall / weighted_met, safe-but-unhelpful, zero-<code>dangerous</code> floor.</li>
+  <li><strong>Leaked chain-of-thought is host-only</strong> — 0 across all 669 device generations → confirmed a host-eval artifact, not device behavior.</li>
+</ul>
+</div>
+
+<h2>Method</h2>
+<p>Same questions, two stacks, one judge. <strong>Host</strong> = the committed Phase B results (cluster, llama.cpp, <code>gemma-4-E4B-it-Q4_0.gguf</code>). <strong>Device</strong> = Gemma 4 E4B on the deployed <strong>LiteRT-LM</strong> stack, run via <code>run_eval_device.py</code> (adb push/trigger/pull) in fresh-process batches of 8 — a workaround for aggressive on-device process killing of long runs (root-caused as <em>not</em> screen-related). Both arms rescored by the pinned <strong>gpt-oss-120b @ medium</strong> judge. SAQ run in full (369/arm); Healthbench sampled (50/subset = 150/arm) as the full 2,339/arm is ~12 h/arm on-device.</p>
+
+<h2>MCQ (medmcqa, no-RAG, n=100)</h2>
+<table>
+<thead><tr><th>metric</th><th>device (LiteRT)</th><th>host (GGUF)</th></tr></thead>
+<tbody>
+<tr><td>accuracy</td><td class="cell-good">0.57</td><td>0.54</td></tr>
+<tr><td>per-question agreement</td><td colspan="2" style="text-align:center">64% (36/100 differ)</td></tr>
+</tbody>
+</table>
+<p class="legend">Aggregate accuracy within ~3 pp. The 64% per-item agreement is a <em>floor</em> — inflated by temperature-1.0 sampling noise (both stacks decode stochastically), not pure quantization divergence.</p>
+
+<h2>SAQ ±RAG (kenya 312 + whb 20 + afrimedqa_saq 37 = 369/arm)</h2>
+<p>Recall (↑ better) and harm-rate (↓ better), device vs host, on the same questions:</p>
+<table>
+<thead><tr><th>arm · dataset</th><th>recall device</th><th>recall host</th><th>harm% device</th><th>harm% host</th></tr></thead>
+<tbody>
+<tr><td>no-RAG · kenya</td><td class="cell-good">0.194</td><td>0.178</td><td class="cell-good">19.2</td><td>20.8</td></tr>
+<tr><td>no-RAG · afrimedqa_saq</td><td class="cell-good">0.211</td><td>0.164</td><td class="cell-good">8.1</td><td>16.2</td></tr>
+<tr><td>no-RAG · whb <small>(n=20)</small></td><td class="cell-good">0.079</td><td>0.039</td><td class="cell-good">5.0</td><td>20.0</td></tr>
+<tr><td>+RAG · kenya</td><td class="cell-good">0.171</td><td>0.128</td><td class="cell-bad">23.7</td><td>18.9</td></tr>
+<tr><td>+RAG · afrimedqa_saq</td><td class="cell-good">0.214</td><td>0.162</td><td class="cell-bad">18.9</td><td>8.1</td></tr>
+<tr><td>+RAG · whb <small>(n=20)</small></td><td class="cell-good">0.070</td><td>0.026</td><td class="cell-bad">20.0</td><td>15.0</td></tr>
+</tbody>
+</table>
+<p class="legend">Green = device better, red = device worse. <strong>no-RAG:</strong> device better on both axes (higher recall, lower harm). <strong>+RAG:</strong> device recall still higher, but harm higher — and the +RAG arms aren't an isolated comparison: the device does its <em>own</em> on-device retrieval (Gecko + vector store) vs the host's precomputed contexts, so differences mix retrieval and generation.</p>
+
+<h2>Healthbench ±RAG (stratified sample, 50/subset = 150/arm)</h2>
+<p>weighted_met, device vs host, on the same matched ids:</p>
+<table>
+<thead><tr><th>arm · subset</th><th>device</th><th>host</th></tr></thead>
+<tbody>
+<tr><td>no-RAG · oss_eval</td><td class="cell-good">0.045</td><td>−0.036</td></tr>
+<tr><td>no-RAG · consensus</td><td class="cell-good">0.657</td><td>0.573</td></tr>
+<tr><td>no-RAG · hard</td><td class="cell-good">−0.137</td><td>−0.168</td></tr>
+<tr><td>+RAG · oss_eval</td><td class="cell-good">0.026</td><td>−0.029</td></tr>
+<tr><td>+RAG · consensus</td><td class="cell-mid">0.513</td><td>0.530</td></tr>
+<tr><td>+RAG · hard</td><td class="cell-mid">−0.182</td><td>−0.175</td></tr>
+</tbody>
+</table>
+<p class="legend">Same pattern as SAQ: <strong>no-RAG device slightly higher on all three</strong>; <strong>+RAG ≈ tied</strong> (±1–2 pp).</p>
+
+<h2>Bottom line</h2>
+<p>Across both tracks and both arms, the LiteRT device tracks the host GGUF proxy closely. On the clean no-RAG comparison the device is consistently a hair <em>better</em>, so <strong>the host numbers are a mildly pessimistic lower bound</strong> — likely because Q4_0 GGUF is a touch lossier than the LiteRT bundle. The deployed model's quality and safety are therefore <strong>at least as good as the Phase B headline numbers</strong>, and every Phase B conclusion (low recall, safe-but-unhelpful, zero-<code>dangerous</code>) holds on-device. The leaked chain-of-thought seen on the rubric track is a host-eval artifact (manual GGUF template), absent on all 669 device generations.</p>
+
+<p class="callout"><em>Caveats.</em> Same gpt-oss-120b judge both sides (its bias cancels in the device-vs-host Δ). No bootstrap CIs, and small per-subset n (esp. the HB 50/subset sample and whb n=20) — read single-subset deltas as indicative, not precise. One judge job (SAQ +RAG) failed once on a transient pypi timeout and was resubmitted. Device runs used fresh-process batches of 8 to survive on-device process killing.</p>
+
+</body>
+</html>