diff --git a/.github/workflows/llm-bench.yml b/.github/workflows/llm-bench.yml new file mode 100644 index 0000000..f50960c --- /dev/null +++ b/.github/workflows/llm-bench.yml @@ -0,0 +1,59 @@ +# V30 — the interpretation bench, on demand. +# +# It is NOT part of `ci.yml` on purpose: it downloads 355 MB of weights and +# spends several minutes answering 55 questions on a CPU, which would add that +# cost to every pull request for a number that only moves when the chat's +# prompt, grammar or parser change. Everything about the corpus that does NOT +# need the model — how the deterministic parser reads all 55 questions, the +# grammar automaton, the token mask — runs in `ci.yml` on every commit, in +# seconds (see src/features/ai/llm/corpus.test.ts). +# +# Run it from the Actions tab, or let it fire on a change to the chat. +name: LLM bench + +on: + workflow_dispatch: + pull_request: + paths: + - 'src/features/ai/llm/**' + - 'src/features/ai/chat/**' + - 'scripts/prepare-llm.mjs' + +permissions: + contents: read + +concurrency: + group: llm-bench-${{ github.ref }} + cancel-in-progress: true + +jobs: + bench: + name: Interpretation bench (CPU) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + # The same pinned files production deploys, laid out flat: onnxruntime-node + # reads them from disk and has no 25 MiB limit to work around. + - name: Cache the pinned weights + id: weights + uses: actions/cache@v4 + with: + path: .llm-cache + key: llm-cache-qwen3-0.6b-dq-q4f16-v1 + - if: steps.weights.outputs.cache-hit != 'true' + run: npm run llm:fetch + - name: Bench, decoding inside the grammar + env: + LABML_LLM_OUT: bench-report.json + run: npm run llm:bench:node + - uses: actions/upload-artifact@v4 + with: + name: llm-bench-report + path: bench-report.json + retention-days: 30 diff --git a/.gitignore b/.gitignore index 92c7702..2e25955 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ test-results # V27: the local language model is fetched at deploy time, never committed. public/llm/ + +# V30: the Node bench keeps a flat copy of the same pinned weights here. +.llm-cache/ diff --git a/PLAN.md b/PLAN.md index d4fcfc7..9c2c7f9 100644 --- a/PLAN.md +++ b/PLAN.md @@ -428,26 +428,26 @@ production on 21/08/2026. Cap 6's guiding thread: the lab meets the real world — real photos, real text, real file sizes, and the question every data budget asks. -| Wave | Content | Why | -| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **V23 — delivered** | **Vision 2**: SqueezeNet (2012) retired for three self-hosted ONNX models — **EfficientNet-Lite4 int8** classification (1,000 ImageNet classes, 77.6% top-1), **YOLOX-Nano** object detection (80 COCO classes; the stronger-but-AGPL YOLOs were ruled out, Apache-2.0 kept) and **UltraFace RFB-320** face detection — boxes drawn on the image, FR/EN class names, plain-language counts ("1 person · 1 face"). Box decoding (grids/strides, exp, IoU, per-class NMS) is hand-written and unit-tested; letterboxed inputs (aspect squashing measurably mislocated face boxes); named thresholds (objects 35%; faces 0.9 — real faces score ≥ 0.95, measured false positives top out at 0.85); ~19 MB total, runtime-cached, offline after first use; verified on real photos (NASA portrait → 1 person + 1 face; German Shepherd → dog + breed at 99.9%) | Owner request (21/08/2026): portraits have no ImageNet class, so the old model answered off-target — and the detector must recognize a whole range of things, not just faces | -| **V24 — delivered** | **Text columns**: free text stops being skipped and enters the pipeline as a hand-written **TF-IDF** block — accent-folding bilingual tokenizer, merged FR/EN stop words, vocabulary capped at 256 terms ranked by document frequency (ties alphabetical, terms seen in a single training document dropped), smoothed IDF, L2-normalized vectors, fitted on the training split only. Features are named `column:word`, so importance, Shapley and the report speak in words; `encodedBlocks` now measures a text block by its real width (counting it as one column silently shifted every block after it). Explanations gained **signed word effects** by occlusion — erase one word from the reviews containing it and average the shift of the answer — because permutation is blind to a redundant vocabulary, and multiclass is refused rather than faked. Export bumped to **format v3** (v2 files still import). Demo `reviews.csv`: 240 bilingual orders where the text carries the signal — baseline 0.52 → 0.92, `review` top of the importance chart, `fast`/`excellent`/`avance` pushing up, `refund`/`cheap` pushing down | Real CSVs have text columns (comments, descriptions) — the lab used to drop them on the floor | -| **V25 — delivered** | **Scale**: the lab now takes 100k–1M-row files without dying, on a measure-first design. Before: a stack overflow killed every run past ~65k rows (`push(...spread)` in the split), and the slow families made big runs unusable anyway (random forest alone: **535 s at 5 000 rows**). Measured first, then fixed: (1) the split rewritten with plain loops; (2) the planned typed-array pipeline rewrite was **descoped on measurement** — the pipeline was never the bottleneck (parse 2.6 s + profile 4.9 s + fit/transform 3.1 s at 1M rows); (3) **announced seeded sampling, never silent** — a global cap of 100 000 usable rows (seeded, stratified, `summary.sampledFrom`) plus measured per-family training caps (forest 1 000 · tree 2 000 · logistic/linear/MLP 20 000 · k-NN 5 000 · GBDT 50 000) drawn as nested prefixes of one seeded order, every capped model scored on the same full test set, and every sample printed on the leaderboard, in the tuning panel and in the HTML report; k-NN's old **silent** internal 5 000-row subsample was folded into the announced mechanism; (4) a **named memory guard**: parsing streams and refuses past 20M cells (rows × columns) with the numbers spelled out, instead of letting the tab die. After, measured: a 1M-row file trains the whole 8-model zoo in **~130 s** (and a 120k-row file in the same ~126 s — cost is flat past the cap), gbdt reaches 0.991 accuracy on the 50 000-row announced sample. Every demo dataset sits under every cap: existing behavior unchanged. 283 unit tests (announced-sampling determinism, stratified nesting, cap recording), 54 e2e (a generated 120k-row CSV trains with the announcement asserted; a 21M-cell file is refused by name) | -| **V26 — delivered** | **Learning curves**: the lab answers the classic budget question — "would more data help this model, or is it time to work on features?" — with one new chart. On demand (like tuning), one model is retrained on growing seeded fractions of the train split: the SAME nested prefixes V25's announced caps draw from (same seed, same order), so for a capped family the last point is exactly the leaderboard model's diet and the verdict says out loud whether the announced cap costs accuracy. A geometric ladder of up to 6 sizes (each at least 16 rows, refused entirely when only one rung fits — one point is not a curve); at every size the pipeline is REFITTED on that prefix only (imputation, encoding, IDF, scaling all see just those rows — the strict no-leakage reading of a learning curve) and the model is scored on the same full held-out test set with a V20 bootstrap 95% band. The verdict is the V20 paired bootstrap applied to the last size step: a decisive gain reads "still climbing — more data would probably help", anything else "flattened — work on features or the model", each with the capped variant ("the cap costs accuracy" / "the cap costs nothing here"). The chart (log-spaced sizes, CI band, one dot per announced size) ships with its numbers table, joins the run artifacts (history, HTML report, share links) and refuses the baseline by name — flat by definition. Measured on Titanic/gbdt: 45 → 713 rows traces 0.753 → 0.820 with the plateau verdict at the last step. 293 unit tests, 55 e2e | -| **V27 — delivered** | **Local chat, upgraded**: a real language model — **Qwen3-0.6B-DQ, 355 MB, Apache-2.0** — running entirely in the browser, offered beside the V6 deterministic interpreter, which stays the DEFAULT and the fallback. The model never computes: it translates a question into a V6 query, and every number still comes from the deterministic engine. Its output is checked against the closed Intent grammar — invented columns, unknown operators, absurd k, a correlation of a column with itself are all REFUSED, and a refusal falls back to the keyword parser, with a badge under each answer naming which engine produced the query. Three constraints shaped the build, all discovered by measurement: (1) Cloudflare Pages refuses assets over 25 MiB, so `scripts/prepare-llm.mjs` fetches the weights at DEPLOY time (never committed — 355 MB in git would slow every clone) and splits them into **15 parts of ≤ 24 MiB**, glued back in the browser through transformers.js's `customCache` hook, with per-part size checks and named refusals (`llm-part-missing`, `llm-part-size`, `llm-short`); (2) the strict CSP forbids the library's CDN default, so its pinned ONNX Runtime build is self-hosted under `/ort-llm/` — the jsep (WebGPU) variant clears the 25 MiB limit by only 0.1 MiB, so re-check it on upgrades; (3) **WebGPU is required, not preferred**: the model's `GatherBlockQuantized` embedding kernel has no WASM implementation and needs `shader-f16`, so a device without it gets a named refusal instead of an unusable download. Verified end to end in a browser: the sharded weights download with correct cumulative progress, reassemble, and build an ONNX Runtime WebGPU session (18 s warm). **Honest limit, stated rather than hidden**: the interpretation-quality bench could NOT be run here — it needs a GPU with `shader-f16`, which the CI and dev runners lack. It ships instead as a repo tool (`npm run llm:bench`, 16 FR/EN questions over Titanic, half of them phrasings the keyword grammar cannot catch) that exercises the real production path, so the number can be measured on real hardware before the model is promoted beyond opt-in. 314 unit tests, 57 e2e | -| **V27.1 — delivered** | **The model earns its place, it does not take it**: the V27 order was wrong, and the measurement said so. With the local model selected it read EVERY question first and won whenever its JSON passed the grammar check — even when the keyword parser had a correct reading of its own. Measured in production on six reference questions over Titanic: the model turned « combien de personnes sont montées à Cherbourg ? » into `embarked = Cherbourg` → **0 rows**, where the deterministic parser had `embark_town = Cherbourg` → **168**; and read « est-ce que les femmes payaient plus cher que les hommes ? » as a plain count (314 female) instead of mean fare grouped by sex. Tally: 2 right, 2 confidently wrong, 2 refusals. The order is now **deterministic first, model as a rescue** (`resolveIntent`, unit-tested): the keyword grammar can only ever name a column that exists and a value that actually occurs in it, so when it understands, nothing overrides it — and the model is asked only about what it gives up on, which is exactly the gap that justifies its 355 MB. On the same six, **measured on the owner's GPU after deploy**: 5 right, 1 wrong, 0 refusals — up from 2 right, 2 wrong, 2 refusals. Two further defects fixed: (1) a refusal was badged « question read by the local model », claiming a reading nobody had made — refusals now name nobody and say whether the model was even consulted; (2) the system prompt had **no groupBy and no top-k example at all**, and no rule tying a filter value to the column whose value list contains it — both added, with FR phrasings and a numeric-threshold example. The bench gains the two shapes that failed (`age < 10`, a top-k) and now reports the **shipped order** as its headline number instead of the two engines separately. 326 unit tests, 57 e2e. **The number that justifies the download**: « combien d'enfants de moins de 10 ans ? » → `count age < 10` = 62 and « à quel âge moyen voyageaient les passagers ? » → `mean age` = 29.699, both of which the keyword grammar refuses outright; and « combien de personnes sont montées à Cherbourg ? » came back as the deterministic engine's 168, the model never consulted. **Still open**: one question of the six — « est-ce que les femmes payaient plus cher que les hommes ? » — is still read as a correlation (fare↔age, a column the question never names); addressed in V27.2. The full bench remains un-runnable here (no `shader-f16`), so its number still has to come from real hardware. | Measured by the owner in production (22/08/2026), the day V27 shipped. A confidently wrong answer costs more trust than a refusal — and V27 produced two of them, including a 0 where the deterministic engine already had the right 168. | -| **V27.2 — delivered** | **Two honesty defects, one measured, one found while reading the measurement**: (1) the comparison question V27.1 left wrong — « est-ce que les femmes payaient plus cher que les hommes ? » read as a correlation between `fare` and `age` — gets a rule that names both halves of the mistake: a question comparing two groups is an aggregate with `groupBy` on the column whose values name them, NEVER a correlation; and never pick a column the question does not mention. A second FR comparison example ships with it, in a **different phrasing** from the failing one, which stays a held-out bench case rather than becoming a memorised answer. (2) The answer sentence said « (sur 891 lignes) » under a mean built from 714 values: `rowsConsidered` counts rows after the filter, while `numericAt` skips missing and unparseable cells. The number was right, the sentence around it was not. Aggregates now carry `valuesUsed` (scalar) and `used` per group, set only when they differ from the row count, and the UI says « 714 valeurs utilisables sur 891 lignes » — matching what the correlation branch already did. This one predates V27 entirely: it has been there since V6. 330 unit tests, 57 e2e. **Measured after deploy**: the rule did stop the correlation — but the model then read the same question as `count fare >= 0`, still wrong. Two prompt attempts, two failure modes; see V27.3 for where that stops. | Measured by the owner on real hardware (22/08/2026): 5 of 6 reference questions right after V27.1. The sixth is a confidently wrong answer to a different question than the one asked, and the « sur 891 lignes » wording was found by reading that same screenshot closely — a right number inside a wrong sentence is exactly what this project refuses to ship. | -| **V27.3 — delivered** | **A `>=` that was quietly an `=`**: retesting the comparison question after V27.2 produced « 0 ligne correspond où fare >= 0 » — impossible on a table where all 891 fares clear zero. Root cause found by reproduction, not by reading: the model emitted `"value": "0"` as a **string**, `asFilter` accepted a string for any operator, and `matchesFilter` took the numeric branch only for `typeof value === 'number'` — so `>=` fell through to the equality branch and tested `fare == "0"` against a column whose zero fares are written `0.0`. Same intent with a real number: 891 rows. The hole is closed on both sides: `asFilter` converts a numeric string and refuses anything else on `<`, `<=`, `>`, `>=` (equality keeps text — that is how categorical filters work), and `matchesFilter` handles the ordering operators apart, throwing the named `filter-not-numeric` rather than passing a bug off as a query with no matches. V6 code, reachable only through the model: the keyword parser always built numbers. **And a limit, recorded rather than papered over**: « est-ce que les femmes payaient plus cher que les hommes ? » is still read wrong — a correlation before V27.2, a vacuous count after. The model finds `fare` every time and the shape never. Two prompt attempts are enough; a third would be sewing the prompt around one sentence, which buys a flattering bench and nothing else. The measured score stands at **5 of 6**, and the sixth is written down as what a 0.6B does not do. 337 unit tests, 57 e2e. | Found by retesting in production (22/08/2026). An arithmetically impossible answer — zero rows for a condition every row satisfies — is worse than a refusal and worse than a wrong reading: it makes the engine itself untrustworthy, which is the one thing LabML sells. | -| **V28 — delivered** | **« Ne nous croyez pas sur parole »** — a `/privacy` route that states the local-only promise once, in full, and then hands the reader the means to check it without trusting a word of it. Four verification steps, ordered by how hard they are to fake: cut the network (DevTools → Network → Offline, or the Wi-Fi switch) and watch the whole lab keep working; watch the Network tab while loading a file and training, and see nothing happen; read `Content-Security-Policy` on the document itself; open Application → IndexedDB and see exactly what was kept. The served policy is **quoted verbatim on the page and pinned to `public/_headers` by a unit test** — a page that claims a protection the site quietly dropped is worse than no page. A schematic of the Network panel is drawn rather than screenshotted (DevTools chrome differs per browser and per locale) and captioned as a diagram, not a capture. A live audit panel counts this page's own resource timings by origin and says, in the same breath, what it cannot see: worker timelines and requests the CSP blocked — a proof that oversells itself is worth less than none. Last section lists what _does_ cross the network (app files, demo datasets on click, vision models on entering Vision, LLM weights on explicit consent) and what never does. FR/EN, prerendered shell, WCAG AA verified by axe including the audit result. 344 unit tests, 60 e2e. | Owner request (22/08/2026): the promise is repeated across the site, but a user has no way to tell a true claim from a comforting one. Verifiability is the product here — anyone can write « your data stays local » in a footer. | -| **V29 — delivered** | **Analytical SQL in the browser (DuckDB-Wasm, MIT)**: the Data Studio gains a real OLAP engine — joins, window functions, aggregations — over the file you just loaded, with no server and no upload. The file is queried **as dropped, before the cleaning recipe**: the recipe belongs to the studio, and a result traceable to nothing the user can reopen would be worse than no SQL at all. Extra CSV / **Parquet** / JSON files can be attached in the same session (Parquet is a new input format for the lab), each exposed as a view named after the file; a result exports to CSV or goes to the ML Lab in one click, through the handoff path V4 already built. Errors show **DuckDB's own message** — it names the line and the token, which no paraphrase of ours would. **The measurement that set the version**: `@duckdb/duckdb-wasm` is pinned to **1.28.0**, not `latest`. From 1.29 the binaries cross Cloudflare Pages' hard 25 MiB per-file limit (eh 34.2 MiB, mvp 39.4 MiB); at 1.28.0 they are **17.3 and 21.1 MiB** and fit. Newer would have meant sharding the wasm and either widening `connect-src` to `blob:` — days after publishing a page that quotes that very directive — or rebuilding the service worker in injectManifest mode. An older engine was the cheaper honest trade, and it is written here so the next upgrade re-measures instead of rediscovering. Self-hosted under `/duckdb/` (the library defaults to jsDelivr, which the CSP refuses), **never precached** — cached on first use like the vision models, so nobody pays 18 MiB before opening the console — and the `coi` threaded build is left out entirely: no COOP/COEP, no SharedArrayBuffer, single-threaded as the assumed mode. Remote S3/HTTP querying stays out, by CSP and by intent. 352 unit tests, 61 e2e. | Owner request (21/08/2026): real analytical SQL on ~100 MB files with zero backend. Delivered after the /privacy page at the owner's request (22/08/2026). | -| V30 | **Chat that reads better, measured before it is made bigger.** The V27.1–V27.3 measurement stands at **5 of 6** reference questions, and the one failure is a _shape_ error, not missing knowledge: the model finds `fare` every time and picks the wrong intent. **First, the numbers that kill the obvious idea** — a « 600 MB model » is not an upgrade: at q4f16, Qwen3-0.6B **non-DQ is 570 MB and the same brain**, only its embeddings unquantised. The real rungs are gemma-3-1b-it **764 MB** (2×), Llama-3.2-1B **1.09 GB**, SmolLM2-1.7B **1.11 GB**, Qwen2.5-1.5B **1.22 GB**, Qwen3-1.7B **1.43 GB** — against 370 MB today. So the plan spends nothing on weights until the cheap levers are exhausted. **(A) A bench worth the name** — 40–60 FR/EN questions including the phrasings that fail, runnable and reported; today's 18 cases cannot run in CI, and without this nothing that follows is measurable. **(B) Constrained decoding** — a hand-written `LogitsProcessor` masking every token outside the grammar _during_ generation: after `{"kind":"` only seven tokens are legal. The shape error becomes unrepresentable rather than caught after the fact, and on this task that can beat a model four times larger. **(C) Examples drawn from the user's own columns** instead of frozen Titanic ones — 0 MB, and it removes the temptation to copy an example column. **(D) Two samples, one vote**, keeping the candidate that validates and invents no column the question never names — 0 MB, 2× the time. **Only then** the bigger model, and as a SECOND announced download (« reinforced model », 764 MB) with Qwen 370 MB staying the default: the V27 sharding infrastructure already handles it (32 parts of 24 MiB). VRAM (~1.2–1.5 GB estimated) and first-token latency to be measured before promising anything. | Owner question (22/08/2026): would a bigger model raise the share of correct answers? The measured failure is structural, so the plan tests that hypothesis for 0 MB before asking a visitor for twice the bandwidth. | -| V31 | **Vision that stops being asked the impossible.** Today's three models weigh **18.6 MB total** (EfficientNet-Lite4 int8 13.6, YOLOX-Nano 3.7, UltraFace 1.3) against 370 MB for the chat model — the headroom is enormous. **The main cause of the mistakes is not the network**: ImageNet-1k has **no « person » class** — 1000 labels, ~120 of them dog breeds, none for a human being — so a photo of someone comes back as « suit » or « jersey ». The model is not wrong; it is being asked a question whose answer is absent from its vocabulary. **(A) Measure first**: 30–50 public-domain images with expected label and expected boxes, replayed in e2e, so « it still makes mistakes » becomes a percentage. **(B) Fix the label space — the real correction**: CLIP ViT-B/32 zero-shot, vision q4f16 **126 MB** + text int8 **64 MB** ≈ **190 MB**, letting the visitor type their own labels (« a cat », « an invoice », « a houseplant »). It repairs the defect and makes a far better demonstration than 1000 frozen classes; open weights, self-hosted, local execution — the doctrine holds. **(C) What costs no download**: check the crop (squashing a 16:9 photo into a square skews everything — `preprocess.ts` is the suspect), average over two crops, recalibrate `OBJECT_THRESHOLD` (0.35) and `FACE_THRESHOLD` (0.9), and above all **refuse below a confidence floor** — « I am not sure » rather than a label picked at random, which is the chat's doctrine applied to pixels. **(D) A better detector**: YOLOX-S (Apache-2.0), ~35 MB, roughly +14 mAP over Nano — with acquisition and licence verified first, as in V23: the YOLOX ONNX files on the Hub are community re-uploads, not official releases. | Owner report (22/08/2026): the vision playground is better than the chat but still makes mistakes. Naming the label-space mismatch is what turns a vague complaint into a fixable defect. | -| V32 | **Documentation, the scaffolding and one finished tutorial.** A `/docs` route, linked from the footer beside « Comment ça marche », built on the **Diátaxis** split — tutorial (learning), how-to (a task), reference (lookup), explanation (the why) — because the usual failure of documentation is mixing all four on one page: a tutorial that pauses to weigh an alternative loses the beginner it was written for. A tutorial offers **no choices** and **guarantees the result**. Five rules specific to this project: **(1) the docs are tested like the code** — everything here is seeded at 42, so « you will get 0.821 accuracy » becomes an assertion in `e2e/docs.spec.ts` and a drifting page **breaks the build**; a documentation that cannot lie is the same promise as the rest of the site. **(2) Screenshots are generated** with Playwright, never hand-taken — one that cannot be regenerated does not ship. **(3) Better than a screenshot, a link that does the thing**: « try it » deep-links landing on the panel with the demo already loaded (needs small URL-parameter support), which never goes stale. **(4) Markdown lives in the repo** (`src/content/docs/**`), compiled at build with prerendered shells like every other route; no Algolia, no third-party doc host — a third-party call on a site that publishes `/privacy` would be indefensible, so search is a local index. **(5) The docs are not PLAN.md**: this file is the engineering record in English with the trade-offs; the docs are for users, FR/EN. Scope of this wave: the route, the Markdown pipeline, the table of contents, local search, and **one** complete tutorial — « premier modèle en 10 minutes » — tested end to end. It is the template every later page copies: tone, length, how figures are quoted. | Owner request (22/08/2026): document every shipped feature across /ml, /data and /ai, linked from the footer. One finished tutorial first, on purpose — writing the full reference before the template is settled means rewriting all of it. | -| V33 | **The reference, and the table of refusals.** Page-per-panel coverage of the three sections: ML Lab (leaderboard, tuning, thresholds, segments, uncertainty, learning curves, run comparison, model export/import, batch scoring), Data Studio (quality score, recipe, forced types, join, drift, anomalies, SQL console) and AI (vision, assistant, the two interpreters). Reference is dry, exhaustive and structured like the software — not prose. The page that no competitor has: **a complete table of the named refusals** — `filter-not-numeric`, `llm-part-missing`, `too-large`, `no-webgpu`, « neither interpreter understood », « the interval is not conclusive » — with what triggers each one, what it means and what to do about it. Refusing well is this project's distinguishing feature; documenting the refusals is the most honest page it can publish. Plus a formats page (CSV, Parquet, JSON, the model manifest). **Honest sizing**: this is 1–2 days of _writing_ for ~25 features in two languages. It does not automate into anything but mush. | A feature nobody can look up is a feature that does not exist for the reader; and a refusal nobody can decode reads as a bug rather than as the design it is. | -| V34 | **Explanations, how-to guides, and the pages that make the project readable as engineering.** The why-pages: why a baseline before anything else, why intervals instead of a single figure, why seed 42 everywhere, why everything runs locally (pointing at `/privacy` rather than repeating it — duplicated prose diverges), and **what LabML does not do, and why** — a project that names its limits reads as a serious one, and the limits are already measured here (the comparison question a 0.6B model does not read, the bench that needs `shader-f16`, SQL over the file before the recipe). Task-shaped how-to guides for readers who already know their way around: score a new batch, compare two runs, read a learning curve, hand a SQL result to the lab. Generated screenshots and the « try it » deep-links land here too. Every page ends with « et ensuite ? » — documentation without a next step is a dead end. | Three audiences, deliberately: the curious visitor (five minutes), the practitioner (one task), and the evaluator judging whether the engineering is rigorous. The explanation pages are what the third one reads. | -| **V35 — delivered** | **ML Lab: the number stops flattering itself.** Two method defects in shipped code, fixed, plus the two additions that follow from them. **(1) The winner was picked on the test set** — the leaderboard sorted nine models by the metric computed on test and crowned `sorted[0]`, which makes the headline figure the optimistic maximum of nine draws. There is now a **third split**: validation is carved out of the train side (64/16/20 with the default ratios), ranking and crowning happen on validation, and the champion line spells out both numbers and the gap between them — « selected on validation at 0.974, scores 0.917 on the untouched test set ». The **test indices are byte-identical** to what the same config produced before V35, so every panel that reads the test set (segments, thresholds, uncertainty, batch compare) is unchanged; below 60 usable rows the third split is refused by name and the lab ranks on test as before. Ranking now lives in ONE module (`ranking.ts`) used by the leaderboard, the history, the run comparison, the report and the auto-selected insights model — the bug that shipped mid-wave was exactly that a fourth site still sorted on test and opened a different model than the one crowned. **(2) The split was always random, even on dated data.** A chronological split (oldest rows train, newest test, rows without a parseable date dropped and counted) and a group split (no group on both sides — the same customer in train and test is the same leak) are offered when a column supports one, and **announced** in the run info. **(3) A predictive leak detector**: V6 caught columns that MAP to the target; this catches the merely predictive one — a one-column stump fitted on train and scored on validation, and a lone column reading the target at ≥ 99% shows as a copper warning with its measured score, never as a victory. **(4) A robust leaderboard on demand**: 5×2 repeated cross-validation over train+validation (the pipeline refitted inside every fold, the test set never touched), reporting a mean, a spread, and how often the leader actually beat the runner-up — « 10 of 10 folds: the order is stable » or « 6 of 10: treat them as tied ». **A defect the wave exposed and named**: with the smaller train split, Gaussian Naive Bayes on ~150 TF-IDF features saturates to exactly 0/1, so V24's word-effect occlusion measured exactly zero for every word and the card simply vanished — reading as « no word matters », which is false. Measured (2 distinct probabilities out of 48 test rows, against 48 for logistic and gbdt), the card now **refuses by name** and points at a model that can answer. 369 unit tests, 65 e2e. | Owner request (22/08/2026), launched 23/08/2026. Two of the four items were defects rather than gaps: a lab that sells honest evaluation cannot ship a headline figure it knows to be optimistic, nor a split that leaks on dated data. | -| **V36 — delivered** | **ML Lab: the gaps that were deliberately left open.** Each item was consciously deferred in an earlier wave rather than forgotten; delivering them together keeps the descopes visible instead of letting them quietly become permanent. **(1) Class imbalance**, descoped by name in V16. Two mechanisms, each NAMED per family rather than hidden behind one word: the loss is weighted where the loss is ours (logistic regression, gradient boosting — the gradient AND the hessian are scaled, since scaling only the gradient inflates leaf values instead of rebalancing), and a **seeded balanced resample** is used for the ml-cart families (tree, forest), which take no sample weights. The minority is upsampled to the majority's size — never the reverse: balancing by trimming the common class throws away real observations to fix a ratio. Off by default, because on a balanced target it changes nothing and a knob that does nothing is worse than no knob; the run announces the majority share, and the leaderboard says so when it crosses 60%. **(2) The ranking metric is a choice.** Accuracy and RMSE were imposed, which is the wrong criterion on an imbalanced problem — a model that never predicts the rare class can top an accuracy ranking and be useless. Rank on F1, recall, precision or ROC-AUC and the order genuinely changes; a model that cannot produce the chosen metric sorts last rather than being dropped. Ranking stays in the single V35 module, so the leaderboard, the history, the comparison, the report and the auto-inspected model all move together. **(3) Multiclass thresholds**, open since V16, read **one-vs-rest**: pick a class, score it against all the others, same PR and calibration curves. What the panel refuses to imply is a complete multiclass decision rule — two classes can both clear their thresholds and nothing here says which wins, so it says that instead. **(4) An ensemble of the best**: the average of the top three, built from models already fitted, so it costs one pass over the test set. The baseline is never a member (averaging a constant predictor drags the result toward the majority class), members are picked by the same V35 ranking rule, and **probabilistic members are preferred** — a mid-wave measurement showed the ensemble winning on iris with a bare vote because k-NN was in the top three, which silently closed the threshold, calibration and word-effect panels on the champion. **What this wave deliberately does not do**: add a tenth model family (nine is plenty; a tenth improves neither honesty nor understanding), build an AutoML « we handle everything » mode (the opposite of a lab that shows its decisions), or bring in tabular deep learning (high cost, no gain at this scale, and no longer hand-written). 387 unit tests, 69 e2e. | Launched 23/08/2026, right after V35. Each item was a named descope, not an oversight — and the ensemble exposed one more silent-disappearance defect, of the same family as the one V35 found. | -| **V37 — delivered** | **ML Lab: speed and the comfort of long sessions.** The wave opened, as the plan demanded, with a measurement — and the measurement moved the wave. **(1) Parallel training** was the headline: the zoo trains sequentially in one worker, so the heavy families were shipped to helper cores. Models cannot cross a worker boundary — `predict` is a closure and structured clone drops functions — so each helper returns `toJSON()` **as a JSON string** and the caller rebuilds through the V22 import path, which makes a parallel model byte-identical to an imported one. That string is not a detail: posting the object instead let structured clone keep shapes JSON drops, and ml-cart's `load()` then rebuilt a tree whose first prediction threw `this.root.classify(...).maxRowIndex is not a function` — a defect that would only have surfaced later, when the user opened insights. Helpers are split by measured cost, heaviest first, to the lightest helper (greedy longest-processing-time); k-NN never leaves the main worker, being the one family with no `toJSON` and also the one that fits in 0 ms; any failure — no Worker support, a helper that throws, an unserialisable family — silently falls back to the sequential trainer, because parallelism may change how long a run takes and never which models it produces. Every family's inference latency is still measured **here**, on the rebuilt predictor: a helper's timing would describe another core under contention, and the column would otherwise read 0 ms for exactly the families that ran in parallel. **(2) The measurement then found the real bottleneck, which was not training at all.** On a 60 000-row run, k-NN inference cost **59.6 s of a 68.8 s wall time — 87% of the whole run** — because the neighbour search allocated 5 000 objects and sorted them for every single prediction, and the scorer asked for labels and then for probabilities, searching twice. Fixed with a bounded top-k insertion over a flat `Float64Array` (ties keep the row seen first, exactly as the stable sort did) and an explicit `predictWithProba` for families where both answers come out of one computation. The old sorted implementation is kept verbatim in the tests as the oracle: the fast path is asserted to predict identically, row for row. **(3) Comparing more than two runs** — V21 compares two; three to six read against the **oldest** of the selection, so the deltas say what the session's changes did rather than what the newest run happens to be. Deliberately not a second diff engine: the matrix is the same V35 ranking and the feature columns are set algebra over the same `summary.featureColumns` V21 reads. **Measured, same machine, same 60 000-row file, same seed** — four arms: | +| Wave | Content | Why | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **V23 — delivered** | **Vision 2**: SqueezeNet (2012) retired for three self-hosted ONNX models — **EfficientNet-Lite4 int8** classification (1,000 ImageNet classes, 77.6% top-1), **YOLOX-Nano** object detection (80 COCO classes; the stronger-but-AGPL YOLOs were ruled out, Apache-2.0 kept) and **UltraFace RFB-320** face detection — boxes drawn on the image, FR/EN class names, plain-language counts ("1 person · 1 face"). Box decoding (grids/strides, exp, IoU, per-class NMS) is hand-written and unit-tested; letterboxed inputs (aspect squashing measurably mislocated face boxes); named thresholds (objects 35%; faces 0.9 — real faces score ≥ 0.95, measured false positives top out at 0.85); ~19 MB total, runtime-cached, offline after first use; verified on real photos (NASA portrait → 1 person + 1 face; German Shepherd → dog + breed at 99.9%) | Owner request (21/08/2026): portraits have no ImageNet class, so the old model answered off-target — and the detector must recognize a whole range of things, not just faces | +| **V24 — delivered** | **Text columns**: free text stops being skipped and enters the pipeline as a hand-written **TF-IDF** block — accent-folding bilingual tokenizer, merged FR/EN stop words, vocabulary capped at 256 terms ranked by document frequency (ties alphabetical, terms seen in a single training document dropped), smoothed IDF, L2-normalized vectors, fitted on the training split only. Features are named `column:word`, so importance, Shapley and the report speak in words; `encodedBlocks` now measures a text block by its real width (counting it as one column silently shifted every block after it). Explanations gained **signed word effects** by occlusion — erase one word from the reviews containing it and average the shift of the answer — because permutation is blind to a redundant vocabulary, and multiclass is refused rather than faked. Export bumped to **format v3** (v2 files still import). Demo `reviews.csv`: 240 bilingual orders where the text carries the signal — baseline 0.52 → 0.92, `review` top of the importance chart, `fast`/`excellent`/`avance` pushing up, `refund`/`cheap` pushing down | Real CSVs have text columns (comments, descriptions) — the lab used to drop them on the floor | +| **V25 — delivered** | **Scale**: the lab now takes 100k–1M-row files without dying, on a measure-first design. Before: a stack overflow killed every run past ~65k rows (`push(...spread)` in the split), and the slow families made big runs unusable anyway (random forest alone: **535 s at 5 000 rows**). Measured first, then fixed: (1) the split rewritten with plain loops; (2) the planned typed-array pipeline rewrite was **descoped on measurement** — the pipeline was never the bottleneck (parse 2.6 s + profile 4.9 s + fit/transform 3.1 s at 1M rows); (3) **announced seeded sampling, never silent** — a global cap of 100 000 usable rows (seeded, stratified, `summary.sampledFrom`) plus measured per-family training caps (forest 1 000 · tree 2 000 · logistic/linear/MLP 20 000 · k-NN 5 000 · GBDT 50 000) drawn as nested prefixes of one seeded order, every capped model scored on the same full test set, and every sample printed on the leaderboard, in the tuning panel and in the HTML report; k-NN's old **silent** internal 5 000-row subsample was folded into the announced mechanism; (4) a **named memory guard**: parsing streams and refuses past 20M cells (rows × columns) with the numbers spelled out, instead of letting the tab die. After, measured: a 1M-row file trains the whole 8-model zoo in **~130 s** (and a 120k-row file in the same ~126 s — cost is flat past the cap), gbdt reaches 0.991 accuracy on the 50 000-row announced sample. Every demo dataset sits under every cap: existing behavior unchanged. 283 unit tests (announced-sampling determinism, stratified nesting, cap recording), 54 e2e (a generated 120k-row CSV trains with the announcement asserted; a 21M-cell file is refused by name) | +| **V26 — delivered** | **Learning curves**: the lab answers the classic budget question — "would more data help this model, or is it time to work on features?" — with one new chart. On demand (like tuning), one model is retrained on growing seeded fractions of the train split: the SAME nested prefixes V25's announced caps draw from (same seed, same order), so for a capped family the last point is exactly the leaderboard model's diet and the verdict says out loud whether the announced cap costs accuracy. A geometric ladder of up to 6 sizes (each at least 16 rows, refused entirely when only one rung fits — one point is not a curve); at every size the pipeline is REFITTED on that prefix only (imputation, encoding, IDF, scaling all see just those rows — the strict no-leakage reading of a learning curve) and the model is scored on the same full held-out test set with a V20 bootstrap 95% band. The verdict is the V20 paired bootstrap applied to the last size step: a decisive gain reads "still climbing — more data would probably help", anything else "flattened — work on features or the model", each with the capped variant ("the cap costs accuracy" / "the cap costs nothing here"). The chart (log-spaced sizes, CI band, one dot per announced size) ships with its numbers table, joins the run artifacts (history, HTML report, share links) and refuses the baseline by name — flat by definition. Measured on Titanic/gbdt: 45 → 713 rows traces 0.753 → 0.820 with the plateau verdict at the last step. 293 unit tests, 55 e2e | +| **V27 — delivered** | **Local chat, upgraded**: a real language model — **Qwen3-0.6B-DQ, 355 MB, Apache-2.0** — running entirely in the browser, offered beside the V6 deterministic interpreter, which stays the DEFAULT and the fallback. The model never computes: it translates a question into a V6 query, and every number still comes from the deterministic engine. Its output is checked against the closed Intent grammar — invented columns, unknown operators, absurd k, a correlation of a column with itself are all REFUSED, and a refusal falls back to the keyword parser, with a badge under each answer naming which engine produced the query. Three constraints shaped the build, all discovered by measurement: (1) Cloudflare Pages refuses assets over 25 MiB, so `scripts/prepare-llm.mjs` fetches the weights at DEPLOY time (never committed — 355 MB in git would slow every clone) and splits them into **15 parts of ≤ 24 MiB**, glued back in the browser through transformers.js's `customCache` hook, with per-part size checks and named refusals (`llm-part-missing`, `llm-part-size`, `llm-short`); (2) the strict CSP forbids the library's CDN default, so its pinned ONNX Runtime build is self-hosted under `/ort-llm/` — the jsep (WebGPU) variant clears the 25 MiB limit by only 0.1 MiB, so re-check it on upgrades; (3) **WebGPU is required, not preferred**: the model's `GatherBlockQuantized` embedding kernel has no WASM implementation and needs `shader-f16`, so a device without it gets a named refusal instead of an unusable download. Verified end to end in a browser: the sharded weights download with correct cumulative progress, reassemble, and build an ONNX Runtime WebGPU session (18 s warm). **Honest limit, stated rather than hidden**: the interpretation-quality bench could NOT be run here — it needs a GPU with `shader-f16`, which the CI and dev runners lack. It ships instead as a repo tool (`npm run llm:bench`, 16 FR/EN questions over Titanic, half of them phrasings the keyword grammar cannot catch) that exercises the real production path, so the number can be measured on real hardware before the model is promoted beyond opt-in. 314 unit tests, 57 e2e | +| **V27.1 — delivered** | **The model earns its place, it does not take it**: the V27 order was wrong, and the measurement said so. With the local model selected it read EVERY question first and won whenever its JSON passed the grammar check — even when the keyword parser had a correct reading of its own. Measured in production on six reference questions over Titanic: the model turned « combien de personnes sont montées à Cherbourg ? » into `embarked = Cherbourg` → **0 rows**, where the deterministic parser had `embark_town = Cherbourg` → **168**; and read « est-ce que les femmes payaient plus cher que les hommes ? » as a plain count (314 female) instead of mean fare grouped by sex. Tally: 2 right, 2 confidently wrong, 2 refusals. The order is now **deterministic first, model as a rescue** (`resolveIntent`, unit-tested): the keyword grammar can only ever name a column that exists and a value that actually occurs in it, so when it understands, nothing overrides it — and the model is asked only about what it gives up on, which is exactly the gap that justifies its 355 MB. On the same six, **measured on the owner's GPU after deploy**: 5 right, 1 wrong, 0 refusals — up from 2 right, 2 wrong, 2 refusals. Two further defects fixed: (1) a refusal was badged « question read by the local model », claiming a reading nobody had made — refusals now name nobody and say whether the model was even consulted; (2) the system prompt had **no groupBy and no top-k example at all**, and no rule tying a filter value to the column whose value list contains it — both added, with FR phrasings and a numeric-threshold example. The bench gains the two shapes that failed (`age < 10`, a top-k) and now reports the **shipped order** as its headline number instead of the two engines separately. 326 unit tests, 57 e2e. **The number that justifies the download**: « combien d'enfants de moins de 10 ans ? » → `count age < 10` = 62 and « à quel âge moyen voyageaient les passagers ? » → `mean age` = 29.699, both of which the keyword grammar refuses outright; and « combien de personnes sont montées à Cherbourg ? » came back as the deterministic engine's 168, the model never consulted. **Still open**: one question of the six — « est-ce que les femmes payaient plus cher que les hommes ? » — is still read as a correlation (fare↔age, a column the question never names); addressed in V27.2. The full bench remains un-runnable here (no `shader-f16`), so its number still has to come from real hardware. | Measured by the owner in production (22/08/2026), the day V27 shipped. A confidently wrong answer costs more trust than a refusal — and V27 produced two of them, including a 0 where the deterministic engine already had the right 168. | +| **V27.2 — delivered** | **Two honesty defects, one measured, one found while reading the measurement**: (1) the comparison question V27.1 left wrong — « est-ce que les femmes payaient plus cher que les hommes ? » read as a correlation between `fare` and `age` — gets a rule that names both halves of the mistake: a question comparing two groups is an aggregate with `groupBy` on the column whose values name them, NEVER a correlation; and never pick a column the question does not mention. A second FR comparison example ships with it, in a **different phrasing** from the failing one, which stays a held-out bench case rather than becoming a memorised answer. (2) The answer sentence said « (sur 891 lignes) » under a mean built from 714 values: `rowsConsidered` counts rows after the filter, while `numericAt` skips missing and unparseable cells. The number was right, the sentence around it was not. Aggregates now carry `valuesUsed` (scalar) and `used` per group, set only when they differ from the row count, and the UI says « 714 valeurs utilisables sur 891 lignes » — matching what the correlation branch already did. This one predates V27 entirely: it has been there since V6. 330 unit tests, 57 e2e. **Measured after deploy**: the rule did stop the correlation — but the model then read the same question as `count fare >= 0`, still wrong. Two prompt attempts, two failure modes; see V27.3 for where that stops. | Measured by the owner on real hardware (22/08/2026): 5 of 6 reference questions right after V27.1. The sixth is a confidently wrong answer to a different question than the one asked, and the « sur 891 lignes » wording was found by reading that same screenshot closely — a right number inside a wrong sentence is exactly what this project refuses to ship. | +| **V27.3 — delivered** | **A `>=` that was quietly an `=`**: retesting the comparison question after V27.2 produced « 0 ligne correspond où fare >= 0 » — impossible on a table where all 891 fares clear zero. Root cause found by reproduction, not by reading: the model emitted `"value": "0"` as a **string**, `asFilter` accepted a string for any operator, and `matchesFilter` took the numeric branch only for `typeof value === 'number'` — so `>=` fell through to the equality branch and tested `fare == "0"` against a column whose zero fares are written `0.0`. Same intent with a real number: 891 rows. The hole is closed on both sides: `asFilter` converts a numeric string and refuses anything else on `<`, `<=`, `>`, `>=` (equality keeps text — that is how categorical filters work), and `matchesFilter` handles the ordering operators apart, throwing the named `filter-not-numeric` rather than passing a bug off as a query with no matches. V6 code, reachable only through the model: the keyword parser always built numbers. **And a limit, recorded rather than papered over**: « est-ce que les femmes payaient plus cher que les hommes ? » is still read wrong — a correlation before V27.2, a vacuous count after. The model finds `fare` every time and the shape never. Two prompt attempts are enough; a third would be sewing the prompt around one sentence, which buys a flattering bench and nothing else. The measured score stands at **5 of 6**, and the sixth is written down as what a 0.6B does not do. 337 unit tests, 57 e2e. | Found by retesting in production (22/08/2026). An arithmetically impossible answer — zero rows for a condition every row satisfies — is worse than a refusal and worse than a wrong reading: it makes the engine itself untrustworthy, which is the one thing LabML sells. | +| **V28 — delivered** | **« Ne nous croyez pas sur parole »** — a `/privacy` route that states the local-only promise once, in full, and then hands the reader the means to check it without trusting a word of it. Four verification steps, ordered by how hard they are to fake: cut the network (DevTools → Network → Offline, or the Wi-Fi switch) and watch the whole lab keep working; watch the Network tab while loading a file and training, and see nothing happen; read `Content-Security-Policy` on the document itself; open Application → IndexedDB and see exactly what was kept. The served policy is **quoted verbatim on the page and pinned to `public/_headers` by a unit test** — a page that claims a protection the site quietly dropped is worse than no page. A schematic of the Network panel is drawn rather than screenshotted (DevTools chrome differs per browser and per locale) and captioned as a diagram, not a capture. A live audit panel counts this page's own resource timings by origin and says, in the same breath, what it cannot see: worker timelines and requests the CSP blocked — a proof that oversells itself is worth less than none. Last section lists what _does_ cross the network (app files, demo datasets on click, vision models on entering Vision, LLM weights on explicit consent) and what never does. FR/EN, prerendered shell, WCAG AA verified by axe including the audit result. 344 unit tests, 60 e2e. | Owner request (22/08/2026): the promise is repeated across the site, but a user has no way to tell a true claim from a comforting one. Verifiability is the product here — anyone can write « your data stays local » in a footer. | +| **V29 — delivered** | **Analytical SQL in the browser (DuckDB-Wasm, MIT)**: the Data Studio gains a real OLAP engine — joins, window functions, aggregations — over the file you just loaded, with no server and no upload. The file is queried **as dropped, before the cleaning recipe**: the recipe belongs to the studio, and a result traceable to nothing the user can reopen would be worse than no SQL at all. Extra CSV / **Parquet** / JSON files can be attached in the same session (Parquet is a new input format for the lab), each exposed as a view named after the file; a result exports to CSV or goes to the ML Lab in one click, through the handoff path V4 already built. Errors show **DuckDB's own message** — it names the line and the token, which no paraphrase of ours would. **The measurement that set the version**: `@duckdb/duckdb-wasm` is pinned to **1.28.0**, not `latest`. From 1.29 the binaries cross Cloudflare Pages' hard 25 MiB per-file limit (eh 34.2 MiB, mvp 39.4 MiB); at 1.28.0 they are **17.3 and 21.1 MiB** and fit. Newer would have meant sharding the wasm and either widening `connect-src` to `blob:` — days after publishing a page that quotes that very directive — or rebuilding the service worker in injectManifest mode. An older engine was the cheaper honest trade, and it is written here so the next upgrade re-measures instead of rediscovering. Self-hosted under `/duckdb/` (the library defaults to jsDelivr, which the CSP refuses), **never precached** — cached on first use like the vision models, so nobody pays 18 MiB before opening the console — and the `coi` threaded build is left out entirely: no COOP/COEP, no SharedArrayBuffer, single-threaded as the assumed mode. Remote S3/HTTP querying stays out, by CSP and by intent. 352 unit tests, 61 e2e. | Owner request (21/08/2026): real analytical SQL on ~100 MB files with zero backend. Delivered after the /privacy page at the owner's request (22/08/2026). | +| **V30 — delivered** | **Chat that reads better, measured before it is made bigger.** The wave began by building the instrument, because V27's stood on 18 cases that needed a GPU with `shader-f16` — one laptop's worth of evidence, re-runnable by nobody. It now stands on **55 reference questions**, French and English, over every shape of the query grammar plus three that no query can answer, where refusing is the only correct outcome. Two harnesses run it: one in CI on every commit with no model at all (the deterministic parser, the grammar automaton, the token mask), and one against the REAL pinned q4f16 weights on a CPU through onnxruntime-node — same files, same prompt, same decoding path as production, minus the GPU. « Measurable » stopped meaning « on one machine ». **The instrument immediately contradicted the wave's own premise.** The failure was not mainly the model: on those 55 questions the shipped app answered **33 right, 15 WRONG, 7 refused** — and **seven of the fifteen wrong came from the deterministic parser**, which runs first and can never be overridden. « Combien de femmes ? » answered 891 instead of 314: the grammar knows `combien`, knows nothing about `femmes`, kept the count and dropped the condition — under the badge that is supposed to mean exact. Four of those seven the local model reads correctly, and never got asked. **So the parser now checks its own coverage**: every word of the question must be accounted for by a lexicon phrase, a column the answer uses, a value it filters on, or one of three closed lists (the table's own furniture, generic row nouns, grammatical filler). A leftover word is a refusal. Measured: **19 right, 0 wrong, 36 refused** — the seven wrong answers became refusals and not one correct answer was lost. `wrong === 0` is now asserted in CI, and the trade is one-directional by construction: an unknown word can cost a refusal where an answer was possible, never a wrong answer where a refusal was right. **(B) Constrained decoding**, hand-written: an automaton over the query grammar and a `LogitsProcessor` that masks, at every token, everything that would leave it. It walks UTF-8 **bytes**, not characters, because Qwen's vocabulary is byte-level BPE and 1 457 of its 151 669 tokens are fragments of a character — a character-level automaton would have made « Île-de-France » unwritable as a filter value. It cost about 16 ms at its most expensive step (`{"kind":"` masks seven letters against seven large buckets) after the first-byte step was hoisted out of the per-token loop, down from 53 ms. **And on its own it made things worse**: model refusals fell 14 → 2 and correct answers rose 29 → 34, but **wrong answers rose 12 → 19**. Forcing a valid answer turns « I could not parse that » into a confident wrong number. That is why the grammar keeps `{"kind":"none"}` reachable — a shape whose only meaning is « I cannot express this », which maps to the refusal V27 already had. **(C) Examples drawn from the user's own columns**, for 0 MB — and the reason turned out to be sharper than the plan's. V27's nine examples were frozen Titanic, and **seven of the 55 corpus questions appear in them verbatim**: the prompt had been fitted to the bench across V27.1 and V27.2, so on those questions the old bench could not tell reading from recitation. (Checked rather than assumed: on those seven, before and after score identically, 5 right / 1 wrong / 1 refused. The defect is methodological, and its measured effect on this comparison is zero.) Generating the examples from the loaded file removes the contamination structurally and deletes the rule that asked the model to ignore what it had just been shown. **Two versions of them were worse than the frozen ones, and both reasons are now in the code.** The first left out the aggregate-WITH-FILTER shape: under constraint, a shape the model has not been shown comes out as a confident wrong answer rather than a refusal, and « prix moyen payé par les survivants » became `count where fare = 1000000000`. The second still picked the FIRST numeric column, which on Titanic is `survived` — so the examples read « average survived » and the model duly reached for that column on questions that never mention it. `ColumnInfo` gained a capped `distinct` count so an example averages a **quantity**, not a 0/1 flag. **(D) Two samples, one vote — dropped, on this wave's own measurements.** Both halves of its tie-break died with (B): constrained decoding guarantees every candidate validates, so « keep the one that validates » no longer discriminates; and « invents no column the question never names » is contradicted by the corpus, where « did women pay more than men? » is correctly answered with `fare`, a column the question never names. Two samples for a vote with no criterion, at twice the latency, is not a trade. **What the wave deliberately does not do**: ship a second, bigger model as a download (the cheap levers were not exhausted when the plan proposed it, and now they are), guess a column by fuzzy name-matching (the refusal is the honest outcome), or let the grammar automaton replace `validateIntent` — it over-approximates in two named places and is a filter, not the authority. 557 unit tests, 79 e2e. | Owner question (22/08/2026): would a bigger model raise the share of correct answers? The wave answers with a measurement rather than an estimate, and the answer has two halves. For **0 MB**, the app went from **33 right / 15 wrong** to **42 right / 7 wrong** out of 55 — nine more correct answers and **fifty-three percent fewer wrong ones**, the single largest piece of which came from the deterministic parser rather than the model. And the bench now takes the model as a parameter (`LABML_LLM_REPO=… npm run llm:bench:node`), so the second half is a command, not an opinion. | +| V31 | **Vision that stops being asked the impossible.** Today's three models weigh **18.6 MB total** (EfficientNet-Lite4 int8 13.6, YOLOX-Nano 3.7, UltraFace 1.3) against 370 MB for the chat model — the headroom is enormous. **The main cause of the mistakes is not the network**: ImageNet-1k has **no « person » class** — 1000 labels, ~120 of them dog breeds, none for a human being — so a photo of someone comes back as « suit » or « jersey ». The model is not wrong; it is being asked a question whose answer is absent from its vocabulary. **(A) Measure first**: 30–50 public-domain images with expected label and expected boxes, replayed in e2e, so « it still makes mistakes » becomes a percentage. **(B) Fix the label space — the real correction**: CLIP ViT-B/32 zero-shot, vision q4f16 **126 MB** + text int8 **64 MB** ≈ **190 MB**, letting the visitor type their own labels (« a cat », « an invoice », « a houseplant »). It repairs the defect and makes a far better demonstration than 1000 frozen classes; open weights, self-hosted, local execution — the doctrine holds. **(C) What costs no download**: check the crop (squashing a 16:9 photo into a square skews everything — `preprocess.ts` is the suspect), average over two crops, recalibrate `OBJECT_THRESHOLD` (0.35) and `FACE_THRESHOLD` (0.9), and above all **refuse below a confidence floor** — « I am not sure » rather than a label picked at random, which is the chat's doctrine applied to pixels. **(D) A better detector**: YOLOX-S (Apache-2.0), ~35 MB, roughly +14 mAP over Nano — with acquisition and licence verified first, as in V23: the YOLOX ONNX files on the Hub are community re-uploads, not official releases. | Owner report (22/08/2026): the vision playground is better than the chat but still makes mistakes. Naming the label-space mismatch is what turns a vague complaint into a fixable defect. | +| V32 | **Documentation, the scaffolding and one finished tutorial.** A `/docs` route, linked from the footer beside « Comment ça marche », built on the **Diátaxis** split — tutorial (learning), how-to (a task), reference (lookup), explanation (the why) — because the usual failure of documentation is mixing all four on one page: a tutorial that pauses to weigh an alternative loses the beginner it was written for. A tutorial offers **no choices** and **guarantees the result**. Five rules specific to this project: **(1) the docs are tested like the code** — everything here is seeded at 42, so « you will get 0.821 accuracy » becomes an assertion in `e2e/docs.spec.ts` and a drifting page **breaks the build**; a documentation that cannot lie is the same promise as the rest of the site. **(2) Screenshots are generated** with Playwright, never hand-taken — one that cannot be regenerated does not ship. **(3) Better than a screenshot, a link that does the thing**: « try it » deep-links landing on the panel with the demo already loaded (needs small URL-parameter support), which never goes stale. **(4) Markdown lives in the repo** (`src/content/docs/**`), compiled at build with prerendered shells like every other route; no Algolia, no third-party doc host — a third-party call on a site that publishes `/privacy` would be indefensible, so search is a local index. **(5) The docs are not PLAN.md**: this file is the engineering record in English with the trade-offs; the docs are for users, FR/EN. Scope of this wave: the route, the Markdown pipeline, the table of contents, local search, and **one** complete tutorial — « premier modèle en 10 minutes » — tested end to end. It is the template every later page copies: tone, length, how figures are quoted. | Owner request (22/08/2026): document every shipped feature across /ml, /data and /ai, linked from the footer. One finished tutorial first, on purpose — writing the full reference before the template is settled means rewriting all of it. | +| V33 | **The reference, and the table of refusals.** Page-per-panel coverage of the three sections: ML Lab (leaderboard, tuning, thresholds, segments, uncertainty, learning curves, run comparison, model export/import, batch scoring), Data Studio (quality score, recipe, forced types, join, drift, anomalies, SQL console) and AI (vision, assistant, the two interpreters). Reference is dry, exhaustive and structured like the software — not prose. The page that no competitor has: **a complete table of the named refusals** — `filter-not-numeric`, `llm-part-missing`, `too-large`, `no-webgpu`, « neither interpreter understood », « the interval is not conclusive » — with what triggers each one, what it means and what to do about it. Refusing well is this project's distinguishing feature; documenting the refusals is the most honest page it can publish. Plus a formats page (CSV, Parquet, JSON, the model manifest). **Honest sizing**: this is 1–2 days of _writing_ for ~25 features in two languages. It does not automate into anything but mush. | A feature nobody can look up is a feature that does not exist for the reader; and a refusal nobody can decode reads as a bug rather than as the design it is. | +| V34 | **Explanations, how-to guides, and the pages that make the project readable as engineering.** The why-pages: why a baseline before anything else, why intervals instead of a single figure, why seed 42 everywhere, why everything runs locally (pointing at `/privacy` rather than repeating it — duplicated prose diverges), and **what LabML does not do, and why** — a project that names its limits reads as a serious one, and the limits are already measured here (the comparison question a 0.6B model does not read, the bench that needs `shader-f16`, SQL over the file before the recipe). Task-shaped how-to guides for readers who already know their way around: score a new batch, compare two runs, read a learning curve, hand a SQL result to the lab. Generated screenshots and the « try it » deep-links land here too. Every page ends with « et ensuite ? » — documentation without a next step is a dead end. | Three audiences, deliberately: the curious visitor (five minutes), the practitioner (one task), and the evaluator judging whether the engineering is rigorous. The explanation pages are what the third one reads. | +| **V35 — delivered** | **ML Lab: the number stops flattering itself.** Two method defects in shipped code, fixed, plus the two additions that follow from them. **(1) The winner was picked on the test set** — the leaderboard sorted nine models by the metric computed on test and crowned `sorted[0]`, which makes the headline figure the optimistic maximum of nine draws. There is now a **third split**: validation is carved out of the train side (64/16/20 with the default ratios), ranking and crowning happen on validation, and the champion line spells out both numbers and the gap between them — « selected on validation at 0.974, scores 0.917 on the untouched test set ». The **test indices are byte-identical** to what the same config produced before V35, so every panel that reads the test set (segments, thresholds, uncertainty, batch compare) is unchanged; below 60 usable rows the third split is refused by name and the lab ranks on test as before. Ranking now lives in ONE module (`ranking.ts`) used by the leaderboard, the history, the run comparison, the report and the auto-selected insights model — the bug that shipped mid-wave was exactly that a fourth site still sorted on test and opened a different model than the one crowned. **(2) The split was always random, even on dated data.** A chronological split (oldest rows train, newest test, rows without a parseable date dropped and counted) and a group split (no group on both sides — the same customer in train and test is the same leak) are offered when a column supports one, and **announced** in the run info. **(3) A predictive leak detector**: V6 caught columns that MAP to the target; this catches the merely predictive one — a one-column stump fitted on train and scored on validation, and a lone column reading the target at ≥ 99% shows as a copper warning with its measured score, never as a victory. **(4) A robust leaderboard on demand**: 5×2 repeated cross-validation over train+validation (the pipeline refitted inside every fold, the test set never touched), reporting a mean, a spread, and how often the leader actually beat the runner-up — « 10 of 10 folds: the order is stable » or « 6 of 10: treat them as tied ». **A defect the wave exposed and named**: with the smaller train split, Gaussian Naive Bayes on ~150 TF-IDF features saturates to exactly 0/1, so V24's word-effect occlusion measured exactly zero for every word and the card simply vanished — reading as « no word matters », which is false. Measured (2 distinct probabilities out of 48 test rows, against 48 for logistic and gbdt), the card now **refuses by name** and points at a model that can answer. 369 unit tests, 65 e2e. | Owner request (22/08/2026), launched 23/08/2026. Two of the four items were defects rather than gaps: a lab that sells honest evaluation cannot ship a headline figure it knows to be optimistic, nor a split that leaks on dated data. | +| **V36 — delivered** | **ML Lab: the gaps that were deliberately left open.** Each item was consciously deferred in an earlier wave rather than forgotten; delivering them together keeps the descopes visible instead of letting them quietly become permanent. **(1) Class imbalance**, descoped by name in V16. Two mechanisms, each NAMED per family rather than hidden behind one word: the loss is weighted where the loss is ours (logistic regression, gradient boosting — the gradient AND the hessian are scaled, since scaling only the gradient inflates leaf values instead of rebalancing), and a **seeded balanced resample** is used for the ml-cart families (tree, forest), which take no sample weights. The minority is upsampled to the majority's size — never the reverse: balancing by trimming the common class throws away real observations to fix a ratio. Off by default, because on a balanced target it changes nothing and a knob that does nothing is worse than no knob; the run announces the majority share, and the leaderboard says so when it crosses 60%. **(2) The ranking metric is a choice.** Accuracy and RMSE were imposed, which is the wrong criterion on an imbalanced problem — a model that never predicts the rare class can top an accuracy ranking and be useless. Rank on F1, recall, precision or ROC-AUC and the order genuinely changes; a model that cannot produce the chosen metric sorts last rather than being dropped. Ranking stays in the single V35 module, so the leaderboard, the history, the comparison, the report and the auto-inspected model all move together. **(3) Multiclass thresholds**, open since V16, read **one-vs-rest**: pick a class, score it against all the others, same PR and calibration curves. What the panel refuses to imply is a complete multiclass decision rule — two classes can both clear their thresholds and nothing here says which wins, so it says that instead. **(4) An ensemble of the best**: the average of the top three, built from models already fitted, so it costs one pass over the test set. The baseline is never a member (averaging a constant predictor drags the result toward the majority class), members are picked by the same V35 ranking rule, and **probabilistic members are preferred** — a mid-wave measurement showed the ensemble winning on iris with a bare vote because k-NN was in the top three, which silently closed the threshold, calibration and word-effect panels on the champion. **What this wave deliberately does not do**: add a tenth model family (nine is plenty; a tenth improves neither honesty nor understanding), build an AutoML « we handle everything » mode (the opposite of a lab that shows its decisions), or bring in tabular deep learning (high cost, no gain at this scale, and no longer hand-written). 387 unit tests, 69 e2e. | Launched 23/08/2026, right after V35. Each item was a named descope, not an oversight — and the ensemble exposed one more silent-disappearance defect, of the same family as the one V35 found. | +| **V37 — delivered** | **ML Lab: speed and the comfort of long sessions.** The wave opened, as the plan demanded, with a measurement — and the measurement moved the wave. **(1) Parallel training** was the headline: the zoo trains sequentially in one worker, so the heavy families were shipped to helper cores. Models cannot cross a worker boundary — `predict` is a closure and structured clone drops functions — so each helper returns `toJSON()` **as a JSON string** and the caller rebuilds through the V22 import path, which makes a parallel model byte-identical to an imported one. That string is not a detail: posting the object instead let structured clone keep shapes JSON drops, and ml-cart's `load()` then rebuilt a tree whose first prediction threw `this.root.classify(...).maxRowIndex is not a function` — a defect that would only have surfaced later, when the user opened insights. Helpers are split by measured cost, heaviest first, to the lightest helper (greedy longest-processing-time); k-NN never leaves the main worker, being the one family with no `toJSON` and also the one that fits in 0 ms; any failure — no Worker support, a helper that throws, an unserialisable family — silently falls back to the sequential trainer, because parallelism may change how long a run takes and never which models it produces. Every family's inference latency is still measured **here**, on the rebuilt predictor: a helper's timing would describe another core under contention, and the column would otherwise read 0 ms for exactly the families that ran in parallel. **(2) The measurement then found the real bottleneck, which was not training at all.** On a 60 000-row run, k-NN inference cost **59.6 s of a 68.8 s wall time — 87% of the whole run** — because the neighbour search allocated 5 000 objects and sorted them for every single prediction, and the scorer asked for labels and then for probabilities, searching twice. Fixed with a bounded top-k insertion over a flat `Float64Array` (ties keep the row seen first, exactly as the stable sort did) and an explicit `predictWithProba` for families where both answers come out of one computation. The old sorted implementation is kept verbatim in the tests as the oracle: the fast path is asserted to predict identically, row for row. **(3) Comparing more than two runs** — V21 compares two; three to six read against the **oldest** of the selection, so the deltas say what the session's changes did rather than what the newest run happens to be. Deliberately not a second diff engine: the matrix is the same V35 ranking and the feature columns are set algebra over the same `summary.featureColumns` V21 reads. **Measured, same machine, same 60 000-row file, same seed** — four arms: | | | k-NN as shipped before | k-NN fixed | | ---------- | ---------------------- | ------------ | @@ -466,7 +466,32 @@ The baseline scores 0.594, so the headroom above it collapses from 0.406 to 0.22 | **V39 — delivered** | **Data Studio: a recipe that works column by column.** `RecipeOptions` applied `missing` and `clipOutliers` to the **whole file** — one strategy for every column, however different they are. A median makes sense for an age and none at all for a postcode. The recipe is now an ordered list of **per-column steps**, with the file-wide settings demoted to **defaults a column may override**: a column with no entry behaves exactly as it did before V39, which is what keeps every previously exported recipe valid and replayable. The missing strategies that were absent are there: **median, mean, most-frequent, a constant used verbatim, and a « MANQUANT » category** for categorical columns — absence becoming a level of its own rather than a guess. Clipping became a per-column decision on the same terms, so a column can opt in or out of the global flag. **The rule the wave exists to enforce: imputing without marking destroys information.** A blank field is rarely blank at random, and the fact of the blank is frequently predictive on its own, so every column may add a `_absent` indicator — and the ordering is the whole point: **every indicator is written before any blank is filled**, so it records where the blanks really were, never where some other column's rule left them. Three consequences fall out of that and are tested: a column with nothing missing gets no indicator (an all-zero column is noise, not information); rows dropped by one column's rule are gathered and removed **once**, so indicators stay aligned instead of being shifted by a sibling column; and when a strategy cannot be honoured — a median over a column holding no parseable number, a constant with no value typed — the blanks are **left blank** rather than filled with something invented. Because the indicator is optional, the studio **announces the columns it filled without marking**, by name, instead of quietly producing a tidier table. **What this wave deliberately does not do**: make the indicator mandatory (it would silently add columns to every existing recipe, and imposing is not the same as announcing), impute from a model (opaque, and it fabricates values that look plausible — already refused for V40), or reorder the recipe's fixed stages, which is what keeps toggling an option from compounding with the last one. 461 unit tests, 75 e2e. | A single global strategy is the kind of default that looks tidy and quietly makes the data worse; per-column steps cost little to build because the recipe was already an object, not a pile of checkboxes. The build confirmed it: the engine change is contained in one stage of `applyRecipe`, and the 17 pre-existing recipe tests passed untouched. | | **V40 — delivered** | **Data Studio: validity, drift, and an auditable diff.** Quality was measured as completeness and consistency of type; what was missing is **validity** — a value can be present, correctly typed and still impossible. Five named rules now say so in plain language: an age outside 0–120, a date in the future, a percentage outside 0–100, a negative amount, a malformed postcode. Then **cross-column consistency**: an end date before its start, a total that is not quantity × price — every cell fine, the ROW impossible. **The plan's choice of engine was wrong, and building it showed why.** It proposed running the consistency rules through V29's DuckDB, since the file is already registered there. But DuckDB is an announced, opt-in 18–22 MB download: routing these checks through it would have made a universally applicable check conditional on a large download most users will decline, leaving the panel empty for them. Comparing two columns of a table already in memory is a loop, so it is a loop, and every user gets it. DuckDB keeps the job it is genuinely needed for — arbitrary SQL, and the new **Parquet export** (`COPY … TO`, one call, bytes straight to a download). Both families of rules obey the two laws V38's reader established: **they fire on evidence, never on a column's name alone** — a column called `age` holding 20 000 is a duration in days, so the rule checks that most of the column is plausible before it flags the rest, and stays silent otherwise — and **they report without ever repairing**, because V39's recipe is the one record of what was done to the data. Three things then make the studio auditable rather than merely helpful. A **before/after diff** naming which rows, which columns and which values changed: the hard part is that a recipe drops rows and adds columns, so `applyRecipe` now returns which SOURCE row each surviving row came from — without it the diff would pair row 7 with a different row 7 and report a screen full of changes that never happened. A **replayable reference profile**, the V22 manifest idea applied to data: bin edges and shares, never rows, so a profile of a payroll file describes the shape of the salary distribution and nobody's salary — which is what makes it safe to commit beside the code, and it scores a new file to the same PSI, to six decimal places, as V11's live two-file comparison. And a **score broken into its parts**, each with its weight and what it actually cost. The weights now sum to **105, not 100**, deliberately: validity brought its own 5 points rather than taking them from an existing part, because redistributing would have quietly changed what every previously published score meant. **What this wave deliberately does not do**: a spreadsheet-style cell editor (hand edits break reproducibility — the recipe is the record), fuzzy deduplication (guaranteed false positives on names and addresses, silently merging two real people), or model-based imputation (opaque, and it fabricates values that look plausible). 502 unit tests, 78 e2e. | Came last because it builds on V38's faithful read and V39's per-column recipe: validity rules on mis-parsed numbers would have flagged the parser, not the data. Closes the Data Studio group. Of the 36 new unit tests, eleven assert that a rule REFUSES to fire — a rule that flags a good file is worse than no rule, because it teaches the reader to ignore the panel. | -**Ordering**: V38 came before V39 and V40, and for the same reason V35 came first in its own group: its headline item is a defect in shipped code, not a feature — a studio that promises honest data cannot silently turn `12,5` into `NaN`. V35, V36 and V37 are delivered. V32 ships one finished tutorial before any reference page — the tutorial is the template the rest copies, and settling it late means rewriting everything. V30 and V31 both start with a bench, because neither « a bigger model » nor « it still makes mistakes » is a measurable statement today; no wave starts without an explicit launch command. V23 first (owner request); V24 keeps its vocabulary capped — +**V30's measurements**, all on the same 55 questions, the same pinned q4f16 +weights and the same CPU runtime. « app » is what a visitor actually gets: the +deterministic parser first, the model only on what it refuses. + +| configuration | model alone | app | +| -------------------------------------------------- | --------------- | --------------- | +| V27.3 as shipped — free decoding, frozen examples | 29 / 12 / 14 | **33 / 15 / 7** | +| V30 parser alone, no model at all | — | 19 / **0** / 36 | +| + constrained decoding, frozen examples | 34 / **19** / 2 | 45 / 9 / 1 ◊ | +| + generated examples, first version | 32 / 12 / 11 | 34 / 11 / 10 | +| + the aggregate-with-filter shape | 34 / 10 / 11 | 37 / 9 / 9 | +| + examples on a quantity, not a flag — **shipped** | **41 / 7 / 7** | **42 / 7 / 6** | + +Read as right / wrong / refused. ◊ That line is **not comparable**: its prompt +contains seven of the 55 questions verbatim, so on those it measures recitation. +It is listed because it is the number the wave would have reported if nobody had +checked, and because it is the only configuration with fewer refusals and more +wrong answers than the one that shipped — which is the trade constrained +decoding makes when refusal is not left reachable. + +Latency is deliberately absent from that table: it was measured on a CPU under +concurrent load, which says nothing about a visitor's WebGPU run. What was +measured in isolation is the mask itself — 16 ms at its most expensive step, +about 1 ms at every other — and that cost does not shrink on a GPU. + +**Ordering**: V38 came before V39 and V40, and for the same reason V35 came first in its own group: its headline item is a defect in shipped code, not a feature — a studio that promises honest data cannot silently turn `12,5` into `NaN`. V35, V36, V37 and V30 are delivered. V32 ships one finished tutorial before any reference page — the tutorial is the template the rest copies, and settling it late means rewriting everything. V30 is delivered, and it started with a bench for exactly the reason written here: « a bigger model » was not a measurable statement. It is one now — the bench takes the model as a parameter — and V31 still has its own instrument to build. No wave starts without an explicit launch command. V23 first (owner request); V24 keeps its vocabulary capped — V25 (delivered) chose announced sampling and a named memory guard over the typed-array rewrite, which measurement showed unnecessary; widening the vocabulary stays possible later. Set aside for now: multiclass thresholds. V12 diff --git a/README.md b/README.md index 2046208..7d3a5ec 100644 --- a/README.md +++ b/README.md @@ -98,16 +98,21 @@ The project follows three non-negotiable principles: reason: from 1.29 its binaries exceed Cloudflare's 25 MiB per-file limit. - **Data assistant** (`/ai/chat`): plain French or English questions about a loaded dataset (averages, counts, top-N, correlations…) answered by a deterministic local - interpreter — when it does not understand, it says so. A **real local language model** - (Qwen3-0.6B, 355 MB, Apache-2.0, self-hosted and split into 24 MiB parts to clear - Cloudflare's limit) can be downloaded on explicit consent to read free-form phrasings: - it only _translates_ the question into a query — the deterministic engine still - computes every number, the translation is validated against a closed grammar, and a - badge under each answer names which engine produced it. The deterministic parser reads - **first** and is never overridden: it can only name a column that exists and a value - that occurs in it, so the model is asked only about the questions it gives up on. - WebGPU required; without it the refusal is named and the deterministic interpreter - stays fully available. + interpreter — when it does not understand, it says so. It only claims to understand + once it has read the **whole** question: a word it cannot account for is a refusal, not + an answer to a shorter question. A **real local language model** (Qwen3-0.6B, 355 MB, + Apache-2.0, self-hosted and split into 24 MiB parts to clear Cloudflare's limit) can be + downloaded on explicit consent to read free-form phrasings: it only _translates_ the + question into a query — the deterministic engine still computes every number, and a + badge under each answer names which engine produced it. The translation is decoded + **inside** the query grammar: a hand-written logits processor masks, at every token, + everything that would leave the grammar, so an invented column, an operator that does + not exist or a category the column does not hold cannot be written in the first place. + One shape stays reachable on purpose — `{"kind":"none"}`, the model's way of saying it + cannot express the question — because forcing a valid answer turns a refusal into a + wrong number. The reading of all 55 reference questions is measured, not asserted: see + **Measuring the assistant** below. WebGPU required; without it the refusal is named and + the deterministic interpreter stays fully available. ## Engineering notes @@ -170,6 +175,23 @@ llm:prepare` downloads it into `public/llm/` and splits it into parts under Clou else works — the assistant simply falls back to its deterministic interpreter, which is the default in any case. +### Measuring the assistant + +`src/features/ai/llm/corpus.ts` holds **55 reference questions**, French and English, +across every shape of the query grammar plus three that no query can answer — where +refusing is the only correct outcome. Two harnesses run the same corpus: + +| Command | Needs | Measures | +| ------------------------------------------------------------- | --------------------------------- | --------------------------------------------------------------- | +| `npm run test` (`corpus.test.ts`) | nothing | the deterministic parser, the grammar automaton, the token mask | +| `npm run llm:fetch && npm run llm:bench:node` | 355 MB on disk, a few CPU minutes | the real model, end to end | +| `V27_BENCH=1 npm run build && node scripts/run-llm-bench.mjs` | a GPU with `shader-f16` | the same, on the shipped WebGPU runtime | + +The CI half runs on every commit and asserts the number that matters most: the +deterministic parser produces **zero wrong answers** on the corpus. The model half is a +separate on-demand workflow (`.github/workflows/llm-bench.yml`) — it downloads 355 MB and +takes minutes, which is not a cost worth adding to every pull request. + ## Deployment CI builds, tests and deploys on every push: pull requests get a Cloudflare Pages preview, diff --git a/e2e/chat.spec.ts b/e2e/chat.spec.ts index 125f7be..8e9adbf 100644 --- a/e2e/chat.spec.ts +++ b/e2e/chat.spec.ts @@ -60,3 +60,37 @@ test('suggestion chips ask a real question and the hub links here', async ({ pag await expect(answer).toContainText('Missing cells', { timeout: 15000 }); await expect(answer).toContainText('deck'); // titanic's famously incomplete column }); + +test('V30 — a question it only half reads is refused, not answered short', async ({ page }) => { + await page.goto('/ai/chat'); + await page.getByRole('button', { name: /titanic\.csv/ }).click(); + await expect(page.getByText('891 rows · 15 columns')).toBeVisible(); + + const input = page.getByLabel('Your question'); + const ask = page.getByRole('button', { name: 'Ask' }); + + // Before V30 this answered 891 — the whole table — because the keyword + // grammar knows "how many" and knows nothing about "women", so it kept the + // count and dropped the condition. The number was wrong and the badge said + // the deterministic interpreter had read the question. + await input.fill('how many women?'); + await ask.click(); + const refused = page.getByTestId('chat-assistant').last(); + await expect(refused).toContainText('did not understand', { timeout: 15000 }); + await expect(refused).not.toContainText('891 rows match'); + + // Same shape in French, and the same refusal. + await input.fill('average age of women'); + await ask.click(); + await expect(page.getByTestId('chat-assistant').last()).toContainText('did not understand', { + timeout: 15000, + }); + + // What it DOES read, it still answers — the guard did not cost the questions + // the grammar genuinely understands. + await input.fill('How many rows where sex is female?'); + await ask.click(); + await expect(page.getByTestId('chat-assistant').last()).toContainText('314 rows match', { + timeout: 15000, + }); +}); diff --git a/package.json b/package.json index aea1af3..c8f74e8 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "preview": "vite preview --host 127.0.0.1 --port 4173 --strictPort", "llm:prepare": "node scripts/prepare-llm.mjs", "llm:bench": "node scripts/run-llm-bench.mjs", + "llm:bench:node": "LABML_LLM_BENCH=1 vitest run src/features/ai/llm/bench.node.test.ts", + "llm:fetch": "node scripts/prepare-llm.mjs .llm-cache --flat", "lint": "eslint .", "format": "prettier --write .", "format:check": "prettier --check .", diff --git a/scripts/prepare-llm.mjs b/scripts/prepare-llm.mjs index c8d84b6..66ffff9 100644 --- a/scripts/prepare-llm.mjs +++ b/scripts/prepare-llm.mjs @@ -6,7 +6,14 @@ * weights are NOT committed: 355 MB of binaries would slow every clone and * every CI checkout, for a model only the opt-in chat engine downloads. * - * node scripts/prepare-llm.mjs # default: dist/llm + * node scripts/prepare-llm.mjs # default: dist/llm + * node scripts/prepare-llm.mjs --flat # unsharded, for the Node bench + * + * V30 — `--flat` writes the same pinned files without splitting them, which is + * what `npm run llm:bench:node` needs: onnxruntime-node reads the weights from + * disk directly and has no 25 MiB limit to work around. It is the same + * download, the same size checks, and the same revision as the deployed copy — + * so the bench measures the model production actually ships, not a lookalike. * * Every file is checked against the byte sizes pinned below. A mismatch is a * hard failure: shipping a truncated model would fail in the browser, later, @@ -72,7 +79,9 @@ async function write(outDir, relative, bytes) { } async function main() { - const outDir = process.argv[2] ?? 'dist/llm'; + const args = process.argv.slice(2).filter((a) => a !== '--flat'); + const flat = process.argv.includes('--flat'); + const outDir = args[0] ?? 'dist/llm'; const root = join(outDir, REPO); const files = []; let totalBytes = 0; @@ -80,7 +89,7 @@ async function main() { for (const path of [...Object.keys(FILES), 'LICENSE']) { const bytes = await download(path); totalBytes += bytes.byteLength; - if (bytes.byteLength <= SHARD_BYTES) { + if (flat || bytes.byteLength <= SHARD_BYTES) { await write(root, path, bytes); console.log(` entier ${path} (${(bytes.byteLength / 1e6).toFixed(1)} Mo)`); continue; @@ -103,6 +112,14 @@ async function main() { await write(outDir, 'manifest.json', JSON.stringify(manifest, null, 2)); console.log(`\nmanifeste écrit — ${(totalBytes / 1e6).toFixed(0)} Mo au total`); + // The 25 MiB guard below is about Cloudflare Pages. A flat copy never goes + // there — it is read from local disk by the bench — so the guard would fail + // on a layout that is correct for its purpose. + if (flat) { + console.log('copie à plat (banc Node) — garde des 25 Mio sans objet ✓'); + return; + } + // A last guard: nothing we just wrote may exceed the platform limit. const oversized = []; const walk = async (dir) => { diff --git a/src/features/ai/chat/EnginePicker.tsx b/src/features/ai/chat/EnginePicker.tsx index 35ab5fb..5c363a4 100644 --- a/src/features/ai/chat/EnginePicker.tsx +++ b/src/features/ai/chat/EnginePicker.tsx @@ -16,6 +16,7 @@ export function EnginePicker() { const { t, i18n } = useTranslation(); const lang = i18n.resolvedLanguage ?? 'en'; const llmStatus = useChatStore((s) => s.llmStatus); + const llmConstrained = useChatStore((s) => s.llmConstrained); const llmBytes = useChatStore((s) => s.llmBytes); const llmProgress = useChatStore((s) => s.llmProgress); const llmError = useChatStore((s) => s.llmError); @@ -121,12 +122,19 @@ export function EnginePicker() { )} {llmStatus === 'ready' && ( -

- - {t('ai.chat.engine.readyTag')} - - {t('ai.chat.engine.readyNote')} -

+
+

+ + {t(llmConstrained ? 'ai.chat.engine.constrainedTag' : 'ai.chat.engine.readyTag')} + + {t( + llmConstrained + ? 'ai.chat.engine.constrainedNote' + : 'ai.chat.engine.unconstrainedNote', + )} +

+

{t('ai.chat.engine.readyNote')}

+
)} {llmStatus === 'failed' && ( diff --git a/src/features/ai/chat/chat-store.ts b/src/features/ai/chat/chat-store.ts index 41b7336..5efd14b 100644 --- a/src/features/ai/chat/chat-store.ts +++ b/src/features/ai/chat/chat-store.ts @@ -42,6 +42,12 @@ interface ChatState { llmBytes: number; llmProgress: { loaded: number; total: number } | null; llmError: string | null; + /** + * V30 — true when the model's answer is decoded INSIDE the query grammar. + * Shown rather than assumed: if the tokenizer does not expose its vocabulary + * the guard cannot run, and the badge must not claim it did. + */ + llmConstrained: boolean; /** The user's choice of interpreter; deterministic stays the default. */ engine: ChatEngine; loadFile: (file: File) => void; @@ -71,6 +77,7 @@ const initialState = { llmBytes: 0, llmProgress: null, llmError: null, + llmConstrained: false, engine: 'deterministic' as ChatEngine, }; @@ -107,7 +114,12 @@ export const useChatStore = create((set, get) => { } else if (message.kind === 'llm-progress') { set({ llmProgress: { loaded: message.loaded, total: message.total } }); } else if (message.kind === 'llm-ready') { - set({ llmStatus: 'ready', llmProgress: null, engine: 'llm' }); + set({ + llmStatus: 'ready', + llmProgress: null, + engine: 'llm', + llmConstrained: message.constrained, + }); } else if (message.kind === 'llm-failed') { set({ llmStatus: 'failed', llmProgress: null, llmError: message.reason }); } else if (message.kind === 'error') { diff --git a/src/features/ai/chat/chat.worker.ts b/src/features/ai/chat/chat.worker.ts index ed03593..c00d9ee 100644 --- a/src/features/ai/chat/chat.worker.ts +++ b/src/features/ai/chat/chat.worker.ts @@ -31,7 +31,7 @@ export type ChatWorkerResponse = | { kind: 'unknown'; by: 'none' | 'none-both' } | { kind: 'llm-capability'; available: boolean; webgpu: boolean; totalBytes: number } | { kind: 'llm-progress'; loaded: number; total: number } - | { kind: 'llm-ready' } + | { kind: 'llm-ready'; constrained: boolean } /** Named refusal: 'no-manifest' | 'no-webgpu' | anything the loader threw. */ | { kind: 'llm-failed'; reason: string } | { kind: 'error'; message: string }; @@ -72,21 +72,21 @@ function resetState() { columnInfo = []; } +/** Enough to tell a 0/1 flag from a quantity; counting further changes nothing. */ +const DISTINCT_CAP = 12; + function buildColumnInfo(): ColumnInfo[] { return header.map((name, i) => { const type = inferColumnType(name, columns[i]); const isNumeric = type === 'numeric' || type === 'id'; - let values: string[] = []; - if (!isNumeric) { - const distinct = new Set(); - for (const cell of columns[i]) { - if (isMissing(cell)) continue; - distinct.add((cell as string).trim()); - if (distinct.size > MAX_VALUES) break; - } - if (distinct.size <= MAX_VALUES) values = [...distinct].sort(); + const distinct = new Set(); + for (const cell of columns[i]) { + if (isMissing(cell)) continue; + distinct.add((cell as string).trim()); + if (distinct.size > Math.max(MAX_VALUES, DISTINCT_CAP)) break; } - return { name, isNumeric, values }; + const values = !isNumeric && distinct.size <= MAX_VALUES ? [...distinct].sort() : []; + return { name, isNumeric, values, distinct: distinct.size }; }); } @@ -173,7 +173,7 @@ self.onmessage = async (event: MessageEvent) => { model = await loadModel(capability.manifest, { onProgress: ({ loaded, total }) => post({ kind: 'llm-progress', loaded, total }), }); - post({ kind: 'llm-ready' }); + post({ kind: 'llm-ready', constrained: model.constrained }); } catch (error) { model = null; post({ diff --git a/src/features/ai/chat/parser.test.ts b/src/features/ai/chat/parser.test.ts index be5659b..b8427a5 100644 --- a/src/features/ai/chat/parser.test.ts +++ b/src/features/ai/chat/parser.test.ts @@ -176,17 +176,16 @@ describe('a condition it cannot read is REFUSED, never dropped', () => { expect(parseQuestion('average age for kids born before 1900', columns, 'en')).toBeNull(); }); - // The guard targets DROPPED conditions, not guessed columns. Here the - // condition IS applied — to `fare`, the only numeric column named — which - // also happens to be the legitimate reading of "average fare below 12". - // Refusing this would cost a real question to catch a fuzzy one. - it('keeps a condition it can attach, even when the wording is loose', () => { - expect(parseQuestion('average fare for kids below 12', columns, 'en')).toMatchObject({ - kind: 'aggregate', - op: 'mean', - column: 'fare', - filter: { column: 'fare', op: '<', value: 12 }, - }); + // V30 reverses this case, deliberately. It used to assert that the parser + // ANSWERS here, on the grounds that `fare < 12` is « also a legitimate + // reading » — but the question says *kids*, and the only reading that word + // supports is `age < 12`. Attaching the threshold to the nearest numeric + // column is a guess, and the guess is delivered under the deterministic + // badge, which is supposed to mean exact. Refusing hands the question to the + // model, which reads it; answering hands the user a number for a question + // nobody asked. + it('refuses when the condition is attached to a column the wording never named', () => { + expect(parseQuestion('average fare for kids below 12', columns, 'en')).toBeNull(); }); it('still answers the same question when the condition names a real column', () => { diff --git a/src/features/ai/chat/parser.ts b/src/features/ai/chat/parser.ts index 235ee54..2442168 100644 --- a/src/features/ai/chat/parser.ts +++ b/src/features/ai/chat/parser.ts @@ -10,6 +10,16 @@ export interface ColumnInfo { isNumeric: boolean; /** Known category values for equality filters (non-numeric columns, capped). */ values: string[]; + /** + * V30 — how many distinct values the column holds, counted up to a cap and + * then abandoned. It exists for one reason: `survived` and `fare` are both + * « numeric », and only one of them is a quantity worth averaging. The + * prompt's worked examples pick a measure with this, rather than taking + * whichever numeric column happens to come first — which on Titanic wrote + * « average survived » and taught the model to reach for that column. + * Absent when it was not counted; never a reason to refuse anything. + */ + distinct?: number; } /** Lowercase, strip accents, unify separators — the space all matching happens in. */ @@ -50,7 +60,7 @@ const EN: Lexicon = { by: ['by', 'per'], correlation: ['correlation', 'correlated', 'relationship between', 'link between'], distribution: ['distribution', 'breakdown', 'histogram'], - shape: ['shape', 'dimensions', 'how big', 'size of'], + shape: ['shape', 'dimensions', 'how big', 'size of', 'rows and columns', 'columns and rows'], missing: ['missing', 'empty cells'], comparators: [ ['>=', ['greater than or equal to', 'at least', '>=']], @@ -76,7 +86,19 @@ const FR: Lexicon = { by: ['par', 'selon'], correlation: ['correlation', 'correle', 'lien entre', 'relation entre'], distribution: ['distribution', 'repartition', 'histogramme', 'ventilation'], - shape: ['taille du jeu', 'dimensions', 'quelle taille'], + shape: [ + 'taille du jeu', + 'dimensions', + 'quelle taille', + 'taille du tableau', + 'taille de la table', + 'taille du fichier', + 'taille des donnees', + 'lignes et de colonnes', + 'lignes et colonnes', + 'colonnes et de lignes', + 'colonnes et lignes', + ], missing: ['manquante', 'manquantes', 'manquants', 'manquant', 'vides'], comparators: [ ['>=', ['superieur ou egal a', 'superieure ou egale a', 'au moins', '>=']], @@ -265,6 +287,344 @@ function droppedCondition( return /\d/.test(rest); } +/** + * V30 — the words the parser did NOT read. + * + * The measured defect this exists for: on « combien de femmes ? » the grammar + * knows `combien` and knows nothing about `femmes`, so it answered the total + * row count — 891 instead of 314 — badged « déterministe », the badge that is + * supposed to mean exact. `droppedCondition` did not catch it, because it only + * looks for an ordering word or a stray digit, and « femmes » is neither. + * + * Measured on the V30 corpus: seven of fifty-five questions were answered with + * a condition silently removed, and four of those seven the local model reads + * correctly — but never gets asked, because the deterministic parser goes + * first and never admits defeat. + * + * So the parser now checks its own coverage. Every word of the question must + * be accounted for by something: a phrase from the lexicon, a column the + * answer uses, a category value the answer filters on, or one of the three + * closed lists below. A leftover word means the question said something the + * grammar did not read, and the honest answer is to refuse and let the model + * (or the user) have a turn. + * + * The trade is deliberate and one-directional: an unknown word can now cost a + * refusal where an answer was possible, and never a wrong answer where a + * refusal was right. A refusal is announced and falls through to the model; a + * wrong answer is delivered with full confidence. + */ + +/** The table's own furniture — never a condition on the rows. */ +const STRUCTURE_WORDS = new Set([ + 'ligne', + 'lignes', + 'rangee', + 'rangees', + 'enregistrement', + 'enregistrements', + 'entree', + 'entrees', + 'observation', + 'observations', + 'colonne', + 'colonnes', + 'champ', + 'champs', + 'valeur', + 'valeurs', + 'cellule', + 'cellules', + 'donnee', + 'donnees', + 'jeu', + 'tableau', + 'table', + 'fichier', + 'taille', + 'total', + 'row', + 'rows', + 'record', + 'records', + 'entry', + 'entries', + 'column', + 'columns', + 'field', + 'fields', + 'value', + 'values', + 'cell', + 'cells', + 'data', + 'dataset', + 'file', + 'size', + 'shape', +]); + +/** + * Generic nouns for « the thing one row is ». Deliberately a short, closed + * list: an entity noun it does not know makes the parser refuse rather than + * guess, which is the safe direction. + */ +const ENTITY_WORDS = new Set([ + 'personne', + 'personnes', + 'gens', + 'individu', + 'individus', + 'passager', + 'passagers', + 'client', + 'clients', + 'utilisateur', + 'utilisateurs', + 'eleve', + 'eleves', + 'element', + 'elements', + 'produit', + 'produits', + 'people', + 'person', + 'persons', + 'individual', + 'individuals', + 'passenger', + 'passengers', + 'customer', + 'customers', + 'user', + 'users', + 'student', + 'students', + 'item', + 'items', + 'product', + 'products', +]); + +/** + * Grammatical filler. Comparison words (plus, moins, more, than, over…) are + * deliberately ABSENT: they are the signal `droppedCondition` reads, and + * treating them as filler would hide exactly what it is looking for. + */ +const FILLER_WORDS = new Set([ + 'le', + 'la', + 'les', + 'l', + 'un', + 'une', + 'des', + 'du', + 'de', + 'd', + 'au', + 'aux', + 'et', + 'ou', + 'a', + 'en', + 'dans', + 'sur', + 'pour', + 'avec', + 'sans', + 'est', + 'sont', + 'etait', + 'etaient', + 'ce', + 'c', + 'cette', + 'ces', + 'qui', + 'que', + 'quoi', + 'quel', + 'quelle', + 'quels', + 'quelles', + 'y', + 'il', + 'elle', + 'on', + 'se', + 'ne', + 'me', + 'son', + 'sa', + 'ses', + 'leur', + 'leurs', + 'notre', + 'nos', + 'votre', + 'vos', + 'moi', + 'the', + 'an', + 'of', + 'in', + 'on', + 'at', + 'by', + 'for', + 'with', + 'without', + 'is', + 'are', + 'was', + 'were', + 'be', + 'been', + 'what', + 'which', + 'who', + 'whom', + 'whose', + 'that', + 'this', + 'these', + 'those', + 'and', + 'or', + 'do', + 'does', + 'did', + 'there', + 'here', + 'it', + 'its', + 'their', + 'his', + 'her', + 'my', + 'your', + 'our', + 'to', + 'from', + 'as', + 'me', + 'each', + 'per', + 'show', + 'give', + 'tell', + 'list', + 'where', + 'when', + 'between', + 'among', + 'have', + 'has', + 'had', + 'being', + 'all', + 'any', + 'some', + 'only', + 'also', + 'just', + 'please', + 'can', + 'could', + 'would', + 'should', + 'will', + 'much', + 'many', + 'avait', + 'avaient', + 'ont', + 'tous', + 'toutes', + 'tout', + 'toute', + 'entre', + 'parmi', + 'seulement', + 'aussi', + 'peut', + 'donne', + 'montre', + 'liste', + 'affiche', +]); + +/** Splits folded text into comparable words, without their trailing marks. */ +function words(text: string): string[] { + return text + .split(' ') + .map((word) => word.replace(/^[.,-]+/, '').replace(/[.,-]+$/, '')) + .filter((word) => word.length > 0); +} + +function markPhrase(tokens: string[], used: boolean[], phrase: string): void { + const parts = words(phrase); + if (parts.length === 0) return; + for (let i = 0; i + parts.length <= tokens.length; i++) { + if (!parts.every((part, k) => tokens[i + k] === part)) continue; + for (let k = 0; k < parts.length; k++) used[i + k] = true; + } +} + +/** Every column the answer actually refers to. */ +function intentColumns(intent: Intent): string[] { + const names: string[] = []; + if ('column' in intent && intent.column) names.push(intent.column); + if ('groupBy' in intent && intent.groupBy) names.push(intent.groupBy); + if ('a' in intent) names.push(intent.a, intent.b); + if ('filter' in intent && intent.filter) names.push(intent.filter.column); + return names; +} + +/** + * The words of `question` that nothing in the answer accounts for. An empty + * result means the parser read the whole question; anything else means it did + * not, and `parseQuestion` refuses instead of answering a shorter question. + */ +export function unreadWords(question: string, intent: Intent, lang: string): string[] { + const lexicon = lang.startsWith('fr') ? FR : EN; + const text = fold(question); + const tokens = words(text); + const used = new Array(tokens.length).fill(false); + + for (const [, phrases] of lexicon.ops) + for (const phrase of phrases) markPhrase(tokens, used, phrase); + for (const [, phrases] of lexicon.comparators) + for (const phrase of phrases) markPhrase(tokens, used, phrase); + for (const list of [ + lexicon.count, + lexicon.top, + lexicon.by, + lexicon.correlation, + lexicon.distribution, + lexicon.shape, + lexicon.missing, + ]) { + for (const phrase of list) markPhrase(tokens, used, phrase); + } + for (const name of intentColumns(intent)) markPhrase(tokens, used, fold(name)); + if ('filter' in intent && intent.filter) + markPhrase(tokens, used, fold(String(intent.filter.value))); + if ('k' in intent) markPhrase(tokens, used, String(intent.k)); + + const leftover: string[] = []; + for (let i = 0; i < tokens.length; i++) { + if (used[i]) continue; + const token = tokens[i]; + // A one- or two-letter leftover is noise, not a condition; a bare number + // is already `droppedCondition`'s business. + if (token.length < 3 || /^\d+(?:[.,]\d+)?$/.test(token)) continue; + if (STRUCTURE_WORDS.has(token) || ENTITY_WORDS.has(token) || FILLER_WORDS.has(token)) continue; + leftover.push(token); + } + return leftover; +} + export function parseQuestion( question: string, columns: ColumnInfo[], @@ -274,18 +634,26 @@ export function parseQuestion( const text = fold(question); if (!text) return null; const mentions = columnMentions(text, columns); + // Every reading below goes through here: an answer that leaves part of the + // question unread is not an answer, it is a different question. + const readOrRefuse = (intent: Intent | null): Intent | null => + intent && unreadWords(question, intent, lang).length === 0 ? intent : null; - if (findAny(text, lexicon.missing)) return { kind: 'missing' }; + if (findAny(text, lexicon.missing)) return readOrRefuse({ kind: 'missing' }); if (findAny(text, lexicon.correlation) && mentions.length >= 2) { - return { kind: 'correlation', a: mentions[0].column.name, b: mentions[1].column.name }; + return readOrRefuse({ + kind: 'correlation', + a: mentions[0].column.name, + b: mentions[1].column.name, + }); } if (findAny(text, lexicon.distribution) && mentions.length >= 1) { - return { kind: 'distribution', column: mentions[0].column.name }; + return readOrRefuse({ kind: 'distribution', column: mentions[0].column.name }); } - if (findAny(text, lexicon.shape)) return { kind: 'shape' }; + if (findAny(text, lexicon.shape)) return readOrRefuse({ kind: 'shape' }); const top = findAny(text, lexicon.top); if (top) { @@ -301,22 +669,22 @@ export function parseQuestion( .find((entry) => entry.mention !== null); const metricMention = mentions.find((m) => m.column.isNumeric && m.column.name !== groupBy); if (opHit && metricMention) { - return { + return readOrRefuse({ kind: 'topk', groupBy, k, op: opHit.op, column: metricMention.column.name, filter: parseFilter(text, mentions, columns, lexicon, new Set([groupBy])), - }; + }); } - return { + return readOrRefuse({ kind: 'topk', groupBy, k, op: 'count', filter: parseFilter(text, mentions, columns, lexicon, new Set([groupBy])), - }; + }); } } @@ -333,24 +701,24 @@ export function parseQuestion( if (groupBy) exclude.add(groupBy); const filter = parseFilter(text, mentions, columns, lexicon, exclude); if (droppedCondition(text, mentions, filter, lexicon)) return null; - return { kind: 'aggregate', op: opHit.op, column, groupBy, filter }; + return readOrRefuse({ kind: 'aggregate', op: opHit.op, column, groupBy, filter }); } } if (findAny(text, lexicon.count)) { const groupBy = groupByColumn(text, mentions, lexicon, new Set()); if (groupBy) { - return { + return readOrRefuse({ kind: 'topk', groupBy, k: 12, op: 'count', filter: parseFilter(text, mentions, columns, lexicon, new Set([groupBy])), - }; + }); } const filter = parseFilter(text, mentions, columns, lexicon, new Set()); if (droppedCondition(text, mentions, filter, lexicon)) return null; - return { kind: 'count', filter }; + return readOrRefuse({ kind: 'count', filter }); } return null; diff --git a/src/features/ai/chat/suggestions.test.ts b/src/features/ai/chat/suggestions.test.ts new file mode 100644 index 0000000..ec8e9ea --- /dev/null +++ b/src/features/ai/chat/suggestions.test.ts @@ -0,0 +1,87 @@ +/** + * V30 — the guard on the coverage rule. + * + * `parseQuestion` now refuses a question it has not read in full. That is the + * right trade for a question a user typed, but it would be a bad joke if the + * app SUGGESTED a question and then refused it: the chips under the input are + * the app's own words, offered as things it can answer. + * + * So every suggestion template, in both languages, filled with real column + * names from the shipped demo datasets, must parse. This is the test that + * fails the day the closed word lists in `parser.ts` drift away from the + * phrasings the product puts in front of people. + */ +import { describe, it, expect } from 'vitest'; +import fr from '@/locales/fr.json'; +import en from '@/locales/en.json'; +import { parseQuestion, type ColumnInfo } from '@/features/ai/chat/parser'; + +/** Column shapes taken from the demo datasets the chat offers. */ +const DATASETS: { name: string; columns: ColumnInfo[] }[] = [ + { + name: 'titanic.csv', + columns: [ + { name: 'survived', isNumeric: true, values: [] }, + { name: 'age', isNumeric: true, values: [] }, + { name: 'fare', isNumeric: true, values: [] }, + { name: 'sex', isNumeric: false, values: ['male', 'female'] }, + { name: 'class', isNumeric: false, values: ['Third', 'First', 'Second'] }, + ], + }, + { + name: 'iris.csv', + columns: [ + { name: 'sepal_length', isNumeric: true, values: [] }, + { name: 'petal_width', isNumeric: true, values: [] }, + { name: 'species', isNumeric: false, values: ['setosa', 'virginica'] }, + ], + }, + { + name: 'cafe-sales.csv', + columns: [ + { name: 'quantity', isNumeric: true, values: [] }, + { name: 'unit_price', isNumeric: true, values: [] }, + { name: 'product', isNumeric: false, values: ['Espresso', 'Latte'] }, + { name: 'payment', isNumeric: false, values: ['card', 'cash'] }, + ], + }, +]; + +/** The same fill-in ChatPage performs, without pulling React into the test. */ +function fill(template: string, values: Record): string { + return template.replace(/\{\{(\w+)\}\}/g, (_, key: string) => values[key] ?? ''); +} + +function suggestionsFor(bundle: typeof fr | typeof en, columns: ColumnInfo[]): string[] { + const suggest = bundle.ai.chat.suggest; + const numeric = columns.filter((c) => c.isNumeric && !/(^|_)id$/i.test(c.name)); + const categorical = columns.filter((c) => !c.isNumeric && c.values.length > 0); + const out: string[] = []; + if (numeric[0] && categorical[0]) { + out.push(fill(suggest.mean, { num: numeric[0].name, cat: categorical[0].name })); + out.push(fill(suggest.count, { cat: categorical[0].name, value: categorical[0].values[0] })); + out.push(fill(suggest.top, { cat: categorical[0].name, num: numeric[0].name })); + } + if (numeric.length >= 2) { + out.push(fill(suggest.corr, { num: numeric[0].name, num2: numeric[1].name })); + } + out.push(suggest.missing); + return out; +} + +describe('le chat sait répondre à ce qu’il propose lui-même', () => { + for (const { name, columns } of DATASETS) { + for (const [lang, bundle] of [ + ['fr', fr], + ['en', en], + ] as const) { + it(`${name} · ${lang}`, () => { + const suggestions = suggestionsFor(bundle, columns); + expect(suggestions.length).toBeGreaterThanOrEqual(4); + for (const question of suggestions) { + expect(parseQuestion(question, columns, lang), `${lang} : ${question}`).not.toBeNull(); + } + }); + } + } +}); diff --git a/src/features/ai/llm/bench.node.test.ts b/src/features/ai/llm/bench.node.test.ts new file mode 100644 index 0000000..c332f06 --- /dev/null +++ b/src/features/ai/llm/bench.node.test.ts @@ -0,0 +1,146 @@ +// @vitest-environment node +/** + * V30 — the bench that runs without a GPU. + * + * V27's bench needed WebGPU with `shader-f16`, so it could only ever run on + * one person's laptop. That is why « 5 of 6 » stayed a claim rather than a + * measurement: nobody else could re-run it, and no automated check ever would. + * + * onnxruntime-node executes the SAME q4f16 graph on the CPU, from the SAME + * pinned files `scripts/prepare-llm.mjs --flat` downloads for production. So + * this file measures the shipped model with the shipped prompt through the + * shipped decoding path — the only difference from a browser is which device + * runs the matrix multiplies. Greedy decoding makes the comparison meaningful: + * with `do_sample: false` there is no sampling noise between the two, though + * the two backends can still differ on a near-tie between two tokens. + * + * It is skipped unless LABML_LLM_BENCH=1, because it needs 355 MB on disk that + * the repo deliberately does not carry: + * + * node scripts/prepare-llm.mjs .llm-cache --flat + * npm run llm:bench:node + */ +import { describe, it, expect } from 'vitest'; +import { existsSync } from 'node:fs'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { BENCH_CASES, BENCH_COLUMNS, score } from '@/features/ai/llm/corpus'; +import { + createConstrainer, + generateIntent, + type Constrainer, + type LogitsModule, + type RawGenerator, +} from '@/features/ai/llm/generate'; +import { formatReport, type BenchReport, type BenchRow } from '@/features/ai/llm/report'; +import { resolveIntent } from '@/features/ai/chat/route'; +import { parseQuestion } from '@/features/ai/chat/parser'; + +const ENABLED = process.env.LABML_LLM_BENCH === '1'; +const MODELS = process.env.LABML_LLM_CACHE ?? resolve(process.cwd(), '.llm-cache'); +/** + * The model under test. It defaults to the one production ships, and is + * overridable so that « would a bigger model read better? » is a one-command + * experiment on the same 55 questions rather than an opinion: + * + * LABML_LLM_REPO=onnx-community/Qwen3-1.7B-ONNX npm run llm:bench:node + * + * The weights have to be under LABML_LLM_CACHE already — `npm run llm:fetch` + * only knows the pinned files of the shipped model, by design: a size check + * that accepts anything is not a size check. + */ +const REPO = process.env.LABML_LLM_REPO ?? 'onnx-community/Qwen3-0.6B-DQ-ONNX'; +const DTYPE = process.env.LABML_LLM_DTYPE ?? 'q4f16'; +const LABEL = process.env.LABML_LLM_LABEL ?? 'banc V30 (CPU, onnxruntime-node)'; +const OUT = process.env.LABML_LLM_OUT; +/** V30 (B): decode inside the grammar. Off reproduces the V27 behaviour exactly. */ +const CONSTRAINED = process.env.LABML_LLM_CONSTRAIN === '1'; +/** A short run while iterating; a full run is the only one worth publishing. */ +const LIMIT = Number(process.env.LABML_LLM_LIMIT ?? BENCH_CASES.length); +/** Loading 355 MB and answering 55 questions on a CPU is minutes, not seconds. */ +const TIMEOUT_MS = 90 * 60 * 1000; + +async function loadCpuModel(): Promise<{ + generator: RawGenerator; + loadMs: number; + constrain: Constrainer | null; +}> { + const module = await import('@huggingface/transformers'); + const { env, pipeline } = module; + env.allowRemoteModels = false; + env.allowLocalModels = true; + env.localModelPath = MODELS.endsWith('/') ? MODELS : `${MODELS}/`; + const started = Date.now(); + const generator = await pipeline('text-generation', REPO, { + dtype: DTYPE as 'q4f16', + device: 'cpu', + }); + const loadMs = Date.now() - started; + const constrain = CONSTRAINED + ? createConstrainer(module as unknown as LogitsModule, generator.tokenizer) + : null; + if (CONSTRAINED && !constrain) throw new Error('constrained-decoding-unavailable'); + return { generator: generator as unknown as RawGenerator, loadMs, constrain }; +} + +describe.skipIf(!ENABLED)("banc d'interprétation V30", () => { + it( + 'mesure le corpus complet contre le modèle réel', + async () => { + const weights = `${MODELS}/${REPO}/onnx/model_${DTYPE}.onnx`; + expect( + existsSync(weights), + `poids absents : ${weights}\nlancer d'abord : node scripts/prepare-llm.mjs .llm-cache --flat`, + ).toBe(true); + + const { generator, loadMs, constrain } = await loadCpuModel(); + if (constrain) + console.log(`décodage contraint actif — ${constrain.usableTokens} jetons utilisables`); + const rows: BenchRow[] = []; + const cases = BENCH_CASES.slice(0, LIMIT); + for (const testCase of cases) { + const deterministic = parseQuestion(testCase.q, BENCH_COLUMNS, testCase.lang); + const result = await generateIntent(generator, testCase.q, BENCH_COLUMNS, { constrain }); + // The shipped order, not a re-implementation of it: the same function + // the chat worker calls decides who answers. + const shipped = await resolveIntent( + () => parseQuestion(testCase.q, BENCH_COLUMNS, testCase.lang), + async () => result.intent, + ); + rows.push({ + q: testCase.q, + lang: testCase.lang, + family: testCase.family, + deterministic: score(testCase, deterministic), + llm: score(testCase, result.intent), + pipeline: score(testCase, shipped.intent), + raw: result.raw.slice(0, 240), + ms: Math.round(result.ms), + }); + const last = rows[rows.length - 1]; + console.log( + `${String(rows.length).padStart(2)}/${cases.length} ${testCase.q.slice(0, 46).padEnd(46)} ` + + `det=${last.deterministic.padEnd(5)} lm=${last.llm.padEnd(5)} app=${last.pipeline.padEnd(5)} (${last.ms} ms)`, + ); + } + await generator.dispose?.(); + + const report: BenchReport = { + label: + `${LABEL} · ${REPO} ${DTYPE}` + + `${CONSTRAINED ? ' — décodage contraint' : ' — décodage libre'}`, + total: cases.length, + loadMs, + rows, + }; + console.log(formatReport(report)); + if (OUT) { + await mkdir(dirname(OUT), { recursive: true }); + await writeFile(OUT, JSON.stringify(report, null, 2)); + console.log(`\nrapport écrit dans ${OUT}`); + } + expect(rows).toHaveLength(cases.length); + }, + TIMEOUT_MS, + ); +}); diff --git a/src/features/ai/llm/bench.ts b/src/features/ai/llm/bench.ts index b240506..affe207 100644 --- a/src/features/ai/llm/bench.ts +++ b/src/features/ai/llm/bench.ts @@ -1,191 +1,26 @@ /** - * V27 — the interpretation bench, run against the REAL production path: - * the sharded model under /llm/, glued by the same custom cache the app uses, - * on the same WebGPU runtime. It measures the only thing that justifies a - * 355 MB download — how often the local model turns a question into a query - * the deterministic parser could not. + * V30 — the browser half of the bench: the same corpus, run against the REAL + * production path. The sharded model under /llm/, glued by the same custom + * cache the app uses, on the same WebGPU runtime. * - * It ships as a repo tool rather than a CI test because it needs a GPU with - * `shader-f16`, which CI runners do not have. See docs in PLAN.md § N (V27). + * Its companion `bench.node.test.ts` runs the identical corpus on the CPU and + * needs no GPU at all. Neither replaces the other: the Node bench is the one + * that can run anywhere and therefore the one that keeps the numbers honest + * between releases; this one is the only one that measures what a visitor's + * browser actually does, latency included. * * npm run llm:prepare -- public/llm * V27_BENCH=1 npm run build && npm run preview * node scripts/run-llm-bench.mjs */ -import { parseQuestion, type ColumnInfo } from '@/features/ai/chat/parser'; +import { BENCH_CASES, BENCH_COLUMNS, score } from '@/features/ai/llm/corpus'; +import { formatReport, type BenchReport, type BenchRow } from '@/features/ai/llm/report'; import { loadModel, probeCapability } from '@/features/ai/llm/interpret'; -import type { Intent } from '@/features/ai/chat/engine'; +import { parseQuestion } from '@/features/ai/chat/parser'; +import { resolveIntent } from '@/features/ai/chat/route'; -/** Titanic's columns, as the chat worker would summarize them. */ -export const BENCH_COLUMNS: ColumnInfo[] = [ - { name: 'survived', isNumeric: true, values: [] }, - { name: 'pclass', isNumeric: true, values: [] }, - { name: 'sex', isNumeric: false, values: ['male', 'female'] }, - { name: 'age', isNumeric: true, values: [] }, - { name: 'fare', isNumeric: true, values: [] }, - { name: 'embarked', isNumeric: false, values: ['S', 'C', 'Q'] }, - { name: 'class', isNumeric: false, values: ['Third', 'First', 'Second'] }, - { name: 'who', isNumeric: false, values: ['man', 'woman', 'child'] }, - { name: 'deck', isNumeric: false, values: ['A', 'B', 'C', 'D', 'E', 'F', 'G'] }, - { name: 'embark_town', isNumeric: false, values: ['Southampton', 'Cherbourg', 'Queenstown'] }, - { name: 'alive', isNumeric: false, values: ['no', 'yes'] }, - { name: 'alone', isNumeric: false, values: ['True', 'False'] }, -]; - -export interface BenchCase { - q: string; - lang: string; - want: Intent; - /** True for phrasings the keyword grammar was never going to catch. */ - beyondKeywords: boolean; -} - -export const BENCH_CASES: BenchCase[] = [ - { q: 'how many rows and columns?', lang: 'en', want: { kind: 'shape' }, beyondKeywords: false }, - { - q: 'combien de lignes et de colonnes ?', - lang: 'fr', - want: { kind: 'shape' }, - beyondKeywords: false, - }, - { - q: 'average age', - lang: 'en', - want: { kind: 'aggregate', op: 'mean', column: 'age' }, - beyondKeywords: false, - }, - { - q: 'moyenne de fare par class', - lang: 'fr', - want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'class' }, - beyondKeywords: false, - }, - { - q: 'distribution of class', - lang: 'en', - want: { kind: 'distribution', column: 'class' }, - beyondKeywords: false, - }, - { - q: 'correlation between age and fare', - lang: 'en', - want: { kind: 'correlation', a: 'age', b: 'fare' }, - beyondKeywords: false, - }, - { q: 'valeurs manquantes', lang: 'fr', want: { kind: 'missing' }, beyondKeywords: false }, - { - q: 'how many female?', - lang: 'en', - want: { kind: 'count', filter: { column: 'sex', op: '=', value: 'female' } }, - beyondKeywords: false, - }, - // --- The gap the model has to justify -------------------------------- - { - q: 'what was the typical age of the people on board?', - lang: 'en', - want: { kind: 'aggregate', op: 'mean', column: 'age' }, - beyondKeywords: true, - }, - { - q: 'a quel age moyen voyageaient les passagers ?', - lang: 'fr', - want: { kind: 'aggregate', op: 'mean', column: 'age' }, - beyondKeywords: true, - }, - { - q: 'did women pay more than men?', - lang: 'en', - want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'sex' }, - beyondKeywords: true, - }, - { - q: 'est-ce que le prix du billet dependait de la classe ?', - lang: 'fr', - want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'class' }, - beyondKeywords: true, - }, - { - q: 'show me how the ticket prices spread out', - lang: 'en', - want: { kind: 'distribution', column: 'fare' }, - beyondKeywords: true, - }, - { - q: 'combien de personnes sont montees a Cherbourg ?', - lang: 'fr', - want: { kind: 'count', filter: { column: 'embark_town', op: '=', value: 'Cherbourg' } }, - beyondKeywords: true, - }, - { - q: 'count the passengers older than 60', - lang: 'en', - want: { kind: 'count', filter: { column: 'age', op: '>', value: 60 } }, - beyondKeywords: true, - }, - { - q: 'y a-t-il un lien entre le prix paye et la survie ?', - lang: 'fr', - want: { kind: 'correlation', a: 'fare', b: 'survived' }, - beyondKeywords: true, - }, - // V27.1: the two shapes the measured failures exposed — an implicit numeric - // threshold ("enfants" is not a column, `age < 10` is), and a top-k, which - // the model had never seen an example of. - { - q: "combien d'enfants de moins de 10 ans ?", - lang: 'fr', - want: { kind: 'count', filter: { column: 'age', op: '<', value: 10 } }, - beyondKeywords: true, - }, - { - q: 'les 3 ponts avec le plus de passagers', - lang: 'fr', - want: { kind: 'topk', groupBy: 'deck', k: 3, op: 'count' }, - beyondKeywords: true, - }, - // V27.2: the one question still wrong after V27.1 — read as a correlation - // between fare and age. Kept in the exact phrasing it failed on, and - // deliberately NOT added to the prompt's examples: the prompt gained a rule - // and a different phrasing, so this stays a held-out case. - { - q: 'est-ce que les femmes payaient plus cher que les hommes ?', - lang: 'fr', - want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'sex' }, - beyondKeywords: true, - }, -]; - -/** Key-order-independent equality — the grammar has no meaningful ordering. */ -export function sameIntent(a: Intent | null, b: Intent): boolean { - if (!a) return false; - const norm = (i: Intent) => JSON.stringify(i, Object.keys(i).sort()); - return norm(a) === norm(b); -} - -export type Outcome = 'ok' | 'wrong' | 'none'; - -export interface BenchRow { - q: string; - lang: string; - beyondKeywords: boolean; - deterministic: Outcome; - llm: Outcome; - /** - * V27.1 — what the app actually answers, in the shipped order: the keyword - * grammar's reading when it has one, the model's only otherwise. This is the - * number that describes the product; the two columns above describe the - * parts. - */ - pipeline: Outcome; - raw: string; - ms: number; -} - -export interface BenchReport { - total: number; - loadMs: number; - rows: BenchRow[]; -} +export type { BenchReport, BenchRow }; +export { BENCH_CASES, BENCH_COLUMNS, formatReport }; export async function runBench(log: (line: string) => void): Promise { const capability = await probeCapability(); @@ -202,18 +37,21 @@ export async function runBench(log: (line: string) => void): Promise - !intent ? 'none' : sameIntent(intent, testCase.want) ? 'ok' : 'wrong'; + // The shipped order, decided by the function the chat worker itself calls. + const shipped = await resolveIntent( + () => parseQuestion(testCase.q, BENCH_COLUMNS, testCase.lang), + async () => result.intent, + ); rows.push({ q: testCase.q, lang: testCase.lang, - beyondKeywords: testCase.beyondKeywords, - deterministic: outcome(det), - llm: outcome(result.intent), - pipeline: det ? outcome(det) : outcome(result.intent), - raw: result.raw.slice(0, 200), + family: testCase.family, + deterministic: score(testCase, deterministic), + llm: score(testCase, result.intent), + pipeline: score(testCase, shipped.intent), + raw: result.raw.slice(0, 240), ms: Math.round(result.ms), }); const last = rows[rows.length - 1]; @@ -222,5 +60,5 @@ export async function runBench(log: (line: string) => void): Promise(); + let next = 0; + for (let b = 0; b < 256; b++) { + if (direct.includes(b)) continue; + moved.set(b, 256 + next); + next += 1; + } + let out = ''; + for (const byte of new TextEncoder().encode(text)) { + out += String.fromCodePoint(moved.get(byte) ?? byte); + } + return out; +} + +/** A tiny vocabulary: id 0 is end-of-turn, the rest are content. */ +const PIECES = [ + '<|im_end|>', + '{', + '"kind"', + ':', + '"shape"', + '"count"', + '"miss', + 'ing"', + '}', + ' ', + '```', + 'json', + '"med', + 'ian"', + '"région"', + '"Île-de-France"', +]; +const VOCAB = PIECES.map((piece, i) => (i === 0 ? piece : spell(piece))); +const EOS = [0]; +const index = buildVocabIndex(VOCAB, EOS); +const grammar = buildGrammar(BENCH_COLUMNS); +const id = (piece: string) => PIECES.indexOf(piece); +const encoder = new TextEncoder(); + +describe('index du vocabulaire', () => { + it('rend chaque jeton en octets, sauf ceux de fin de tour', () => { + expect(index.usable).toBe(PIECES.length - 1); + expect(index.bytes[id('<|im_end|>')]).toBeNull(); + expect([...index.bytes[id('{')]!]).toEqual([...encoder.encode('{')]); + // A multi-byte character survives, which is the reason the automaton walks + // bytes rather than characters. + expect([...index.bytes[id('"région"')]!]).toEqual([...encoder.encode('"région"')]); + }); + + it('range les jetons par premier octet', () => { + expect(index.byFirstByte[0x7b]).toContain(id('{')); + expect(index.byFirstByte[0x22]).toContain(id('"kind"')); + }); + + it('recolle les octets déjà générés', () => { + const bytes = generatedBytes(index, [id('{'), id('"kind"'), id(':')]); + expect(new TextDecoder().decode(bytes)).toBe('{"kind":'); + }); +}); + +describe('jetons autorisés', () => { + it("n'ouvre la réponse que par une accolade", () => { + const ids = allowedTokens(grammar, index, new Uint8Array()); + expect(ids).toEqual([id('{')]); + // The two failure modes measured in V27 — a markdown fence and a leading + // space — are simply not choosable. + expect(ids).not.toContain(id('```')); + expect(ids).not.toContain(id(' ')); + }); + + it('ne propose que des formes réelles après la clé kind', () => { + const ids = allowedTokens(grammar, index, encoder.encode('{"kind":')); + expect(ids).toContain(id('"shape"')); + expect(ids).toContain(id('"count"')); + expect(ids).toContain(id('"miss')); + // `median` is an operator, never a kind. V27 emitted {"kind":"median"} and + // the answer was thrown away; here the token cannot be picked at all. + expect(ids).not.toContain(id('"med')); + }); + + it('exige la fin de tour dès que la requête est complète', () => { + const ids = allowedTokens(grammar, index, encoder.encode('{"kind":"shape"}')); + expect(ids).toEqual(EOS); + }); + + it('permet de finir ou de continuer quand les deux sont légaux', () => { + const ids = allowedTokens(grammar, index, encoder.encode('{"kind":"count"')); + expect(ids).toContain(id('}')); + expect(ids).not.toContain(EOS[0]); + }); + + it('finit le tour plutôt que de rester coincé sur une impasse', () => { + // Not reachable through the automaton, but if it ever were, ending the + // turn keeps the run finite and the answer becomes an honest refusal. + const ids = allowedTokens(grammar, index, encoder.encode('{"kind":"zzz')); + expect(ids).toEqual(EOS); + }); +}); + +describe('processeur de logits', () => { + class FakeBase implements LogitsProcessorLike { + _call(_inputIds: bigint[][], logits: LogitsTensor): LogitsTensor { + return logits; + } + } + + function run(prompt: number[], generated: number[]): Float32Array { + const processor = createGrammarProcessor(FakeBase, grammar, index); + const data = new Float32Array(VOCAB.length).fill(1); + const logits: LogitsTensor = { data, dims: [1, VOCAB.length] }; + const ids = [...prompt, ...generated].map((value) => BigInt(value)); + // First call carries the prompt alone — that is how the processor learns + // where the prompt ends. + processor._call([prompt.map((value) => BigInt(value))], { + data: new Float32Array(VOCAB.length).fill(1), + dims: [1, VOCAB.length], + }); + processor._call([ids], logits); + return data; + } + + it('rend le tenseur — la bibliothèque chaîne les processeurs', () => { + const processor = createGrammarProcessor(FakeBase, grammar, index); + const logits: LogitsTensor = { data: new Float32Array(VOCAB.length), dims: [1, VOCAB.length] }; + expect(processor._call([[]], logits)).toBe(logits); + }); + + it('laisse passer ce qui est légal et coupe le reste', () => { + const data = run([9, 9, 9], [id('{'), id('"kind"'), id(':')]); + expect(data[id('"shape"')]).toBe(1); + expect(data[id('"count"')]).toBe(1); + expect(data[id('"med')]).toBe(-Infinity); + expect(data[id('```')]).toBe(-Infinity); + }); + + it('garde la préférence du modèle entre les suites légales', () => { + // Masking is a veto, not a vote: an allowed token keeps its own score. + const processor = createGrammarProcessor(FakeBase, grammar, index); + const data = new Float32Array(VOCAB.length).fill(0); + data[id('"shape"')] = 3.5; + data[id('"count"')] = 1.25; + const logits: LogitsTensor = { data, dims: [1, VOCAB.length] }; + const ids = [id('{'), id('"kind"'), id(':')].map((value) => BigInt(value)); + processor._call([[]], { data: new Float32Array(VOCAB.length), dims: [1, VOCAB.length] }); + processor._call([ids], logits); + expect(data[id('"shape"')]).toBe(3.5); + expect(data[id('"count"')]).toBe(1.25); + }); +}); + +describe('lecture du vocabulaire du tokenizer', () => { + it('accepte les deux formes que la bibliothèque a utilisées', () => { + const asObject = readVocab({ + _tokenizerJSON: { model: { vocab: { '{': 1, '"kind"': 0 } } }, + all_special_ids: [5], + eos_token_id: 5, + }); + expect(asObject?.vocab).toEqual(['"kind"', '{']); + expect(asObject?.eosIds).toEqual([5]); + const asArray = readVocab({ _tokenizer: { model: { vocab: ['a', 'b'] } }, eos_token_id: 1 }); + expect(asArray?.vocab).toEqual(['a', 'b']); + }); + + it("renvoie null quand le vocabulaire n'est pas exposé", () => { + // Constrained decoding is then announced as OFF rather than silently + // skipped: an unconstrained answer badged as constrained would be a lie. + expect(readVocab({})).toBeNull(); + expect(readVocab({ _tokenizerJSON: { model: { vocab: {} } } })).toBeNull(); + }); +}); diff --git a/src/features/ai/llm/constrain.ts b/src/features/ai/llm/constrain.ts new file mode 100644 index 0000000..77765a9 --- /dev/null +++ b/src/features/ai/llm/constrain.ts @@ -0,0 +1,254 @@ +/** + * V30 — the hand-written logits processor: the grammar, applied DURING + * generation rather than checked after it. + * + * At every step the model proposes a probability for each of Qwen's 151 669 + * tokens. This masks out every token that would take the answer outside the + * query grammar, so `{"kind":"` can only be followed by one of the eight + * shapes, `"column":` only by a column the table actually has, and a filter + * value only by something that column can hold. The model still chooses — it + * simply cannot choose something unwritable. + * + * Two implementation notes that are the difference between working and + * unusably slow: + * + * 1. **Tokens are runs of BYTES, taken from the vocabulary's byte-level BPE + * form.** `decode()` would give text, and text loses the 1 457 tokens that + * are fragments of a multi-byte character. The GPT-2 byte↔unicode table + * below is the exact inverse of the encoding the vocabulary file uses. + * 2. **Candidates are bucketed by first byte.** Walking all 151 669 tokens at + * every step would cost more than the model does. The automaton first says + * which bytes may come next — usually one or two — and only those buckets + * are tested. + */ +import { + accepting, + allowedBytes, + startState, + stepByte, + stepBytes, + type Grammar, +} from '@/features/ai/llm/grammar'; + +/** + * GPT-2's byte↔unicode table, the encoding every byte-level BPE vocabulary is + * written in: printable ASCII and Latin-1 keep their own code point, and the + * remaining 68 bytes are moved to U+0100 and up so that no vocabulary entry + * ever contains a control character. + */ +function byteDecoder(): Map { + const direct: number[] = []; + for (let b = 0x21; b <= 0x7e; b++) direct.push(b); + for (let b = 0xa1; b <= 0xac; b++) direct.push(b); + for (let b = 0xae; b <= 0xff; b++) direct.push(b); + const map = new Map(); + for (const b of direct) map.set(String.fromCodePoint(b), b); + let next = 0; + for (let b = 0; b < 256; b++) { + if (direct.includes(b)) continue; + map.set(String.fromCodePoint(256 + next), b); + next += 1; + } + return map; +} + +export interface VocabIndex { + /** Byte form of each token id; null for a token that may never appear. */ + bytes: (Uint8Array | null)[]; + /** Token ids by first byte — 256 buckets, most of them empty. */ + byFirstByte: number[][]; + /** Ids that end the turn. Allowed only where the grammar is complete. */ + eosIds: number[]; + /** How many ids carry usable bytes — reported so a broken vocab is visible. */ + usable: number; +} + +/** + * Builds the index from the raw vocabulary, in the byte-level form the + * tokenizer file stores. A token whose spelling contains a character outside + * the table is a special token (`<|im_end|>` and friends) and is excluded from + * the content vocabulary — it can only ever be chosen as an end-of-turn. + */ +export function buildVocabIndex(vocab: readonly string[], eosIds: readonly number[]): VocabIndex { + const decoder = byteDecoder(); + const bytes: (Uint8Array | null)[] = new Array(vocab.length).fill(null); + const byFirstByte: number[][] = Array.from({ length: 256 }, () => []); + const eos = new Set(eosIds); + let usable = 0; + + for (let id = 0; id < vocab.length; id++) { + if (eos.has(id)) continue; + const spelling = vocab[id]; + if (typeof spelling !== 'string' || spelling.length === 0) continue; + const out = new Uint8Array(spelling.length); + let length = 0; + let ok = true; + for (const char of spelling) { + const byte = decoder.get(char); + if (byte === undefined) { + ok = false; + break; + } + out[length++] = byte; + } + if (!ok || length === 0) continue; + const trimmed = out.subarray(0, length); + bytes[id] = trimmed; + byFirstByte[trimmed[0]].push(id); + usable += 1; + } + return { bytes, byFirstByte, eosIds: [...eosIds], usable }; +} + +/** + * The ids that keep the answer inside the grammar, given what has been + * generated so far. When the answer is already complete, ending the turn is + * allowed too — and when it is complete and nothing may follow, ending is the + * only thing allowed, which is what stops the model from writing a second + * object after the first. + */ +export function allowedTokens( + grammar: Grammar, + index: VocabIndex, + generated: Uint8Array, +): number[] { + let state = startState(grammar); + state = stepBytes(grammar, state, generated); + const ids: number[] = []; + if (state.size === 0) return index.eosIds.slice(); + for (const byte of allowedBytes(grammar, state)) { + // The first byte is stepped ONCE for the whole bucket, not once per token. + // At `{"kind":"` the state holds seventy positions and the bucket holds + // thousands of tokens; repeating that first step per token was measured at + // 53 ms for this single position, against 3 ms for the whole rest. + const afterFirst = stepByte(grammar, state, byte); + for (const id of index.byFirstByte[byte]) { + const token = index.bytes[id]; + if (!token) continue; + if (token.length === 1) { + ids.push(id); + continue; + } + if (stepBytes(grammar, afterFirst, token.subarray(1)).size > 0) ids.push(id); + } + } + if (accepting(grammar, state)) ids.push(...index.eosIds); + // A dead end can only happen if the grammar admits a prefix with no + // continuation at all. Ending the turn keeps the run finite; the answer then + // fails `validateIntent` and becomes an honest refusal. + return ids.length > 0 ? ids : index.eosIds.slice(); +} + +/** The bytes generated so far, from the ids the model has already chosen. */ +export function generatedBytes(index: VocabIndex, ids: readonly number[]): Uint8Array { + const parts: Uint8Array[] = []; + let total = 0; + for (const id of ids) { + const token = index.bytes[id]; + if (!token) continue; + parts.push(token); + total += token.length; + } + const out = new Uint8Array(total); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} + +/** The minimum of a transformers.js Tensor this module touches. */ +export interface LogitsTensor { + data: Float32Array | Float64Array; + dims: number[]; +} + +/** + * The library chains processors — `toReturn = processor(ids, toReturn)` — so + * `_call` MUST hand the tensor back. Returning nothing leaves the next + * processor, and then the sampler, holding `undefined`. + */ +export interface LogitsProcessorLike { + _call(inputIds: bigint[][], logits: LogitsTensor): LogitsTensor; +} + +/** + * Builds the processor. `Base` is the library's own `LogitsProcessor` class, + * passed in rather than imported: this module must stay out of the main bundle, + * and the library is only ever loaded on demand. + * + * The prompt length is learned rather than passed. The first call carries the + * prompt and nothing else, so whatever arrives then is the prompt — which also + * makes the processor correct if it is reused for a second generation. + */ +export function createGrammarProcessor( + Base: new () => LogitsProcessorLike, + grammar: Grammar, + index: VocabIndex, +): LogitsProcessorLike { + let promptLength = -1; + return new (class extends Base { + _call(inputIds: bigint[][], logits: LogitsTensor): LogitsTensor { + if (promptLength < 0) promptLength = inputIds[0]?.length ?? 0; + const vocabSize = logits.dims[logits.dims.length - 1]; + const data = logits.data; + for (let batch = 0; batch < inputIds.length; batch++) { + const row = inputIds[batch] ?? []; + const generated = row.slice(promptLength).map((id) => Number(id)); + const ids = allowedTokens(grammar, index, generatedBytes(index, generated)); + const start = batch * vocabSize; + // Save, blank, restore: the allowed logits keep their own values, so + // the model's own preference still decides between the legal + // continuations. Masking is a veto, never a vote. + const keep = new Float64Array(ids.length); + for (let i = 0; i < ids.length; i++) keep[i] = data[start + ids[i]]; + data.fill(-Infinity, start, start + vocabSize); + for (let i = 0; i < ids.length; i++) data[start + ids[i]] = keep[i]; + } + return logits; + } + })(); +} + +/** + * Digs the byte-level vocabulary out of a loaded tokenizer. + * + * transformers.js keeps the parsed `tokenizer.json` on the instance, which is + * the form we want: an object mapping each token's byte-level spelling to its + * id. Two shapes are accepted because the library has used both, and a + * tokenizer that offers neither returns null — the caller then announces that + * constrained decoding is off rather than silently generating unconstrained. + */ +export function readVocab(tokenizer: unknown): { vocab: string[]; eosIds: number[] } | null { + const owner = tokenizer as { + _tokenizerJSON?: { model?: { vocab?: unknown } }; + _tokenizer?: { model?: { vocab?: unknown }; all_special_ids?: number[] }; + all_special_ids?: number[]; + eos_token_id?: number; + }; + const raw = owner._tokenizerJSON?.model?.vocab ?? owner._tokenizer?.model?.vocab; + let vocab: string[] | null = null; + if (Array.isArray(raw)) { + vocab = raw.map((entry) => (typeof entry === 'string' ? entry : '')); + } else if (raw && typeof raw === 'object') { + const entries = Object.entries(raw as Record); + const size = entries.reduce((max, [, id]) => Math.max(max, id), -1) + 1; + if (size > 0) { + const built = new Array(size).fill(''); + for (const [token, id] of entries) built[id] = token; + vocab = built; + } + } + if (!vocab || vocab.length === 0) return null; + + // Every special id ends the content, not just the configured EOS: a model + // that emits <|im_end|> or <|endoftext|> has stopped either way. + const specials = owner.all_special_ids ?? owner._tokenizer?.all_special_ids ?? []; + const eosIds = [ + ...new Set([...(specials ?? []), ...(owner.eos_token_id != null ? [owner.eos_token_id] : [])]), + ] + .filter((id) => typeof id === 'number' && id >= 0) + .sort((a, b) => a - b); + return { vocab, eosIds }; +} diff --git a/src/features/ai/llm/corpus.test.ts b/src/features/ai/llm/corpus.test.ts new file mode 100644 index 0000000..2e294f4 --- /dev/null +++ b/src/features/ai/llm/corpus.test.ts @@ -0,0 +1,123 @@ +/** + * V30 — the half of the bench that runs in CI. + * + * The model needs 355 MB and a GPU (or several minutes of CPU), so its half + * lives in `bench.node.test.ts` and runs on demand. Everything else about the + * corpus is checkable here, in two seconds, on every commit: that the corpus + * is well-formed, and exactly how the deterministic parser reads it. + * + * The second one is the regression guard V30 exists to install. Before this + * wave the parser answered seven of these fifty-five questions with a + * condition silently removed — « combien de femmes ? » answered 891 instead of + * 314, under the badge that says the reading is exact. The `wrong` count below + * is asserted to be ZERO, and that assertion is the whole point: it fails the + * build the day the grammar starts answering a question nobody asked. + */ +import { describe, it, expect } from 'vitest'; +import { BENCH_CASES, BENCH_COLUMNS, score, sameIntent } from '@/features/ai/llm/corpus'; +import { validateIntent } from '@/features/ai/llm/prompt'; +import { isComplete, buildGrammar } from '@/features/ai/llm/grammar'; +import { parseQuestion } from '@/features/ai/chat/parser'; +import type { Outcome } from '@/features/ai/llm/corpus'; + +function deterministicTally(): Record { + const counts: Record = { ok: 0, wrong: 0, none: 0 }; + for (const testCase of BENCH_CASES) { + counts[score(testCase, parseQuestion(testCase.q, BENCH_COLUMNS, testCase.lang))] += 1; + } + return counts; +} + +describe('corpus V30 — le corpus lui-même', () => { + it('compte entre 40 et 60 questions, dans les deux langues', () => { + expect(BENCH_CASES.length).toBeGreaterThanOrEqual(40); + expect(BENCH_CASES.length).toBeLessThanOrEqual(60); + expect(BENCH_CASES.filter((c) => c.lang === 'fr').length).toBeGreaterThanOrEqual(20); + expect(BENCH_CASES.filter((c) => c.lang === 'en').length).toBeGreaterThanOrEqual(20); + }); + + it('ne pose jamais deux fois la même question', () => { + const seen = new Set(BENCH_CASES.map((c) => c.q)); + expect(seen.size).toBe(BENCH_CASES.length); + }); + + it('couvre les sept formes de la grammaire, plus le refus', () => { + const families = new Set(BENCH_CASES.map((c) => c.family)); + for (const family of [ + 'shape', + 'missing', + 'count', + 'aggregate', + 'distribution', + 'correlation', + 'topk', + 'refuse', + ]) { + expect(families.has(family as never), family).toBe(true); + } + }); + + it('attend des réponses que le validateur et la grammaire acceptent', () => { + // A corpus whose expected answers are themselves out of grammar would + // measure the measuring instrument, not the app. + const grammar = buildGrammar(BENCH_COLUMNS); + for (const testCase of BENCH_CASES) { + for (const intent of [testCase.want, ...(testCase.alsoOk ?? [])]) { + if (intent === null) continue; + expect(validateIntent(intent, BENCH_COLUMNS), testCase.q).not.toBeNull(); + expect(isComplete(grammar, JSON.stringify(intent)), `${testCase.q} → grammaire`).toBe(true); + } + } + }); + + it('ne compte juste que la lecture attendue', () => { + const shape = BENCH_CASES.find((c) => c.family === 'shape')!; + expect(score(shape, { kind: 'shape' })).toBe('ok'); + expect(score(shape, { kind: 'missing' })).toBe('wrong'); + expect(score(shape, null)).toBe('none'); + const refuse = BENCH_CASES.find((c) => c.family === 'refuse')!; + expect(score(refuse, null)).toBe('ok'); + expect(score(refuse, { kind: 'shape' })).toBe('wrong'); + }); + + it("compare les intentions sans tenir compte de l'ordre des clés", () => { + expect( + sameIntent({ kind: 'aggregate', op: 'mean', column: 'age' }, { + column: 'age', + op: 'mean', + kind: 'aggregate', + } as never), + ).toBe(true); + expect(sameIntent(null, null)).toBe(true); + expect(sameIntent(null, { kind: 'shape' })).toBe(false); + }); +}); + +describe('corpus V30 — lecture déterministe', () => { + it('ne répond JAMAIS à côté : zéro réponse fausse', () => { + // The V30 invariant. A wrong deterministic answer cannot be rescued — the + // parser runs first and the model is never consulted — so it is the only + // outcome the grammar is forbidden to produce. + expect(deterministicTally().wrong).toBe(0); + }); + + it('lit 19 questions sur 55 et refuse les 36 autres', () => { + // Measured, not aimed at: the exact split the wave shipped with. A change + // that moves either number is a change in what the chat understands, and + // should be an explicit decision rather than a surprise. + expect(deterministicTally()).toEqual({ ok: 19, wrong: 0, none: 36 }); + }); + + it("refuse plutôt que de laisser tomber une condition qu'il ne lit pas", () => { + // The four questions that used to be answered short, by name. + for (const q of [ + 'combien de femmes ?', + 'average age of women', + 'combien de passagers en première classe ?', + 'quel âge avaient les passagers en moyenne, par classe ?', + ]) { + const testCase = BENCH_CASES.find((c) => c.q === q)!; + expect(parseQuestion(q, BENCH_COLUMNS, testCase.lang), q).toBeNull(); + } + }); +}); diff --git a/src/features/ai/llm/corpus.ts b/src/features/ai/llm/corpus.ts new file mode 100644 index 0000000..0f7cd0e --- /dev/null +++ b/src/features/ai/llm/corpus.ts @@ -0,0 +1,453 @@ +/** + * V30 — the reference corpus: every question the chat is measured on. + * + * V27 shipped 18 cases that lived inside the browser bench and could only run + * on a machine with WebGPU. That made every claim about the chat unfalsifiable + * in practice: nobody could re-run it, so « 5 of 6 » was a number in a document + * rather than a measurement anyone could reproduce. This module exists to fix + * that first, because nothing else in V30 is measurable until it does. + * + * Three rules shape it: + * + * 1. **No hand-labelled difficulty.** V27's corpus carried a `beyondKeywords` + * flag written by hand — a PREDICTION about what the deterministic parser + * would fail at. Predictions belong in the plan, not in the measuring + * instrument: the split is now computed from the parser's actual behaviour + * at report time, so it cannot be wrong. + * 2. **A question may have more than one right answer.** « combien de + * passagers en première classe ? » is correctly read as `class = First` or + * as `pclass = 1`. Forcing one would score a correct reading as a failure, + * so `alsoOk` names the other acceptable readings explicitly. + * 3. **Refusing is an answer.** Three cases here are not queries at all. The + * only correct behaviour is a refusal, and a run that invents a query for + * them is scored wrong — which is exactly how V30's constrained decoding + * gets held to account, since constraining the output makes refusal harder, + * not easier. + */ +import type { Intent } from '@/features/ai/chat/engine'; +import type { ColumnInfo } from '@/features/ai/chat/parser'; + +/** Titanic's columns, as the chat worker summarizes them for the model. */ +export const BENCH_COLUMNS: ColumnInfo[] = [ + // `distinct` is what the chat worker would report for titanic.csv, capped at + // 13 the way it caps it: it is what lets the prompt's examples pick `age` + // rather than `survived` as the quantity to average. + { name: 'survived', isNumeric: true, values: [], distinct: 2 }, + { name: 'pclass', isNumeric: true, values: [], distinct: 3 }, + { name: 'sex', isNumeric: false, values: ['male', 'female'], distinct: 2 }, + { name: 'age', isNumeric: true, values: [], distinct: 13 }, + { name: 'fare', isNumeric: true, values: [], distinct: 13 }, + { name: 'embarked', isNumeric: false, values: ['S', 'C', 'Q'] }, + { name: 'class', isNumeric: false, values: ['Third', 'First', 'Second'] }, + { name: 'who', isNumeric: false, values: ['man', 'woman', 'child'] }, + { name: 'deck', isNumeric: false, values: ['A', 'B', 'C', 'D', 'E', 'F', 'G'] }, + { name: 'embark_town', isNumeric: false, values: ['Southampton', 'Cherbourg', 'Queenstown'] }, + { name: 'alive', isNumeric: false, values: ['no', 'yes'] }, + { name: 'alone', isNumeric: false, values: ['True', 'False'] }, +]; + +export interface BenchCase { + q: string; + lang: 'fr' | 'en'; + /** The intent kind expected, or 'refuse' when no query is the right answer. */ + family: Intent['kind'] | 'refuse'; + /** The canonical correct reading. Null means: the only right answer is none. */ + want: Intent | null; + /** Other readings that are also correct — a question can be honestly ambiguous. */ + alsoOk?: Intent[]; +} + +export const BENCH_CASES: BenchCase[] = [ + // --- shape ----------------------------------------------------------- + { q: 'how many rows and columns?', lang: 'en', family: 'shape', want: { kind: 'shape' } }, + { + q: 'combien de lignes et de colonnes ?', + lang: 'fr', + family: 'shape', + want: { kind: 'shape' }, + }, + { q: 'what is the size of this table?', lang: 'en', family: 'shape', want: { kind: 'shape' } }, + { q: 'quelle est la taille du tableau ?', lang: 'fr', family: 'shape', want: { kind: 'shape' } }, + + // --- missing --------------------------------------------------------- + { q: 'valeurs manquantes', lang: 'fr', family: 'missing', want: { kind: 'missing' } }, + { + q: 'which columns have missing values?', + lang: 'en', + family: 'missing', + want: { kind: 'missing' }, + }, + { + q: 'y a-t-il des trous dans les données ?', + lang: 'fr', + family: 'missing', + want: { kind: 'missing' }, + }, + + // --- count ----------------------------------------------------------- + { + q: 'how many female?', + lang: 'en', + family: 'count', + want: { kind: 'count', filter: { column: 'sex', op: '=', value: 'female' } }, + }, + { + q: 'combien de femmes ?', + lang: 'fr', + family: 'count', + want: { kind: 'count', filter: { column: 'sex', op: '=', value: 'female' } }, + alsoOk: [{ kind: 'count', filter: { column: 'who', op: '=', value: 'woman' } }], + }, + { + q: 'count the passengers older than 60', + lang: 'en', + family: 'count', + want: { kind: 'count', filter: { column: 'age', op: '>', value: 60 } }, + }, + { + q: "combien d'enfants de moins de 10 ans ?", + lang: 'fr', + family: 'count', + want: { kind: 'count', filter: { column: 'age', op: '<', value: 10 } }, + }, + { + q: 'combien de personnes sont montées à Cherbourg ?', + lang: 'fr', + family: 'count', + want: { kind: 'count', filter: { column: 'embark_town', op: '=', value: 'Cherbourg' } }, + alsoOk: [{ kind: 'count', filter: { column: 'embarked', op: '=', value: 'C' } }], + }, + { + q: 'how many passengers boarded at Southampton?', + lang: 'en', + family: 'count', + want: { kind: 'count', filter: { column: 'embark_town', op: '=', value: 'Southampton' } }, + alsoOk: [{ kind: 'count', filter: { column: 'embarked', op: '=', value: 'S' } }], + }, + { q: 'combien de lignes ?', lang: 'fr', family: 'count', want: { kind: 'count' } }, + { + q: 'how many people travelled alone?', + lang: 'en', + family: 'count', + want: { kind: 'count', filter: { column: 'alone', op: '=', value: 'True' } }, + }, + { + q: 'combien de passagers en première classe ?', + lang: 'fr', + family: 'count', + want: { kind: 'count', filter: { column: 'class', op: '=', value: 'First' } }, + alsoOk: [{ kind: 'count', filter: { column: 'pclass', op: '=', value: 1 } }], + }, + { + q: 'combien de passagers avaient plus de 30 ans ?', + lang: 'fr', + family: 'count', + want: { kind: 'count', filter: { column: 'age', op: '>', value: 30 } }, + }, + + // --- aggregate, one number ------------------------------------------- + { + q: 'average age', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age' }, + }, + { + q: 'âge moyen', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age' }, + }, + { + q: 'what was the typical age of the people on board?', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age' }, + alsoOk: [{ kind: 'aggregate', op: 'median', column: 'age' }], + }, + { + q: 'à quel âge moyen voyageaient les passagers ?', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age' }, + }, + { + q: 'median fare', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'median', column: 'fare' }, + }, + { + q: 'prix médian du billet', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'median', column: 'fare' }, + }, + { + q: 'highest fare paid', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'max', column: 'fare' }, + }, + { + q: 'le prix le plus bas payé', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'min', column: 'fare' }, + }, + { + q: 'total of all fares', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'sum', column: 'fare' }, + }, + { + q: "écart-type de l'âge", + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'std', column: 'age' }, + }, + + // --- aggregate, by group --------------------------------------------- + { + q: 'moyenne de fare par class', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'class' }, + }, + { + q: 'average fare by class', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'class' }, + alsoOk: [{ kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'pclass' }], + }, + { + q: 'did women pay more than men?', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'sex' }, + }, + { + q: 'est-ce que les femmes payaient plus cher que les hommes ?', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'sex' }, + }, + { + q: 'est-ce que le prix du billet dépendait de la classe ?', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'class' }, + alsoOk: [{ kind: 'aggregate', op: 'mean', column: 'fare', groupBy: 'pclass' }], + }, + { + q: 'les hommes voyageaient-ils plus jeunes que les femmes ?', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age', groupBy: 'sex' }, + }, + { + q: 'average age per deck', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age', groupBy: 'deck' }, + }, + { + q: 'survival rate by sex', + lang: 'en', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'survived', groupBy: 'sex' }, + }, + { + q: 'quel âge avaient les passagers en moyenne, par classe ?', + lang: 'fr', + family: 'aggregate', + want: { kind: 'aggregate', op: 'mean', column: 'age', groupBy: 'class' }, + alsoOk: [{ kind: 'aggregate', op: 'mean', column: 'age', groupBy: 'pclass' }], + }, + + // --- aggregate, filtered --------------------------------------------- + { + q: 'average age of women', + lang: 'en', + family: 'aggregate', + want: { + kind: 'aggregate', + op: 'mean', + column: 'age', + filter: { column: 'sex', op: '=', value: 'female' }, + }, + alsoOk: [ + { + kind: 'aggregate', + op: 'mean', + column: 'age', + filter: { column: 'who', op: '=', value: 'woman' }, + }, + ], + }, + { + q: 'âge moyen des hommes', + lang: 'fr', + family: 'aggregate', + want: { + kind: 'aggregate', + op: 'mean', + column: 'age', + filter: { column: 'sex', op: '=', value: 'male' }, + }, + alsoOk: [ + { + kind: 'aggregate', + op: 'mean', + column: 'age', + filter: { column: 'who', op: '=', value: 'man' }, + }, + ], + }, + { + q: 'how much did a ticket cost on average in third class?', + lang: 'en', + family: 'aggregate', + want: { + kind: 'aggregate', + op: 'mean', + column: 'fare', + filter: { column: 'class', op: '=', value: 'Third' }, + }, + alsoOk: [ + { + kind: 'aggregate', + op: 'mean', + column: 'fare', + filter: { column: 'pclass', op: '=', value: 3 }, + }, + ], + }, + { + q: 'prix moyen payé par les survivants', + lang: 'fr', + family: 'aggregate', + want: { + kind: 'aggregate', + op: 'mean', + column: 'fare', + filter: { column: 'alive', op: '=', value: 'yes' }, + }, + alsoOk: [ + { + kind: 'aggregate', + op: 'mean', + column: 'fare', + filter: { column: 'survived', op: '=', value: 1 }, + }, + ], + }, + + // --- distribution ---------------------------------------------------- + { + q: 'distribution of class', + lang: 'en', + family: 'distribution', + want: { kind: 'distribution', column: 'class' }, + }, + { + q: 'répartition des classes', + lang: 'fr', + family: 'distribution', + want: { kind: 'distribution', column: 'class' }, + alsoOk: [{ kind: 'distribution', column: 'pclass' }], + }, + { + q: 'show me how the ticket prices spread out', + lang: 'en', + family: 'distribution', + want: { kind: 'distribution', column: 'fare' }, + }, + { + q: 'comment se répartissent les âges ?', + lang: 'fr', + family: 'distribution', + want: { kind: 'distribution', column: 'age' }, + }, + + // --- correlation ----------------------------------------------------- + { + q: 'correlation between age and fare', + lang: 'en', + family: 'correlation', + want: { kind: 'correlation', a: 'age', b: 'fare' }, + }, + { + q: 'y a-t-il un lien entre le prix payé et la survie ?', + lang: 'fr', + family: 'correlation', + want: { kind: 'correlation', a: 'fare', b: 'survived' }, + }, + { + q: 'is age related to how much people paid?', + lang: 'en', + family: 'correlation', + want: { kind: 'correlation', a: 'age', b: 'fare' }, + }, + { + q: "corrélation entre l'âge et la classe", + lang: 'fr', + family: 'correlation', + want: { kind: 'correlation', a: 'age', b: 'pclass' }, + }, + + // --- top-k ----------------------------------------------------------- + { + q: 'les 3 ponts avec le plus de passagers', + lang: 'fr', + family: 'topk', + want: { kind: 'topk', groupBy: 'deck', k: 3, op: 'count' }, + }, + { + q: 'top 3 embarkation towns by number of passengers', + lang: 'en', + family: 'topk', + want: { kind: 'topk', groupBy: 'embark_town', k: 3, op: 'count' }, + }, + { + q: 'which 5 decks have the highest average fare?', + lang: 'en', + family: 'topk', + want: { kind: 'topk', groupBy: 'deck', k: 5, op: 'mean', column: 'fare' }, + }, + { + q: "les 2 classes où l'âge moyen est le plus élevé", + lang: 'fr', + family: 'topk', + want: { kind: 'topk', groupBy: 'class', k: 2, op: 'mean', column: 'age' }, + alsoOk: [{ kind: 'topk', groupBy: 'pclass', k: 2, op: 'mean', column: 'age' }], + }, + + // --- questions the grammar cannot express: refusing IS the answer ------ + { q: 'predict who would survive', lang: 'en', family: 'refuse', want: null }, + { q: 'trace-moi un graphique en camembert', lang: 'fr', family: 'refuse', want: null }, + { q: 'quelle est la capitale de la France ?', lang: 'fr', family: 'refuse', want: null }, +]; + +/** Key-order-independent equality — the grammar has no meaningful ordering. */ +export function sameIntent(a: Intent | null, b: Intent | null): boolean { + if (a === null || b === null) return a === b; + const norm = (i: Intent) => JSON.stringify(i, Object.keys(i).sort()); + return norm(a) === norm(b); +} + +export type Outcome = 'ok' | 'wrong' | 'none'; + +/** + * Scores one reading against a case. A refusal case is passed by refusing; + * everything else is passed by matching the canonical reading or one of the + * readings the case declares equally correct. + */ +export function score(testCase: BenchCase, got: Intent | null): Outcome { + if (testCase.want === null) return got === null ? 'ok' : 'wrong'; + if (got === null) return 'none'; + if (sameIntent(got, testCase.want)) return 'ok'; + return (testCase.alsoOk ?? []).some((alt) => sameIntent(got, alt)) ? 'ok' : 'wrong'; +} diff --git a/src/features/ai/llm/examples.test.ts b/src/features/ai/llm/examples.test.ts new file mode 100644 index 0000000..c019d73 --- /dev/null +++ b/src/features/ai/llm/examples.test.ts @@ -0,0 +1,130 @@ +import { describe, it, expect } from 'vitest'; +import { buildExamples, pickExampleColumns } from '@/features/ai/llm/examples'; +import { validateIntent } from '@/features/ai/llm/prompt'; +import { buildGrammar, isComplete } from '@/features/ai/llm/grammar'; +import { BENCH_COLUMNS } from '@/features/ai/llm/corpus'; +import type { ColumnInfo } from '@/features/ai/chat/parser'; + +const SALES: ColumnInfo[] = [ + { name: 'montant', isNumeric: true, values: [], distinct: 40 }, + { name: 'quantite', isNumeric: true, values: [], distinct: 9 }, + { name: 'boutique', isNumeric: false, values: ['Nord', 'Sud'] }, + { name: 'paiement', isNumeric: false, values: ['carte', 'especes'] }, + { name: 'commentaire', isNumeric: false, values: [] }, +]; + +/** The JSON on the right-hand side of each `Q: … -> …` line. */ +function payloads(columns: ColumnInfo[]): unknown[] { + return buildExamples(columns).map((line) => JSON.parse(line.slice(line.indexOf('-> ') + 3))); +} + +describe('exemples tirés des colonnes du fichier', () => { + it('choisit deux nombres et deux catégories utilisables', () => { + const picked = pickExampleColumns(SALES); + expect(picked.numeric?.name).toBe('montant'); + expect(picked.otherNumeric?.name).toBe('quantite'); + expect(picked.categorical?.name).toBe('boutique'); + expect(picked.otherCategorical?.name).toBe('paiement'); + }); + + it('ignore une colonne de texte sans valeurs connues', () => { + // `commentaire` has no listed values, so it can neither be filtered on in + // an example nor be the group of a top-k. + expect(pickExampleColumns(SALES).categorical?.name).not.toBe('commentaire'); + }); + + it("n'écrit que des colonnes qui existent, dans chaque exemple", () => { + const known = new Set(SALES.map((c) => c.name)); + for (const line of buildExamples(SALES)) { + for (const [, name] of line.matchAll(/"(?:column|groupBy|a|b)":"([^"]+)"/g)) { + expect(known.has(name), line).toBe(true); + } + } + // The Titanic names V27 hard-coded must not survive anywhere. + const text = buildExamples(SALES).join('\n'); + for (const ghost of ['fare', 'embark_town', 'pclass', 'sex']) { + expect(text).not.toContain(`"${ghost}"`); + } + }); + + it('préfère une grandeur à un drapeau 0/1', () => { + // Measured on the corpus: taking the first numeric column gave `survived` + // on Titanic, and « average survived » in an example is a column the model + // then reached for on questions that never mention it. + const titanicish: ColumnInfo[] = [ + { name: 'survived', isNumeric: true, values: [], distinct: 2 }, + { name: 'age', isNumeric: true, values: [], distinct: 13 }, + { name: 'sex', isNumeric: false, values: ['male', 'female'], distinct: 2 }, + ]; + expect(pickExampleColumns(titanicish).numeric?.name).toBe('age'); + // With nothing but flags, the examples still get built rather than vanish. + const flagsOnly: ColumnInfo[] = [{ name: 'ok', isNumeric: true, values: [], distinct: 2 }]; + expect(pickExampleColumns(flagsOnly).numeric?.name).toBe('ok'); + // A column counted by nobody keeps the benefit of the doubt. + const uncounted: ColumnInfo[] = [{ name: 'x', isNumeric: true, values: [] }]; + expect(pickExampleColumns(uncounted).numeric?.name).toBe('x'); + }); + + it('produit des requêtes que le validateur et la grammaire acceptent', () => { + // An example outside the grammar would teach the model to write something + // the constrained decoder then forbids — the worst of both. + const grammar = buildGrammar(SALES); + for (const intent of payloads(SALES)) { + const kind = (intent as { kind: string }).kind; + expect(isComplete(grammar, JSON.stringify(intent)), JSON.stringify(intent)).toBe(true); + if (kind === 'none') continue; + expect(validateIntent(intent, SALES), JSON.stringify(intent)).not.toBeNull(); + } + }); + + it('montre le refus une fois, et pas en dernier', () => { + // Measured: two refusal examples at the END of the list took the model + // from two refusals out of 55 to eleven. A small model imitates the last + // examples hardest, and refusing is not the habit to teach it. + const kinds = payloads(SALES).map((intent) => (intent as { kind: string }).kind); + expect(kinds.filter((kind) => kind === 'none').length).toBe(1); + expect(kinds[kinds.length - 1]).not.toBe('none'); + }); + + it("montre l'agrégat avec filtre — la forme dont l'absence a coûté onze réponses", () => { + const withFilter = payloads(SALES).filter( + (intent) => + (intent as { kind: string }).kind === 'aggregate' && + (intent as { filter?: unknown }).filter !== undefined, + ); + expect(withFilter.length).toBeGreaterThanOrEqual(1); + }); + + it('reste utilisable sur une table sans aucune colonne numérique', () => { + const textOnly: ColumnInfo[] = [{ name: 'ville', isNumeric: false, values: ['Paris', 'Lyon'] }]; + const lines = buildExamples(textOnly); + expect(lines.length).toBeGreaterThan(3); + const grammar = buildGrammar(textOnly); + for (const intent of payloads(textOnly)) { + expect(isComplete(grammar, JSON.stringify(intent)), JSON.stringify(intent)).toBe(true); + } + // No numeric column means no example that would need one. + expect(lines.join('\n')).not.toContain('"correlation"'); + }); + + it('reste utilisable sur une table vide de tout', () => { + expect(buildExamples([]).length).toBeGreaterThan(0); + }); + + it('couvre toutes les formes sur une table riche', () => { + const kinds = new Set( + payloads(BENCH_COLUMNS).map((intent) => (intent as { kind: string }).kind), + ); + for (const kind of [ + 'count', + 'shape', + 'aggregate', + 'topk', + 'distribution', + 'correlation', + 'none', + ]) { + expect(kinds.has(kind), kind).toBe(true); + } + }); +}); diff --git a/src/features/ai/llm/examples.ts b/src/features/ai/llm/examples.ts new file mode 100644 index 0000000..2efef14 --- /dev/null +++ b/src/features/ai/llm/examples.ts @@ -0,0 +1,164 @@ +/** + * V30 (C) — examples built from the user's OWN columns, for 0 MB. + * + * V27's prompt ended with nine worked examples written against Titanic: + * `age`, `fare`, `sex`, `embark_town`. Every user got them, whatever their + * file held — and the prompt then had to spend a rule (« the examples below + * describe a DIFFERENT table, never reuse a column name from them ») asking + * the model to ignore what it had just been shown. Measured on the V30 corpus, + * that rule did not always hold: on « which 5 decks have the highest average + * fare? » the model answered with a filter on `deck` rather than a top-k, and + * on several questions it reached for a column the question never mentioned. + * + * Asking a 0.6B model to ignore its most recent, most concrete input is a bad + * bet. So the examples are now written in the user's own vocabulary: the + * columns in them are columns that exist, and copying one is no longer a + * mistake. It costs nothing to download and it deletes a rule. + * + * When a table has no numeric column, or no small categorical one, the + * examples that would need one are simply left out. A made-up column in an + * example is exactly the failure this module exists to remove. + */ +import type { ColumnInfo } from '@/features/ai/chat/parser'; + +/** A category set worth quoting: small enough to list, big enough to filter on. */ +const MAX_VALUES = 12; +/** + * Above this many distinct values a numeric column is a QUANTITY — something + * an average is about. Below it, it is a flag or a code: on Titanic, taking the + * first numeric column gave `survived`, so the examples read « average + * survived » and « survived moyen par sex », and the model duly answered other + * questions with that column. Measured on the corpus before the fix: three of + * the nine remaining wrong answers named `survived` for no reason in the + * question. + */ +const MEASURE_DISTINCT = 6; + +export interface ExampleColumns { + numeric: ColumnInfo | null; + otherNumeric: ColumnInfo | null; + categorical: ColumnInfo | null; + otherCategorical: ColumnInfo | null; +} + +export function pickExampleColumns(columns: ColumnInfo[]): ExampleColumns { + const allNumbers = columns.filter((column) => column.isNumeric); + // A column whose distinct count was never taken is given the benefit of the + // doubt: the field is an improvement to the examples, never a gate. + const measures = allNumbers.filter( + (column) => column.distinct === undefined || column.distinct > MEASURE_DISTINCT, + ); + const numbers = measures.length > 0 ? measures : allNumbers; + const categories = columns.filter( + (column) => + !column.isNumeric && column.values.length >= 2 && column.values.length <= MAX_VALUES, + ); + return { + numeric: numbers[0] ?? null, + otherNumeric: numbers[1] ?? null, + categorical: categories[0] ?? null, + otherCategorical: categories[1] ?? null, + }; +} + +const json = (value: unknown) => JSON.stringify(value); + +/** + * The worked examples, in the user's own column names. Each line is + * `Q: -> `; the JSON is built rather than written out, so an + * example can never drift out of the grammar the answer is validated against. + */ +export function buildExamples(columns: ColumnInfo[]): string[] { + const { numeric, otherNumeric, categorical, otherCategorical } = pickExampleColumns(columns); + const lines: string[] = []; + const add = (question: string, intent: unknown) => + lines.push(`Q: ${question} -> ${json(intent)}`); + + // Order and coverage are both measured, not chosen by taste. The first + // version of this list left out the aggregate-WITH-FILTER shape, which V27's + // hand-written examples had, and the bench fell from 45 to 34 correct out of + // 55: without an example of « an average over a subset », the model reached + // for a count with an invented threshold — `{"column":"fare","op":"=", + // "value":1000000000}` on « prix moyen payé par les survivants ». Under + // constrained decoding it CANNOT write malformed JSON, so a shape it has not + // been shown comes out as a confident wrong answer instead of a refusal. + if (numeric) { + const name = numeric.name; + add(`average ${name}`, { kind: 'aggregate', op: 'mean', column: name }); + if (categorical) { + add(`average ${name} for ${categorical.values[0]}`, { + kind: 'aggregate', + op: 'mean', + column: name, + filter: { column: categorical.name, op: '=', value: categorical.values[0] }, + }); + add(`${name} moyen par ${categorical.name}`, { + kind: 'aggregate', + op: 'mean', + column: name, + groupBy: categorical.name, + }); + // The shape V27.2 had to add a rule for: a question comparing two groups + // is a grouped aggregate, and the example now says so in the user's own + // column names rather than in Titanic's. + add(`est-ce que ${name} change selon ${categorical.name} ?`, { + kind: 'aggregate', + op: 'mean', + column: name, + groupBy: categorical.name, + }); + } + } + + add('combien de lignes ?', { kind: 'count' }); + if (categorical) { + add(`how many rows where ${categorical.name} is ${categorical.values[0]}?`, { + kind: 'count', + filter: { column: categorical.name, op: '=', value: categorical.values[0] }, + }); + } + if (numeric) { + add(`combien de lignes où ${numeric.name} est inférieur à 10 ?`, { + kind: 'count', + filter: { column: numeric.name, op: '<', value: 10 }, + }); + } + + // One refusal example, in the middle. Two of them, at the end, was measured + // at eleven refusals out of 55 against two — the last examples are the ones + // a small model imitates hardest, and refusing is not what it should imitate. + add('quelle est la capitale de la France ?', { kind: 'none' }); + + if (categorical) { + add(`répartition de ${categorical.name}`, { + kind: 'distribution', + column: categorical.name, + }); + } + if (numeric && otherNumeric) { + add(`lien entre ${numeric.name} et ${otherNumeric.name}`, { + kind: 'correlation', + a: numeric.name, + b: otherNumeric.name, + }); + } + if (numeric && categorical) { + add(`top 3 ${categorical.name} par ${numeric.name} moyen`, { + kind: 'topk', + groupBy: categorical.name, + k: 3, + op: 'mean', + column: numeric.name, + }); + } + if (categorical && otherCategorical) { + add(`les 3 ${otherCategorical.name} les plus fréquents`, { + kind: 'topk', + groupBy: otherCategorical.name, + k: 3, + op: 'count', + }); + } + add('how many rows and columns?', { kind: 'shape' }); + return lines; +} diff --git a/src/features/ai/llm/generate.ts b/src/features/ai/llm/generate.ts new file mode 100644 index 0000000..9b14ad7 --- /dev/null +++ b/src/features/ai/llm/generate.ts @@ -0,0 +1,130 @@ +/** + * V30 — the generation core, shared by the browser and by the bench. + * + * V27 built the prompt, called the pipeline and decoded the answer inside + * `loadModel`, which only runs on WebGPU. The bench therefore had to rebuild + * the same three steps for itself, and any drift between the two copies would + * have made every measured number describe something the product does not do. + * The steps live here now, exactly once: the browser and the bench differ only + * in how the weights are loaded, never in what is asked of them or how the + * answer is read. + */ +import { buildSystemPrompt, buildUserPrompt, intentFromCompletion } from '@/features/ai/llm/prompt'; +import { buildGrammar } from '@/features/ai/llm/grammar'; +import { + buildVocabIndex, + createGrammarProcessor, + readVocab, + type LogitsProcessorLike, + type VocabIndex, +} from '@/features/ai/llm/constrain'; +import type { Intent } from '@/features/ai/chat/engine'; +import type { ColumnInfo } from '@/features/ai/chat/parser'; + +/** Enough for the longest valid query; the JSON we want is far shorter. */ +export const MAX_NEW_TOKENS = 96; + +/** + * The shape of a transformers.js text-generation pipeline, described + * structurally so this module never imports the library — it is loaded on + * demand in the browser and must not be pulled into the main bundle. + */ +export interface RawGenerator { + (input: string, options: Record): Promise<{ generated_text: string }[]>; + tokenizer: { + apply_chat_template(messages: unknown, options: Record): string | unknown; + }; + dispose?: () => Promise; +} + +/** + * V30 — constrained decoding, built once per loaded model. + * + * The library's `LogitsProcessor` classes are passed in rather than imported: + * `@huggingface/transformers` is 355 MB of model away from being wanted, and + * this module must never pull it into the main bundle. + */ +export interface Constrainer { + processorList(columns: ColumnInfo[]): unknown; + /** How many vocabulary entries carry usable bytes — reported, not assumed. */ + usableTokens: number; +} + +export interface LogitsModule { + LogitsProcessor: new () => LogitsProcessorLike; + LogitsProcessorList: new () => { push(processor: LogitsProcessorLike): void }; +} + +/** + * Returns null when the tokenizer does not expose its byte-level vocabulary. + * Constrained decoding is then OFF and says so, rather than being silently + * skipped — an unconstrained answer badged as constrained would be the exact + * dishonesty the rest of this codebase refuses. + */ +export function createConstrainer(module: LogitsModule, tokenizer: unknown): Constrainer | null { + const read = readVocab(tokenizer); + if (!read) return null; + const index: VocabIndex = buildVocabIndex(read.vocab, read.eosIds); + if (index.usable === 0) return null; + return { + usableTokens: index.usable, + processorList(columns: ColumnInfo[]) { + // Built per call: for a dozen columns this is well under a millisecond, + // and a cache keyed on anything less than the columns' full content is a + // staleness bug waiting for the first dataset that changes shape. + const list = new module.LogitsProcessorList(); + list.push(createGrammarProcessor(module.LogitsProcessor, buildGrammar(columns), index)); + return list; + }, + }; +} + +export interface InterpretResult { + /** Null when the model's answer failed the grammar check — a refusal. */ + intent: Intent | null; + /** What the model actually produced, kept for the honest "why" panel. */ + raw: string; + ms: number; +} + +/** + * Qwen3 is a reasoning model: left to itself it opens a block and + * spends the whole token budget arguing with itself before answering. The chat + * template turns that off when `enable_thinking` is explicitly false — so we + * apply the template ourselves instead of handing the pipeline a message list + * and hoping. Without this the model NEVER emits the JSON and every question + * silently falls back to the parser. + */ +export function buildPrompt( + generator: RawGenerator, + question: string, + columns: ColumnInfo[], +): string { + return generator.tokenizer.apply_chat_template( + [ + { role: 'system', content: buildSystemPrompt(columns) }, + { role: 'user', content: buildUserPrompt(question) }, + ], + // `enable_thinking` is not in the library's option type, but every unknown + // key is spread into the Jinja template — which is where Qwen3 reads it. + { tokenize: false, add_generation_prompt: true, enable_thinking: false }, + ) as string; +} + +/** One question, one query — or a refusal. Greedy: same question, same answer. */ +export async function generateIntent( + generator: RawGenerator, + question: string, + columns: ColumnInfo[], + options: { constrain?: Constrainer | null } = {}, +): Promise { + const started = performance.now(); + const output = await generator(buildPrompt(generator, question, columns), { + max_new_tokens: MAX_NEW_TOKENS, + do_sample: false, + return_full_text: false, + ...(options.constrain ? { logits_processor: options.constrain.processorList(columns) } : {}), + }); + const raw = output[0]?.generated_text ?? ''; + return { intent: intentFromCompletion(raw, columns), raw, ms: performance.now() - started }; +} diff --git a/src/features/ai/llm/grammar.test.ts b/src/features/ai/llm/grammar.test.ts new file mode 100644 index 0000000..6a9a781 --- /dev/null +++ b/src/features/ai/llm/grammar.test.ts @@ -0,0 +1,223 @@ +import { describe, it, expect } from 'vitest'; +import { + buildGrammar, + isComplete, + isLegalPrefix, + startState, + step, + MAX_FILTER_COLUMNS, +} from '@/features/ai/llm/grammar'; +import { validateIntent } from '@/features/ai/llm/prompt'; +import { BENCH_COLUMNS } from '@/features/ai/llm/corpus'; +import type { ColumnInfo } from '@/features/ai/chat/parser'; + +const grammar = buildGrammar(BENCH_COLUMNS); + +/** Every character the automaton would allow next, over the printable ASCII set. */ +function allowedNext(text: string): string[] { + let state = startState(grammar); + for (const char of text) state = step(grammar, state, char); + const allowed: string[] = []; + for (let code = 32; code < 127; code++) { + const char = String.fromCharCode(code); + if (step(grammar, state, char).size > 0) allowed.push(char); + } + return allowed; +} + +describe('grammaire — ce qui est écrivable', () => { + it('accepte les sept formes de la grammaire', () => { + for (const text of [ + '{"kind":"shape"}', + '{"kind":"missing"}', + '{"kind":"none"}', + '{"kind":"count"}', + '{"kind":"distribution","column":"class"}', + '{"kind":"correlation","a":"age","b":"fare"}', + '{"kind":"aggregate","op":"mean","column":"age"}', + '{"kind":"aggregate","op":"mean","column":"fare","groupBy":"sex"}', + '{"kind":"topk","groupBy":"deck","k":3,"op":"count"}', + '{"kind":"topk","groupBy":"deck","k":5,"op":"mean","column":"fare"}', + ]) { + expect(isComplete(grammar, text), text).toBe(true); + } + }); + + it('accepte les filtres, typés par leur propre colonne', () => { + for (const text of [ + '{"kind":"count","filter":{"column":"sex","op":"=","value":"female"}}', + '{"kind":"count","filter":{"column":"age","op":">","value":60}}', + '{"kind":"count","filter":{"column":"age","op":"<","value":10}}', + '{"kind":"count","filter":{"column":"fare","op":">=","value":-12.5}}', + '{"kind":"aggregate","op":"mean","column":"age","filter":{"column":"sex","op":"=","value":"male"}}', + '{"kind":"aggregate","op":"mean","column":"fare","groupBy":"class","filter":{"column":"alive","op":"!=","value":"no"}}', + ]) { + expect(isComplete(grammar, text), text).toBe(true); + } + }); + + it("refuse une valeur que la colonne n'a pas", () => { + // `sex` holds male and female. « femme » is not one of them, and under the + // constraint it is not merely refused after the fact: it cannot be typed. + expect( + isLegalPrefix(grammar, '{"kind":"count","filter":{"column":"sex","op":"=","value":"f'), + ).toBe(true); + expect( + isLegalPrefix(grammar, '{"kind":"count","filter":{"column":"sex","op":"=","value":"fe'), + ).toBe(true); + expect( + isLegalPrefix(grammar, '{"kind":"count","filter":{"column":"sex","op":"=","value":"femm'), + ).toBe(false); + }); + + it('refuse une colonne inventée dès la première lettre qui diverge', () => { + expect(isLegalPrefix(grammar, '{"kind":"distribution","column":"cl')).toBe(true); + expect(isLegalPrefix(grammar, '{"kind":"distribution","column":"cli')).toBe(false); + }); + + it('refuse un seuil ordonné sur une colonne de texte', () => { + // `>` between two strings is not a comparison — the same rule V27.3 had to + // enforce after the fact, enforced here before a character is written. + expect(isLegalPrefix(grammar, '{"kind":"count","filter":{"column":"sex","op":">')).toBe(false); + expect(isLegalPrefix(grammar, '{"kind":"count","filter":{"column":"age","op":">')).toBe(true); + }); + + it('refuse un nombre incomplet et un k hors bornes', () => { + expect( + isComplete(grammar, '{"kind":"count","filter":{"column":"age","op":">","value":-}}'), + ).toBe(false); + expect( + isComplete(grammar, '{"kind":"count","filter":{"column":"age","op":">","value":1.}}'), + ).toBe(false); + expect(isComplete(grammar, '{"kind":"topk","groupBy":"deck","k":50,"op":"count"}')).toBe(true); + expect(isComplete(grammar, '{"kind":"topk","groupBy":"deck","k":51,"op":"count"}')).toBe(false); + expect(isComplete(grammar, '{"kind":"topk","groupBy":"deck","k":0,"op":"count"}')).toBe(false); + }); + + it('interdit tout ce qui suit une requête complète', () => { + expect(isLegalPrefix(grammar, '{"kind":"shape"} ')).toBe(false); + expect(isLegalPrefix(grammar, '```json')).toBe(false); + expect(isLegalPrefix(grammar, ' {')).toBe(false); + expect(isLegalPrefix(grammar, '{"kind":"shape"}{')).toBe(false); + }); + + it('ne laisse que sept suites après {"kind":"', () => { + // The claim V30 is built on, checked rather than asserted: once the shape + // key is open, the model chooses between the kinds and nothing else. + const next = allowedNext('{"kind":"'); + expect(next.sort()).toEqual(['a', 'c', 'd', 'm', 'n', 's', 't']); + }); + + it("n'accepte que { comme tout premier caractère", () => { + expect(allowedNext('')).toEqual(['{']); + }); +}); + +describe('grammaire — accord avec le validateur', () => { + /** + * One complete string per alternative, built by walking its atoms and taking + * the option at `pick` (clamped). Exhaustive over the alternatives, and + * linear — a depth-first search over characters was exact too, but it took + * five seconds and timed out under load, which makes it a worse test than a + * slightly narrower one that always runs. + */ + function sample(pick: number): string[] { + const decoder = new TextDecoder(); + return grammar.alternatives.map((atoms) => { + // The option index advances at every choice, so `correlation` gets two + // DIFFERENT columns rather than the same one twice. + let choice = pick; + return atoms + .map((atom) => { + if (atom.kind === 'lit') return decoder.decode(atom.bytes); + if (atom.kind === 'oneOf') { + const at = choice++ % atom.options.length; + return decoder.decode(atom.options[at]); + } + return atom.kind === 'number' ? '-12.5' : '"libre"'; + }) + .join(''); + }); + } + + it('écrit une requête complète pour chaque alternative', () => { + for (const pick of [0, 1, 99]) { + for (const text of sample(pick)) { + expect(isComplete(grammar, text), text).toBe(true); + } + } + }); + + it('accepte une corrélation dégénérée que le validateur refuse', () => { + // The one over-approximation the module's header declares: writing `a` and + // `b` as the SAME column would need one alternative per pair, and the + // validator already refuses it. Pinned here so the gap stays a decision + // rather than a surprise. + expect(isComplete(grammar, '{"kind":"correlation","a":"age","b":"age"}')).toBe(true); + expect(validateIntent({ kind: 'correlation', a: 'age', b: 'age' }, BENCH_COLUMNS)).toBeNull(); + }); + + it('toute requête complète passe validateIntent', () => { + // The automaton is a filter, not the validator. This proves the two agree + // on everything the automaton can write, apart from the one gap above. + for (const pick of [0, 1, 99]) { + for (const text of sample(pick)) { + const parsed: unknown = JSON.parse(text); + if ((parsed as { kind: string }).kind === 'none') continue; // not an Intent + expect(validateIntent(parsed, BENCH_COLUMNS), text).not.toBeNull(); + } + } + }); +}); + +describe('grammaire — tables réelles', () => { + it("plafonne les colonnes filtrables et n'explose pas", () => { + const wide: ColumnInfo[] = Array.from({ length: MAX_FILTER_COLUMNS + 20 }, (_, i) => ({ + name: `c${i}`, + isNumeric: true, + values: [], + })); + const big = buildGrammar(wide); + // Past the cap a column is still aggregable and groupable... + expect(isComplete(big, '{"kind":"aggregate","op":"mean","column":"c80"}')).toBe(true); + // ...but no longer filterable, which is the announced trade. + expect(isComplete(big, '{"kind":"count","filter":{"column":"c80","op":">","value":1}}')).toBe( + false, + ); + expect(isComplete(big, '{"kind":"count","filter":{"column":"c1","op":">","value":1}}')).toBe( + true, + ); + }); + + it('accepte une valeur accentuée, octet par octet', () => { + // The reason the automaton walks bytes: « Côte d'Ivoire » is not a + // sequence of whole-character tokens in Qwen's byte-level vocabulary, and + // a character-level automaton would have made it unwritable. + const accented = buildGrammar([ + { name: 'région', isNumeric: false, values: ["Côte d'Ivoire", 'Île-de-France'] }, + { name: 'n', isNumeric: true, values: [] }, + ]); + expect( + isComplete( + accented, + '{"kind":"count","filter":{"column":"région","op":"=","value":"Île-de-France"}}', + ), + ).toBe(true); + expect( + isComplete( + accented, + '{"kind":"count","filter":{"column":"région","op":"=","value":"Ile-de-France"}}', + ), + ).toBe(false); + }); + + it('accepte une valeur libre sur une colonne de texte sans valeurs connues', () => { + const free = buildGrammar([ + { name: 'city', isNumeric: false, values: [] }, + { name: 'n', isNumeric: true, values: [] }, + ]); + expect( + isComplete(free, '{"kind":"count","filter":{"column":"city","op":"=","value":"Paris"}}'), + ).toBe(true); + }); +}); diff --git a/src/features/ai/llm/grammar.ts b/src/features/ai/llm/grammar.ts new file mode 100644 index 0000000..0ed44f6 --- /dev/null +++ b/src/features/ai/llm/grammar.ts @@ -0,0 +1,424 @@ +/** + * V30 — the query grammar as an automaton, so an invalid query cannot be + * WRITTEN rather than being caught after the fact. + * + * V27 let the model produce whatever it liked and then checked the result + * (`validateIntent`). That catches every malformed answer, but catching is not + * the same as preventing: a refused answer is a question the app cannot + * answer, and the measured V27 failures were overwhelmingly SHAPE errors — + * the model had found the right column and then wrapped it in the wrong + * envelope, or in three paragraphs of markdown. + * + * This module describes the set of legal query strings as a small + * non-deterministic automaton over CHARACTERS. Given the text generated so + * far, it answers two questions: which characters may come next, and is what + * we have a complete query. A logits processor (`constrain.ts`) turns those + * two answers into a mask over the model's vocabulary, so the only tokens the + * model can pick are ones that keep the answer inside the grammar. + * + * Three deliberate limits, stated rather than hidden: + * + * 1. **It is a filter, not the validator.** The automaton over-approximates in + * two places where exactness would multiply its size for no gain + * (`correlation` with a === b, and a k outside its bounds is impossible but + * a nonsense combination of op and column is not). Everything it produces + * still goes through `validateIntent`, which stays the authority. + * 2. **Refusing must stay possible.** Constraining the output makes a refusal + * HARDER, not easier: a model that can only emit valid queries will emit a + * valid query for « what is the capital of France? ». So the grammar has a + * `{"kind":"none"}` shape whose only meaning is « I cannot express this », + * and it maps to the same refusal V27 already had. + * A note on units: the automaton steps over UTF-8 BYTES, not characters. + * Qwen's vocabulary is byte-level BPE, so 1 457 of its 151 669 tokens are + * fragments of a multi-byte character rather than a character. Stepping over + * characters would have meant masking those tokens out, and with them any + * category value the tokenizer happens to split mid-character — « Côte + * d'Ivoire » in a French file, for instance. Bytes cost about thirty lines and + * remove the whole class of problem. + * + * 3. **A filter value is checked against its own column.** The prompt asked + * for this in prose (rule 2) and nothing enforced it. Here, choosing + * `"column":"sex"` restricts what may follow `"value":` to the values `sex` + * actually has. A column with no known values (free text) still accepts an + * arbitrary string — constraining what we do not know would remove a + * capability rather than add correctness. + */ +import type { ColumnInfo } from '@/features/ai/chat/parser'; + +/** Aggregation operators, in the order the prompt lists them. */ +export const GRAMMAR_OPS = ['count', 'mean', 'median', 'min', 'max', 'sum', 'std'] as const; +export const GRAMMAR_FILTER_OPS = ['>', '>=', '<', '<=', '=', '!='] as const; +/** `validateIntent` refuses a k outside 1–50; the grammar cannot write one. */ +export const MAX_K = 50; +/** A free-text filter value never needs to be longer than this. */ +const MAX_FREE_STRING = 64; +/** + * Only the first columns of a wide table get their own filter shape. Each one + * adds five alternatives to the automaton (its value list has to be tied to + * its own column, which is the whole point), so an uncapped table would make + * the mask below slower with every column. Past the cap a column can still be + * aggregated, grouped and described — it simply cannot be filtered on, and + * `buildGrammar` reports how many were left out rather than pretending. + */ +export const MAX_FILTER_COLUMNS = 64; + +export type Atom = + | { kind: 'lit'; bytes: Uint8Array } + | { kind: 'oneOf'; options: Uint8Array[] } + | { kind: 'number' } + | { kind: 'freeString' }; + +const encoder = new TextEncoder(); +export function lit(text: string): Atom { + return { kind: 'lit', bytes: encoder.encode(text) }; +} +function oneOf(options: string[]): Atom { + return { kind: 'oneOf', options: options.map((option) => encoder.encode(option)) }; +} + +export interface Grammar { + alternatives: Atom[][]; +} + +// --- position packing ------------------------------------------------- +// A position is (alternative, atom, option, offset). Packed into one number so +// a state is a Set: the mask below walks the whole vocabulary at every +// generated token, and object churn there is the difference between a +// millisecond and a second. +const ATOMS_PER_ALT = 64; +const OPTIONS = 4096; +const OFFSETS = 256; +/** + * Sub-states of a `number` atom, stored in the option field. Only IN_INT and + * IN_FRAC may end the atom, so `-` and `1.` can never be a complete value. + */ +const NUM_START = 4000; +const NUM_AFTER_SIGN = 4001; +const NUM_IN_INT = 4002; +const NUM_AFTER_DOT = 4003; +const NUM_IN_FRAC = 4004; +/** Sub-states of a `freeString` atom. */ +const STR_OPEN = 4010; +const STR_BODY = 4011; + +function pack(alt: number, atom: number, option: number, offset: number): number { + return ((alt * ATOMS_PER_ALT + atom) * OPTIONS + option) * OFFSETS + offset; +} + +/** + * The packing above only holds if every field stays inside its field width. + * `alt` is unbounded (a number carries 53 bits and the alternatives are + * capped by MAX_FILTER_COLUMNS anyway); the other three are checked once, when + * the grammar is built, so a long column name can never silently corrupt a + * position into a different one. + */ +function assertPackable(grammar: Grammar): void { + for (const atoms of grammar.alternatives) { + if (atoms.length >= ATOMS_PER_ALT) throw new Error('grammar-too-long'); + for (const atom of atoms) { + if (atom.kind === 'lit' && atom.bytes.length >= OFFSETS) throw new Error('grammar-atom-long'); + if (atom.kind !== 'oneOf') continue; + if (atom.options.length >= 3000) throw new Error('grammar-too-many-options'); + for (const option of atom.options) { + if (option.length >= OFFSETS) throw new Error('grammar-option-long'); + } + } + } +} +function unpack(position: number): { alt: number; atom: number; option: number; offset: number } { + const offset = position % OFFSETS; + const rest = (position - offset) / OFFSETS; + const option = rest % OPTIONS; + const alt = (rest - option) / OPTIONS; + return { alt: Math.floor(alt / ATOMS_PER_ALT), atom: alt % ATOMS_PER_ALT, option, offset }; +} + +// --- building the grammar --------------------------------------------- + +const json = (value: string) => JSON.stringify(value); + +function filterVariants(columns: ColumnInfo[]): Atom[][] { + const variants: Atom[][] = []; + for (const column of columns) { + // A category whose JSON form does not fit a position's offset field is + // dropped rather than truncated: half a value is a value that does not + // exist. A column left with none of them falls back to a free string. + const listed = column.values.map(json).filter((option) => encoder.encode(option).length < 200); + const value: Atom = column.isNumeric + ? { kind: 'number' } + : listed.length > 0 + ? oneOf(listed) + : { kind: 'freeString' }; + // An ordering comparison only means something between numbers: on a text + // column the grammar offers equality and inequality and nothing else, + // which is the rule `asFilter` already enforces after the fact. + const ops = column.isNumeric ? GRAMMAR_FILTER_OPS : (['=', '!='] as const); + variants.push([ + lit(`{"column":${json(column.name)},"op":`), + oneOf(ops.map(json)), + lit(',"value":'), + value, + lit('}'), + ]); + } + return variants; +} + +export function buildGrammar(columns: ColumnInfo[]): Grammar { + // A name whose JSON form does not fit a position's offset field is left out + // of the grammar entirely rather than truncated into a column that does not + // exist. 200 bytes is far past any real header. + const usable = columns.filter( + (c) => c.name.length > 0 && encoder.encode(json(c.name)).length < 200, + ); + const numeric = usable.filter((c) => c.isNumeric); + // A table with no numeric column can still be counted and described; the + // aggregate target then falls back to every column rather than to none. + const measurable = (numeric.length > 0 ? numeric : usable).map((c) => json(c.name)); + const anyColumn = usable.map((c) => json(c.name)); + const ops = GRAMMAR_OPS.map(json); + const opsWithoutCount = GRAMMAR_OPS.filter((op) => op !== 'count').map(json); + const ks = Array.from({ length: MAX_K }, (_, i) => String(i + 1)); + const filters = filterVariants(usable.slice(0, MAX_FILTER_COLUMNS)); + + const alternatives: Atom[][] = [ + [lit('{"kind":"shape"}')], + [lit('{"kind":"missing"}')], + // The escape hatch — see the header. Not an Intent: it maps to a refusal. + [lit('{"kind":"none"}')], + [lit('{"kind":"distribution","column":'), oneOf(anyColumn), lit('}')], + [ + lit('{"kind":"correlation","a":'), + oneOf(measurable), + lit(',"b":'), + oneOf(measurable), + lit('}'), + ], + ]; + + const tails: (Atom[] | null)[] = [null, ...filters]; + for (const tail of tails) { + const withFilter = (head: Atom[]): Atom[] => + tail === null ? [...head, lit('}')] : [...head, lit(',"filter":'), ...tail, lit('}')]; + + alternatives.push(withFilter([lit('{"kind":"count"')])); + + for (const grouped of [false, true]) { + const head: Atom[] = [ + lit('{"kind":"aggregate","op":'), + oneOf(ops), + lit(',"column":'), + oneOf(measurable), + ]; + if (grouped) { + head.push(lit(',"groupBy":'), oneOf(anyColumn)); + } + alternatives.push(withFilter(head)); + } + + // Counting the rows of each group needs no measured column; every other + // operator needs one, so the two are separate shapes rather than one shape + // with an optional key the validator has to police. + alternatives.push( + withFilter([ + lit('{"kind":"topk","groupBy":'), + oneOf(anyColumn), + lit(',"k":'), + oneOf(ks), + lit(',"op":"count"'), + ]), + ); + alternatives.push( + withFilter([ + lit('{"kind":"topk","groupBy":'), + oneOf(anyColumn), + lit(',"k":'), + oneOf(ks), + lit(',"op":'), + oneOf(opsWithoutCount), + lit(',"column":'), + oneOf(measurable), + ]), + ); + } + const grammar: Grammar = { alternatives }; + assertPackable(grammar); + return grammar; +} + +// --- walking the automaton -------------------------------------------- + +/** The set of positions the automaton may be in. Empty means: dead. */ +export type State = ReadonlySet; + +/** Every position reachable without consuming a character. */ +function expand(grammar: Grammar, alt: number, atom: number, out: Set): void { + const atoms = grammar.alternatives[alt]; + if (atom >= atoms.length) { + // Complete: parked one past the last atom. `accepting` looks for this. + out.add(pack(alt, atom, 0, 0)); + return; + } + const current = atoms[atom]; + switch (current.kind) { + case 'lit': + if (current.bytes.length === 0) expand(grammar, alt, atom + 1, out); + else out.add(pack(alt, atom, 0, 0)); + return; + case 'oneOf': + for (let option = 0; option < current.options.length; option++) { + if (current.options[option].length === 0) expand(grammar, alt, atom + 1, out); + else out.add(pack(alt, atom, option, 0)); + } + return; + case 'number': + out.add(pack(alt, atom, NUM_START, 0)); + return; + case 'freeString': + out.add(pack(alt, atom, STR_OPEN, 0)); + return; + } +} + +export function startState(grammar: Grammar): State { + const out = new Set(); + for (let alt = 0; alt < grammar.alternatives.length; alt++) expand(grammar, alt, 0, out); + return out; +} + +export function accepting(grammar: Grammar, state: State): boolean { + for (const position of state) { + const { alt, atom } = unpack(position); + if (atom >= grammar.alternatives[alt].length) return true; + } + return false; +} + +const BYTE_ZERO = 0x30; +const BYTE_NINE = 0x39; +const BYTE_MINUS = 0x2d; +const BYTE_DOT = 0x2e; +const BYTE_QUOTE = 0x22; +const BYTE_BACKSLASH = 0x5c; +const BYTE_SPACE = 0x20; + +function isDigit(byte: number): boolean { + return byte >= BYTE_ZERO && byte <= BYTE_NINE; +} + +/** A byte allowed inside a JSON string without escaping. */ +function plainStringByte(byte: number): boolean { + return byte !== BYTE_QUOTE && byte !== BYTE_BACKSLASH && byte >= BYTE_SPACE; +} + +function stepPosition(grammar: Grammar, position: number, byte: number, out: Set): void { + const { alt, atom, option, offset } = unpack(position); + const atoms = grammar.alternatives[alt]; + if (atom >= atoms.length) return; // complete: nothing may follow + const current = atoms[atom]; + + const advanceLiteral = (bytes: Uint8Array): void => { + if (bytes[offset] !== byte) return; + if (offset + 1 === bytes.length) expand(grammar, alt, atom + 1, out); + else out.add(pack(alt, atom, option, offset + 1)); + }; + + switch (current.kind) { + case 'lit': + advanceLiteral(current.bytes); + return; + case 'oneOf': + advanceLiteral(current.options[option]); + return; + case 'number': { + // Reaching a sub-state that may end the number also opens whatever comes + // after it, which is what lets `,` or `}` close the value without the + // number atom having to know what follows it. + const enter = (next: number): void => { + out.add(pack(alt, atom, next, 0)); + if (next === NUM_IN_INT || next === NUM_IN_FRAC) expand(grammar, alt, atom + 1, out); + }; + if (option === NUM_START) { + if (byte === BYTE_MINUS) enter(NUM_AFTER_SIGN); + else if (isDigit(byte)) enter(NUM_IN_INT); + return; + } + if (option === NUM_AFTER_SIGN) { + if (isDigit(byte)) enter(NUM_IN_INT); + return; + } + if (option === NUM_IN_INT) { + if (isDigit(byte)) enter(NUM_IN_INT); + else if (byte === BYTE_DOT) enter(NUM_AFTER_DOT); + return; + } + if (option === NUM_AFTER_DOT) { + if (isDigit(byte)) enter(NUM_IN_FRAC); + return; + } + if (option === NUM_IN_FRAC && isDigit(byte)) enter(NUM_IN_FRAC); + return; + } + case 'freeString': { + if (option === STR_OPEN) { + if (byte === BYTE_QUOTE) out.add(pack(alt, atom, STR_BODY, 0)); + return; + } + if (byte === BYTE_QUOTE) { + expand(grammar, alt, atom + 1, out); + return; + } + if (offset < MAX_FREE_STRING && plainStringByte(byte)) { + out.add(pack(alt, atom, STR_BODY, offset + 1)); + } + return; + } + } +} + +/** One byte. Returns the new state; an empty set means the byte is illegal. */ +export function stepByte(grammar: Grammar, state: State, byte: number): State { + const out = new Set(); + for (const position of state) stepPosition(grammar, position, byte, out); + return out; +} + +/** A run of bytes — what the mask uses, since a token is a run of bytes. */ +export function stepBytes(grammar: Grammar, state: State, bytes: Uint8Array): State { + let current = state; + for (const byte of bytes) { + if (current.size === 0) return current; + current = stepByte(grammar, current, byte); + } + return current; +} + +/** One character, for readable call sites; multi-byte characters step twice. */ +export function step(grammar: Grammar, state: State, char: string): State { + return stepBytes(grammar, state, encoder.encode(char)); +} + +/** Every byte the automaton would accept next — the mask's first cut. */ +export function allowedBytes(grammar: Grammar, state: State): Set { + const allowed = new Set(); + for (let byte = 0; byte < 256; byte++) { + if (stepByte(grammar, state, byte).size > 0) allowed.add(byte); + } + return allowed; +} + +/** A whole string. */ +export function advance(grammar: Grammar, state: State, text: string): State { + return stepBytes(grammar, state, encoder.encode(text)); +} + +/** True when `text` is a complete, legal query for this grammar. */ +export function isComplete(grammar: Grammar, text: string): boolean { + const state = advance(grammar, startState(grammar), text); + return state.size > 0 && accepting(grammar, state); +} + +/** True when `text` could still become one. */ +export function isLegalPrefix(grammar: Grammar, text: string): boolean { + return advance(grammar, startState(grammar), text).size > 0; +} diff --git a/src/features/ai/llm/interpret.ts b/src/features/ai/llm/interpret.ts index b818ad6..7351b80 100644 --- a/src/features/ai/llm/interpret.ts +++ b/src/features/ai/llm/interpret.ts @@ -8,19 +8,25 @@ * and an answer that fails the grammar check. Each one falls back to the * deterministic parser, which stays the default engine. */ -import { buildSystemPrompt, buildUserPrompt, intentFromCompletion } from '@/features/ai/llm/prompt'; +import { + createConstrainer, + generateIntent, + type Constrainer, + type InterpretResult, + type LogitsModule, + type RawGenerator, +} from '@/features/ai/llm/generate'; import { createShardCache, type DownloadProgress, type LlmManifest, } from '@/features/ai/llm/shards'; -import type { Intent } from '@/features/ai/chat/engine'; import type { ColumnInfo } from '@/features/ai/chat/parser'; /** Where the build script (scripts/prepare-llm.mjs) puts the sharded model. */ export const LLM_BASE = '/llm/'; -/** Enough for the longest valid query; the JSON we want is far shorter. */ -const MAX_NEW_TOKENS = 96; + +export type { InterpretResult }; export interface LlmCapability { /** Measured, never assumed: WebGPU decides whether this is usable at all. */ @@ -61,27 +67,29 @@ export async function probeCapability(baseUrl = LLM_BASE): Promise; + /** + * V30 — whether the answer is being decoded inside the query grammar. False + * means the tokenizer did not expose its vocabulary, and the UI says so: + * the model still answers, its answers are still validated, but the guard + * that makes a malformed one impossible is not running. + */ + constrained: boolean; dispose(): Promise; } -export interface InterpretResult { - /** Null when the model's answer failed the grammar check — a refusal. */ - intent: Intent | null; - /** What the model actually produced, kept for the honest "why" panel. */ - raw: string; - ms: number; -} - export async function loadModel( manifest: LlmManifest, options: { baseUrl?: string; onProgress?: (progress: DownloadProgress) => void; signal?: AbortSignal; + /** V30 — off only to reproduce the pre-V30 behaviour on the bench. */ + constrained?: boolean; } = {}, ): Promise { const baseUrl = options.baseUrl ?? LLM_BASE; - const { env, pipeline } = await import('@huggingface/transformers'); + const library = await import('@huggingface/transformers'); + const { env, pipeline } = library; // Everything self-hosted: the strict CSP forbids the library's CDN default. env.allowRemoteModels = false; env.allowLocalModels = true; @@ -101,38 +109,15 @@ export async function loadModel( device: 'webgpu', }); + const constrain: Constrainer | null = + options.constrained === false + ? null + : createConstrainer(library as unknown as LogitsModule, generator.tokenizer); + return { - async generate(question, columns) { - const started = performance.now(); - // Qwen3 is a reasoning model: left to itself it opens a block and - // spends the whole token budget arguing with itself before answering. - // The chat template turns that off when `enable_thinking` is explicitly - // false — so we apply the template ourselves instead of handing the - // pipeline a message list and hoping. Without this the model NEVER emits - // the JSON and every question silently falls back to the parser. - const prompt = generator.tokenizer.apply_chat_template( - [ - { role: 'system', content: buildSystemPrompt(columns) }, - { role: 'user', content: buildUserPrompt(question) }, - ], - // `enable_thinking` is not in the library's option type, but every - // unknown key is spread into the Jinja template — which is exactly - // where Qwen3 reads it. - { - tokenize: false, - add_generation_prompt: true, - enable_thinking: false, - } as unknown as { tokenize: false; add_generation_prompt: boolean }, - ) as string; - const output = (await generator(prompt, { - // Greedy: the same question must give the same query, every time. - max_new_tokens: MAX_NEW_TOKENS, - do_sample: false, - return_full_text: false, - })) as { generated_text: string }[]; - const raw = output[0]?.generated_text ?? ''; - return { intent: intentFromCompletion(raw, columns), raw, ms: performance.now() - started }; - }, + constrained: constrain !== null, + generate: (question, columns) => + generateIntent(generator as unknown as RawGenerator, question, columns, { constrain }), async dispose() { await generator.dispose?.(); }, diff --git a/src/features/ai/llm/prompt.test.ts b/src/features/ai/llm/prompt.test.ts index a3f69ae..a231c94 100644 --- a/src/features/ai/llm/prompt.test.ts +++ b/src/features/ai/llm/prompt.test.ts @@ -38,7 +38,25 @@ describe('buildSystemPrompt', () => { it('ties a filter value to the column whose values list contains it', () => { const prompt = buildSystemPrompt(COLUMNS); expect(prompt).toContain('must be one of the values listed for THAT column'); - expect(prompt).toContain('DIFFERENT table'); + }); + + // V30 — the rule this replaces read « the examples below describe a + // DIFFERENT table, never reuse a column name from them ». The examples are + // built from the caller's own columns now, so there is no foreign name left + // to warn about: every column named anywhere in the prompt exists. + it("n'écrit jamais dans les exemples une colonne qui n'existe pas", () => { + const prompt = buildSystemPrompt(COLUMNS); + const known = new Set(COLUMNS.map((c) => c.name)); + for (const [, name] of prompt.matchAll(/"(?:column|groupBy|a|b)":"([^"]+)"/g)) { + expect(known.has(name), name).toBe(true); + } + expect(prompt).not.toContain('DIFFERENT table'); + }); + + it('montre au modèle comment refuser', () => { + const prompt = buildSystemPrompt(COLUMNS); + expect(prompt).toContain('{"kind":"none"}'); + expect(prompt).toContain('better than a query that answers a different question'); }); // V27.2 — measured on real hardware: « est-ce que les femmes payaient plus diff --git a/src/features/ai/llm/prompt.ts b/src/features/ai/llm/prompt.ts index a8d4bc1..c9630be 100644 --- a/src/features/ai/llm/prompt.ts +++ b/src/features/ai/llm/prompt.ts @@ -9,6 +9,7 @@ * is a REFUSAL that falls back to the deterministic parser, never a guess. */ import { parseNumber } from '@/features/ml/data/infer'; +import { buildExamples } from '@/features/ai/llm/examples'; import type { AggOp, Filter, FilterOp, Intent } from '@/features/ai/chat/engine'; import type { ColumnInfo } from '@/features/ai/chat/parser'; @@ -45,6 +46,7 @@ Reply with ONE JSON object and nothing else. Allowed shapes: {"kind":"correlation","a":COL,"b":COL} {"kind":"shape"} {"kind":"missing"} +{"kind":"none"} OP is one of: ${AGG_OPS.join(', ')}. FILTER is {"column":COL,"op":CMP,"value":V} where CMP is one of: ${FILTER_OPS.join(', ')}. @@ -56,18 +58,11 @@ Rules: 3. Comparing two groups is not a count: use aggregate with groupBy set to the column that defines the groups. 4. An age or price threshold is a filter with < or >, on the numeric column it refers to. 5. A question that compares two groups (women vs men, one class against another) is an aggregate with groupBy on the column whose values name those groups — NEVER a correlation. Correlation relates two number columns and nothing else; never pick a column the question does not mention. -6. The examples below describe a DIFFERENT table. Never reuse a column name from them unless that exact name is in the list above. +6. A filter VALUE must come from the question itself: a number the question states, or a category listed above. If the question names neither, write no filter at all — never invent a threshold. +7. If the question is not something this table can answer — a prediction, a drawing, a fact about the world — reply {"kind":"none"}. That is a real answer, and it is better than a query that answers a different question. -Examples: -Q: average age of women -> {"kind":"aggregate","op":"mean","column":"age","filter":{"column":"sex","op":"=","value":"female"}} -Q: a quel age moyen voyageaient les passagers ? -> {"kind":"aggregate","op":"mean","column":"age"} -Q: did women pay more than men? -> {"kind":"aggregate","op":"mean","column":"fare","groupBy":"sex"} -Q: les hommes voyageaient-ils plus jeunes que les femmes ? -> {"kind":"aggregate","op":"mean","column":"age","groupBy":"sex"} -Q: combien d'enfants de moins de 10 ans ? -> {"kind":"count","filter":{"column":"age","op":"<","value":10}} -Q: combien de lignes ? -> {"kind":"count"} -Q: top 3 des ports par nombre de passagers -> {"kind":"topk","groupBy":"embark_town","k":3,"op":"count"} -Q: repartition des classes -> {"kind":"distribution","column":"pclass"} -Q: lien entre age et prix -> {"kind":"correlation","a":"age","b":"fare"}`; +Examples (this table, these columns): +${buildExamples(columns).join('\n')}`; } export function buildUserPrompt(question: string): string { @@ -144,6 +139,12 @@ export function validateIntent(parsed: unknown, columns: ColumnInfo[]): Intent | return { kind: 'shape' }; case 'missing': return { kind: 'missing' }; + // V30: the model's own way of saying « this table cannot answer that ». + // It is not an Intent, and null is exactly what the caller does with a + // refusal — but the case is written out so that a reader sees the refusal + // is DESIGNED, not the default branch catching an unknown kind. + case 'none': + return null; case 'count': return withFilter({ kind: 'count' } as Intent); case 'distribution': { diff --git a/src/features/ai/llm/report.ts b/src/features/ai/llm/report.ts new file mode 100644 index 0000000..5f530d4 --- /dev/null +++ b/src/features/ai/llm/report.ts @@ -0,0 +1,100 @@ +/** + * V30 — one shape for a bench result, and one way of printing it. + * + * The browser bench (WebGPU, the shipped runtime) and the Node bench (CPU, the + * one that runs without a GPU) must be comparable line by line, so both fill in + * the same row type and both print through the function below. A difference + * between two runs is then a difference in the model or the code — never in how + * the two harnesses happened to count. + */ +import type { BenchCase, Outcome } from '@/features/ai/llm/corpus'; + +export interface BenchRow { + q: string; + lang: string; + family: BenchCase['family']; + /** The keyword grammar's reading. */ + deterministic: Outcome; + /** The model's reading, asked of every question regardless of order. */ + llm: Outcome; + /** + * What the app actually answers, in the shipped order: the keyword grammar + * when it has a reading, the model only otherwise. This is the number that + * describes the product; the two above describe its parts. + */ + pipeline: Outcome; + raw: string; + ms: number; +} + +export interface BenchReport { + label: string; + total: number; + loadMs: number; + rows: BenchRow[]; +} + +export function tally(rows: readonly BenchRow[], key: 'deterministic' | 'llm' | 'pipeline') { + const counts: Record = { ok: 0, wrong: 0, none: 0 }; + for (const row of rows) counts[row[key]] += 1; + return counts; +} + +function line(name: string, counts: Record, total: number): string { + const pct = ((counts.ok / total) * 100).toFixed(0); + return `${name.padEnd(22)} ${String(counts.ok).padStart(3)}/${total} justes (${pct} %) · ${counts.wrong} faux · ${counts.none} sans réponse`; +} + +/** + * The printed report. Wrong answers are listed in full and refusals are not: + * a refusal is announced to the user as one, while a wrong answer is delivered + * with the same confidence as a right one, and is the only outcome that costs + * trust. + */ +export function formatReport(report: BenchReport): string { + const out: string[] = []; + const { rows, total } = report; + out.push(`\n${report.label} — ${total} questions, modèle chargé en ${report.loadMs} ms`); + out.push(line('déterministe', tally(rows, 'deterministic'), total)); + out.push(line('modèle local', tally(rows, 'llm'), total)); + out.push(line('appli (ordre livré)', tally(rows, 'pipeline'), total)); + + // The split V27 hand-labelled is computed here instead: the questions the + // keyword grammar gives up on ARE the gap the download has to justify. + const gap = rows.filter((row) => row.deterministic === 'none'); + if (gap.length > 0) { + const rescued = gap.filter((row) => row.llm === 'ok').length; + out.push( + `\nlà où le déterministe déclare forfait (${gap.length} questions) : ` + + `modèle juste ${rescued}, faux ${gap.filter((r) => r.llm === 'wrong').length}, ` + + `refus ${gap.filter((r) => r.llm === 'none').length}`, + ); + } + + const misread = rows.filter((row) => row.deterministic === 'wrong'); + if (misread.length > 0) { + out.push( + `\nlues de travers par le déterministe (${misread.length}) — jamais rattrapables, ` + + `il passe en premier :`, + ); + for (const row of misread) out.push(` ${row.q}`); + } + + const wrong = rows.filter((row) => row.pipeline === 'wrong'); + if (wrong.length > 0) { + out.push( + `\nréponses fausses de l'appli (${wrong.length}) — les seules qui coûtent la confiance :`, + ); + for (const row of wrong) + out.push(` ${row.q}\n ${row.raw.replace(/\n/g, ' ').slice(0, 160)}`); + } + + const times = rows.map((row) => row.ms).sort((a, b) => a - b); + if (times.length > 0) { + const median = Math.round(times[Math.floor(times.length / 2)]); + out.push( + `\nlatence modèle : médiane ${median} ms, max ${Math.round(times[times.length - 1])} ms`, + ); + } + return out.join('\n'); +} diff --git a/src/locales/en.json b/src/locales/en.json index 3eafd3c..98bd800 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -978,14 +978,17 @@ "downloadNote": "Downloaded once, then kept by your browser and available offline. Nothing is uploaded: it runs on your machine, on your data — and the deterministic interpreter keeps working whether you download it or not.", "downloading": "Downloading the model… {{percent}}%", "readyTag": "loaded", - "readyNote": "How the two work together: the deterministic interpreter reads your question first — it can only ever name a column that exists and a value that really occurs in it, so when it understands, nothing overrides it. When it does not, the local model takes over: it only READS your question and turns it into a calculation, it never computes anything itself, so no number ever comes from a language model. If neither manages, you are told so — never an invented answer. The line under each answer says who replied.", + "readyNote": "How the two work together: the deterministic interpreter reads your question first — it can only ever name a column that exists and a value that really occurs in it, so when it understands, nothing overrides it. And it only claims to understand once it has read the WHOLE question: if one word escapes it, it refuses rather than answer a shorter question. The local model then takes over: it only READS your question and turns it into a calculation, it never computes anything itself, so no number ever comes from a language model. If neither manages, you are told so — never an invented answer. The line under each answer says who replied.", "failed": "The local model could not be loaded ({{reason}}). The deterministic interpreter is unaffected.", "by": { "deterministic": "answered by the deterministic interpreter", "llm": "question read by the local model · computed by the deterministic engine", "none": "the deterministic interpreter did not understand", "none-both": "neither the deterministic interpreter nor the local model understood" - } + }, + "constrainedTag": "grammar-constrained", + "constrainedNote": "The model writes its query under constraint: at every token, anything outside the query grammar is made impossible to pick. It cannot invent a column, an operator, or a value your column does not hold. It keeps one way of saying no, deliberately: forcing a valid answer would turn « I did not understand » into a wrong number.", + "unconstrainedNote": "Grammar-constrained decoding is unavailable for this model: its answers are still checked before they run, but nothing stops them from being malformed in the first place." }, "thinking": "Computing…", "honesty": { diff --git a/src/locales/fr.json b/src/locales/fr.json index c93ab6b..dbbd2fc 100644 --- a/src/locales/fr.json +++ b/src/locales/fr.json @@ -978,14 +978,17 @@ "downloadNote": "Téléchargé une seule fois, puis conservé par votre navigateur et disponible hors ligne. Rien n'est envoyé : il s'exécute sur votre machine, sur vos données — et l'interpréteur déterministe fonctionne que vous le téléchargiez ou non.", "downloading": "Téléchargement du modèle… {{percent}} %", "readyTag": "chargé", - "readyNote": "Comment les deux travaillent ensemble : l'interpréteur déterministe lit votre question en premier — il ne peut nommer qu'une colonne qui existe et une valeur qui s'y trouve vraiment, donc quand il comprend, personne ne le contredit. Quand il ne comprend pas, le modèle local prend le relais : il se contente de LIRE votre question et de la transformer en calcul, il ne calcule jamais lui-même, donc aucun chiffre ne sort d'un modèle de langage. Si aucun des deux n'y arrive, on vous le dit — jamais de réponse inventée. La mention sous chaque réponse indique qui a répondu.", + "readyNote": "Comment les deux travaillent ensemble : l'interpréteur déterministe lit votre question en premier — il ne peut nommer qu'une colonne qui existe et une valeur qui s'y trouve vraiment, donc quand il comprend, personne ne le contredit. Et il ne dit comprendre que s'il a lu TOUTE la question : si un mot lui échappe, il refuse au lieu de répondre à une question plus courte. Le modèle local prend alors le relais : il se contente de LIRE votre question et de la transformer en calcul, il ne calcule jamais lui-même, donc aucun chiffre ne sort d'un modèle de langage. Si aucun des deux n'y arrive, on vous le dit — jamais de réponse inventée. La mention sous chaque réponse indique qui a répondu.", "failed": "Le modèle local n'a pas pu être chargé ({{reason}}). L'interpréteur déterministe n'est pas affecté.", "by": { "deterministic": "répondu par l'interpréteur déterministe", "llm": "question lue par le modèle local · calculé par le moteur déterministe", "none": "l'interpréteur déterministe n'a pas compris", "none-both": "ni l'interpréteur déterministe, ni le modèle local n'ont compris" - } + }, + "constrainedTag": "sous grammaire", + "constrainedNote": "Le modèle écrit sa requête sous contrainte : à chaque jeton, tout ce qui sortirait de la grammaire de requêtes est rendu impossible à choisir. Il ne peut donc pas inventer une colonne, un opérateur ni une valeur que votre colonne n'a pas. Il lui reste un mot pour dire non, et c'est délibéré : forcer une réponse valable transformerait un « je n'ai pas compris » en un chiffre faux.", + "unconstrainedNote": "Décodage sous grammaire indisponible sur ce modèle : ses réponses sont toujours vérifiées avant d'être exécutées, mais rien ne les empêche d'être malformées." }, "thinking": "Calcul…", "honesty": {