fix: preserve fallback emoji highlights and complete cleanup

.changeset/fallback-emoji-highlights.md

@@ -0,0 +1,5 @@
+---
+"webgpu-search": patch
+---
+
+Preserve whole emoji ZWJ sequences and regional-indicator flag pairs in highlight ranges when Intl.Segmenter is unavailable.

.gitignore

@@ -6,7 +6,7 @@ logs
 benchmark_*.json
 # Phase 5 frozen baseline (Issue #11): the checked-in headless matrix is the
 # exception to the benchmark_*.json ignore above.
-!benchmark_snapshot_matrix.json
+!docs/benchmarks/benchmark_snapshot_matrix.json
 fuzzy-benchmark-results.json
 chart_*.png
 *chart*.png

CONTRIBUTING.md

@@ -110,7 +110,7 @@ on `127.0.0.1:5173` before running `scripts/test-regression.ts`.
    restore, CRC, caps, getters, hooks, and profile guards in
    `bun run test:snapshot`.
 4. For benchmark fixture changes, bump `BENCHMARK_FIXTURE_VERSION`, regenerate
-   `benchmark_snapshot_matrix.json` + `.md`, and update `docs/benchmarks.md` §3.
+   `docs/benchmarks/benchmark_snapshot_matrix.json` + `.md`, and update `docs/benchmarks.md` §3.
 5. Run the relevant gates before opening a PR (at minimum `check:shaders`,
    `typecheck`, `build`, `test:mock`, plus any suite covering your area).
 

apps/benchmark/index.html

@@ -22,13 +22,7 @@
       <section class="result-panel" aria-label="Benchmark setup">
         <h2>1. Choose a run</h2>
         <div class="benchmark-controls">
-          <div class="field"><span class="field-label">Dataset sizes</span><div class="size-options">
-            <label><input type="checkbox" name="size" value="10000" checked /> 10k</label>
-            <label><input type="checkbox" name="size" value="100000" checked /> 100k</label>
-            <label><input type="checkbox" name="size" value="500000" /> 500k</label>
-            <label><input type="checkbox" name="size" value="1000000" /> 1M</label>
-            <label><input type="checkbox" name="size" value="2000000" /> 2M</label>
-          </div></div>
+          <div class="field"><span class="field-label">Dataset sizes</span><div class="size-options"></div></div>
           <div class="field"><label for="benchmark-corpus">Text corpus</label><select id="benchmark-corpus"><option value="ascii">ASCII code paths</option><option value="cjk">CJK</option><option value="emoji">Emoji + mixed script</option></select></div>
           <div class="field"><label for="benchmark-mode">Search mode</label><select id="benchmark-mode"><option value="substring">Substring</option><option value="fuzzy">Fuzzy</option></select></div>
           <div class="field"><label for="benchmark-query">Query</label><input id="benchmark-query" value="AuthController" /></div>

apps/benchmark/src/benchmark.ts

@@ -1,3 +1,4 @@
+import { BENCHMARK_DATASET_SIZES } from '../../../scripts/bench-data/dataset-sizes';
 /**
  * Full Matrix Benchmark Engine & Performance Characterization Runner.
  *
@@ -282,7 +283,7 @@ export class BenchmarkRunner {
    * execution, median + p95 statistics, and real UI thread telemetry.
    */
   async runBenchmark(
-    sizes: number[] = [10_000, 100_000, 500_000, 1_000_000, 2_000_000],
+    sizes: number[] = [...BENCHMARK_DATASET_SIZES],
     query: string = 'AuthController',
     mode: 'substring' | 'fuzzy' = 'substring',
     iterationsOrOptions?: number | BenchmarkOptions,

apps/benchmark/src/main.ts

@@ -1,3 +1,4 @@
+import { BENCHMARK_DATASET_SIZES } from '../../../scripts/bench-data/dataset-sizes';
 import { CPUEngine, WebGPUEngine } from 'webgpu-search';
 import './style.css';
 import { BenchmarkRunner, type BenchmarkRowResult } from './benchmark';
@@ -24,6 +25,18 @@ const exportCsv = document.querySelector<HTMLButtonElement>('#export-csv');
 const exportMarkdown = document.querySelector<HTMLButtonElement>('#export-markdown');
 const exportChart = document.querySelector<HTMLButtonElement>('#export-chart');
 
+document.querySelector('.size-options')?.replaceChildren(...BENCHMARK_DATASET_SIZES.map(size => {
+  const label = document.createElement('label');
+  const input = document.createElement('input');
+  input.type = 'checkbox';
+  input.name = 'size';
+  input.value = String(size);
+  input.checked = size <= 100_000;
+  const text = size >= 1_000_000 ? `${size / 1_000_000}M` : `${size / 1000}k`;
+  label.append(input, ` ${text}`);
+  return label;
+}));
+
 let currentRows: BenchmarkRowResult[] = [];
 let currentMode: 'substring' | 'fuzzy' = 'substring';
 let currentCorpus: CorpusType = 'ascii';

apps/site/index.html

@@ -64,7 +64,7 @@ <h1 id="hero-title">Find the right thing.<br /><span>Keep the UI fast.</span></h
           <div class="playground-top"><div class="playground-title">WebGPU vs CPU vs popular libs</div><span class="preview-label">live</span></div>
           <div class="search-controls">
             <div class="search-input-wrap"><span class="search-icon" aria-hidden="true">⌕</span><input id="qb-query" class="search-input" type="search" value="AuthController" aria-label="Benchmark query" autocomplete="off" /></div>
-            <select id="qb-size" class="control-select" aria-label="Dataset size"><option value="100000">100,000 records</option><option value="200000">200,000 records</option><option value="500000" selected>500,000 records</option><option value="1000000">1,000,000 records</option><option value="2000000">2,000,000 records</option></select>
+            <select id="qb-size" class="control-select" aria-label="Dataset size"></select>
             <button id="qb-run" class="button button-small" type="button">Run benchmark</button>
           </div>
           <div class="result-summary"><span id="qb-status" role="status" aria-live="polite">Ready.</span><span>7 timed searches / engine</span></div>

apps/site/src/corpus.ts

@@ -2,11 +2,10 @@ import './fonts.css';
 import './style.css';
 import { normalizeText, scoreExactMatches, type SearchResultItem } from 'webgpu-search';
 import { corpusRow } from './corpus-gen';
+import { SITE_DATASET_SIZES } from '../../../scripts/bench-data/dataset-sizes';
 
 const PAGE_SIZE = 100;
 const MAX_MATCHES_SHOWN = 1000;
-/** Sizes the homepage benchmark offers (apps/site/index.html #qb-size). */
-const ALLOWED_SIZES = [100000, 200000, 500000, 1000000, 2000000] as const;
 const DEFAULT_SIZE = 100000;
 const BUILD_CHUNK = 50000;
 
@@ -15,8 +14,8 @@ function parseSize(raw: string | null): number {
   if (raw === null || raw.trim() === '') return DEFAULT_SIZE;
   const requested = Number(raw);
   if (!Number.isFinite(requested) || requested <= 0) return DEFAULT_SIZE;
-  let size: number = ALLOWED_SIZES[0];
-  for (const allowed of ALLOWED_SIZES) if (allowed <= requested) size = allowed;
+  let size: number = SITE_DATASET_SIZES[0]!;
+  for (const allowed of SITE_DATASET_SIZES) if (allowed <= requested) size = allowed;
   return size;
 }
 

apps/site/src/main.ts

@@ -1,10 +1,18 @@
+import { SITE_DATASET_SIZES } from '../../../scripts/bench-data/dataset-sizes';
 import './fonts.css';
 import './style.css';
 import { formatMs } from './format';
 
 const copyButton = document.querySelector<HTMLButtonElement>('#copy-install');
 
 const qbSize = document.querySelector<HTMLSelectElement>('#qb-size');
+if (qbSize) {
+  qbSize.replaceChildren(...SITE_DATASET_SIZES.map(size => {
+    const option = new Option(`${size.toLocaleString('en-US')} records`, String(size));
+    option.selected = size === 500_000;
+    return option;
+  }));
+}
 const qbQuery = document.querySelector<HTMLInputElement>('#qb-query');
 const qbRun = document.querySelector<HTMLButtonElement>('#qb-run');
 const qbStatus = document.querySelector<HTMLElement>('#qb-status');

docs/IMPROVEMENT-PLAN.md

@@ -0,0 +1,121 @@
+# Post-1.0 Improvement Plan
+
+> **Created**: 2026-10-11
+> **Baseline**: `webgpu-search@1.0.1` (commit `bee5fda`)
+> **Status**: Proposed
+
+This plan lists the next improvements, most valuable first. It comes from a quick look at the repo layout, tests, open issues and recent commits, not a full audit.
+
+---
+
+## Phase 1: Quick cleanup (≈ half a day)
+
+Low-risk housekeeping that clears the way for the bigger phases.
+
+| # | Task | Done when |
+|---|------|-----------|
+| 1.1 | **Implemented:** Keep one shared dataset-size list in `scripts/bench-data`, consumed by `apps/site` and `apps/benchmark`. | Only one definition exists; issue #38 closes when the cleanup PR merges. |
+| 1.2 | **Done:** Move the benchmark snapshot JSON and Markdown to `docs/benchmarks/` and update the runner, checks, and docs. | Both artifacts live under `docs/benchmarks/`. |
+| 1.3 | **Done:** Delete the two finished implementation plans and remove stale references. | Remaining documentation references resolve. |
+
+---
+
+## Phase 2: Tests for the 1.0 promises (≈ 2–3 days)
+
+There are about 300 tests, but several core modules have none of their own. This phase covers the guarantees that 1.0 commits to.
+
+### 2.1 Tests that GPU and CPU return the same scores
+- Add `test/scoring-parity.test.ts`. It runs a fixed set of queries through the mock WebGPU path and through `cpu-reference.ts` / `exact-scorer.ts`.
+- Check that both return the same `SearchResultItem[]`: same order, same integer scores, scores in descending order.
+- Cover ASCII, CJK, emoji/ZWJ, mixed case, and queries at the maximum length.
+- **Why:** AGENTS.md §2.4 requires the two paths to score the same, and nothing enforces it today.
+
+### 2.2 Snapshot codec tests
+- Add `test/snapshot-codec.test.ts`:
+  - Saving an index and loading it back gives identical search results.
+  - Bad input is rejected with the right typed error: wrong magic bytes, wrong version, truncated data, a corrupted checksum, a mismatched text profile.
+  - A checked-in snapshot from 1.0.x still loads (a "golden" fixture).
+- **Why:** the format in `docs/snapshot-format.md` is a public promise to keep old snapshots loading.
+
+### 2.3 Text pipeline tests
+- **Existing coverage verified:** normalization and case folding pass 98 checks in both default and Turkish locales; the highlighting suite covers NFC/NFD, folding expansions, surrogate pairs, and emoji.
+- **Fallback bug fixed:** without `Intl.Segmenter`, highlights could split joined emoji and flags. Shared fallback boundaries now preserve ZWJ sequences and regional-indicator pairs; regression cases run with segmentation disabled before the normal suite. See `docs/public-api.md` §2.5.
+- Add `test/text-normalization.test.ts`, covering NFC, case folding with `CaseFolding-16.0.0` (ß→ss, the Turkish dotless i, Greek final sigma), surrogate pairs, and `u32` packing.
+- Add `test/highlight.test.ts`, checking that highlight ranges land on whole characters in multi-byte text.
+
+### 2.4 Run in CI
+- Make sure `bun run test:mock` in `.github/workflows/ci.yml` picks up the new files.
+
+---
+
+## Phase 3: Public results page for the collected benchmarks (≈ 3–4 days)
+
+Benchmark results already go to Upstash automatically (the opt-in `api/submit-benchmark.ts` intake), but nobody can see them.
+
+| # | Task |
+|---|------|
+| 3.1 | Add a read-only, cached API route in `api/` that returns aggregated results (median speedup grouped by GPU type, browser and dataset size). Return only aggregates, never raw rows. |
+| 3.2 | Add a results section or page to `apps/site`: a chart of WebGPU vs CPU vs uFuzzy vs Fuse.js, with integrated and discrete GPUs shown separately. |
+| 3.3 | Use the data to set the default `threshold` (currently 30,000) from measured crossover points. Consider a separate default for integrated GPUs. |
+| 3.4 | Show a minimum sample count on each group, and hide groups with too few samples. |
+
+**Done when:** the page is live, and the README links to it in place of the old benchmark numbers.
+
+---
+
+## Phase 4: Break up the largest files (≈ 3–5 days)
+
+`document-index.ts` has 3,331 lines, almost 30% of the library. `webgpu-engine.ts` has 1,093.
+
+### 4.1 Split `document-index.ts`
+Proposed modules under `src/document/`:
+- `build.ts`: ingesting records and building the index
+- `query.ts`: running searches and merging results
+- `filter-facet.ts`: wiring in the code from `filtering/` and `faceting/`
+- `snapshot.ts`: saving and loading with `snapshot-codec.ts`
+- `document-index.ts`: a thin public class that delegates to the above
+
+### 4.2 Split `webgpu-engine.ts`
+- Move buffer allocation and resizing, pipeline/shader setup, and dispatch/readback into separate modules.
+
+### Rules for this phase
+- Behavior must not change. The Phase 2 tests must pass before and after.
+- The public API in `src/index.ts` must not change. Compare `dist/index.d.ts` before and after.
+- Run `bun run check:shaders`, `typecheck`, `build` and `test:mock`, then do the subagent review that AGENTS.md requires.
+
+---
+
+## Phase 5: Bundle size (≈ 1–2 days)
+
+| # | Task |
+|---|------|
+| 5.1 | Add `size-limit` (or a small script) to CI with a budget for each entry point (`.` and `./worker`). |
+| 5.2 | Measure how much `case-fold-table.ts` and `fold-table.ts` add to the bundle. |
+| 5.3 | If they are large, load the full Unicode tables only for the `unicode-default` profile, so ASCII-only users don't pay for them. This must not change the snapshot format. |
+
+---
+
+## Phase 6: New features (later, one at a time)
+
+Rank these again once Phases 1–5 are done.
+
+1. **Add and remove records without a full rebuild.** This would need GPU buffers that can grow and a way to mark deleted records ("tombstones"). It needs its own design doc first.
+2. **Official framework bindings** (`webgpu-search/react`, `/vue`), built from the recipes in `examples/`.
+3. **A Node example on the `webgpu` npm package (Dawn)**, to back up the claim that search runs on the server too. Add it to `docs/support-matrix.md`.
+
+---
+
+## Order and dependencies
+
+```
+Phase 1 ──► Phase 2 ──► Phase 4 (refactor needs the tests first)
+               │
+               └──────► Phase 5
+Phase 3 can run in parallel with everything else
+Phase 6 starts after Phase 4
+```
+
+## Release notes
+- Phases 1–3 and 5 are patch or minor releases (add a changeset).
+- Phase 4 should produce a `patch` changeset with no public API changes.
+- Phase 6 items are minor releases, each with its own changeset.