detect: rank only systems whose source script matches the input

The detect walk loaded all 288 maps per request. On the Workers free
plan (10ms CPU) that intermittently dies with 1102 — production
returns anywhere between 47 and 96 partial rankings per call, or 503s.
Loading every map to rank unrelated scripts was also semantically
empty: Georgian input ranked 280 irrelevant systems at distance 9.

detectInputScript() picks the majority Unicode script of the input
(non-Latin beats Latin when tied); a build-time generated
maps/scripts.js maps each system to its source script, and detect
walks only the matching family (Hangul input also matches the mixed
Kore sources, Han input also matches Hans/Kore/Hrkt). Georgian input
now loads 8 systems instead of 288 and the top hits are unchanged.

Test fixtures: the detect fixture records the prefiltered ranking.

Author: ronaldtse

maps/scripts.d.ts

@@ -0,0 +1 @@
+export const mapScripts: Record<string, string | undefined>

maps/scripts.js

@@ -50,5 +50,22 @@ writeFileSync(
 )
 writeFileSync(join(repoMaps, "manifest.d.ts"), "export const systemCodes: readonly string[]\n")
 
+const mapScripts = {}
+for (const name of readdirSync(outMaps)) {
+  const code = basename(name, ".json")
+  if (manifest.systems.includes(code)) {
+    mapScripts[code] = JSON.parse(readFileSync(join(outMaps, name), "utf8")).metadata?.source_script
+  }
+}
+writeFileSync(
+  join(repoMaps, "scripts.js"),
+  "// Generated by scripts/build-maps.mjs from interscript/maps — source script per system. Do not edit.\n" +
+    `export const mapScripts = ${JSON.stringify(mapScripts)};\n`,
+)
+writeFileSync(
+  join(repoMaps, "scripts.d.ts"),
+  "export const mapScripts: Record<string, string | undefined>\n",
+)
+
 rmSync(outDir, { recursive: true, force: true })
 console.log(`bundled ${manifest.count} maps into maps/`)

scripts/build-maps.mjs

@@ -4,6 +4,8 @@
  */
 import { configure, transliterateAsync, type SystemCode } from "interscript"
 import { bundledSystemCodes, detectableSystemCodes, loadMap, type MapAssets } from "./maps.js"
+import { mapScripts } from "../maps/scripts.js"
+import { detectInputScript, sourceMatchesScript } from "./scripts.js"
 import type { LoadStrategy } from "interscript"
 
 let configuredFor: MapAssets | undefined
@@ -43,18 +45,26 @@ function levenshtein(a: string, b: string): number {
   return prev[b.length]!
 }
 
-// Mirrors interscript-ruby's detector: transliterate the input through
-// every system, rank by Levenshtein distance to the output. Systems
-// that fail (including ML-powered maps with no runtime configured) are
-// skipped, exactly as the Ruby detector rescues per-map errors.
+// Transliterate the input through every system whose source script
+// matches the input, rank by Levenshtein distance to the output.
+// Systems that fail (including ML-powered maps with no runtime
+// configured) are skipped, exactly as the Ruby detector rescues
+// per-map errors. The full-corpus walk is gone deliberately: it
+// exceeded the Workers CPU budget (1102s in production).
 export async function detect(
   assets: MapAssets,
   input: string,
   output: string,
 ): Promise<{ mapName: string; distance: number }[]> {
   ensureEngine(assets)
+  // Only systems whose source script matches the input can rank: walking
+  // all 288 maps exhausts the Workers CPU budget (1102s in production).
+  const detected = detectInputScript(input)
+  const codes = detected
+    ? detectableSystemCodes().filter((code) => sourceMatchesScript(mapScripts[code], detected))
+    : []
   const candidates: { mapName: string; distance: number }[] = []
-  for (const code of detectableSystemCodes()) {
+  for (const code of codes) {
     try {
       const transliterated = await transliterateAsync(code, input)
       candidates.push({ mapName: code, distance: levenshtein(transliterated, output) })

src/engine.ts

@@ -7,7 +7,8 @@
  *   GET  /v1/maps             every bundled transliteration system code
  *   GET  /v1/maps/:code       one system's compiled map (JSON IR)
  *   POST /v1/transliterate    {system, input} -> {output}
- *   POST /v1/detect           {input, output} -> ranked systems
+ *   POST /v1/detect           {input, output} -> systems whose source
+ *                             script matches the input, ranked
  *   GET  /v1/models           the neural model index (IMF v1)
  *   GET  /v1/models/:id       one model's full entry
  *   POST /v1/infer            {model, input} -> {output} (Modal proxy)

src/rest.ts

@@ -0,0 +1,152 @@
+/**
+ * Input-script detection for detect(): the walk only transliterates
+ * systems whose source script matches the input — loading all 288 maps
+ * per request exceeds the Workers CPU budget (1102s in production).
+ * Ranges cover exactly the source scripts present in the corpus.
+ */
+
+type ScriptName = string
+
+const SCRIPT_RANGES: ReadonlyArray<
+  readonly [ScriptName, ReadonlyArray<readonly [number, number]>]
+> = [
+  [
+    "Latn",
+    [
+      [0x41, 0x5a],
+      [0x61, 0x7a],
+      [0xc0, 0xff],
+      [0x100, 0x24f],
+    ],
+  ],
+  [
+    "Grek",
+    [
+      [0x370, 0x3ff],
+      [0x1f00, 0x1fff],
+    ],
+  ],
+  ["Cyrl", [[0x400, 0x52f]]],
+  ["Armn", [[0x530, 0x58f]]],
+  ["Hebr", [[0x590, 0x5ff]]],
+  [
+    "Arab",
+    [
+      [0x600, 0x6ff],
+      [0x750, 0x77f],
+      [0xfb50, 0xfdff],
+      [0xfe70, 0xfeff],
+    ],
+  ],
+  ["Deva", [[0x900, 0x97f]]],
+  ["Beng", [[0x980, 0x9ff]]],
+  ["Guru", [[0xa00, 0xa7f]]],
+  ["Gujr", [[0xa80, 0xaff]]],
+  ["Orya", [[0xb00, 0xb7f]]],
+  ["Taml", [[0xb80, 0xbff]]],
+  ["Telu", [[0xc00, 0xc7f]]],
+  ["Kana", [[0xc80, 0xcff]]],
+  ["Mlym", [[0xd00, 0xd7f]]],
+  ["Sinh", [[0xd80, 0xdff]]],
+  ["Thaa", [[0x780, 0x7bf]]],
+  ["Thai", [[0xe00, 0xe7f]]],
+  [
+    "Geor",
+    [
+      [0x10a0, 0x10ff],
+      [0x1c90, 0x1cbf],
+    ],
+  ],
+  ["Ethi", [[0x1200, 0x137f]]],
+  ["Mong", [[0x1800, 0x18af]]],
+  [
+    "Hang",
+    [
+      [0x1100, 0x11ff],
+      [0x3130, 0x318f],
+      [0xac00, 0xd7af],
+    ],
+  ],
+  [
+    "Hrkt",
+    [
+      [0x3040, 0x309f],
+      [0x30a0, 0x30ff],
+      [0x31f0, 0x31ff],
+    ],
+  ],
+  [
+    "Hani",
+    [
+      [0x2e80, 0x2eff],
+      [0x3400, 0x4dbf],
+      [0x4e00, 0x9fff],
+      [0xf900, 0xfaff],
+      [0x20000, 0x2a6df],
+    ],
+  ],
+]
+
+function scriptOfCodePoint(cp: number): ScriptName | null {
+  for (const [script, ranges] of SCRIPT_RANGES) {
+    for (const [lo, hi] of ranges) {
+      if (cp >= lo && cp <= hi) return script
+    }
+  }
+  return null
+}
+
+export function detectInputScript(input: string): ScriptName | null {
+  const counts = new Map<ScriptName, number>()
+  for (const ch of input) {
+    const script = scriptOfCodePoint(ch.codePointAt(0)!)
+    if (script) counts.set(script, (counts.get(script) ?? 0) + 1)
+  }
+  let best: ScriptName | null = null
+  let bestWeighted = 0
+  for (const [script, n] of counts) {
+    // A Latin majority is the weakest signal: one stray non-Latin letter
+    // outranks any amount of Latin.
+    const weighted = script === "Latn" ? n - 0.5 : n
+    if (weighted > bestWeighted) {
+      best = script
+      bestWeighted = weighted
+    }
+  }
+  return best
+}
+
+// Detected script → corpus source_script values that can legitimately
+// process it ("Kana" is Kannada in this corpus; "Kore" is the mixed
+// Hangul+Latin jamo family; "Hrkt" is Japanese kana).
+const FAMILIES: Readonly<Record<ScriptName, readonly string[]>> = {
+  Latn: ["Latn"],
+  Grek: ["Grek"],
+  Cyrl: ["Cyrl"],
+  Armn: ["Armn"],
+  Hebr: ["Hebr"],
+  Arab: ["Arab"],
+  Deva: ["Deva"],
+  Beng: ["Beng"],
+  Guru: ["Guru"],
+  Gujr: ["Gujr"],
+  Orya: ["Orya"],
+  Taml: ["Taml"],
+  Telu: ["Telu"],
+  Kana: ["Kana"],
+  Mlym: ["Mlym"],
+  Sinh: ["Sinh"],
+  Thai: ["Thai"],
+  Thaa: ["Thaa"],
+  Ethi: ["Ethi"],
+  Mong: ["Mong"],
+  Geor: ["Geor"],
+  Hang: ["Hang", "Kore"],
+  Hrkt: ["Hrkt", "Kore"],
+  Hani: ["Hani", "Hans", "Kore", "Hrkt"],
+}
+
+export function sourceMatchesScript(sourceScript: string | undefined, detected: string): boolean {
+  const family = FAMILIES[detected]
+  return !!sourceScript && !!family && family.includes(sourceScript)
+}

src/scripts.ts

@@ -0,0 +1,71 @@
+import { readFileSync } from "node:fs"
+import { describe, expect, it } from "vitest"
+import { detect } from "../src/engine.js"
+import { detectInputScript } from "../src/scripts.js"
+import type { MapAssets } from "../src/maps.js"
+
+const fetched: string[] = []
+const env: MapAssets = {
+  fetch: async (url) => {
+    const path = typeof url === "string" ? url : url.url
+    const code = path.substring(path.lastIndexOf("/") + 1).replace(".json", "")
+    fetched.push(code)
+    try {
+      return new Response(readFileSync(`maps/${code}.json`), { status: 200 })
+    } catch {
+      return new Response(null, { status: 404 })
+    }
+  },
+}
+
+describe("detectInputScript", () => {
+  it("detects the majority script of the input", () => {
+    expect(detectInputScript("ქართული")).toBe("Geor")
+    expect(detectInputScript("한국")).toBe("Hang")
+    expect(detectInputScript("Привет")).toBe("Cyrl")
+    expect(detectInputScript(" தமிழ் ")).toBe("Taml")
+  })
+
+  it("prefers the non-Latin script in mixed input", () => {
+    expect(detectInputScript("한국a")).toBe("Hang")
+    expect(detectInputScript("київ word")).toBe("Cyrl")
+  })
+
+  it("falls back to Latin for pure ASCII", () => {
+    expect(detectInputScript("hello world")).toBe("Latn")
+  })
+
+  it("returns null when no script characters are present", () => {
+    expect(detectInputScript("123 !?")).toBeNull()
+    expect(detectInputScript("")).toBeNull()
+  })
+})
+
+describe("detect prefilter", () => {
+  it("only loads maps whose source script matches the input", async () => {
+    fetched.length = 0
+    const results = await detect(env, "ქართული", "kʻartʻuli")
+    // 8 Georgian-source systems, not the full 288-map walk.
+    expect(fetched.length).toBeLessThanOrEqual(10)
+    expect(results[0]).toMatchObject({ mapName: "alalc-kat-Geor-Latn-1997", distance: 0 })
+  })
+
+  it("matches Korean-family sources for Hangul input", async () => {
+    fetched.length = 0
+    const results = await detect(env, "한국", "Hanguk")
+    expect(results[0]!.distance).toBe(0)
+    expect(results.map((r) => r.mapName)).toContain("var-kor-Kore-Latn-mr-1939")
+  })
+
+  it("ranks Han-source systems for Han input", async () => {
+    const results = await detect(env, "阜康", "Fukang")
+    expect(results[0]).toMatchObject({ mapName: "acadsin-zho-Hani-Latn-2002" })
+  })
+
+  it("returns an empty ranking for script-less input", async () => {
+    fetched.length = 0
+    const results = await detect(env, "123 !?", "x")
+    expect(results).toEqual([])
+    expect(fetched).toEqual([])
+  })
+})