From 0684a37b5961393f84db8dd767729064f47def98 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 1 Oct 2026 19:51:44 +0800 Subject: [PATCH] fix: quote escapes, named dependency stages, not_word, library string classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four conformance gaps, each measured against Ruby: - The expression unescaper decoded only \uXXXX, so a result round- tripping through expression source kept \" as a backslash-quote: gost-rus maps the hard sign to a double quote and съезд came out s\"ezd. Standard escapes now decode. - run map..stage. executed the dependency's main stage, ignoring the named one. mvd-rus-2010 runs the 2008 dependency's translit stage precisely to skip its trailing compose, which would precompose l+U+0301 into ĺ before the postrule strips the acute. - not_word joins the stdlib aliases (odni-che guards the digit 1 with it); it is Onigmo's ASCII \W, [^a-zA-Z0-9_]. - A library alias holding a bare string acts as a character class inside any() — the unicode library's greek — while an in-map string alias is a literal sequence (both measured with synthetic maps). alalc-ell's gamma nasalization rules guard on any(greek); the look- behind had compiled to the literal 300-character alphabet. Direct corpus sweep: 16 failures / 10 maps -> 4 / 3 — alalc-ell-2010, bas-rus, bgnpcgn-rus-1947, both gost-rus, mvd-rus-2010 and odni-che fully healed. Only odni-ara 2004/2015 (4 examples) and the rababa map (needs the ML model; excluded by design) remain. --- src/interscript/engine.py | 10 +++++++++- src/interscript/expr.py | 13 ++++++++++--- src/interscript/isc.py | 18 ++++++++++++++---- tests/test_engine.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/test_isc.py | 2 +- 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/interscript/engine.py b/src/interscript/engine.py index 14a1899..89b40b3 100644 --- a/src/interscript/engine.py +++ b/src/interscript/engine.py @@ -139,7 +139,15 @@ def _run_op(self, op: dict, text: str) -> str: target = deps[alias] if self._loader is None: raise ExecutionError(f"run {op['map']!r}: no map loader configured") - return self._loader(target).transliterate(text) + dep = self._loader(target) + stage_name = op.get("stage") + if not stage_name: + return dep.transliterate(text) + out = text + for st in dep.tree.get("stages", []): + if st.get("name") == stage_name: + out = dep._run_stage(st, out) + return out if kind == "separate": separator = op.get("separator", " ") return separator.join(text) diff --git a/src/interscript/expr.py b/src/interscript/expr.py index 5885a77..f229bb5 100644 --- a/src/interscript/expr.py +++ b/src/interscript/expr.py @@ -17,7 +17,7 @@ r'|any\(\s*"(?P(?:[^"\\]|\\.)*)"\s*\.\.\s*"(?P(?:[^"\\]|\\.)*)"\s*\)' r'|any\(\s*"(?P(?:[^"\\]|\\.)*)"\s*\)' r"|(?P\bspace\b)|(?P\bboundary\b)" - r"|(?P\b(?:alpha|digit|word|any_character)\b)" + r"|(?P\b(?:alpha|digit|word|not_word|any_character)\b)" r"|(?P\bnon_word_boundary\b)" r'|capture\(\s*(?P(?:[^()\\]|\\.|\([^()]*\))*)\s*\)' r"|(?P\bline_end\b)|(?P\bline_start\b)" @@ -74,7 +74,7 @@ def _read_bracketed(expr: str, pos: int) -> tuple[str, int]: return expr[pos + 1 : i], i + 1 i += 1 raise ValueError(f"unterminated any([ in {expr!r}") -_UNESC = re.compile(r"\\u([0-9a-fA-F]{4})") +_UNESC = re.compile(r"\\(u[0-9a-fA-F]{4}|.)", re.S) SPACE = re.escape(" ") @@ -84,6 +84,7 @@ def _read_bracketed(expr: str, pos: int) -> tuple[str, int]: "alpha": "[a-zA-Z]", "digit": "[0-9]", "word": "[a-zA-Z0-9_]", + "not_word": "[^a-zA-Z0-9_]", "any_character": ".", } @@ -100,7 +101,13 @@ def _read_bracketed(expr: str, pos: int) -> tuple[str, int]: def _unesc(s: str) -> str: - return _UNESC.sub(lambda m: chr(int(m.group(1), 16)), s) + def repl(m: re.Match) -> str: + g = m.group(1) + if g[0] == "u" and len(g) == 5: + return chr(int(g[1:], 16)) + return {"n": "\n", "t": "\t", "r": "\r"}.get(g, g) + + return _UNESC.sub(repl, s) _ANY_LIST = re.compile(r"any\(\s*\[") diff --git a/src/interscript/isc.py b/src/interscript/isc.py index 12b2355..ea7ed84 100644 --- a/src/interscript/isc.py +++ b/src/interscript/isc.py @@ -59,7 +59,7 @@ def __init__(self, subs: list[dict], capture_rules: list[dict]) -> None: _CONSTRAINTS = {"before", "after", "not_before", "not_after"} # Stdlib aliases usable as bare names (Ruby Stdlib::ALIASES, resolved # before doc-local aliases, mirroring the interpreter's lookup order). -_STDLIB_EXPR = {"alpha", "digit", "word", "any_character"} +_STDLIB_EXPR = {"alpha", "digit", "word", "not_word", "any_character"} # Tokens that terminate an item inside a rule; a bare word equal to one # of these is a keyword, never an alias reference. _KEYWORDS = _CONSTRAINTS | {"to", "from", "note"} @@ -768,7 +768,7 @@ def _regex_of(item: dict, aliases: dict[str, str]) -> str: if item.get("map"): return _qualified_regex(item["map"], name) if name in _STDLIB_EXPR: - return {"alpha": "[a-zA-Z]", "digit": "[0-9]", "word": "[a-zA-Z0-9_]", "any_character": "."}[name] + return {"alpha": "[a-zA-Z]", "digit": "[0-9]", "word": "[a-zA-Z0-9_]", "not_word": "[^a-zA-Z0-9_]", "any_character": "."}[name] if name in _imported: return _qualified_regex(None, name) if name not in aliases: @@ -875,7 +875,10 @@ def _stage_tree( dep = body_item.get("dependency") if dep: target = dep_aliases.get(dep, dep) - return {"kind": "run", "map": target} + op = {"kind": "run", "map": target} + if body_item.get("stage"): + op["stage"] = body_item["stage"] + return op return {"kind": "run_stage", "name": body_item["stage"]} if kind in ("compose", "decompose"): return {"kind": kind} @@ -979,7 +982,14 @@ def _library_aliases(name: str, libs_dir: Path) -> dict[str, str]: # Library aliases may reference aliases defined earlier # in the same file (var-kor: jamo = any([jamo_leading_cons, # ...])); substitute them, outside string literals only. - aliases[m.group(1)] = _subst_alias_refs(value, aliases) + value = _subst_alias_refs(value, aliases) + # A library alias holding a bare string acts as a + # character class inside any() (measured: the unicode + # library's greek); an in-map string alias would be a + # literal sequence instead. + if re.fullmatch(r'"(?:[^"\\]|\\.)*"', value): + value = f"any({value})" + aliases[m.group(1)] = value _LIB_CACHE[name] = aliases return aliases diff --git a/tests/test_engine.py b/tests/test_engine.py index 2564cbb..1e7dbb0 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -357,3 +357,42 @@ def test_any_character_renders(): e = _load("kp-kor-Hang-Latn-2002") assert e.transliterate("고비리") == "Kobi-ri" assert e.transliterate("교구동") == "Kyogu-dong" + + +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_quote_escape_survives_the_round_trip(): + """gost-rus maps the hard sign to a double quote (to "\\""); the + value round-trips through expression source, where the unescaper + only decoded \\uXXXX — \\" stayed a backslash-quote and съезд came + out s\\"ezd.""" + e = _load("gost-rus-Cyrl-Latn-7.79-2000-2002") + assert e.transliterate("съезд") == 's"ezd' + + +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_run_executes_the_named_dependency_stage(): + """mvd-rus-2010 runs the 2008 dependency's translit stage — not its + main, which ends in compose. Running main precomposed l+U+0301 into + ĺ before the postrule could strip the acute (Vasiĺeva).""" + e = _load("mvd-rus-Cyrl-Latn-2010") + assert e.transliterate("Васiльева") == "Vasileva" + + +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_not_word_stdlib_alias_renders(): + """odni-che maps the digit 1 (a palochka stand-in) guarded by + not_word; the alias had no stdlib entry and the rules were + dropped — Ахмадк1ант kept its 1.""" + e = _load("odni-che-Cyrl-Latn-2015") + assert e.transliterate("Ахмадк1ант") == "Akhmadkant" + + +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_library_string_alias_is_a_class_inside_any(): + """Measured in Ruby: an alias imported from a library whose value + is a plain string acts as a CHARACTER CLASS inside any() (the + unicode library's greek), while an in-map string alias is a literal + sequence. alalc-ell's γ-nasal rules guard on any(greek).""" + e = _load("alalc-ell-Grek-Latn-2010") + assert e.transliterate("γκέγκε") == "gkenke" + assert e.transliterate("Λαγκαδάς") == "Lankadas" diff --git a/tests/test_isc.py b/tests/test_isc.py index 0bb8b0d..4b83e78 100644 --- a/tests/test_isc.py +++ b/tests/test_isc.py @@ -72,7 +72,7 @@ def test_tests_render_as_pairs(): def test_dependencies_resolve_alias_for_run(): tree = isc_to_tree(SAMPLE) stage_children = tree["stages"][0]["children"] - assert stage_children[0] == {"kind": "run", "map": "other-map"} + assert stage_children[0] == {"kind": "run", "map": "other-map", "stage": "main"} def test_parallel_subs_render_as_expr_source():