Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/interscript/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,15 @@ def _run_op(self, op: dict, text: str) -> str:
target = deps[alias]
if self._loader is None:
raise ExecutionError(f"run {op['map']!r}: no map loader configured")
return self._loader(target).transliterate(text)
dep = self._loader(target)
stage_name = op.get("stage")
if not stage_name:
return dep.transliterate(text)
out = text
for st in dep.tree.get("stages", []):
if st.get("name") == stage_name:
out = dep._run_stage(st, out)
return out
if kind == "separate":
separator = op.get("separator", " ")
return separator.join(text)
Expand Down
13 changes: 10 additions & 3 deletions src/interscript/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
r'|any\(\s*"(?P<rlo>(?:[^"\\]|\\.)*)"\s*\.\.\s*"(?P<rhi>(?:[^"\\]|\\.)*)"\s*\)'
r'|any\(\s*"(?P<cls>(?:[^"\\]|\\.)*)"\s*\)'
r"|(?P<space>\bspace\b)|(?P<boundary>\bboundary\b)"
r"|(?P<stdin_kw>\b(?:alpha|digit|word|any_character)\b)"
r"|(?P<stdin_kw>\b(?:alpha|digit|word|not_word|any_character)\b)"
r"|(?P<nwb>\bnon_word_boundary\b)"
r'|capture\(\s*(?P<grp>(?:[^()\\]|\\.|\([^()]*\))*)\s*\)'
r"|(?P<line_end>\bline_end\b)|(?P<line_start>\bline_start\b)"
Expand Down Expand Up @@ -74,7 +74,7 @@ def _read_bracketed(expr: str, pos: int) -> tuple[str, int]:
return expr[pos + 1 : i], i + 1
i += 1
raise ValueError(f"unterminated any([ in {expr!r}")
_UNESC = re.compile(r"\\u([0-9a-fA-F]{4})")
_UNESC = re.compile(r"\\(u[0-9a-fA-F]{4}|.)", re.S)

SPACE = re.escape(" ")

Expand All @@ -84,6 +84,7 @@ def _read_bracketed(expr: str, pos: int) -> tuple[str, int]:
"alpha": "[a-zA-Z]",
"digit": "[0-9]",
"word": "[a-zA-Z0-9_]",
"not_word": "[^a-zA-Z0-9_]",
"any_character": ".",
}

Expand All @@ -100,7 +101,13 @@ def _read_bracketed(expr: str, pos: int) -> tuple[str, int]:


def _unesc(s: str) -> str:
return _UNESC.sub(lambda m: chr(int(m.group(1), 16)), s)
def repl(m: re.Match) -> str:
g = m.group(1)
if g[0] == "u" and len(g) == 5:
return chr(int(g[1:], 16))
return {"n": "\n", "t": "\t", "r": "\r"}.get(g, g)

return _UNESC.sub(repl, s)


_ANY_LIST = re.compile(r"any\(\s*\[")
Expand Down
18 changes: 14 additions & 4 deletions src/interscript/isc.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def __init__(self, subs: list[dict], capture_rules: list[dict]) -> None:
_CONSTRAINTS = {"before", "after", "not_before", "not_after"}
# Stdlib aliases usable as bare names (Ruby Stdlib::ALIASES, resolved
# before doc-local aliases, mirroring the interpreter's lookup order).
_STDLIB_EXPR = {"alpha", "digit", "word", "any_character"}
_STDLIB_EXPR = {"alpha", "digit", "word", "not_word", "any_character"}
# Tokens that terminate an item inside a rule; a bare word equal to one
# of these is a keyword, never an alias reference.
_KEYWORDS = _CONSTRAINTS | {"to", "from", "note"}
Expand Down Expand Up @@ -768,7 +768,7 @@ def _regex_of(item: dict, aliases: dict[str, str]) -> str:
if item.get("map"):
return _qualified_regex(item["map"], name)
if name in _STDLIB_EXPR:
return {"alpha": "[a-zA-Z]", "digit": "[0-9]", "word": "[a-zA-Z0-9_]", "any_character": "."}[name]
return {"alpha": "[a-zA-Z]", "digit": "[0-9]", "word": "[a-zA-Z0-9_]", "not_word": "[^a-zA-Z0-9_]", "any_character": "."}[name]
if name in _imported:
return _qualified_regex(None, name)
if name not in aliases:
Expand Down Expand Up @@ -875,7 +875,10 @@ def _stage_tree(
dep = body_item.get("dependency")
if dep:
target = dep_aliases.get(dep, dep)
return {"kind": "run", "map": target}
op = {"kind": "run", "map": target}
if body_item.get("stage"):
op["stage"] = body_item["stage"]
return op
return {"kind": "run_stage", "name": body_item["stage"]}
if kind in ("compose", "decompose"):
return {"kind": kind}
Expand Down Expand Up @@ -979,7 +982,14 @@ def _library_aliases(name: str, libs_dir: Path) -> dict[str, str]:
# Library aliases may reference aliases defined earlier
# in the same file (var-kor: jamo = any([jamo_leading_cons,
# ...])); substitute them, outside string literals only.
aliases[m.group(1)] = _subst_alias_refs(value, aliases)
value = _subst_alias_refs(value, aliases)
# A library alias holding a bare string acts as a
# character class inside any() (measured: the unicode
# library's greek); an in-map string alias would be a
# literal sequence instead.
if re.fullmatch(r'"(?:[^"\\]|\\.)*"', value):
value = f"any({value})"
aliases[m.group(1)] = value
_LIB_CACHE[name] = aliases
return aliases

Expand Down
39 changes: 39 additions & 0 deletions tests/test_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,3 +357,42 @@ def test_any_character_renders():
e = _load("kp-kor-Hang-Latn-2002")
assert e.transliterate("고비리") == "Kobi-ri"
assert e.transliterate("교구동") == "Kyogu-dong"


@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present")
def test_quote_escape_survives_the_round_trip():
"""gost-rus maps the hard sign to a double quote (to "\\""); the
value round-trips through expression source, where the unescaper
only decoded \\uXXXX — \\" stayed a backslash-quote and съезд came
out s\\"ezd."""
e = _load("gost-rus-Cyrl-Latn-7.79-2000-2002")
assert e.transliterate("съезд") == 's"ezd'


@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present")
def test_run_executes_the_named_dependency_stage():
"""mvd-rus-2010 runs the 2008 dependency's translit stage — not its
main, which ends in compose. Running main precomposed l+U+0301 into
ĺ before the postrule could strip the acute (Vasiĺeva)."""
e = _load("mvd-rus-Cyrl-Latn-2010")
assert e.transliterate("Васiльева") == "Vasileva"


@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present")
def test_not_word_stdlib_alias_renders():
"""odni-che maps the digit 1 (a palochka stand-in) guarded by
not_word; the alias had no stdlib entry and the rules were
dropped — Ахмадк1ант kept its 1."""
e = _load("odni-che-Cyrl-Latn-2015")
assert e.transliterate("Ахмадк1ант") == "Akhmadkant"


@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present")
def test_library_string_alias_is_a_class_inside_any():
"""Measured in Ruby: an alias imported from a library whose value
is a plain string acts as a CHARACTER CLASS inside any() (the
unicode library's greek), while an in-map string alias is a literal
sequence. alalc-ell's γ-nasal rules guard on any(greek)."""
e = _load("alalc-ell-Grek-Latn-2010")
assert e.transliterate("γκέγκε") == "gkenke"
assert e.transliterate("Λαγκαδάς") == "Lankadas"
2 changes: 1 addition & 1 deletion tests/test_isc.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def test_tests_render_as_pairs():
def test_dependencies_resolve_alias_for_run():
tree = isc_to_tree(SAMPLE)
stage_children = tree["stages"][0]["children"]
assert stage_children[0] == {"kind": "run", "map": "other-map"}
assert stage_children[0] == {"kind": "run", "map": "other-map", "stage": "main"}


def test_parallel_subs_render_as_expr_source():
Expand Down
Loading