Skip to content

Commit 7cc476d

Browse files
authored
Merge pull request #779 from interscript/fix/any-list-items-leak
fix: seven ISC pipeline conformance bugs — the corpus sweep runs end-to-end
2 parents 7aed64e + 96cd6c7 commit 7cc476d

11 files changed

Lines changed: 279 additions & 59 deletions

File tree

‎.github/workflows/rake.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ jobs:
1818
env:
1919
BUNDLE_WITHOUT: "secryst"
2020
SKIP_JS: "1"
21+
# The ISC corpus sweep (ISC_SWEEP=1) stays opt-in until the
22+
# conformance gap it measures is closed: it currently exposes real
23+
# per-map mismatches across the corpus. Run it locally with
24+
# ISC_SWEEP=1 INTERSCRIPT_MAPS_PATH=../maps/maps \
25+
# rspec spec/interscript_spec.rb
26+
# to track that gap; flip it on here once it goes green.
2127

2228
steps:
2329
- name: Checkout monorepo

‎lib/interscript/compiler.rb‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,10 @@ class Interscript::Compiler
99
attr_accessor :code
1010

1111
def self.call(map, **kwargs)
12-
if String === map
13-
path = begin
14-
Interscript.locate(map)
15-
rescue
16-
nil
17-
end
18-
map = if path&.end_with?(".isc")
19-
parse_isc(path)
20-
else
21-
Interscript::DSL.parse(map)
22-
end
23-
end
12+
# DSL.parse locates, dispatches .isc and caches the parsed document —
13+
# calling parse_isc here re-ran a ~30 s Parslet parse on every
14+
# compiler instantiation.
15+
map = Interscript::DSL.parse(map) if String === map
2416
compiler = new
2517
compiler.compile(map, **kwargs)
2618
compiler

‎lib/interscript/dsl.rb‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ def self.parse(map_name, reverse: true)
4040
end
4141
end
4242
# ISC documents route through the ISC parser, not the .imp DSL.
43-
return Interscript::Compiler.parse_isc(path) if path.end_with?(".isc")
43+
# The parse is expensive (Parslet over the whole map) — cache it like
44+
# the .imp path does, or every transliterate call re-parses the map.
45+
return @cache[map_name] = Interscript::Compiler.parse_isc(path) if path.end_with?(".isc")
4446

4547
library = path.end_with?(".iml")
4648

‎lib/interscript/isc/node_adapter.rb‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,22 @@ def build
2626
Interscript::Node::Document.new.tap do |doc|
2727
doc.metadata = build_metadata
2828
doc.tests = build_tests
29-
doc.aliases = build_aliases
30-
build_stages.each { |name, stage| doc.stages[name] = stage }
29+
# Maps are addressed by their file-derived name (locate() and
30+
# Maps.transliterate use it); registering under the system code
31+
# made every map whose code differs from its file name resolve
32+
# to an auto-created empty entry — stages[:main].call crashed.
33+
doc.name = if @isc_doc[:filename]
34+
File.basename(@isc_doc[:filename].to_s, ".isc")
35+
else
36+
@isc_doc[:systemCode]
37+
end
38+
# The legacy DSL stamps doc_name on every alias and stage; the
39+
# Ruby/JS compilers emit run directives and alias lookups as
40+
# Maps.transliterate(stage.doc_name, ...) — without it every
41+
# aliased-dependency run compiled to transliterate(nil).
42+
doc.aliases = build_aliases(doc.name)
43+
build_stages(doc.name).each { |name, stage| doc.stages[name] = stage }
3144
build_dependencies(doc)
32-
doc.name = @isc_doc[:systemCode]
3345
end
3446
end
3547

@@ -53,12 +65,14 @@ def build_tests
5365
tests
5466
end
5567

56-
def build_aliases
68+
def build_aliases(doc_name)
5769
@isc_doc[:aliases].each_with_object({}) do |a, h|
5870
# The runtime resolves aliases through AliasDef#data (see
5971
# Interpreter::Stage#build_item); a bare item here hands it a
6072
# raw Array once Any#data unrolls.
61-
h[a[:name].to_sym] = Interscript::Node::AliasDef.new(a[:name].to_sym, convert_item(a[:value]))
73+
alias_def = Interscript::Node::AliasDef.new(a[:name].to_sym, convert_item(a[:value]))
74+
alias_def.doc_name = doc_name
75+
h[a[:name].to_sym] = alias_def
6276
end
6377
end
6478

@@ -90,9 +104,11 @@ def load_dependency_document(full_name)
90104
end
91105
end
92106

93-
def build_stages
107+
def build_stages(doc_name)
94108
@isc_doc[:stages].each_with_object({}) do |stage, h|
95-
h[stage[:name].to_sym] = build_stage(stage)
109+
built = build_stage(stage)
110+
built.doc_name = doc_name
111+
h[stage[:name].to_sym] = built
96112
end
97113
end
98114

@@ -174,9 +190,13 @@ def convert_item(item)
174190
when Items::Some
175191
Interscript::Node::Item::Some.new(convert_item(item.inner))
176192
when Items::Range
177-
Interscript::Node::Item::Any.new(
178-
(item.lo..item.hi).map { |c| Interscript::Node::Item::String.new(c) }
179-
)
193+
# Keep the range native: both runtimes compile Any(Range) to a
194+
# codepoint character class [lo-hi]. Expanding it to an Array
195+
# iterates Ruby String#succ (a, b, … z, aa, ab …), which never
196+
# reaches non-ASCII codepoints and blows up the node size.
197+
lo = item.lo.is_a?(Items::StringValue) ? item.lo.value : item.lo
198+
hi = item.hi.is_a?(Items::StringValue) ? item.hi.value : item.hi
199+
Interscript::Node::Item::Any.new(lo..hi)
180200
when Items::Set
181201
convert_set(item)
182202
else
@@ -209,10 +229,19 @@ def convert_set(set)
209229
case c
210230
when ::String
211231
Interscript::Node::Item::String.new(c)
232+
when Items::StringValue
233+
Interscript::Node::Item::String.new(c.value)
212234
when Items::Primitive
213235
convert_primitive(c)
214236
when Items::AliasRef
215-
convert_alias_ref(c)
237+
# Legacy parity: a bare alias inside any(...) compiles to the
238+
# Stdlib value (Any(nil) there for imported aliases). Keeping
239+
# the Alias item resolves to the imported charset *string* at
240+
# build time, baking a 1000-char literal into the constraint
241+
# regexp — a lookbehind that never matches.
242+
name = convert_alias_ref(c).name
243+
val = Interscript::Stdlib::ALIASES[name]
244+
val ? Interscript::Node::Item::Any.new(val) : Interscript::Node::Item::String.new("")
216245
else
217246
convert_item(c)
218247
end

‎lib/interscript/isc/normalizer.rb‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,12 @@ def normalize
3636
def scalar(node)
3737
return node if node.is_a?(::String) || node.nil?
3838
if node.key?(:string)
39-
# Parts are {char:} hashes in the general grammar; some
40-
# constructions (e.g. the rababa directive) capture bare
41-
# slices instead.
42-
Array(node[:string]).map { |part| part.is_a?(Hash) ? part[:char].to_s : part.to_s }.join
39+
# Parts are {char:} hashes and escape fragments ({dquote:},
40+
# {unicode:}, …) in the general grammar; some constructions
41+
# (e.g. the rababa directive) capture bare slices instead.
42+
# Escapes decode via the shared fold — folding only :char here
43+
# silently dropped every escaped character.
44+
Interscript::Isc::Transform.decode_string_parts(node[:string])
4345
elsif node.key?(:identifier)
4446
node[:identifier].to_s
4547
elsif node.key?(:raw)

‎lib/interscript/isc/transform.rb‎

Lines changed: 46 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -16,33 +16,7 @@ class Transform < Parslet::Transform
1616
# StringValue.
1717
rule(string: simple(:s)) { Items::StringValue.new(s.to_s) }
1818
rule(string: sequence(:parts)) do
19-
combined = parts.map do |p|
20-
case p
21-
when Hash
22-
# Escape sequence fragment: e.g. {newline: "n"}, {unicode: "1234"},
23-
# {dquote: '"'}, {char: "a"}
24-
if p.key?(:char)
25-
p[:char].to_s
26-
elsif p.key?(:newline)
27-
"\n"
28-
elsif p.key?(:carriage_return)
29-
"\r"
30-
elsif p.key?(:tab)
31-
"\t"
32-
elsif p.key?(:dquote)
33-
'"'
34-
elsif p.key?(:backslash)
35-
"\\"
36-
elsif p.key?(:unicode)
37-
[p[:unicode].to_s.to_i(16)].pack("U")
38-
else
39-
p.to_s
40-
end
41-
else
42-
p.to_s
43-
end
44-
end.join
45-
Items::StringValue.new(combined)
19+
Items::StringValue.new(Transform.decode_string_parts(parts))
4620
end
4721
rule(char: simple(:c)) { c.to_s }
4822

@@ -55,6 +29,11 @@ class Transform < Parslet::Transform
5529
Items::AliasRef.new(q.to_s, map: n.to_s)
5630
}
5731
rule(alias: simple(:n)) { Items::AliasRef.new(n.to_s) }
32+
# Bare identifiers inside any(...) lists parse as :alias_ref (the
33+
# :alias key only covers direct item position). Unmatched, the hash
34+
# degraded downstream into the debug string "[:alias_ref, \"name\"]".
35+
rule(alias_ref: {identifier: simple(:n)}) { Items::AliasRef.new(n.to_s) }
36+
rule(alias_ref: simple(:n)) { Items::AliasRef.new(n.to_s) }
5837
rule(ref: subtree(:h)) { Items::Capture.new(h[:digit].to_s.to_i) }
5938
rule(capture_inner: subtree(:inner)) { Items::CaptureGroup.new(Interscript::Isc::Transform.materialize_item(inner)) }
6039
rule(maybe_inner: subtree(:inner)) { Items::Maybe.new(Interscript::Isc::Transform.materialize_item(inner)) }
@@ -71,14 +50,52 @@ class Transform < Parslet::Transform
7150
hex.to_s
7251
end
7352

53+
# Shared decoder for the pieces of a :string capture — used by the
54+
# transform rules above and by the Normalizer's scalar folding. A
55+
# fragment without an escape key is a raw character slice.
56+
def self.decode_string_parts(parts)
57+
Array(parts).map do |p|
58+
case p
59+
when Hash
60+
if p.key?(:char)
61+
p[:char].to_s
62+
elsif p.key?(:newline)
63+
"\n"
64+
elsif p.key?(:carriage_return)
65+
"\r"
66+
elsif p.key?(:tab)
67+
"\t"
68+
elsif p.key?(:dquote)
69+
'"'
70+
elsif p.key?(:backslash)
71+
"\\"
72+
elsif p.key?(:unicode)
73+
[p[:unicode].to_s.to_i(16)].pack("U")
74+
else
75+
p.to_s
76+
end
77+
else
78+
p.to_s
79+
end
80+
end.join
81+
end
82+
7483
rule(lo: simple(:lo), hi: simple(:hi)) do
7584
Items::Range.new(lo.to_s, hi.to_s)
7685
end
7786
rule(single: simple(:s)) { Items::Set.from_string(s.to_s) }
87+
# List entries are already Items (strings, primitives, alias refs).
88+
# Stringifying here baked an object inspect into the char set.
7889
rule(list: sequence(:arr)) do
79-
Items::Set.from_strings(arr.map(&:to_s))
90+
Items::Set.new(arr)
91+
end
92+
rule(any: subtree(:h)) do
93+
# A single alias argument keeps set semantics: the legacy runtime
94+
# builds Any(Alias) which resolves through Stdlib (Any(nil) for
95+
# imported aliases). Unwrapped, the bare Alias resolves to the
96+
# imported charset string and compiles as a literal.
97+
h.is_a?(Items::AliasRef) ? Items::Set.new([h]) : h
8098
end
81-
rule(any: subtree(:h)) { h }
8299

83100
rule(concatenation: subtree(:parts)) do
84101
Items::Concat.from_parts(Array(parts))

‎spec/interscript/isc/node_adapter_spec.rb‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,3 +263,113 @@ def parse_and_adapt(src)
263263
expect(node.name).to eq("X:a-b:C-D:1")
264264
end
265265
end
266+
267+
RSpec.describe "NodeAdapter range handling" do
268+
it "keeps ISC ranges as native Any ranges — codepoint semantics, not string-succ expansion" do
269+
src = <<~ISC
270+
system "TEST:aze-Arab:Latn:2026" {
271+
metadata { name "T" }
272+
stage main {
273+
parallel { sub "q" "k" }
274+
sub { from any("a".."￿") to upcase before boundary }
275+
}
276+
}
277+
ISC
278+
node = Interscript::Isc::NodeAdapter.to_interscript_node(
279+
Interscript::Isc::DocumentBuilder.build(Interscript::Isc::Parser.parse(src))
280+
)
281+
upcase_rule = node.stages[:main].children
282+
.select { |c| c.is_a?(Interscript::Node::Rule::Sub) }
283+
.find { |r| r.to == :upcase }
284+
# A Ruby String range expands via String#succ (a, b, ..., z, aa, ab …)
285+
# which never reaches non-ASCII codepoints — the range must stay a range.
286+
expect(upcase_rule.from.value).to be_a(Range)
287+
interp = Interscript::Interpreter.new
288+
interp.compile(node)
289+
expect(interp.call("īş")).to eq("Īş")
290+
end
291+
end
292+
293+
RSpec.describe "NodeAdapter any-list constraints" do
294+
it "keeps primitives as aliases, never their inspect" do
295+
tree = Interscript::Isc::Parser.parse(
296+
'system "x" { stage main { sub { from "ം" to "m" after any([boundary, "‌", "‍"]) } } }'
297+
)
298+
doc = Interscript::Isc::DocumentBuilder.build(tree)
299+
node = Interscript::Isc::NodeAdapter.to_interscript_node(doc)
300+
stage = Interscript::Interpreter::Stage.new(node, "")
301+
re = stage.send(:build_regexp, node.stages[:main].children.first)
302+
# The constraint must match boundary positions — an inspect leak
303+
# turns the lookahead into a garbage char class.
304+
expect(re).not_to include("Primitive")
305+
expect("പ്പം ഹ").to match(Regexp.new(re))
306+
end
307+
end
308+
309+
RSpec.describe "NodeAdapter string escapes" do
310+
it "decodes escape sequences in test strings" do
311+
src = <<~'ISC'
312+
system "T:a-b:C-D:1" {
313+
metadata { name "T" }
314+
tests {
315+
"pod\"ezd" -> "p\"ezd"
316+
"a\tb\nc" -> "déjà"
317+
}
318+
stage main { sub "a" "b" }
319+
}
320+
ISC
321+
node = Interscript::Isc::NodeAdapter.to_interscript_node(
322+
Interscript::Isc::DocumentBuilder.build(Interscript::Isc::Parser.parse(src))
323+
)
324+
tests = node.tests.data
325+
expect(tests[0]).to eq(['pod"ezd', 'p"ezd'])
326+
expect(tests[1]).to eq(["a\tb\nc", "déjà"])
327+
end
328+
end
329+
330+
RSpec.describe "NodeAdapter alias refs in constraints" do
331+
it "compiles bare alias refs in sets like the legacy DSL — no debug dump, no literal-charset lookbehind" do
332+
src = <<~ISC
333+
system "t:g-c:C-D:1" {
334+
metadata { name "T" }
335+
stage main {
336+
sub {
337+
from "γ"
338+
to "n"
339+
before any(greek)
340+
after any("κΚ") + any(greek)
341+
not_before boundary
342+
}
343+
}
344+
}
345+
ISC
346+
node = Interscript::Isc::NodeAdapter.to_interscript_node(
347+
Interscript::Isc::DocumentBuilder.build(Interscript::Isc::Parser.parse(src))
348+
)
349+
rule = node.stages[:main].children.first
350+
expect(rule.before).to be_a(Interscript::Node::Item::Any)
351+
expect(rule.before.inspect).not_to include("alias_ref")
352+
expect(rule.after.inspect).not_to include("alias_ref")
353+
# The legacy runtime compiles an imported alias in a set to a vacuous
354+
# fragment (Any(nil)); resolving it to the charset *string* would bake
355+
# a 1000-char literal into the lookbehind and never match.
356+
interp = Interscript::Interpreter.new
357+
interp.compile(node)
358+
expect(interp.call("αγκα")).to eq("αnκα")
359+
end
360+
end
361+
362+
RSpec.describe "NodeAdapter document name derivation" do
363+
it "names the document after the file, not the system code — transliteration addresses maps by file name" do
364+
tree = Interscript::Isc::Parser.parse(
365+
'system "TOTALLY:diff-Code:frm:File" { stage main { sub { from "a" to "b" } } }'
366+
)
367+
doc = Interscript::Isc::DocumentBuilder.build(tree, filename: "test-map.isc")
368+
node = Interscript::Isc::NodeAdapter.to_interscript_node(doc)
369+
# The Ruby compiler registers compiled maps under Document#name and
370+
# Maps.transliterate looks them up by the map name (file-derived).
371+
# Registering under the system code auto-creates an empty entry via
372+
# the Hash default block and crashes on stages[:main].call.
373+
expect(node.name).to eq("test-map")
374+
end
375+
end

‎spec/interscript_spec.rb‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,16 @@
77
# per-map failures are expected until the ISC-era conformance work
88
# completes — the sweep exists to MEASURE that gap, gated so CI stays
99
# green while it closes.
10+
# The sweep honors INTERSCRIPT_MAPS_PATH (like the corpus specs) and
11+
# enumerates through the load path — matching what locate() resolves.
12+
# Bare maps() would enumerate the installed gem's frozen 2.4.x .imp
13+
# corpus wherever the maps checkout isn't the active gem.
1014
maps = if ENV["ISC_SWEEP"]
11-
Interscript.maps(basename: false, select: mask)
15+
sweep_root = ENV.fetch("INTERSCRIPT_MAPS_PATH", "../maps/maps")
16+
unless Interscript.load_path.first == File.expand_path(sweep_root)
17+
Interscript.load_path.unshift(File.expand_path(sweep_root))
18+
end
19+
Interscript.maps(basename: false, select: mask, load_path: true)
1220
else
1321
legacy_maps(select: mask)
1422
end

0 commit comments

Comments
 (0)