Skip to content

Commit 6a27046

Browse files
authored
Merge pull request #774 from interscript/feat/python-bridge-isc-generator
feat: Python bridge via ISC generation
2 parents d208772 + 287ddc5 commit 6a27046

6 files changed

Lines changed: 324 additions & 12 deletions

File tree

‎.github/workflows/rake.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,13 @@ jobs:
3838
- name: Install Python helper and gems
3939
working-directory: ruby
4040
run: |
41-
pip install regex
4241
bundle config set --local with jsexec
4342
bundle install
43+
# PyCall binds whichever libpython it was built against, which
44+
# is not necessarily the python that bare `pip` targets —
45+
# install the regex helper into that interpreter.
46+
PYBIN=$(bundle exec ruby -e 'require "pycall"; v = PyCall.sys.version[/\d+\.\d+/]; puts "python#{v}"')
47+
"$PYBIN" -m pip install regex
4448
4549
- name: RSpec
4650
working-directory: ruby

‎lib/interscript/compiler/python.rb‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -290,10 +290,14 @@ def compile_item i, doc = @map, target = nil
290290
class << self
291291
attr_accessor :maps_loaded
292292
attr_accessor :ctx
293+
attr_accessor :maps_dir
293294
end
294295

295296
def load
296-
if !self.class.maps_loaded[@map.name]
297+
# Keyed by document identity: synthetic spec documents reuse names
298+
# (example-1, example-2, ...), and a stale engine under a reused
299+
# name would silently serve the previous document.
300+
if self.class.maps_loaded[@map.name] != @map.object_id
297301
@map.dependencies.each do |dep|
298302
dep = dep.full_name
299303
if !self.class.maps_loaded[dep]
@@ -302,27 +306,42 @@ def load
302306
end
303307

304308
ctx = self.class.ctx
305-
python_src_path = File.join(__dir__, "..", "..", "..", "..", "python", "src")
309+
python_src_path = ENV["INTERSCRIPT_PYTHON_SRC"] ||
310+
File.join(__dir__, "..", "..", "..", "..", "python", "src")
306311
unless ctx
307-
PyCall.sys.path.append(python_src_path)
312+
# Insert at 0: an editable install of interscript-python may
313+
# shadow the monorepo checkout on sys.path.
314+
PyCall.sys.path.insert(0, python_src_path)
308315
self.class.ctx = PyCall.import_module("interscript")
316+
ctx = self.class.ctx
309317
end
310-
# puts @code
311-
begin
312-
Dir.mkdir("#{python_src_path}/interscript/maps")
313-
rescue
314-
nil
318+
# The rewritten python runtime executes ISC sources from its load
319+
# paths, so in-memory documents cross the boundary as generated
320+
# ISC text in a per-process temp dir — never inside the python
321+
# package tree.
322+
unless self.class.maps_dir
323+
require "tmpdir" unless defined?(Dir.mktmpdir)
324+
self.class.maps_dir = Dir.mktmpdir("interscript-python")
325+
ctx.add_load_path(self.class.maps_dir)
315326
end
316-
File.write("#{python_src_path}/interscript/maps/#{@map.name}.py", @code)
327+
File.write(File.join(self.class.maps_dir, "#{@map.name}.isc"),
328+
Interscript::Isc::Generator.generate(@map))
317329
self.class.ctx.load_map(@map.name)
318330

319-
self.class.maps_loaded[@map.name] = true
331+
if self.class.maps_loaded.key?(@map.name)
332+
self.class.ctx.unload_map(@map.name)
333+
end
334+
self.class.maps_loaded[@map.name] = @map.object_id
320335
end
321336
end
322337

323338
def call(str, stage = :main)
324339
load
325-
self.class.ctx.transliterate(@map.name, str, stage.to_s)
340+
unless stage.to_s == "main"
341+
raise Interscript::MapLogicError,
342+
"the python runtime transliterates the whole map; stage #{stage} can't be selected"
343+
end
344+
self.class.ctx.transliterate(@map.name, str)
326345
end
327346

328347
def self.read_debug_data

‎lib/interscript/isc.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ module Isc
1515
autoload :YamlBridge, "interscript/isc/yaml_bridge"
1616
autoload :Serializer, "interscript/isc/serializer"
1717
autoload :Normalizer, "interscript/isc/normalizer"
18+
autoload :Generator, "interscript/isc/generator"
1819

1920
SCHEMA_VERSION = 1
2021

‎lib/interscript/isc/generator.rb‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# frozen_string_literal: true
2+
3+
module Interscript
4+
module Isc
5+
# Serializes a Node::Document back to ISC source — the inverse of
6+
# Parser + DocumentBuilder + NodeAdapter. The Python compiler
7+
# bridge generates ISC from in-memory documents because the
8+
# rewritten python runtime executes ISC sources, not compiled
9+
# modules. Every construct the generator emits must round-trip:
10+
# spec/interscript/isc/generator_spec.rb pins behavior equality.
11+
class Generator
12+
def self.generate(doc)
13+
new(doc).generate
14+
end
15+
16+
def initialize(doc)
17+
@doc = doc
18+
end
19+
20+
def generate
21+
out = "system #{quote(@doc.name.to_s)} {\n"
22+
out << generate_aliases
23+
out << generate_dependencies
24+
@doc.stages.each do |name, group|
25+
out << " stage #{name} {\n" << generate_group(group, 2) << " }\n"
26+
end
27+
out << "}\n"
28+
end
29+
30+
private
31+
32+
def generate_aliases
33+
return "" if @doc.aliases.empty?
34+
35+
out = +" aliases {\n"
36+
@doc.aliases.each_value do |alias_def|
37+
out << " #{alias_def.name} = #{item(alias_def.data)}\n"
38+
end
39+
out << " }\n"
40+
end
41+
42+
def generate_dependencies
43+
return "" if @doc.dependencies.empty?
44+
45+
out = +""
46+
@doc.dependencies.each do |dep|
47+
out << " dependency #{quote(dep.full_name.to_s)}"
48+
out << " as #{dep.name}" if dep.name
49+
out << "\n"
50+
end
51+
out
52+
end
53+
54+
def generate_group(group, indent)
55+
Array(group.children).map { |child| emit(child, indent) }.join
56+
end
57+
58+
def emit(child, indent)
59+
pad = " " * indent
60+
case child
61+
when Interscript::Node::Group::Parallel
62+
"#{pad}parallel {\n#{generate_group(child, indent + 2)}#{pad}}\n"
63+
when Interscript::Node::Group
64+
generate_group(child, indent)
65+
when Interscript::Node::Rule::Sub
66+
rule(child, pad)
67+
when Interscript::Node::Rule::Run
68+
"#{pad}run #{run_target(child)}\n"
69+
when Interscript::Node::Rule::Funcall
70+
funcall(child, pad)
71+
else
72+
raise UnsupportedConstruct, "cannot generate #{child.class} as a stage item"
73+
end
74+
end
75+
76+
def rule(sub, pad)
77+
out = "#{pad}sub {\n"
78+
out << "#{pad} from #{item(sub.from)}\n"
79+
out << "#{pad} to #{target(sub.to)}\n"
80+
{before: sub.before, after: sub.after,
81+
not_before: sub.not_before, not_after: sub.not_after}.each do |kw, value|
82+
out << "#{pad} #{kw} #{item(value)}\n" if value
83+
end
84+
out << "#{pad}}\n"
85+
end
86+
87+
def target(to)
88+
return to.to_s if %i[upcase downcase].include?(to)
89+
90+
item(to)
91+
end
92+
93+
def run_target(run)
94+
stage = run.stage
95+
stage.map ? "map.#{stage.map}.stage.#{stage.name}" : "stage.#{stage.name}"
96+
end
97+
98+
def funcall(rule, pad)
99+
unless rule.kwargs.empty?
100+
raise UnsupportedConstruct,
101+
"cannot generate #{rule.name}(...) with kwargs as ISC"
102+
end
103+
104+
"#{pad}#{rule.name}\n"
105+
end
106+
107+
# -- items --
108+
109+
def item(i)
110+
case i
111+
when Interscript::Node::Item::String
112+
quote(i.data)
113+
when Interscript::Node::Item::Any
114+
any(i)
115+
when Interscript::Node::Item::Alias
116+
identifier(i.name)
117+
when Interscript::Node::Item::CaptureGroup
118+
"capture(#{item(i.data)})"
119+
when Interscript::Node::Item::CaptureRef
120+
"ref(#{i.id})"
121+
when Interscript::Node::Item::Group
122+
i.children.map { |c| item(c) }.join(" + ")
123+
when Interscript::Node::Item::Maybe
124+
"maybe(#{item(i.data)})"
125+
when Interscript::Node::Item::Some
126+
"some(#{item(i.data)})"
127+
when ::String
128+
quote(i)
129+
when ::Symbol
130+
identifier(i)
131+
else
132+
raise UnsupportedConstruct, "cannot generate #{i.class} as an item"
133+
end
134+
end
135+
136+
def any(node)
137+
case node.value
138+
when ::String
139+
"any(#{quote(node.value)})"
140+
when ::Range
141+
"any(#{quote(node.value.first)}..#{quote(node.value.last)})"
142+
when ::Array
143+
inner = node.value.map { |v| item(v) }
144+
return "any(#{inner.first})" if inner.size == 1
145+
146+
"any([#{inner.join(", ")}])"
147+
else
148+
raise UnsupportedConstruct, "cannot generate Any of #{node.value.class}"
149+
end
150+
end
151+
152+
def identifier(sym)
153+
name = sym.to_s
154+
raise UnsupportedConstruct, "#{name.inspect} is not a bare identifier" unless /\A[A-Za-z_][A-Za-z0-9_]*\z/.match?(name)
155+
156+
name
157+
end
158+
159+
ESCAPE_MAP = {
160+
"\\" => "\\\\",
161+
"\"" => "\\\""
162+
}.freeze
163+
164+
def quote(str)
165+
escaped = str.to_s.gsub(/[\\"]/) { |c| ESCAPE_MAP.fetch(c) }
166+
# Only control and DEL characters need escaping; both parsers
167+
# read raw UTF-8, including astral code points (surrogate-pair
168+
# \u escapes are NOT re-joined).
169+
escaped = escaped.each_char.map { |c|
170+
(c.ord < 0x20 || c.ord == 0x7f) ? format("\\u%04x", c.ord) : c
171+
}.join
172+
"\"" + escaped + "\""
173+
end
174+
175+
class UnsupportedConstruct < StandardError; end
176+
end
177+
end
178+
end
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# frozen_string_literal: true
2+
3+
# Generator round-trip: every document the DSL can build must survive a
4+
# generate → parse → adapt cycle with identical runtime behavior. This
5+
# is the contract the Python bridge relies on — the rewritten python
6+
# runtime executes ISC sources, so in-memory documents cross the
7+
# boundary as generated ISC text.
8+
require "spec_helper"
9+
10+
RSpec.describe Interscript::Isc::Generator do
11+
def round_trip(doc)
12+
isc = described_class.generate(doc)
13+
tree = Interscript::Isc::Parser.parse(isc)
14+
built = Interscript::Isc::DocumentBuilder.build(tree)
15+
Interscript::Isc::NodeAdapter.to_interscript_node(built)
16+
end
17+
18+
def round_trip_stage(&block)
19+
original = stage(&block)
20+
reparsed = round_trip(original)
21+
[original, reparsed]
22+
end
23+
24+
it "generates parseable ISC for a stage document" do
25+
doc = stage { sub "b", "e" }
26+
isc = described_class.generate(doc)
27+
expect(isc).to start_with(%(system "example-))
28+
expect { Interscript::Isc::Parser.parse(isc) }.not_to raise_error
29+
end
30+
31+
it "round-trips basic substitution" do
32+
original, reparsed = round_trip_stage { sub "b", "e" }
33+
expect(reparsed.call("abcd")).to eq(original.call("abcd"))
34+
end
35+
36+
it "round-trips substitution with all four constraints" do
37+
original, reparsed = round_trip_stage {
38+
sub "a", "A", before: space
39+
sub "a", "B", not_before: space
40+
sub "a", "C", after: space
41+
sub "a", "D", not_after: space
42+
}
43+
expect(reparsed.call("abcda abcda abada")).to eq(original.call("abcda abcda abada"))
44+
end
45+
46+
it "round-trips a casing target" do
47+
original, reparsed = round_trip_stage { sub "b", :upcase }
48+
expect(reparsed.call("aba")).to eq(original.call("aba"))
49+
end
50+
51+
it "round-trips parallel blocks" do
52+
original, reparsed = round_trip_stage {
53+
parallel {
54+
sub "ab", "X"
55+
sub "a", "Y"
56+
}
57+
}
58+
expect(reparsed.call("aba")).to eq(original.call("aba"))
59+
end
60+
61+
it "round-trips any() sets and ranges" do
62+
original, reparsed = round_trip_stage {
63+
sub any("abc"), "X"
64+
sub "z", "W", before: any("aeiou")
65+
}
66+
expect(reparsed.call("zaza zab")).to eq(original.call("zaza zab"))
67+
end
68+
69+
it "round-trips capture groups and references" do
70+
original, reparsed = round_trip_stage {
71+
sub capture("b") + capture("c"), ref(2) + ref(1)
72+
}
73+
expect(reparsed.call("xbc")).to eq(original.call("xbc"))
74+
end
75+
76+
it "round-trips document-level aliases" do
77+
original = document {
78+
aliases do
79+
def_alias :vowel, any("aeiou")
80+
end
81+
stage {
82+
sub "v", "V", before: vowel
83+
}
84+
}
85+
reparsed = round_trip(original)
86+
expect(reparsed.call("ava eve")).to eq(original.call("ava eve"))
87+
end
88+
89+
it "round-trips multiple stages with a stage run" do
90+
original = document {
91+
stage :first do
92+
sub "b", "e"
93+
end
94+
stage {
95+
run Interscript::Node::Item::Stage.new(:first)
96+
}
97+
}
98+
reparsed = round_trip(original)
99+
expect(reparsed.call("abcd")).to eq(original.call("abcd"))
100+
end
101+
end

‎spec/spec_helper.rb‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,15 @@
1919
# Disable RSpec exposing methods globally on `Module` and `main`
2020
config.disable_monkey_patching!
2121

22+
# dsl_stage_spec sets the $compiler global per example; without a
23+
# reset it leaks into every file that runs afterwards, silently
24+
# rerouting suites that never opted into a specific compiler.
25+
# standard:disable Style/GlobalVars (deliberate $DEBUG / -d-flag debug idiom)
26+
config.before(:example) do
27+
$compiler = nil
28+
end
29+
# standard:enable Style/GlobalVars
30+
2231
config.expect_with :rspec do |c|
2332
c.syntax = :expect
2433
end

0 commit comments

Comments
 (0)