feat: Interscript::ML — built-in neural runtime (secryst crystal folded in)

.github/workflows/assets.yml

@@ -10,7 +10,7 @@ jobs:
   assets:
     runs-on: ubuntu-latest
     env:
-      BUNDLE_WITHOUT: "secryst:jsexec"
+      BUNDLE_WITHOUT: "jsexec"
       SKIP_JS: "1"
     steps:
       - name: Checkout monorepo
@@ -30,7 +30,7 @@ jobs:
 
       - name: Install bundle
         working-directory: ruby
-        run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst
+        run: bundle install --jobs 4 --retry 3 --with jsexec
 
       - name: Generate visualization JSON
         working-directory: ruby

.github/workflows/rake.yml

@@ -16,7 +16,7 @@ jobs:
         ruby: ["3.3", "3.4"]
         os: [ubuntu-latest, windows-latest, macos-latest]
     env:
-      BUNDLE_WITHOUT: "secryst"
+      BUNDLE_WITHOUT: "jsexec"
       SKIP_JS: "1"
       ISC_SWEEP: "1"
       # The ISC corpus sweep: every map's own tests through every

.github/workflows/release.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Install gems
         working-directory: ruby
-        run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst
+        run: bundle install --jobs 4 --retry 3 --with jsexec
 
       - name: Test Ruby package
         working-directory: ruby

Gemfile

@@ -16,12 +16,10 @@ else
   gem "interscript-maps"
 end
 
-group :secryst do
-  if File.exist?("../../secryst")
-    gem "secryst", path: "../../secryst"
-  else
-    gem "secryst"
-  end
+# The neural layer (Interscript::ML) - optional at runtime, loaded for specs
+group :ml do
+  gem "rubyzip"
+  gem "onnxruntime"
 end
 
 gem "regexp_parser"

lib/interscript.rb

@@ -12,6 +12,7 @@ module Interscript
   autoload :Node, "interscript/node"
   autoload :Detector, "interscript/detector"
   autoload :Isc, "interscript/isc"
+  autoload :ML, "interscript/ml"
 
   # An error caused by a lack of some map
   class MapNotFoundError < StandardError; end

lib/interscript/ml.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# The neural layer: IMF v1 artifact runtime + index provisioning for
+# byte-level ByT5 models, ported from the secryst crystal (1.1.1) into
+# interscript proper. Runtime dependencies rubyzip and onnxruntime are
+# OPTIONAL: they load lazily with a helpful error when absent.
+#
+#   Interscript::ML::Translator.new(model_file: "khm-latn-1.0")
+#     .translate("ភាសា")
+module Interscript::ML
+  autoload :IMF, "interscript/ml/imf"
+  autoload :Provisioning, "interscript/ml/provisioning"
+  autoload :Translator, "interscript/ml/translator"
+  autoload :Model, "interscript/ml/model"
+  autoload :Vocab, "interscript/ml/vocab"
+  autoload :Byt5Onnx, "interscript/ml/byt5_onnx"
+
+  # Expected to be loaded before IMF/Byt5Onnx entry points.
+  def self.require_optional!(gem_name, feature: gem_name)
+    require feature
+  rescue LoadError => e
+    raise Interscript::ExternalUtilError,
+      "the neural layer needs the #{gem_name} gem: gem install #{gem_name} (or use a bundle including it)"
+  end
+end

lib/interscript/ml/byt5_onnx.rb

@@ -0,0 +1,102 @@
+require 'tmpdir'
+
+module Interscript::ML
+  # Byte-level seq2seq (ByT5 family) inference over ONNX sessions from an
+  # IMF v1 model zip: sha256-verified graphs, greedy KV-cache decode when
+  # the zip ships decoder-kv.onnx (default), plain full-recompute
+  # fallback otherwise. Tokenization is the canonical ByT5 table
+  # (byte b -> id b+3, trailing EOS) — see Interscript::ML::IMF.
+  class Byt5Onnx
+    def initialize(zip_path)
+      Interscript::ML.require_optional!("onnxruntime")
+      @manifest = IMF.manifest(zip_path)
+      graphs = IMF.verify_and_read(zip_path)
+      @tmpdir = Dir.mktmpdir('secryst-imf')
+      @encoder = build_session(graphs, 'encoder.onnx')
+      @kv = @manifest['decoder'] == 'kv' && graphs.key?('decoder-kv.onnx')
+      @decoder = build_session(graphs, @kv ? 'decoder-kv.onnx' : 'decoder.onnx')
+      @pasts = @kv ? zero_pasts : {}
+    end
+
+    def translate(text, max_seq_length: 256)
+      ids = IMF.encode(text)
+      return '' if ids.length == 1 # only the trailing EOS: empty input
+
+      hidden = @encoder.predict({ input_ids: [ids] })['last_hidden_state']
+      tokens = @kv ? greedy_kv(hidden, max_seq_length) : greedy_plain(hidden, max_seq_length)
+      IMF.decode(tokens)
+    end
+
+    def id
+      @manifest['id']
+    end
+
+    private
+
+    # The onnxruntime gem loads from paths, not bytes: verified bytes go
+    # to a private tmpdir.
+    def build_session(graphs, name)
+      path = File.join(@tmpdir, name)
+      File.binwrite(path, graphs.fetch(name))
+      OnnxRuntime::Model.new(path)
+    end
+
+    def zero_pasts
+      @decoder.inputs.each_with_object({}) do |meta, pasts|
+        next unless meta[:name].start_with?('past_')
+        shape = meta[:shape] # [batch, heads, past_seq, d_kv]; dynamic dims are Strings
+        heads = shape[1].is_a?(Integer) ? shape[1] : 4
+        d_kv = shape[3].is_a?(Integer) ? shape[3] : 8
+        # Numo carries the explicit [1, heads, 0, d_kv] shape through;
+        # nested Ruby arrays cannot express a zero-length dim.
+        pasts[meta[:name]] = Numo::DFloat.zeros(1, heads, 0, d_kv)
+      end
+    end
+
+    def argmax(row)
+      best = 0
+      best_val = -Float::INFINITY
+      row.each_with_index do |value, index|
+        if value > best_val
+          best_val = value
+          best = index
+        end
+      end
+      best
+    end
+
+    def greedy_kv(hidden, max_seq_length)
+      pasts = @pasts.dup
+      current = [IMF::PAD_ID]
+      generated = []
+      max_seq_length.times do
+        results = @decoder.predict(
+          { input_ids: [current], encoder_hidden_states: hidden }.merge(pasts)
+        )
+        token = argmax(results['logits'].first.last)
+        break if token == IMF::EOS_ID
+
+        generated << token
+        pasts = pasts.keys.to_h { |name| [name, results[name.sub('past_', 'present_')]] }
+        current = [token]
+      end
+      generated
+    end
+
+    def greedy_plain(hidden, max_seq_length)
+      decoder_ids = [IMF::PAD_ID]
+      generated = []
+      max_seq_length.times do
+        logits = @decoder.predict(
+          { input_ids: [decoder_ids], encoder_hidden_states: hidden }
+        )['logits']
+        token = argmax(logits.first.last)
+        break if token == IMF::EOS_ID
+
+        generated << token
+        decoder_ids = decoder_ids + [token]
+      end
+      generated
+    end
+  end
+end

lib/interscript/ml/imf.rb

@@ -0,0 +1,182 @@
+require "yaml"
+require "digest/sha2"
+require "fileutils"
+require "open-uri"
+
+module Interscript::ML
+  # Interscript Model Format v1 — the byte-level runtime contract shared
+  # with the Python (interscript-ml) and TypeScript (npm: secryst)
+  # runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3,
+  # trailing EOS; pad=0, unk=2. Ids are NOT raw byte values.
+  module IMF
+    BYTE_OFFSET = 3
+    PAD_ID = 0
+    EOS_ID = 1
+    UNK_ID = 2
+
+    DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml"
+
+    class FormatError < StandardError; end
+    class RegistryError < StandardError; end
+
+    class << self
+      def require_zip!
+        Interscript::ML.require_optional!("rubyzip", feature: "zip")
+      end
+
+      def encode(text)
+        text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID]
+      end
+
+      def decode(token_ids)
+        out = +""
+        token_ids.each do |token|
+          break if token == EOS_ID
+          next if token == PAD_ID || token == UNK_ID
+          out << ((token - BYTE_OFFSET) % 256).chr
+        end
+        out.force_encoding(Encoding::UTF_8)
+      end
+
+      def manifest(zip_path)
+        require_zip!
+        Zip::File.open(zip_path) do |zf|
+          raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml")
+          meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false)
+          raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1"
+          if meta["tokenizer"] != "bytes"
+            raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only"
+          end
+          %w[encoder.onnx decoder.onnx].each do |required|
+            raise FormatError, "missing #{required}" unless zf.find_entry(required)
+          end
+          meta
+        end
+      end
+
+      # Reads every .onnx member after verifying its sha256 against the
+      # manifest — corrupt zips fail loudly, before any session loads.
+      def verify_and_read(zip_path)
+        meta = manifest(zip_path)
+        sha = meta.fetch("sha256", {})
+        graphs = {}
+        require_zip!
+        Zip::File.open(zip_path) do |zf|
+          zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry|
+            recorded = sha[entry.name]
+            raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded
+            bytes = entry.get_input_stream.read
+            actual = Digest::SHA256.hexdigest(bytes)
+            if actual != recorded
+              raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}"
+            end
+            graphs[entry.name] = bytes
+          end
+        end
+        graphs
+      end
+
+      def cache_dir
+        ENV["INTERSCRIPT_ML_CACHE"] || ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "interscript-ml")
+      end
+
+      # models.yaml resolution: cache hit (re-verified), or download ->
+      # verify whole-file sha256 -> atomic install into the cache.
+      # Entries with `parts` (GitHub's 2 GiB per-asset cap) are streamed
+      # in order, each part sha256-verified as it lands, then the
+      # assembled file is checked against the whole-file sha256.
+      def resolve(model_id, index_url: nil)
+        source = index_url || ENV["INTERSCRIPT_ML_INDEX"] || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL
+        entries = load_index(source)
+        entry = entries[model_id]
+        raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry
+
+        target = File.join(cache_dir, "models", model_id, entry["filename"])
+        if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"]
+          return target
+        end
+
+        FileUtils.mkdir_p(File.dirname(target))
+        tmp = target + ".part.#{Process.pid}"
+        if entry["parts"]
+          download_parts(entry, tmp)
+        else
+          channel = entry["url"]
+          if channel.start_with?("file://")
+            FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp)
+          else
+            URI.open(channel) { |remote| IO.copy_stream(remote, tmp) }
+          end
+        end
+        actual = Digest::SHA256.file(tmp).hexdigest
+        unless actual == entry["sha256"]
+          File.delete(tmp)
+          raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}"
+        end
+        File.rename(tmp, target)
+        target
+      end
+
+      private
+
+      def download_parts(entry, tmp)
+        File.open(tmp, "wb") do |out|
+          entry["parts"].each_with_index do |part, index|
+            digest = Digest::SHA256.new
+            source = part["url"].start_with?("file://") ? part["url"].sub(%r{\Afile://}, "") : part["url"]
+            open_stream = lambda do |io|
+              while (chunk = io.read(1024 * 1024))
+                out.write(chunk)
+                digest.update(chunk)
+              end
+            end
+            if part["url"].start_with?("file://")
+              File.open(source, "rb", &open_stream)
+            else
+              URI.open(source, "rb", &open_stream)
+            end
+            unless digest.hexdigest == part["sha256"]
+              raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}"
+            end
+          end
+        end
+      rescue StandardError
+        File.delete(tmp) if File.file?(tmp)
+        raise
+      end
+
+      def load_index(source)
+        text = if source.start_with?("http://", "https://")
+          fetch_index_with_sidecar(source)
+        else
+          File.read(source)
+        end
+        raw = YAML.safe_load(text, permitted_classes: [], aliases: false)
+        raise RegistryError, "index must have version: 1" if raw["version"] != 1
+        raw.fetch("models", {})
+      end
+      public :load_index
+
+      # HTTP sources must ship a sibling .sha256 sidecar; verify the
+      # body against it before parsing (GitHub Releases index assets do).
+      def fetch_index_with_sidecar(source)
+        body = URI.open(source, "rb") { |remote| remote.read }
+        sidecar =
+          begin
+            URI.open("#{source}.sha256") { |remote| remote.read }
+          rescue OpenURI::HTTPError => e
+            raise RegistryError, "index sha256 sidecar missing: #{source}.sha256 (#{e.message})"
+          end
+        expected = sidecar.to_s.strip.split(/\s+/).first.to_s
+        unless expected.match?(/\A[0-9a-fA-F]{64}\z/)
+          raise RegistryError, "index sha256 sidecar malformed: #{source}.sha256"
+        end
+        actual = Digest::SHA256.hexdigest(body)
+        unless actual == expected.downcase
+          raise RegistryError, "index sha256 mismatch: got #{actual}, sidecar says #{expected.downcase}"
+        end
+        body.force_encoding(Encoding::UTF_8)
+      end
+    end
+  end
+end