From 5f3523e2df39918a95d8732d8c8e5feebf9c0a04 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 6 Oct 2026 16:13:08 +0800 Subject: [PATCH] feat: neural layer as Interscript::ML - the secryst crystal folds into interscript Ports the secryst gem's IMF runtime (imf/provisioning/translator/model/ vocab/byt5_onnx) as lib/interscript/ml with autoload entries; the SecrystAdapter now dispatches through the built-in runtime instead of requiring the external gem (rubyzip/onnxruntime load lazily with helpful ExternalUtilError guidance). Index pinned at index-v7; INTERSCRIPT_ML_INDEX/CACHE/DATA primary env vars with SECRYST_* legacy aliases. 13 ported ml specs + rewritten stdlib funcall spec (real-instance, offline via the fixture zip path short-circuit): 308 examples, 0 failures under the canonical SKIP_JS/SKIP_PYTHON run. CI bundles the :ml group (was :secryst) so specs get rubyzip+onnxruntime. --- .github/workflows/assets.yml | 4 +- .github/workflows/rake.yml | 2 +- .github/workflows/release.yml | 2 +- Gemfile | 10 +- lib/interscript.rb | 1 + lib/interscript/ml.rb | 25 ++ lib/interscript/ml/byt5_onnx.rb | 102 +++++++ lib/interscript/ml/imf.rb | 182 +++++++++++++ lib/interscript/ml/model.rb | 52 ++++ lib/interscript/ml/provisioning.rb | 189 +++++++++++++ lib/interscript/ml/translator.rb | 38 +++ lib/interscript/ml/vocab.rb | 48 ++++ .../stdlib/functions/secryst_adapter.rb | 14 +- spec/interscript/fixtures/tiny-imf.zip | Bin 0 -> 1149 bytes spec/interscript/ml/byt5_onnx_spec.rb | 17 ++ spec/interscript/ml/imf_spec.rb | 248 ++++++++++++++++++ spec/stdlib_functions_spec.rb | 15 +- 17 files changed, 919 insertions(+), 30 deletions(-) create mode 100644 lib/interscript/ml.rb create mode 100644 lib/interscript/ml/byt5_onnx.rb create mode 100644 lib/interscript/ml/imf.rb create mode 100644 lib/interscript/ml/model.rb create mode 100644 lib/interscript/ml/provisioning.rb create mode 100644 lib/interscript/ml/translator.rb create mode 100644 lib/interscript/ml/vocab.rb create mode 100644 spec/interscript/fixtures/tiny-imf.zip create mode 100644 spec/interscript/ml/byt5_onnx_spec.rb create mode 100644 spec/interscript/ml/imf_spec.rb diff --git a/.github/workflows/assets.yml b/.github/workflows/assets.yml index 1ec6e948..8686ed9f 100644 --- a/.github/workflows/assets.yml +++ b/.github/workflows/assets.yml @@ -10,7 +10,7 @@ jobs: assets: runs-on: ubuntu-latest env: - BUNDLE_WITHOUT: "secryst:jsexec" + BUNDLE_WITHOUT: "jsexec" SKIP_JS: "1" steps: - name: Checkout monorepo @@ -30,7 +30,7 @@ jobs: - name: Install bundle working-directory: ruby - run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst + run: bundle install --jobs 4 --retry 3 --with jsexec - name: Generate visualization JSON working-directory: ruby diff --git a/.github/workflows/rake.yml b/.github/workflows/rake.yml index a53e300b..d22ed01d 100644 --- a/.github/workflows/rake.yml +++ b/.github/workflows/rake.yml @@ -16,7 +16,7 @@ jobs: ruby: ["3.3", "3.4"] os: [ubuntu-latest, windows-latest, macos-latest] env: - BUNDLE_WITHOUT: "secryst" + BUNDLE_WITHOUT: "jsexec" SKIP_JS: "1" ISC_SWEEP: "1" # The ISC corpus sweep: every map's own tests through every diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76279b00..dae2bcaf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: - name: Install gems working-directory: ruby - run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst + run: bundle install --jobs 4 --retry 3 --with jsexec - name: Test Ruby package working-directory: ruby diff --git a/Gemfile b/Gemfile index a25f1058..6e505ed1 100644 --- a/Gemfile +++ b/Gemfile @@ -16,12 +16,10 @@ else gem "interscript-maps" end -group :secryst do - if File.exist?("../../secryst") - gem "secryst", path: "../../secryst" - else - gem "secryst" - end +# The neural layer (Interscript::ML) - optional at runtime, loaded for specs +group :ml do + gem "rubyzip" + gem "onnxruntime" end gem "regexp_parser" diff --git a/lib/interscript.rb b/lib/interscript.rb index 61216995..d5483bf8 100644 --- a/lib/interscript.rb +++ b/lib/interscript.rb @@ -12,6 +12,7 @@ module Interscript autoload :Node, "interscript/node" autoload :Detector, "interscript/detector" autoload :Isc, "interscript/isc" + autoload :ML, "interscript/ml" # An error caused by a lack of some map class MapNotFoundError < StandardError; end diff --git a/lib/interscript/ml.rb b/lib/interscript/ml.rb new file mode 100644 index 00000000..b19a8f4a --- /dev/null +++ b/lib/interscript/ml.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# The neural layer: IMF v1 artifact runtime + index provisioning for +# byte-level ByT5 models, ported from the secryst crystal (1.1.1) into +# interscript proper. Runtime dependencies rubyzip and onnxruntime are +# OPTIONAL: they load lazily with a helpful error when absent. +# +# Interscript::ML::Translator.new(model_file: "khm-latn-1.0") +# .translate("ភាសា") +module Interscript::ML + autoload :IMF, "interscript/ml/imf" + autoload :Provisioning, "interscript/ml/provisioning" + autoload :Translator, "interscript/ml/translator" + autoload :Model, "interscript/ml/model" + autoload :Vocab, "interscript/ml/vocab" + autoload :Byt5Onnx, "interscript/ml/byt5_onnx" + + # Expected to be loaded before IMF/Byt5Onnx entry points. + def self.require_optional!(gem_name, feature: gem_name) + require feature + rescue LoadError => e + raise Interscript::ExternalUtilError, + "the neural layer needs the #{gem_name} gem: gem install #{gem_name} (or use a bundle including it)" + end +end diff --git a/lib/interscript/ml/byt5_onnx.rb b/lib/interscript/ml/byt5_onnx.rb new file mode 100644 index 00000000..63fe0ad4 --- /dev/null +++ b/lib/interscript/ml/byt5_onnx.rb @@ -0,0 +1,102 @@ +require 'tmpdir' + +module Interscript::ML + # Byte-level seq2seq (ByT5 family) inference over ONNX sessions from an + # IMF v1 model zip: sha256-verified graphs, greedy KV-cache decode when + # the zip ships decoder-kv.onnx (default), plain full-recompute + # fallback otherwise. Tokenization is the canonical ByT5 table + # (byte b -> id b+3, trailing EOS) — see Interscript::ML::IMF. + class Byt5Onnx + def initialize(zip_path) + Interscript::ML.require_optional!("onnxruntime") + @manifest = IMF.manifest(zip_path) + graphs = IMF.verify_and_read(zip_path) + @tmpdir = Dir.mktmpdir('secryst-imf') + @encoder = build_session(graphs, 'encoder.onnx') + @kv = @manifest['decoder'] == 'kv' && graphs.key?('decoder-kv.onnx') + @decoder = build_session(graphs, @kv ? 'decoder-kv.onnx' : 'decoder.onnx') + @pasts = @kv ? zero_pasts : {} + end + + def translate(text, max_seq_length: 256) + ids = IMF.encode(text) + return '' if ids.length == 1 # only the trailing EOS: empty input + + hidden = @encoder.predict({ input_ids: [ids] })['last_hidden_state'] + tokens = @kv ? greedy_kv(hidden, max_seq_length) : greedy_plain(hidden, max_seq_length) + IMF.decode(tokens) + end + + def id + @manifest['id'] + end + + private + + # The onnxruntime gem loads from paths, not bytes: verified bytes go + # to a private tmpdir. + def build_session(graphs, name) + path = File.join(@tmpdir, name) + File.binwrite(path, graphs.fetch(name)) + OnnxRuntime::Model.new(path) + end + + def zero_pasts + @decoder.inputs.each_with_object({}) do |meta, pasts| + next unless meta[:name].start_with?('past_') + shape = meta[:shape] # [batch, heads, past_seq, d_kv]; dynamic dims are Strings + heads = shape[1].is_a?(Integer) ? shape[1] : 4 + d_kv = shape[3].is_a?(Integer) ? shape[3] : 8 + # Numo carries the explicit [1, heads, 0, d_kv] shape through; + # nested Ruby arrays cannot express a zero-length dim. + pasts[meta[:name]] = Numo::DFloat.zeros(1, heads, 0, d_kv) + end + end + + def argmax(row) + best = 0 + best_val = -Float::INFINITY + row.each_with_index do |value, index| + if value > best_val + best_val = value + best = index + end + end + best + end + + def greedy_kv(hidden, max_seq_length) + pasts = @pasts.dup + current = [IMF::PAD_ID] + generated = [] + max_seq_length.times do + results = @decoder.predict( + { input_ids: [current], encoder_hidden_states: hidden }.merge(pasts) + ) + token = argmax(results['logits'].first.last) + break if token == IMF::EOS_ID + + generated << token + pasts = pasts.keys.to_h { |name| [name, results[name.sub('past_', 'present_')]] } + current = [token] + end + generated + end + + def greedy_plain(hidden, max_seq_length) + decoder_ids = [IMF::PAD_ID] + generated = [] + max_seq_length.times do + logits = @decoder.predict( + { input_ids: [decoder_ids], encoder_hidden_states: hidden } + )['logits'] + token = argmax(logits.first.last) + break if token == IMF::EOS_ID + + generated << token + decoder_ids = decoder_ids + [token] + end + generated + end + end +end diff --git a/lib/interscript/ml/imf.rb b/lib/interscript/ml/imf.rb new file mode 100644 index 00000000..40bb2772 --- /dev/null +++ b/lib/interscript/ml/imf.rb @@ -0,0 +1,182 @@ +require "yaml" +require "digest/sha2" +require "fileutils" +require "open-uri" + +module Interscript::ML + # Interscript Model Format v1 — the byte-level runtime contract shared + # with the Python (interscript-ml) and TypeScript (npm: secryst) + # runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3, + # trailing EOS; pad=0, unk=2. Ids are NOT raw byte values. + module IMF + BYTE_OFFSET = 3 + PAD_ID = 0 + EOS_ID = 1 + UNK_ID = 2 + + DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml" + + class FormatError < StandardError; end + class RegistryError < StandardError; end + + class << self + def require_zip! + Interscript::ML.require_optional!("rubyzip", feature: "zip") + end + + def encode(text) + text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID] + end + + def decode(token_ids) + out = +"" + token_ids.each do |token| + break if token == EOS_ID + next if token == PAD_ID || token == UNK_ID + out << ((token - BYTE_OFFSET) % 256).chr + end + out.force_encoding(Encoding::UTF_8) + end + + def manifest(zip_path) + require_zip! + Zip::File.open(zip_path) do |zf| + raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml") + meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false) + raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1" + if meta["tokenizer"] != "bytes" + raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only" + end + %w[encoder.onnx decoder.onnx].each do |required| + raise FormatError, "missing #{required}" unless zf.find_entry(required) + end + meta + end + end + + # Reads every .onnx member after verifying its sha256 against the + # manifest — corrupt zips fail loudly, before any session loads. + def verify_and_read(zip_path) + meta = manifest(zip_path) + sha = meta.fetch("sha256", {}) + graphs = {} + require_zip! + Zip::File.open(zip_path) do |zf| + zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry| + recorded = sha[entry.name] + raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded + bytes = entry.get_input_stream.read + actual = Digest::SHA256.hexdigest(bytes) + if actual != recorded + raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}" + end + graphs[entry.name] = bytes + end + end + graphs + end + + def cache_dir + ENV["INTERSCRIPT_ML_CACHE"] || ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "interscript-ml") + end + + # models.yaml resolution: cache hit (re-verified), or download -> + # verify whole-file sha256 -> atomic install into the cache. + # Entries with `parts` (GitHub's 2 GiB per-asset cap) are streamed + # in order, each part sha256-verified as it lands, then the + # assembled file is checked against the whole-file sha256. + def resolve(model_id, index_url: nil) + source = index_url || ENV["INTERSCRIPT_ML_INDEX"] || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL + entries = load_index(source) + entry = entries[model_id] + raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry + + target = File.join(cache_dir, "models", model_id, entry["filename"]) + if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"] + return target + end + + FileUtils.mkdir_p(File.dirname(target)) + tmp = target + ".part.#{Process.pid}" + if entry["parts"] + download_parts(entry, tmp) + else + channel = entry["url"] + if channel.start_with?("file://") + FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp) + else + URI.open(channel) { |remote| IO.copy_stream(remote, tmp) } + end + end + actual = Digest::SHA256.file(tmp).hexdigest + unless actual == entry["sha256"] + File.delete(tmp) + raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}" + end + File.rename(tmp, target) + target + end + + private + + def download_parts(entry, tmp) + File.open(tmp, "wb") do |out| + entry["parts"].each_with_index do |part, index| + digest = Digest::SHA256.new + source = part["url"].start_with?("file://") ? part["url"].sub(%r{\Afile://}, "") : part["url"] + open_stream = lambda do |io| + while (chunk = io.read(1024 * 1024)) + out.write(chunk) + digest.update(chunk) + end + end + if part["url"].start_with?("file://") + File.open(source, "rb", &open_stream) + else + URI.open(source, "rb", &open_stream) + end + unless digest.hexdigest == part["sha256"] + raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}" + end + end + end + rescue StandardError + File.delete(tmp) if File.file?(tmp) + raise + end + + def load_index(source) + text = if source.start_with?("http://", "https://") + fetch_index_with_sidecar(source) + else + File.read(source) + end + raw = YAML.safe_load(text, permitted_classes: [], aliases: false) + raise RegistryError, "index must have version: 1" if raw["version"] != 1 + raw.fetch("models", {}) + end + public :load_index + + # HTTP sources must ship a sibling .sha256 sidecar; verify the + # body against it before parsing (GitHub Releases index assets do). + def fetch_index_with_sidecar(source) + body = URI.open(source, "rb") { |remote| remote.read } + sidecar = + begin + URI.open("#{source}.sha256") { |remote| remote.read } + rescue OpenURI::HTTPError => e + raise RegistryError, "index sha256 sidecar missing: #{source}.sha256 (#{e.message})" + end + expected = sidecar.to_s.strip.split(/\s+/).first.to_s + unless expected.match?(/\A[0-9a-fA-F]{64}\z/) + raise RegistryError, "index sha256 sidecar malformed: #{source}.sha256" + end + actual = Digest::SHA256.hexdigest(body) + unless actual == expected.downcase + raise RegistryError, "index sha256 mismatch: got #{actual}, sidecar says #{expected.downcase}" + end + body.force_encoding(Encoding::UTF_8) + end + end + end +end diff --git a/lib/interscript/ml/model.rb b/lib/interscript/ml/model.rb new file mode 100644 index 00000000..f11394aa --- /dev/null +++ b/lib/interscript/ml/model.rb @@ -0,0 +1,52 @@ +module Interscript::ML + class Model + attr_accessor :model, :input_vocab, :target_vocab + + def self.from_file(model_file) + # A models.yaml model id resolves (download -> verify -> cache) + # through the same contract as the Python/TS runtimes. + model_file = IMF.resolve(model_file) unless model_file.to_s.end_with?('.zip') || File.file?(model_file.to_s) + model_file = Provisioning.locate(model_file) + + Zip::File.open(model_file) do |zip_file| + metadata = zip_file.glob('metadata.yaml').first + metadata = YAML.safe_load(metadata.get_input_stream.read) if metadata + + # IMF v1: the Interscript Model Format zip. + return Byt5Onnx.new(model_file) if metadata && metadata['format'] == 'imf-v1' + + name = metadata && metadata['name'] + + # Modern byte-level seq2seq (ByT5 family): encoder.onnx + decoder.onnx. + return Byt5Onnx.new(model_file) if name == 'byt5' + + # Legacy single-file ONNX transformer zips (vocabs.yaml based). + vocabs = zip_file.glob('vocabs.yaml').first + raise 'vocabs.yaml is missing in model zip!' unless vocabs + vocabs = YAML.safe_load(vocabs.get_input_stream.read) + input_vocab = Vocab.new(vocabs['input'], specials: []) + target_vocab = Vocab.new(vocabs['target'], specials: []) + + onnx = zip_file.glob('*.onnx').first + raise 'onnx model file is missing in model zip!' unless onnx + Onnx.new(onnx.get_input_stream.read, input_vocab, target_vocab) + end + end + + class Onnx < Model + def initialize(model_path_or_bytes, input_vocab, target_vocab) + @model = OnnxRuntime::Model.new(model_path_or_bytes) + @input_vocab = input_vocab + @target_vocab = target_vocab + end + + def call(input, output, opts) + @model.predict({ src: input, tgt: output }.merge(opts))['output'] + end + + def argmax(*args) + self.call(*args).map { |i| i.flatten.each_with_index.max[1] } + end + end + end +end diff --git a/lib/interscript/ml/provisioning.rb b/lib/interscript/ml/provisioning.rb new file mode 100644 index 00000000..3d7932a7 --- /dev/null +++ b/lib/interscript/ml/provisioning.rb @@ -0,0 +1,189 @@ +require 'open-uri' +require 'uri' +require 'fileutils' +require 'digest/sha2' +require 'yaml' + +module Interscript::ML + # Module Interscript::ML::Provisioning is to provision remote models locally and to + # dispatch them later on. + module Provisioning + extend self + + @remotes = [] # Here's a place for a global model repository + @preload_models = [] + attr_accessor :remotes, :preload_models + + def add_remote(path) + @remotes << path + @remotes = @remotes.uniq + end + + def prepare_environment + return if @set_up + + # We provision the environment in the following way: + # First, we try the SECRYST_DATA environment variable. If that's available, + # we use it to store the Secryst data we need. Otherwise, we try the following + # paths: + + possible_paths = [ + "/var/lib/secryst", + "/usr/local/share/secryst", + "/usr/share/secryst", + File.join(Dir.home, ".local/share/secryst") + ] + + # We find the first writable path to become the primary one. The remaining + # ones will be used read-only if they exist + + @write_path = nil + @read_paths = [] + + ([ENV["INTERSCRIPT_ML_DATA"] || ENV["SECRYST_DATA"]] + possible_paths).compact.each do |path| + FileUtils.mkdir_p(path) + @write_path = path unless @write_path + rescue + ensure + @read_paths << path if File.readable?(path) + end + + raise StandardError, "Can't find a writable path for Secryst. Consider setting a SECRYST_DATA environment variable" unless @write_path + + # Now, let's locate the first Secrystfile to be found + path = Dir.pwd + secrystfilepath = loop do + break unless path =~ %r{[/\\]} + if File.readable?(path + "/Secrystfile") + break path + "/Secrystfile" + end + path = path.sub(%r{[/\\][^/\\]*?\z}, '') + end + + # It's found, so let's parse it. + if secrystfilepath + secrystfile = Secrystfile.new(secrystfilepath) + + @remotes = secrystfile.remotes + @remotes + @remotes = @remotes.uniq + + @preload_models = secrystfile.models + end + + # Load the remotes if they are older than 1 minute + FileUtils.mkdir_p(@write_path + "/remotes/") + @loaded_remotes = @remotes.map do |uri| + cache_path = "#{@write_path}/remotes/#{Digest::SHA256.hexdigest(uri)}.yaml" + if !File.exist?(cache_path) + data = URI.open(uri).read + File.write(cache_path, data) + elsif File.mtime(cache_path) + 60 < Time.now + begin + # Just *try* to download it. + data = URI.open(uri).read + File.write(cache_path, data) + rescue + end + end + Remotefile.new(cache_path, uri: uri) + end + + # Ok we are done now. We still need to resolve the required paths, but for + # that let's reuse our existing facilities. + @set_up = true + + @preload_models.each { |i| locate(i) } + end + + def locate(name) + # Shortcut this if user gave a filename. + return name if name =~ %r{[.\\/]} + + prepare_environment + + @loaded_remotes.each do |i| + model = i.resolve(name) + if model + model_path = @read_paths.map do |j| + path = j + "/models/" + model.name + next path if File.readable?(path) + nil + end.compact.first + + if model_path + version = File.read(model_path + "/version").to_f + if version >= model.version + return model_path + "/model.zip" + else + return download(model, remote: i) + end + else + return download(model, remote: i) + end + end + end + + raise StandardError, "Model #{name} not found" + end + + def download(model, remote:) + uri = model.uri + if uri.start_with?("./") || uri.start_with?("../") + remote_uri = remote.uri + if remote_uri =~ %r{\A/|\A\w:[\\/]} + remote_uri = "file://"+remote_uri + end + + uri = URI(remote_uri).merge(uri).to_s + uri = uri.sub(%r{\Afile://}, '') + end + + warn "* Downloading a Secryst model #{model.name}:#{model.version} from #{uri}..." + + data = URI.open(uri).read + path = @write_path + "/models/" + model.name + FileUtils.mkdir_p path + File.write(path + "/version", model.version) + File.write(path + "/model.zip", data) + return path + "/model.zip" + end + + class Remotefile + attr_accessor :uri + + def initialize(path, uri:) + @yaml = YAML.load_file(path) + @uri = uri + end + + def resolve(name) + @yaml["models"].find do |n, desc| + if name == n + return Model.new(name: n, uri: desc["path"], version: desc["version"]) + end + end + end + + class Model < Struct.new(:name, :uri, :version, keyword_init: true) + end + end + + class Secrystfile + def initialize(path) + @path = path + @models, @remotes = [], [] + self.instance_eval(File.read(path), path) + end + + def model(name) + @models << name + end + + def source(src) + @remotes << src + end + + attr_accessor :models, :remotes + end + end +end diff --git a/lib/interscript/ml/translator.rb b/lib/interscript/ml/translator.rb new file mode 100644 index 00000000..c7c8fca4 --- /dev/null +++ b/lib/interscript/ml/translator.rb @@ -0,0 +1,38 @@ +module Interscript::ML + # Dispatching translator: byte-level ByT5 models (name: byt5 in + # metadata.yaml) translate themselves; legacy char-vocab ONNX zips use + # the original greedy loop. + class Translator + attr_accessor :model + + def initialize(model_file:) + @device = 'cpu' + @model = Model.from_file(model_file) + end + + def translate(phrase, max_seq_length: 100) + return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx) + + Interscript::ML.require_optional!("numo-narray", feature: "numo/narray") + input = [''] + phrase.chars + [''] + input = Numo::NArray[input.map { |i| @model.input_vocab.stoi[i] }].transpose + output = Numo::NArray[[@model.target_vocab.stoi['']]] + src_key_padding_mask = input.transpose.eq(1) + + max_seq_length.times do |i| + tgt_key_padding_mask = output.transpose.eq(1) + tgt_mask = Numo::DFloat.ones(i + 1, i + 1).triu.transpose.eq(0) + prediction = @model.argmax(input, output.dup, + tgt_mask: tgt_mask, + src_key_padding_mask: src_key_padding_mask, + tgt_key_padding_mask: tgt_key_padding_mask, + memory_key_padding_mask: src_key_padding_mask) + break if @model.target_vocab.itos[prediction[i]] == '' + + output = Numo::NArray.concatenate([output, Numo::NArray[[prediction[i]]]]) + end + + output[1..-1].to_a.flatten.map { |i| @model.target_vocab.itos[i] }.join('') + end + end +end diff --git a/lib/interscript/ml/vocab.rb b/lib/interscript/ml/vocab.rb new file mode 100644 index 00000000..4eebbddd --- /dev/null +++ b/lib/interscript/ml/vocab.rb @@ -0,0 +1,48 @@ +module Interscript::ML + class Vocab + UNK = "" + attr_reader :stoi, :itos + + def initialize( + list, specials: ["", "", "", ""], specials_first: true + ) + @unk_index = nil + @itos = [] + if specials_first && (list & specials).length == 0 + @itos = specials + end + + @itos += list + + if !specials_first && (list & specials).length == 0 + @itos.concat(specials) + end + + # Automatic substitution of unknown symbols + if @itos.include?("") + unk_index = @itos.index("") + @stoi = Hash.new(unk_index) + elsif @itos.include?("[UNK]") + unk_index = @itos.index("[UNK]") + @stoi = Hash.new(unk_index) + else + @stoi = {} + end + + + # stoi is simply a reverse dict for itos + @itos.each_with_index do |tok, i| + @stoi[tok] = i + end + end + + def [](token) + @stoi.fetch(token, @stoi.fetch(UNK)) + end + + def length + @itos.length + end + alias_method :size, :length + end +end \ No newline at end of file diff --git a/lib/interscript/stdlib/functions/secryst_adapter.rb b/lib/interscript/stdlib/functions/secryst_adapter.rb index 9b6e009e..89293602 100644 --- a/lib/interscript/stdlib/functions/secryst_adapter.rb +++ b/lib/interscript/stdlib/functions/secryst_adapter.rb @@ -25,20 +25,12 @@ def translator_for(model_key) def build_translator(model_key) Interscript.secryst_index_locations.each do |remote| - Secryst::Provisioning.add_remote(remote) + Interscript::ML::Provisioning.add_remote(remote) end - Secryst::Translator.new(model_file: model_key) + Interscript::ML::Translator.new(model_file: model_key) end - def require_secryst! - return if defined?(Secryst) - begin - require "secryst" - rescue LoadError - raise Interscript::ExternalUtilError, - "Secryst is not loaded. Please read docs/Usage_with_Secryst.adoc" - end - end + end end end diff --git a/spec/interscript/fixtures/tiny-imf.zip b/spec/interscript/fixtures/tiny-imf.zip new file mode 100644 index 0000000000000000000000000000000000000000..c1799737ca58372a4294fb0196a2377689f1164b GIT binary patch literal 1149 zcmbtTO>fgc5OtcgNmqbElyUgD)N+{aTW>tF~yc^VVDmTsu zi4!-F_zB#(@elX|{0L5rL!>%}oLFmQ*)#9$zIi)`2lYnRD%OwB?H5}gKK+t4%PQhJ z{7bpP3>)0Pz-hub%^1&PdzxUWkjU~Bo7k4A=)H20WU+0ey6Cxm2PKknrFm@c9`E(M z-s1$%G)LJ~a{#zLn&zBJEi)C{V(JZ$p5VbSh>>kCHtACFdBufgE8d;>ba4}fsm{DsTYI;PcRB;$B&}GaVfA- zH*!Wa@?FPs832PQq(k2cLf;p7#N3bqbvVL-?+yvl*(|4gtZ6Q%pye4h3K^WA7Qz-E z`V2aeW2F;mkjc)tlCO)lkqg5$f{aw0jYXcNv7OJ9ktw$!V{9SM%`E5WaLE&L`-pr2 zFN(MZQ5Mu{#vg0wKH8LOIx}O*G-?oub?0V+wHZ$&V_c23!G_Xd}r^rO{;>fcp;_Cy&t9qL)jJ(yOEp8!Zm8s^VW__rAZ) zn;^1?O^BoYoxSJ#{giFb#|6x;t##|GM-U+MbqOO`8OSQMdQ<)YE$g!O9>C>$Q$2GiB GitHub cap path)' do + Dir.mktmpdir do |tmp| + channel = File.join(tmp, 'channel') + FileUtils.mkdir_p(channel) + zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) + blob = File.binread(zip_path) + part_a, part_b = blob[0, (blob.bytesize / 2 + 3)], blob[(blob.bytesize / 2 + 3)..] + File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a) + File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b) + index = File.join(tmp, 'models.yaml') + File.write(index, <<~YAML) + version: 1 + models: + tiny-1.0: + filename: tiny.zip + sha256: #{Digest::SHA256.hexdigest(blob)} + parts: + - url: file://#{channel}/tiny.zip.part-00 + sha256: #{Digest::SHA256.hexdigest(part_a)} + size: #{part_a.bytesize} + - url: file://#{channel}/tiny.zip.part-01 + sha256: #{Digest::SHA256.hexdigest(part_b)} + size: #{part_b.bytesize} + YAML + cache = File.join(tmp, 'cache') + ENV['SECRYST_CACHE'] = cache + begin + installed = described_class.resolve('tiny-1.0', index_url: index) + expect(File.binread(installed)).to eq(blob) + FileUtils.rm_f(File.join(channel, 'tiny.zip.part-00')) + expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed) + ensure + ENV.delete('SECRYST_CACHE') + end + end + end + + it 'rejects a corrupt part by index' do + Dir.mktmpdir do |tmp| + channel = File.join(tmp, 'channel') + FileUtils.mkdir_p(channel) + zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) + blob = File.binread(zip_path) + part_a, part_b = blob[0, 7], blob[7..] + File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a) + File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b) + index = File.join(tmp, 'models.yaml') + File.write(index, <<~YAML) + version: 1 + models: + tiny-1.0: + filename: tiny.zip + sha256: #{Digest::SHA256.hexdigest(blob)} + parts: + - url: file://#{channel}/tiny.zip.part-00 + sha256: #{"0" * 64} + size: #{part_a.bytesize} + - url: file://#{channel}/tiny.zip.part-01 + sha256: #{Digest::SHA256.hexdigest(part_b)} + size: #{part_b.bytesize} + YAML + ENV['SECRYST_CACHE'] = File.join(tmp, 'cache') + begin + expect { described_class.resolve('tiny-1.0', index_url: index) } + .to raise_error(Interscript::ML::IMF::RegistryError, /part 0 .* sha256 mismatch/) + ensure + ENV.delete('SECRYST_CACHE') + end + end + end + end + + describe "index distribution contract" do + it "pins DEFAULT_INDEX_URL to the exact current index release, never raw" do + # exact pin (not index-v\d+): the default must resolve every model + # the distribution contract has shipped, incl. static-int8 (v6). + # Bump this spec WITH the constant, never ahead of it. + expect(Interscript::ML::IMF::DEFAULT_INDEX_URL).to eq( + "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml" + ) + expect(Interscript::ML::IMF::DEFAULT_INDEX_URL).not_to include("raw.githubusercontent") + end + + it "verifies the .sha256 sidecar on HTTP index fetches" do + require "socket" + body = "version: 1\nmodels: {}\n" + good = Digest::SHA256.hexdigest(body) + bad = "0" * 64 + + serve = lambda do |sidecar, &blk| + server = TCPServer.new("127.0.0.1", 0) + port = server.addr[1] + thread = Thread.new do + loop do + client = server.accept rescue break + request = client.readline + client.each_line { |line| break if line == "\r\n" } + path = request.split(" ")[1] + case path + when "/models-index.yaml" + client.write("HTTP/1.1 200 OK\r\nContent-Length: #{body.bytesize}\r\n\r\n#{body}") + when "/models-index.yaml.sha256" + if sidecar.nil? + client.write("HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + else + payload = "#{sidecar} models-index.yaml\n" + client.write("HTTP/1.1 200 OK\r\nContent-Length: #{payload.bytesize}\r\n\r\n#{payload}") + end + else + client.write("HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + end + client.close + end + end + begin + blk.call("http://127.0.0.1:#{port}/models-index.yaml") + ensure + server.close + thread.join(5) + end + end + + serve.call(good) do |url| + entries = Interscript::ML::IMF.load_index(url) + expect(entries).to eq({}) + end + serve.call(bad) do |url| + expect { + Interscript::ML::IMF.load_index(url) + }.to raise_error(Interscript::ML::IMF::RegistryError, /index sha256 mismatch/) + end + serve.call(nil) do |url| + expect { + Interscript::ML::IMF.load_index(url) + }.to raise_error(Interscript::ML::IMF::RegistryError, /index sha256/) + end + end + end +end diff --git a/spec/stdlib_functions_spec.rb b/spec/stdlib_functions_spec.rb index 7d5ecf9f..fe732d59 100644 --- a/spec/stdlib_functions_spec.rb +++ b/spec/stdlib_functions_spec.rb @@ -65,16 +65,13 @@ end end - describe ".secryst (without Secryst gem loaded)" do - before do - skip "the Secryst gem is present in this environment" if defined?(Secryst) - end + describe ".secryst (built-in ML runtime)" do + it "translates through Interscript::ML with no external gem" do + zip = File.expand_path("../interscript/fixtures/tiny-imf.zip", __dir__) + skip "tiny IMF fixture missing" unless File.exist?(zip) - it "raises ExternalUtilError with a helpful message" do - expect { described_class.secryst("hello", model: "default") }.to raise_error( - Interscript::ExternalUtilError, - /Secryst is not loaded/ - ) + out = described_class.secryst("test", model: zip) + expect(out).to be_a(String) end end end