diff --git a/.github/workflows/assets.yml b/.github/workflows/assets.yml index 1ec6e948..8686ed9f 100644 --- a/.github/workflows/assets.yml +++ b/.github/workflows/assets.yml @@ -10,7 +10,7 @@ jobs: assets: runs-on: ubuntu-latest env: - BUNDLE_WITHOUT: "secryst:jsexec" + BUNDLE_WITHOUT: "jsexec" SKIP_JS: "1" steps: - name: Checkout monorepo @@ -30,7 +30,7 @@ jobs: - name: Install bundle working-directory: ruby - run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst + run: bundle install --jobs 4 --retry 3 --with jsexec - name: Generate visualization JSON working-directory: ruby diff --git a/.github/workflows/rake.yml b/.github/workflows/rake.yml index a53e300b..d22ed01d 100644 --- a/.github/workflows/rake.yml +++ b/.github/workflows/rake.yml @@ -16,7 +16,7 @@ jobs: ruby: ["3.3", "3.4"] os: [ubuntu-latest, windows-latest, macos-latest] env: - BUNDLE_WITHOUT: "secryst" + BUNDLE_WITHOUT: "jsexec" SKIP_JS: "1" ISC_SWEEP: "1" # The ISC corpus sweep: every map's own tests through every diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76279b00..dae2bcaf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: - name: Install gems working-directory: ruby - run: bundle install --jobs 4 --retry 3 --with jsexec --without secryst + run: bundle install --jobs 4 --retry 3 --with jsexec - name: Test Ruby package working-directory: ruby diff --git a/Gemfile b/Gemfile index a25f1058..6e505ed1 100644 --- a/Gemfile +++ b/Gemfile @@ -16,12 +16,10 @@ else gem "interscript-maps" end -group :secryst do - if File.exist?("../../secryst") - gem "secryst", path: "../../secryst" - else - gem "secryst" - end +# The neural layer (Interscript::ML) - optional at runtime, loaded for specs +group :ml do + gem "rubyzip" + gem "onnxruntime" end gem "regexp_parser" diff --git a/lib/interscript.rb b/lib/interscript.rb index 61216995..d5483bf8 100644 --- a/lib/interscript.rb +++ b/lib/interscript.rb @@ -12,6 +12,7 @@ module Interscript autoload :Node, "interscript/node" autoload :Detector, "interscript/detector" autoload :Isc, "interscript/isc" + autoload :ML, "interscript/ml" # An error caused by a lack of some map class MapNotFoundError < StandardError; end diff --git a/lib/interscript/ml.rb b/lib/interscript/ml.rb new file mode 100644 index 00000000..b19a8f4a --- /dev/null +++ b/lib/interscript/ml.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# The neural layer: IMF v1 artifact runtime + index provisioning for +# byte-level ByT5 models, ported from the secryst crystal (1.1.1) into +# interscript proper. Runtime dependencies rubyzip and onnxruntime are +# OPTIONAL: they load lazily with a helpful error when absent. +# +# Interscript::ML::Translator.new(model_file: "khm-latn-1.0") +# .translate("ភាសា") +module Interscript::ML + autoload :IMF, "interscript/ml/imf" + autoload :Provisioning, "interscript/ml/provisioning" + autoload :Translator, "interscript/ml/translator" + autoload :Model, "interscript/ml/model" + autoload :Vocab, "interscript/ml/vocab" + autoload :Byt5Onnx, "interscript/ml/byt5_onnx" + + # Expected to be loaded before IMF/Byt5Onnx entry points. + def self.require_optional!(gem_name, feature: gem_name) + require feature + rescue LoadError => e + raise Interscript::ExternalUtilError, + "the neural layer needs the #{gem_name} gem: gem install #{gem_name} (or use a bundle including it)" + end +end diff --git a/lib/interscript/ml/byt5_onnx.rb b/lib/interscript/ml/byt5_onnx.rb new file mode 100644 index 00000000..63fe0ad4 --- /dev/null +++ b/lib/interscript/ml/byt5_onnx.rb @@ -0,0 +1,102 @@ +require 'tmpdir' + +module Interscript::ML + # Byte-level seq2seq (ByT5 family) inference over ONNX sessions from an + # IMF v1 model zip: sha256-verified graphs, greedy KV-cache decode when + # the zip ships decoder-kv.onnx (default), plain full-recompute + # fallback otherwise. Tokenization is the canonical ByT5 table + # (byte b -> id b+3, trailing EOS) — see Interscript::ML::IMF. + class Byt5Onnx + def initialize(zip_path) + Interscript::ML.require_optional!("onnxruntime") + @manifest = IMF.manifest(zip_path) + graphs = IMF.verify_and_read(zip_path) + @tmpdir = Dir.mktmpdir('secryst-imf') + @encoder = build_session(graphs, 'encoder.onnx') + @kv = @manifest['decoder'] == 'kv' && graphs.key?('decoder-kv.onnx') + @decoder = build_session(graphs, @kv ? 'decoder-kv.onnx' : 'decoder.onnx') + @pasts = @kv ? zero_pasts : {} + end + + def translate(text, max_seq_length: 256) + ids = IMF.encode(text) + return '' if ids.length == 1 # only the trailing EOS: empty input + + hidden = @encoder.predict({ input_ids: [ids] })['last_hidden_state'] + tokens = @kv ? greedy_kv(hidden, max_seq_length) : greedy_plain(hidden, max_seq_length) + IMF.decode(tokens) + end + + def id + @manifest['id'] + end + + private + + # The onnxruntime gem loads from paths, not bytes: verified bytes go + # to a private tmpdir. + def build_session(graphs, name) + path = File.join(@tmpdir, name) + File.binwrite(path, graphs.fetch(name)) + OnnxRuntime::Model.new(path) + end + + def zero_pasts + @decoder.inputs.each_with_object({}) do |meta, pasts| + next unless meta[:name].start_with?('past_') + shape = meta[:shape] # [batch, heads, past_seq, d_kv]; dynamic dims are Strings + heads = shape[1].is_a?(Integer) ? shape[1] : 4 + d_kv = shape[3].is_a?(Integer) ? shape[3] : 8 + # Numo carries the explicit [1, heads, 0, d_kv] shape through; + # nested Ruby arrays cannot express a zero-length dim. + pasts[meta[:name]] = Numo::DFloat.zeros(1, heads, 0, d_kv) + end + end + + def argmax(row) + best = 0 + best_val = -Float::INFINITY + row.each_with_index do |value, index| + if value > best_val + best_val = value + best = index + end + end + best + end + + def greedy_kv(hidden, max_seq_length) + pasts = @pasts.dup + current = [IMF::PAD_ID] + generated = [] + max_seq_length.times do + results = @decoder.predict( + { input_ids: [current], encoder_hidden_states: hidden }.merge(pasts) + ) + token = argmax(results['logits'].first.last) + break if token == IMF::EOS_ID + + generated << token + pasts = pasts.keys.to_h { |name| [name, results[name.sub('past_', 'present_')]] } + current = [token] + end + generated + end + + def greedy_plain(hidden, max_seq_length) + decoder_ids = [IMF::PAD_ID] + generated = [] + max_seq_length.times do + logits = @decoder.predict( + { input_ids: [decoder_ids], encoder_hidden_states: hidden } + )['logits'] + token = argmax(logits.first.last) + break if token == IMF::EOS_ID + + generated << token + decoder_ids = decoder_ids + [token] + end + generated + end + end +end diff --git a/lib/interscript/ml/imf.rb b/lib/interscript/ml/imf.rb new file mode 100644 index 00000000..40bb2772 --- /dev/null +++ b/lib/interscript/ml/imf.rb @@ -0,0 +1,182 @@ +require "yaml" +require "digest/sha2" +require "fileutils" +require "open-uri" + +module Interscript::ML + # Interscript Model Format v1 — the byte-level runtime contract shared + # with the Python (interscript-ml) and TypeScript (npm: secryst) + # runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3, + # trailing EOS; pad=0, unk=2. Ids are NOT raw byte values. + module IMF + BYTE_OFFSET = 3 + PAD_ID = 0 + EOS_ID = 1 + UNK_ID = 2 + + DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml" + + class FormatError < StandardError; end + class RegistryError < StandardError; end + + class << self + def require_zip! + Interscript::ML.require_optional!("rubyzip", feature: "zip") + end + + def encode(text) + text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID] + end + + def decode(token_ids) + out = +"" + token_ids.each do |token| + break if token == EOS_ID + next if token == PAD_ID || token == UNK_ID + out << ((token - BYTE_OFFSET) % 256).chr + end + out.force_encoding(Encoding::UTF_8) + end + + def manifest(zip_path) + require_zip! + Zip::File.open(zip_path) do |zf| + raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml") + meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false) + raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1" + if meta["tokenizer"] != "bytes" + raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only" + end + %w[encoder.onnx decoder.onnx].each do |required| + raise FormatError, "missing #{required}" unless zf.find_entry(required) + end + meta + end + end + + # Reads every .onnx member after verifying its sha256 against the + # manifest — corrupt zips fail loudly, before any session loads. + def verify_and_read(zip_path) + meta = manifest(zip_path) + sha = meta.fetch("sha256", {}) + graphs = {} + require_zip! + Zip::File.open(zip_path) do |zf| + zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry| + recorded = sha[entry.name] + raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded + bytes = entry.get_input_stream.read + actual = Digest::SHA256.hexdigest(bytes) + if actual != recorded + raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}" + end + graphs[entry.name] = bytes + end + end + graphs + end + + def cache_dir + ENV["INTERSCRIPT_ML_CACHE"] || ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "interscript-ml") + end + + # models.yaml resolution: cache hit (re-verified), or download -> + # verify whole-file sha256 -> atomic install into the cache. + # Entries with `parts` (GitHub's 2 GiB per-asset cap) are streamed + # in order, each part sha256-verified as it lands, then the + # assembled file is checked against the whole-file sha256. + def resolve(model_id, index_url: nil) + source = index_url || ENV["INTERSCRIPT_ML_INDEX"] || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL + entries = load_index(source) + entry = entries[model_id] + raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry + + target = File.join(cache_dir, "models", model_id, entry["filename"]) + if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"] + return target + end + + FileUtils.mkdir_p(File.dirname(target)) + tmp = target + ".part.#{Process.pid}" + if entry["parts"] + download_parts(entry, tmp) + else + channel = entry["url"] + if channel.start_with?("file://") + FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp) + else + URI.open(channel) { |remote| IO.copy_stream(remote, tmp) } + end + end + actual = Digest::SHA256.file(tmp).hexdigest + unless actual == entry["sha256"] + File.delete(tmp) + raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}" + end + File.rename(tmp, target) + target + end + + private + + def download_parts(entry, tmp) + File.open(tmp, "wb") do |out| + entry["parts"].each_with_index do |part, index| + digest = Digest::SHA256.new + source = part["url"].start_with?("file://") ? part["url"].sub(%r{\Afile://}, "") : part["url"] + open_stream = lambda do |io| + while (chunk = io.read(1024 * 1024)) + out.write(chunk) + digest.update(chunk) + end + end + if part["url"].start_with?("file://") + File.open(source, "rb", &open_stream) + else + URI.open(source, "rb", &open_stream) + end + unless digest.hexdigest == part["sha256"] + raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}" + end + end + end + rescue StandardError + File.delete(tmp) if File.file?(tmp) + raise + end + + def load_index(source) + text = if source.start_with?("http://", "https://") + fetch_index_with_sidecar(source) + else + File.read(source) + end + raw = YAML.safe_load(text, permitted_classes: [], aliases: false) + raise RegistryError, "index must have version: 1" if raw["version"] != 1 + raw.fetch("models", {}) + end + public :load_index + + # HTTP sources must ship a sibling .sha256 sidecar; verify the + # body against it before parsing (GitHub Releases index assets do). + def fetch_index_with_sidecar(source) + body = URI.open(source, "rb") { |remote| remote.read } + sidecar = + begin + URI.open("#{source}.sha256") { |remote| remote.read } + rescue OpenURI::HTTPError => e + raise RegistryError, "index sha256 sidecar missing: #{source}.sha256 (#{e.message})" + end + expected = sidecar.to_s.strip.split(/\s+/).first.to_s + unless expected.match?(/\A[0-9a-fA-F]{64}\z/) + raise RegistryError, "index sha256 sidecar malformed: #{source}.sha256" + end + actual = Digest::SHA256.hexdigest(body) + unless actual == expected.downcase + raise RegistryError, "index sha256 mismatch: got #{actual}, sidecar says #{expected.downcase}" + end + body.force_encoding(Encoding::UTF_8) + end + end + end +end diff --git a/lib/interscript/ml/model.rb b/lib/interscript/ml/model.rb new file mode 100644 index 00000000..f11394aa --- /dev/null +++ b/lib/interscript/ml/model.rb @@ -0,0 +1,52 @@ +module Interscript::ML + class Model + attr_accessor :model, :input_vocab, :target_vocab + + def self.from_file(model_file) + # A models.yaml model id resolves (download -> verify -> cache) + # through the same contract as the Python/TS runtimes. + model_file = IMF.resolve(model_file) unless model_file.to_s.end_with?('.zip') || File.file?(model_file.to_s) + model_file = Provisioning.locate(model_file) + + Zip::File.open(model_file) do |zip_file| + metadata = zip_file.glob('metadata.yaml').first + metadata = YAML.safe_load(metadata.get_input_stream.read) if metadata + + # IMF v1: the Interscript Model Format zip. + return Byt5Onnx.new(model_file) if metadata && metadata['format'] == 'imf-v1' + + name = metadata && metadata['name'] + + # Modern byte-level seq2seq (ByT5 family): encoder.onnx + decoder.onnx. + return Byt5Onnx.new(model_file) if name == 'byt5' + + # Legacy single-file ONNX transformer zips (vocabs.yaml based). + vocabs = zip_file.glob('vocabs.yaml').first + raise 'vocabs.yaml is missing in model zip!' unless vocabs + vocabs = YAML.safe_load(vocabs.get_input_stream.read) + input_vocab = Vocab.new(vocabs['input'], specials: []) + target_vocab = Vocab.new(vocabs['target'], specials: []) + + onnx = zip_file.glob('*.onnx').first + raise 'onnx model file is missing in model zip!' unless onnx + Onnx.new(onnx.get_input_stream.read, input_vocab, target_vocab) + end + end + + class Onnx < Model + def initialize(model_path_or_bytes, input_vocab, target_vocab) + @model = OnnxRuntime::Model.new(model_path_or_bytes) + @input_vocab = input_vocab + @target_vocab = target_vocab + end + + def call(input, output, opts) + @model.predict({ src: input, tgt: output }.merge(opts))['output'] + end + + def argmax(*args) + self.call(*args).map { |i| i.flatten.each_with_index.max[1] } + end + end + end +end diff --git a/lib/interscript/ml/provisioning.rb b/lib/interscript/ml/provisioning.rb new file mode 100644 index 00000000..3d7932a7 --- /dev/null +++ b/lib/interscript/ml/provisioning.rb @@ -0,0 +1,189 @@ +require 'open-uri' +require 'uri' +require 'fileutils' +require 'digest/sha2' +require 'yaml' + +module Interscript::ML + # Module Interscript::ML::Provisioning is to provision remote models locally and to + # dispatch them later on. + module Provisioning + extend self + + @remotes = [] # Here's a place for a global model repository + @preload_models = [] + attr_accessor :remotes, :preload_models + + def add_remote(path) + @remotes << path + @remotes = @remotes.uniq + end + + def prepare_environment + return if @set_up + + # We provision the environment in the following way: + # First, we try the SECRYST_DATA environment variable. If that's available, + # we use it to store the Secryst data we need. Otherwise, we try the following + # paths: + + possible_paths = [ + "/var/lib/secryst", + "/usr/local/share/secryst", + "/usr/share/secryst", + File.join(Dir.home, ".local/share/secryst") + ] + + # We find the first writable path to become the primary one. The remaining + # ones will be used read-only if they exist + + @write_path = nil + @read_paths = [] + + ([ENV["INTERSCRIPT_ML_DATA"] || ENV["SECRYST_DATA"]] + possible_paths).compact.each do |path| + FileUtils.mkdir_p(path) + @write_path = path unless @write_path + rescue + ensure + @read_paths << path if File.readable?(path) + end + + raise StandardError, "Can't find a writable path for Secryst. Consider setting a SECRYST_DATA environment variable" unless @write_path + + # Now, let's locate the first Secrystfile to be found + path = Dir.pwd + secrystfilepath = loop do + break unless path =~ %r{[/\\]} + if File.readable?(path + "/Secrystfile") + break path + "/Secrystfile" + end + path = path.sub(%r{[/\\][^/\\]*?\z}, '') + end + + # It's found, so let's parse it. + if secrystfilepath + secrystfile = Secrystfile.new(secrystfilepath) + + @remotes = secrystfile.remotes + @remotes + @remotes = @remotes.uniq + + @preload_models = secrystfile.models + end + + # Load the remotes if they are older than 1 minute + FileUtils.mkdir_p(@write_path + "/remotes/") + @loaded_remotes = @remotes.map do |uri| + cache_path = "#{@write_path}/remotes/#{Digest::SHA256.hexdigest(uri)}.yaml" + if !File.exist?(cache_path) + data = URI.open(uri).read + File.write(cache_path, data) + elsif File.mtime(cache_path) + 60 < Time.now + begin + # Just *try* to download it. + data = URI.open(uri).read + File.write(cache_path, data) + rescue + end + end + Remotefile.new(cache_path, uri: uri) + end + + # Ok we are done now. We still need to resolve the required paths, but for + # that let's reuse our existing facilities. + @set_up = true + + @preload_models.each { |i| locate(i) } + end + + def locate(name) + # Shortcut this if user gave a filename. + return name if name =~ %r{[.\\/]} + + prepare_environment + + @loaded_remotes.each do |i| + model = i.resolve(name) + if model + model_path = @read_paths.map do |j| + path = j + "/models/" + model.name + next path if File.readable?(path) + nil + end.compact.first + + if model_path + version = File.read(model_path + "/version").to_f + if version >= model.version + return model_path + "/model.zip" + else + return download(model, remote: i) + end + else + return download(model, remote: i) + end + end + end + + raise StandardError, "Model #{name} not found" + end + + def download(model, remote:) + uri = model.uri + if uri.start_with?("./") || uri.start_with?("../") + remote_uri = remote.uri + if remote_uri =~ %r{\A/|\A\w:[\\/]} + remote_uri = "file://"+remote_uri + end + + uri = URI(remote_uri).merge(uri).to_s + uri = uri.sub(%r{\Afile://}, '') + end + + warn "* Downloading a Secryst model #{model.name}:#{model.version} from #{uri}..." + + data = URI.open(uri).read + path = @write_path + "/models/" + model.name + FileUtils.mkdir_p path + File.write(path + "/version", model.version) + File.write(path + "/model.zip", data) + return path + "/model.zip" + end + + class Remotefile + attr_accessor :uri + + def initialize(path, uri:) + @yaml = YAML.load_file(path) + @uri = uri + end + + def resolve(name) + @yaml["models"].find do |n, desc| + if name == n + return Model.new(name: n, uri: desc["path"], version: desc["version"]) + end + end + end + + class Model < Struct.new(:name, :uri, :version, keyword_init: true) + end + end + + class Secrystfile + def initialize(path) + @path = path + @models, @remotes = [], [] + self.instance_eval(File.read(path), path) + end + + def model(name) + @models << name + end + + def source(src) + @remotes << src + end + + attr_accessor :models, :remotes + end + end +end diff --git a/lib/interscript/ml/translator.rb b/lib/interscript/ml/translator.rb new file mode 100644 index 00000000..c7c8fca4 --- /dev/null +++ b/lib/interscript/ml/translator.rb @@ -0,0 +1,38 @@ +module Interscript::ML + # Dispatching translator: byte-level ByT5 models (name: byt5 in + # metadata.yaml) translate themselves; legacy char-vocab ONNX zips use + # the original greedy loop. + class Translator + attr_accessor :model + + def initialize(model_file:) + @device = 'cpu' + @model = Model.from_file(model_file) + end + + def translate(phrase, max_seq_length: 100) + return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx) + + Interscript::ML.require_optional!("numo-narray", feature: "numo/narray") + input = [''] + phrase.chars + [''] + input = Numo::NArray[input.map { |i| @model.input_vocab.stoi[i] }].transpose + output = Numo::NArray[[@model.target_vocab.stoi['']]] + src_key_padding_mask = input.transpose.eq(1) + + max_seq_length.times do |i| + tgt_key_padding_mask = output.transpose.eq(1) + tgt_mask = Numo::DFloat.ones(i + 1, i + 1).triu.transpose.eq(0) + prediction = @model.argmax(input, output.dup, + tgt_mask: tgt_mask, + src_key_padding_mask: src_key_padding_mask, + tgt_key_padding_mask: tgt_key_padding_mask, + memory_key_padding_mask: src_key_padding_mask) + break if @model.target_vocab.itos[prediction[i]] == '' + + output = Numo::NArray.concatenate([output, Numo::NArray[[prediction[i]]]]) + end + + output[1..-1].to_a.flatten.map { |i| @model.target_vocab.itos[i] }.join('') + end + end +end diff --git a/lib/interscript/ml/vocab.rb b/lib/interscript/ml/vocab.rb new file mode 100644 index 00000000..4eebbddd --- /dev/null +++ b/lib/interscript/ml/vocab.rb @@ -0,0 +1,48 @@ +module Interscript::ML + class Vocab + UNK = "" + attr_reader :stoi, :itos + + def initialize( + list, specials: ["", "", "", ""], specials_first: true + ) + @unk_index = nil + @itos = [] + if specials_first && (list & specials).length == 0 + @itos = specials + end + + @itos += list + + if !specials_first && (list & specials).length == 0 + @itos.concat(specials) + end + + # Automatic substitution of unknown symbols + if @itos.include?("") + unk_index = @itos.index("") + @stoi = Hash.new(unk_index) + elsif @itos.include?("[UNK]") + unk_index = @itos.index("[UNK]") + @stoi = Hash.new(unk_index) + else + @stoi = {} + end + + + # stoi is simply a reverse dict for itos + @itos.each_with_index do |tok, i| + @stoi[tok] = i + end + end + + def [](token) + @stoi.fetch(token, @stoi.fetch(UNK)) + end + + def length + @itos.length + end + alias_method :size, :length + end +end \ No newline at end of file diff --git a/lib/interscript/stdlib/functions/secryst_adapter.rb b/lib/interscript/stdlib/functions/secryst_adapter.rb index 9b6e009e..89293602 100644 --- a/lib/interscript/stdlib/functions/secryst_adapter.rb +++ b/lib/interscript/stdlib/functions/secryst_adapter.rb @@ -25,20 +25,12 @@ def translator_for(model_key) def build_translator(model_key) Interscript.secryst_index_locations.each do |remote| - Secryst::Provisioning.add_remote(remote) + Interscript::ML::Provisioning.add_remote(remote) end - Secryst::Translator.new(model_file: model_key) + Interscript::ML::Translator.new(model_file: model_key) end - def require_secryst! - return if defined?(Secryst) - begin - require "secryst" - rescue LoadError - raise Interscript::ExternalUtilError, - "Secryst is not loaded. Please read docs/Usage_with_Secryst.adoc" - end - end + end end end diff --git a/spec/interscript/fixtures/tiny-imf.zip b/spec/interscript/fixtures/tiny-imf.zip new file mode 100644 index 00000000..c1799737 Binary files /dev/null and b/spec/interscript/fixtures/tiny-imf.zip differ diff --git a/spec/interscript/ml/byt5_onnx_spec.rb b/spec/interscript/ml/byt5_onnx_spec.rb new file mode 100644 index 00000000..efc810e8 --- /dev/null +++ b/spec/interscript/ml/byt5_onnx_spec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' +require "interscript/ml/byt5_onnx" + +RSpec.describe Interscript::ML::Byt5Onnx do + it 'uses the ByT5 byte conventions' do + expect(Interscript::ML::IMF::PAD_ID).to eq(0) + expect(Interscript::ML::IMF::EOS_ID).to eq(1) + end + + describe 'byte round-trip (as used for generated output)' do + it 'packs generated bytes back into UTF-8 text' do + text = 'ភាសាខ្មែរ' + bytes = text.bytes + expect(bytes.pack('C*').force_encoding(Encoding::UTF_8)).to eq(text) + end + end +end diff --git a/spec/interscript/ml/imf_spec.rb b/spec/interscript/ml/imf_spec.rb new file mode 100644 index 00000000..0f32da6a --- /dev/null +++ b/spec/interscript/ml/imf_spec.rb @@ -0,0 +1,248 @@ +require "spec_helper" +require "zip" +require 'interscript/ml' +require 'fileutils' +require 'tmpdir' +require 'json' + +RSpec.describe Interscript::ML::IMF do + it 'uses the canonical ByT5 table (byte + 3, trailing EOS)' do + expect(described_class.encode('rok')).to eq([117, 114, 110, 1]) + expect(described_class.decode([117, 114, 110])).to eq('rok') + expect(described_class.decode([117, 1, 114])).to eq('r') + end + + describe 'with the tiny IMF fixture zip' do + let(:zip_path) { File.expand_path('../fixtures/tiny-imf.zip', __dir__) } + + it 'parses the manifest and verifies graph checksums' do + meta = described_class.manifest(zip_path) + expect(meta['format']).to eq('imf-v1') + expect(meta['tokenizer']).to eq('bytes') + graphs = described_class.verify_and_read(zip_path) + expect(graphs.keys.sort).to eq(['decoder.onnx', 'encoder.onnx']) + end + + it 'loads sessions from verified bytes' do + model = Interscript::ML::Byt5Onnx.new(zip_path) + expect(model.id).to eq('tiny-1.0') + end + + it 'decodes a real zip against the shared golden set', :e2e do + zip = ENV['SECRYST_E2E_ZIP'] + golden = ENV['SECRYST_GOLDEN'] + skip 'set SECRYST_E2E_ZIP and SECRYST_GOLDEN for the end-to-end run' unless zip && golden + translator = Interscript::ML::Translator.new(model_file: zip) + ok = 0 + total = 0 + File.readlines(golden).each do |line| + row = JSON.parse(line) + total += 1 + ok += 1 if translator.translate(row['input'], max_seq_length: 128) == row['output'] + end + expect(ok).to eq(total) + end + end + + it 'rejects a tampered zip loudly' do + Dir.mktmpdir do |tmp| + source = File.expand_path('../fixtures/tiny-imf.zip', __dir__) + tampered = File.join(tmp, 'tampered.zip') + require 'zip' + Zip::File.open(source) do |src| + Zip::File.open(tampered, create: true) do |dst| + src.entries.each do |e| + if e.name == 'encoder.onnx' + dst.get_output_stream(e.name) { |io| io.write('corrupted-bytes') } + else + dst.get_output_stream(e.name) { |io| io.write(e.get_input_stream.read) } + end + end + end + end + expect { described_class.verify_and_read(tampered) } + .to raise_error(Interscript::ML::IMF::FormatError, /sha256 mismatch/) + end + end + + describe '.resolve' do + it 'installs a verified copy into the cache from a local index' do + Dir.mktmpdir do |tmp| + channel = File.join(tmp, 'channel') + FileUtils.mkdir_p(channel) + zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) + FileUtils.cp(zip_path, File.join(channel, 'tiny.zip')) + require 'digest' + index = File.join(tmp, 'models.yaml') + File.write(index, <<~YAML) + version: 1 + models: + tiny-1.0: + filename: tiny.zip + url: file://#{channel}/tiny.zip + sha256: #{Digest::SHA256.file(zip_path).hexdigest} + YAML + cache = File.join(tmp, 'cache') + result = described_class.resolve('tiny-1.0', index_url: index) if false + # env-based cache (the public API reads ENV at call time) + ENV['SECRYST_CACHE'] = cache + begin + installed = described_class.resolve('tiny-1.0', index_url: index) + expect(installed).to eq(File.join(cache, 'models', 'tiny-1.0', 'tiny.zip')) + expect(File.file?(installed)).to be(true) + FileUtils.rm_f(File.join(channel, 'tiny.zip')) + expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed) + ensure + ENV.delete('SECRYST_CACHE') + end + end + end + + it 'raises for unknown ids' do + Dir.mktmpdir do |tmp| + index = File.join(tmp, 'models.yaml') + File.write(index, "version: 1\nmodels: {}\n") + expect { described_class.resolve('nope-1.0', index_url: index) } + .to raise_error(Interscript::ML::IMF::RegistryError, /unknown model id/) + end + end + + it 'assembles and verifies split parts (the >2GiB GitHub cap path)' do + Dir.mktmpdir do |tmp| + channel = File.join(tmp, 'channel') + FileUtils.mkdir_p(channel) + zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) + blob = File.binread(zip_path) + part_a, part_b = blob[0, (blob.bytesize / 2 + 3)], blob[(blob.bytesize / 2 + 3)..] + File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a) + File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b) + index = File.join(tmp, 'models.yaml') + File.write(index, <<~YAML) + version: 1 + models: + tiny-1.0: + filename: tiny.zip + sha256: #{Digest::SHA256.hexdigest(blob)} + parts: + - url: file://#{channel}/tiny.zip.part-00 + sha256: #{Digest::SHA256.hexdigest(part_a)} + size: #{part_a.bytesize} + - url: file://#{channel}/tiny.zip.part-01 + sha256: #{Digest::SHA256.hexdigest(part_b)} + size: #{part_b.bytesize} + YAML + cache = File.join(tmp, 'cache') + ENV['SECRYST_CACHE'] = cache + begin + installed = described_class.resolve('tiny-1.0', index_url: index) + expect(File.binread(installed)).to eq(blob) + FileUtils.rm_f(File.join(channel, 'tiny.zip.part-00')) + expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed) + ensure + ENV.delete('SECRYST_CACHE') + end + end + end + + it 'rejects a corrupt part by index' do + Dir.mktmpdir do |tmp| + channel = File.join(tmp, 'channel') + FileUtils.mkdir_p(channel) + zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) + blob = File.binread(zip_path) + part_a, part_b = blob[0, 7], blob[7..] + File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a) + File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b) + index = File.join(tmp, 'models.yaml') + File.write(index, <<~YAML) + version: 1 + models: + tiny-1.0: + filename: tiny.zip + sha256: #{Digest::SHA256.hexdigest(blob)} + parts: + - url: file://#{channel}/tiny.zip.part-00 + sha256: #{"0" * 64} + size: #{part_a.bytesize} + - url: file://#{channel}/tiny.zip.part-01 + sha256: #{Digest::SHA256.hexdigest(part_b)} + size: #{part_b.bytesize} + YAML + ENV['SECRYST_CACHE'] = File.join(tmp, 'cache') + begin + expect { described_class.resolve('tiny-1.0', index_url: index) } + .to raise_error(Interscript::ML::IMF::RegistryError, /part 0 .* sha256 mismatch/) + ensure + ENV.delete('SECRYST_CACHE') + end + end + end + end + + describe "index distribution contract" do + it "pins DEFAULT_INDEX_URL to the exact current index release, never raw" do + # exact pin (not index-v\d+): the default must resolve every model + # the distribution contract has shipped, incl. static-int8 (v6). + # Bump this spec WITH the constant, never ahead of it. + expect(Interscript::ML::IMF::DEFAULT_INDEX_URL).to eq( + "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml" + ) + expect(Interscript::ML::IMF::DEFAULT_INDEX_URL).not_to include("raw.githubusercontent") + end + + it "verifies the .sha256 sidecar on HTTP index fetches" do + require "socket" + body = "version: 1\nmodels: {}\n" + good = Digest::SHA256.hexdigest(body) + bad = "0" * 64 + + serve = lambda do |sidecar, &blk| + server = TCPServer.new("127.0.0.1", 0) + port = server.addr[1] + thread = Thread.new do + loop do + client = server.accept rescue break + request = client.readline + client.each_line { |line| break if line == "\r\n" } + path = request.split(" ")[1] + case path + when "/models-index.yaml" + client.write("HTTP/1.1 200 OK\r\nContent-Length: #{body.bytesize}\r\n\r\n#{body}") + when "/models-index.yaml.sha256" + if sidecar.nil? + client.write("HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + else + payload = "#{sidecar} models-index.yaml\n" + client.write("HTTP/1.1 200 OK\r\nContent-Length: #{payload.bytesize}\r\n\r\n#{payload}") + end + else + client.write("HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + end + client.close + end + end + begin + blk.call("http://127.0.0.1:#{port}/models-index.yaml") + ensure + server.close + thread.join(5) + end + end + + serve.call(good) do |url| + entries = Interscript::ML::IMF.load_index(url) + expect(entries).to eq({}) + end + serve.call(bad) do |url| + expect { + Interscript::ML::IMF.load_index(url) + }.to raise_error(Interscript::ML::IMF::RegistryError, /index sha256 mismatch/) + end + serve.call(nil) do |url| + expect { + Interscript::ML::IMF.load_index(url) + }.to raise_error(Interscript::ML::IMF::RegistryError, /index sha256/) + end + end + end +end diff --git a/spec/stdlib_functions_spec.rb b/spec/stdlib_functions_spec.rb index 7d5ecf9f..fe732d59 100644 --- a/spec/stdlib_functions_spec.rb +++ b/spec/stdlib_functions_spec.rb @@ -65,16 +65,13 @@ end end - describe ".secryst (without Secryst gem loaded)" do - before do - skip "the Secryst gem is present in this environment" if defined?(Secryst) - end + describe ".secryst (built-in ML runtime)" do + it "translates through Interscript::ML with no external gem" do + zip = File.expand_path("../interscript/fixtures/tiny-imf.zip", __dir__) + skip "tiny IMF fixture missing" unless File.exist?(zip) - it "raises ExternalUtilError with a helpful message" do - expect { described_class.secryst("hello", model: "default") }.to raise_error( - Interscript::ExternalUtilError, - /Secryst is not loaded/ - ) + out = described_class.secryst("test", model: zip) + expect(out).to be_a(String) end end end