|
| 1 | +require "yaml" |
| 2 | +require "digest/sha2" |
| 3 | +require "fileutils" |
| 4 | +require "open-uri" |
| 5 | + |
| 6 | +module Interscript::ML |
| 7 | + # Interscript Model Format v1 — the byte-level runtime contract shared |
| 8 | + # with the Python (interscript-ml) and TypeScript (npm: secryst) |
| 9 | + # runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3, |
| 10 | + # trailing EOS; pad=0, unk=2. Ids are NOT raw byte values. |
| 11 | + module IMF |
| 12 | + BYTE_OFFSET = 3 |
| 13 | + PAD_ID = 0 |
| 14 | + EOS_ID = 1 |
| 15 | + UNK_ID = 2 |
| 16 | + |
| 17 | + DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml" |
| 18 | + |
| 19 | + class FormatError < StandardError; end |
| 20 | + class RegistryError < StandardError; end |
| 21 | + |
| 22 | + class << self |
| 23 | + def require_zip! |
| 24 | + Interscript::ML.require_optional!("rubyzip", feature: "zip") |
| 25 | + end |
| 26 | + |
| 27 | + def encode(text) |
| 28 | + text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID] |
| 29 | + end |
| 30 | + |
| 31 | + def decode(token_ids) |
| 32 | + out = +"" |
| 33 | + token_ids.each do |token| |
| 34 | + break if token == EOS_ID |
| 35 | + next if token == PAD_ID || token == UNK_ID |
| 36 | + out << ((token - BYTE_OFFSET) % 256).chr |
| 37 | + end |
| 38 | + out.force_encoding(Encoding::UTF_8) |
| 39 | + end |
| 40 | + |
| 41 | + def manifest(zip_path) |
| 42 | + require_zip! |
| 43 | + Zip::File.open(zip_path) do |zf| |
| 44 | + raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml") |
| 45 | + meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false) |
| 46 | + raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1" |
| 47 | + if meta["tokenizer"] != "bytes" |
| 48 | + raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only" |
| 49 | + end |
| 50 | + %w[encoder.onnx decoder.onnx].each do |required| |
| 51 | + raise FormatError, "missing #{required}" unless zf.find_entry(required) |
| 52 | + end |
| 53 | + meta |
| 54 | + end |
| 55 | + end |
| 56 | + |
| 57 | + # Reads every .onnx member after verifying its sha256 against the |
| 58 | + # manifest — corrupt zips fail loudly, before any session loads. |
| 59 | + def verify_and_read(zip_path) |
| 60 | + meta = manifest(zip_path) |
| 61 | + sha = meta.fetch("sha256", {}) |
| 62 | + graphs = {} |
| 63 | + require_zip! |
| 64 | + Zip::File.open(zip_path) do |zf| |
| 65 | + zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry| |
| 66 | + recorded = sha[entry.name] |
| 67 | + raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded |
| 68 | + bytes = entry.get_input_stream.read |
| 69 | + actual = Digest::SHA256.hexdigest(bytes) |
| 70 | + if actual != recorded |
| 71 | + raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}" |
| 72 | + end |
| 73 | + graphs[entry.name] = bytes |
| 74 | + end |
| 75 | + end |
| 76 | + graphs |
| 77 | + end |
| 78 | + |
| 79 | + def cache_dir |
| 80 | + ENV["INTERSCRIPT_ML_CACHE"] || ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "interscript-ml") |
| 81 | + end |
| 82 | + |
| 83 | + # models.yaml resolution: cache hit (re-verified), or download -> |
| 84 | + # verify whole-file sha256 -> atomic install into the cache. |
| 85 | + # Entries with `parts` (GitHub's 2 GiB per-asset cap) are streamed |
| 86 | + # in order, each part sha256-verified as it lands, then the |
| 87 | + # assembled file is checked against the whole-file sha256. |
| 88 | + def resolve(model_id, index_url: nil) |
| 89 | + source = index_url || ENV["INTERSCRIPT_ML_INDEX"] || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL |
| 90 | + entries = load_index(source) |
| 91 | + entry = entries[model_id] |
| 92 | + raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry |
| 93 | + |
| 94 | + target = File.join(cache_dir, "models", model_id, entry["filename"]) |
| 95 | + if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"] |
| 96 | + return target |
| 97 | + end |
| 98 | + |
| 99 | + FileUtils.mkdir_p(File.dirname(target)) |
| 100 | + tmp = target + ".part.#{Process.pid}" |
| 101 | + if entry["parts"] |
| 102 | + download_parts(entry, tmp) |
| 103 | + else |
| 104 | + channel = entry["url"] |
| 105 | + if channel.start_with?("file://") |
| 106 | + FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp) |
| 107 | + else |
| 108 | + URI.open(channel) { |remote| IO.copy_stream(remote, tmp) } |
| 109 | + end |
| 110 | + end |
| 111 | + actual = Digest::SHA256.file(tmp).hexdigest |
| 112 | + unless actual == entry["sha256"] |
| 113 | + File.delete(tmp) |
| 114 | + raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}" |
| 115 | + end |
| 116 | + File.rename(tmp, target) |
| 117 | + target |
| 118 | + end |
| 119 | + |
| 120 | + private |
| 121 | + |
| 122 | + def download_parts(entry, tmp) |
| 123 | + File.open(tmp, "wb") do |out| |
| 124 | + entry["parts"].each_with_index do |part, index| |
| 125 | + digest = Digest::SHA256.new |
| 126 | + source = part["url"].start_with?("file://") ? part["url"].sub(%r{\Afile://}, "") : part["url"] |
| 127 | + open_stream = lambda do |io| |
| 128 | + while (chunk = io.read(1024 * 1024)) |
| 129 | + out.write(chunk) |
| 130 | + digest.update(chunk) |
| 131 | + end |
| 132 | + end |
| 133 | + if part["url"].start_with?("file://") |
| 134 | + File.open(source, "rb", &open_stream) |
| 135 | + else |
| 136 | + URI.open(source, "rb", &open_stream) |
| 137 | + end |
| 138 | + unless digest.hexdigest == part["sha256"] |
| 139 | + raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}" |
| 140 | + end |
| 141 | + end |
| 142 | + end |
| 143 | + rescue StandardError |
| 144 | + File.delete(tmp) if File.file?(tmp) |
| 145 | + raise |
| 146 | + end |
| 147 | + |
| 148 | + def load_index(source) |
| 149 | + text = if source.start_with?("http://", "https://") |
| 150 | + fetch_index_with_sidecar(source) |
| 151 | + else |
| 152 | + File.read(source) |
| 153 | + end |
| 154 | + raw = YAML.safe_load(text, permitted_classes: [], aliases: false) |
| 155 | + raise RegistryError, "index must have version: 1" if raw["version"] != 1 |
| 156 | + raw.fetch("models", {}) |
| 157 | + end |
| 158 | + public :load_index |
| 159 | + |
| 160 | + # HTTP sources must ship a sibling .sha256 sidecar; verify the |
| 161 | + # body against it before parsing (GitHub Releases index assets do). |
| 162 | + def fetch_index_with_sidecar(source) |
| 163 | + body = URI.open(source, "rb") { |remote| remote.read } |
| 164 | + sidecar = |
| 165 | + begin |
| 166 | + URI.open("#{source}.sha256") { |remote| remote.read } |
| 167 | + rescue OpenURI::HTTPError => e |
| 168 | + raise RegistryError, "index sha256 sidecar missing: #{source}.sha256 (#{e.message})" |
| 169 | + end |
| 170 | + expected = sidecar.to_s.strip.split(/\s+/).first.to_s |
| 171 | + unless expected.match?(/\A[0-9a-fA-F]{64}\z/) |
| 172 | + raise RegistryError, "index sha256 sidecar malformed: #{source}.sha256" |
| 173 | + end |
| 174 | + actual = Digest::SHA256.hexdigest(body) |
| 175 | + unless actual == expected.downcase |
| 176 | + raise RegistryError, "index sha256 mismatch: got #{actual}, sidecar says #{expected.downcase}" |
| 177 | + end |
| 178 | + body.force_encoding(Encoding::UTF_8) |
| 179 | + end |
| 180 | + end |
| 181 | + end |
| 182 | +end |
0 commit comments