|
1 | 1 | require "spec_helper" |
2 | 2 | require "zip" |
3 | | -require 'interscript/ml' |
4 | | -require 'fileutils' |
5 | | -require 'tmpdir' |
6 | | -require 'json' |
| 3 | +require "interscript/ml" |
| 4 | +require "fileutils" |
| 5 | +require "tmpdir" |
| 6 | +require "json" |
7 | 7 |
|
8 | 8 | RSpec.describe Interscript::ML::IMF do |
9 | | - it 'uses the canonical ByT5 table (byte + 3, trailing EOS)' do |
10 | | - expect(described_class.encode('rok')).to eq([117, 114, 110, 1]) |
11 | | - expect(described_class.decode([117, 114, 110])).to eq('rok') |
12 | | - expect(described_class.decode([117, 1, 114])).to eq('r') |
| 9 | + it "uses the canonical ByT5 table (byte + 3, trailing EOS)" do |
| 10 | + expect(described_class.encode("rok")).to eq([117, 114, 110, 1]) |
| 11 | + expect(described_class.decode([117, 114, 110])).to eq("rok") |
| 12 | + expect(described_class.decode([117, 1, 114])).to eq("r") |
13 | 13 | end |
14 | 14 |
|
15 | | - describe 'with the tiny IMF fixture zip' do |
16 | | - let(:zip_path) { File.expand_path('../fixtures/tiny-imf.zip', __dir__) } |
| 15 | + describe "with the tiny IMF fixture zip" do |
| 16 | + let(:zip_path) { File.expand_path("../fixtures/tiny-imf.zip", __dir__) } |
17 | 17 |
|
18 | | - it 'parses the manifest and verifies graph checksums' do |
| 18 | + it "parses the manifest and verifies graph checksums" do |
19 | 19 | meta = described_class.manifest(zip_path) |
20 | | - expect(meta['format']).to eq('imf-v1') |
21 | | - expect(meta['tokenizer']).to eq('bytes') |
| 20 | + expect(meta["format"]).to eq("imf-v1") |
| 21 | + expect(meta["tokenizer"]).to eq("bytes") |
22 | 22 | graphs = described_class.verify_and_read(zip_path) |
23 | | - expect(graphs.keys.sort).to eq(['decoder.onnx', 'encoder.onnx']) |
| 23 | + expect(graphs.keys.sort).to eq(["decoder.onnx", "encoder.onnx"]) |
24 | 24 | end |
25 | 25 |
|
26 | | - it 'loads sessions from verified bytes' do |
| 26 | + it "loads sessions from verified bytes" do |
27 | 27 | model = Interscript::ML::Byt5Onnx.new(zip_path) |
28 | | - expect(model.id).to eq('tiny-1.0') |
| 28 | + expect(model.id).to eq("tiny-1.0") |
29 | 29 | end |
30 | 30 |
|
31 | | - it 'decodes a real zip against the shared golden set', :e2e do |
32 | | - zip = ENV['SECRYST_E2E_ZIP'] |
33 | | - golden = ENV['SECRYST_GOLDEN'] |
34 | | - skip 'set SECRYST_E2E_ZIP and SECRYST_GOLDEN for the end-to-end run' unless zip && golden |
| 31 | + it "decodes a real zip against the shared golden set", :e2e do |
| 32 | + zip = ENV["SECRYST_E2E_ZIP"] |
| 33 | + golden = ENV["SECRYST_GOLDEN"] |
| 34 | + skip "set SECRYST_E2E_ZIP and SECRYST_GOLDEN for the end-to-end run" unless zip && golden |
35 | 35 | translator = Interscript::ML::Translator.new(model_file: zip) |
36 | 36 | ok = 0 |
37 | 37 | total = 0 |
38 | 38 | File.readlines(golden).each do |line| |
39 | 39 | row = JSON.parse(line) |
40 | 40 | total += 1 |
41 | | - ok += 1 if translator.translate(row['input'], max_seq_length: 128) == row['output'] |
| 41 | + ok += 1 if translator.translate(row["input"], max_seq_length: 128) == row["output"] |
42 | 42 | end |
43 | 43 | expect(ok).to eq(total) |
44 | 44 | end |
45 | 45 | end |
46 | 46 |
|
47 | | - it 'rejects a tampered zip loudly' do |
| 47 | + it "rejects a tampered zip loudly" do |
48 | 48 | Dir.mktmpdir do |tmp| |
49 | | - source = File.expand_path('../fixtures/tiny-imf.zip', __dir__) |
50 | | - tampered = File.join(tmp, 'tampered.zip') |
51 | | - require 'zip' |
| 49 | + source = File.expand_path("../fixtures/tiny-imf.zip", __dir__) |
| 50 | + tampered = File.join(tmp, "tampered.zip") |
| 51 | + require "zip" |
52 | 52 | Zip::File.open(source) do |src| |
53 | 53 | Zip::File.open(tampered, create: true) do |dst| |
54 | 54 | src.entries.each do |e| |
55 | | - if e.name == 'encoder.onnx' |
56 | | - dst.get_output_stream(e.name) { |io| io.write('corrupted-bytes') } |
| 55 | + if e.name == "encoder.onnx" |
| 56 | + dst.get_output_stream(e.name) { |io| io.write("corrupted-bytes") } |
57 | 57 | else |
58 | 58 | dst.get_output_stream(e.name) { |io| io.write(e.get_input_stream.read) } |
59 | 59 | end |
|
65 | 65 | end |
66 | 66 | end |
67 | 67 |
|
68 | | - describe '.resolve' do |
69 | | - it 'installs a verified copy into the cache from a local index' do |
| 68 | + describe ".resolve" do |
| 69 | + it "installs a verified copy into the cache from a local index" do |
70 | 70 | Dir.mktmpdir do |tmp| |
71 | | - channel = File.join(tmp, 'channel') |
| 71 | + channel = File.join(tmp, "channel") |
72 | 72 | FileUtils.mkdir_p(channel) |
73 | | - zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) |
74 | | - FileUtils.cp(zip_path, File.join(channel, 'tiny.zip')) |
75 | | - require 'digest' |
76 | | - index = File.join(tmp, 'models.yaml') |
| 73 | + zip_path = File.expand_path("../fixtures/tiny-imf.zip", __dir__) |
| 74 | + FileUtils.cp(zip_path, File.join(channel, "tiny.zip")) |
| 75 | + require "digest" |
| 76 | + index = File.join(tmp, "models.yaml") |
77 | 77 | File.write(index, <<~YAML) |
78 | 78 | version: 1 |
79 | 79 | models: |
|
82 | 82 | url: file://#{channel}/tiny.zip |
83 | 83 | sha256: #{Digest::SHA256.file(zip_path).hexdigest} |
84 | 84 | YAML |
85 | | - cache = File.join(tmp, 'cache') |
86 | | - result = described_class.resolve('tiny-1.0', index_url: index) if false |
| 85 | + cache = File.join(tmp, "cache") |
| 86 | + |
87 | 87 | # env-based cache (the public API reads ENV at call time) |
88 | | - ENV['SECRYST_CACHE'] = cache |
| 88 | + ENV["SECRYST_CACHE"] = cache |
89 | 89 | begin |
90 | | - installed = described_class.resolve('tiny-1.0', index_url: index) |
91 | | - expect(installed).to eq(File.join(cache, 'models', 'tiny-1.0', 'tiny.zip')) |
| 90 | + installed = described_class.resolve("tiny-1.0", index_url: index) |
| 91 | + expect(installed).to eq(File.join(cache, "models", "tiny-1.0", "tiny.zip")) |
92 | 92 | expect(File.file?(installed)).to be(true) |
93 | | - FileUtils.rm_f(File.join(channel, 'tiny.zip')) |
94 | | - expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed) |
| 93 | + FileUtils.rm_f(File.join(channel, "tiny.zip")) |
| 94 | + expect(described_class.resolve("tiny-1.0", index_url: index)).to eq(installed) |
95 | 95 | ensure |
96 | | - ENV.delete('SECRYST_CACHE') |
| 96 | + ENV.delete("SECRYST_CACHE") |
97 | 97 | end |
98 | 98 | end |
99 | 99 | end |
100 | 100 |
|
101 | | - it 'raises for unknown ids' do |
| 101 | + it "raises for unknown ids" do |
102 | 102 | Dir.mktmpdir do |tmp| |
103 | | - index = File.join(tmp, 'models.yaml') |
| 103 | + index = File.join(tmp, "models.yaml") |
104 | 104 | File.write(index, "version: 1\nmodels: {}\n") |
105 | | - expect { described_class.resolve('nope-1.0', index_url: index) } |
| 105 | + expect { described_class.resolve("nope-1.0", index_url: index) } |
106 | 106 | .to raise_error(Interscript::ML::IMF::RegistryError, /unknown model id/) |
107 | 107 | end |
108 | 108 | end |
109 | 109 |
|
110 | | - it 'assembles and verifies split parts (the >2GiB GitHub cap path)' do |
| 110 | + it "assembles and verifies split parts (the >2GiB GitHub cap path)" do |
111 | 111 | Dir.mktmpdir do |tmp| |
112 | | - channel = File.join(tmp, 'channel') |
| 112 | + channel = File.join(tmp, "channel") |
113 | 113 | FileUtils.mkdir_p(channel) |
114 | | - zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) |
| 114 | + zip_path = File.expand_path("../fixtures/tiny-imf.zip", __dir__) |
115 | 115 | blob = File.binread(zip_path) |
116 | 116 | part_a, part_b = blob[0, (blob.bytesize / 2 + 3)], blob[(blob.bytesize / 2 + 3)..] |
117 | | - File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a) |
118 | | - File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b) |
119 | | - index = File.join(tmp, 'models.yaml') |
| 117 | + File.binwrite(File.join(channel, "tiny.zip.part-00"), part_a) |
| 118 | + File.binwrite(File.join(channel, "tiny.zip.part-01"), part_b) |
| 119 | + index = File.join(tmp, "models.yaml") |
120 | 120 | File.write(index, <<~YAML) |
121 | 121 | version: 1 |
122 | 122 | models: |
|
131 | 131 | sha256: #{Digest::SHA256.hexdigest(part_b)} |
132 | 132 | size: #{part_b.bytesize} |
133 | 133 | YAML |
134 | | - cache = File.join(tmp, 'cache') |
135 | | - ENV['SECRYST_CACHE'] = cache |
| 134 | + cache = File.join(tmp, "cache") |
| 135 | + ENV["SECRYST_CACHE"] = cache |
136 | 136 | begin |
137 | | - installed = described_class.resolve('tiny-1.0', index_url: index) |
| 137 | + installed = described_class.resolve("tiny-1.0", index_url: index) |
138 | 138 | expect(File.binread(installed)).to eq(blob) |
139 | | - FileUtils.rm_f(File.join(channel, 'tiny.zip.part-00')) |
140 | | - expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed) |
| 139 | + FileUtils.rm_f(File.join(channel, "tiny.zip.part-00")) |
| 140 | + expect(described_class.resolve("tiny-1.0", index_url: index)).to eq(installed) |
141 | 141 | ensure |
142 | | - ENV.delete('SECRYST_CACHE') |
| 142 | + ENV.delete("SECRYST_CACHE") |
143 | 143 | end |
144 | 144 | end |
145 | 145 | end |
146 | 146 |
|
147 | | - it 'rejects a corrupt part by index' do |
| 147 | + it "rejects a corrupt part by index" do |
148 | 148 | Dir.mktmpdir do |tmp| |
149 | | - channel = File.join(tmp, 'channel') |
| 149 | + channel = File.join(tmp, "channel") |
150 | 150 | FileUtils.mkdir_p(channel) |
151 | | - zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__) |
| 151 | + zip_path = File.expand_path("../fixtures/tiny-imf.zip", __dir__) |
152 | 152 | blob = File.binread(zip_path) |
153 | 153 | part_a, part_b = blob[0, 7], blob[7..] |
154 | | - File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a) |
155 | | - File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b) |
156 | | - index = File.join(tmp, 'models.yaml') |
| 154 | + File.binwrite(File.join(channel, "tiny.zip.part-00"), part_a) |
| 155 | + File.binwrite(File.join(channel, "tiny.zip.part-01"), part_b) |
| 156 | + index = File.join(tmp, "models.yaml") |
157 | 157 | File.write(index, <<~YAML) |
158 | 158 | version: 1 |
159 | 159 | models: |
|
168 | 168 | sha256: #{Digest::SHA256.hexdigest(part_b)} |
169 | 169 | size: #{part_b.bytesize} |
170 | 170 | YAML |
171 | | - ENV['SECRYST_CACHE'] = File.join(tmp, 'cache') |
| 171 | + ENV["SECRYST_CACHE"] = File.join(tmp, "cache") |
172 | 172 | begin |
173 | | - expect { described_class.resolve('tiny-1.0', index_url: index) } |
| 173 | + expect { described_class.resolve("tiny-1.0", index_url: index) } |
174 | 174 | .to raise_error(Interscript::ML::IMF::RegistryError, /part 0 .* sha256 mismatch/) |
175 | 175 | ensure |
176 | | - ENV.delete('SECRYST_CACHE') |
| 176 | + ENV.delete("SECRYST_CACHE") |
177 | 177 | end |
178 | 178 | end |
179 | 179 | end |
|
201 | 201 | port = server.addr[1] |
202 | 202 | thread = Thread.new do |
203 | 203 | loop do |
204 | | - client = server.accept rescue break |
| 204 | + client = begin |
| 205 | + server.accept |
| 206 | + rescue |
| 207 | + break |
| 208 | + end |
205 | 209 | request = client.readline |
206 | 210 | client.each_line { |line| break if line == "\r\n" } |
207 | 211 | path = request.split(" ")[1] |
|
0 commit comments