diff --git a/.changeset/fix-arraybuffer-serialization.md b/.changeset/fix-arraybuffer-serialization.md new file mode 100644 index 00000000..71399cc6 --- /dev/null +++ b/.changeset/fix-arraybuffer-serialization.md @@ -0,0 +1,5 @@ +--- +"capnweb": patch +--- + +Support exact ArrayBuffer, DataView, and typed array serialization over RPC. diff --git a/README.md b/README.md index 963e4327..bfe4ae98 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ The following types can be passed over RPC (in arguments or return values), and * Arrays * `bigint` * `Date` -* `Uint8Array` +* `ArrayBuffer`, `DataView`, and typed arrays * `Error` and its well-known subclasses * `Blob` * `ReadableStream` and `WritableStream`, with automatic flow control. @@ -207,7 +207,6 @@ The following types can be passed over RPC (in arguments or return values), and The following types are not supported as of this writing, but may be added in the future: * `Map` and `Set` -* `ArrayBuffer` and typed arrays other than `Uint8Array` * `RegExp` The following are intentionally NOT supported: diff --git a/__tests__/index.test.ts b/__tests__/index.test.ts index fdf41b33..3123da5b 100644 --- a/__tests__/index.test.ts +++ b/__tests__/index.test.ts @@ -160,6 +160,16 @@ describe("simple serialization", () => { expect(new Uint8Array(deserialized)).toStrictEqual(bytes); }) + it("can serialize Uint8Array as legacy bytes without a type marker", () => { + let bytes = new Uint8Array([72, 101, 108, 108, 111]); + let serialized = serialize(bytes); + expect(serialized).toBe('["bytes","SGVsbG8"]'); + + let deserialized = deserialize(serialized) as Uint8Array; + expect(deserialized).toBeInstanceOf(Uint8Array); + expect(new Uint8Array(deserialized)).toStrictEqual(bytes); + }) + it("can serialize Node.js Buffer as bytes", () => { if (typeof Buffer === "undefined") return; // skip in browsers let buf = Buffer.from("hello!"); @@ -170,6 +180,73 @@ describe("simple serialization", () => { expect(new Uint8Array(deserialized)).toStrictEqual(new Uint8Array(buf)); }) + it("can serialize ArrayBuffer as bytes with an ArrayBuffer marker", () => { + let bytes = new Uint8Array([72, 101, 108, 108, 111]); + let serialized = serialize(bytes.buffer); + expect(serialized).toBe('["bytes","SGVsbG8","ArrayBuffer"]'); + + let deserialized = deserialize(serialized); + expect(deserialized).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(deserialized as ArrayBuffer)).toStrictEqual(bytes); + }) + + it("can serialize typed array views as bytes with type markers", () => { + let cases = [ + { + name: "DataView", + elementSize: 1, + makeView: (buffer: ArrayBuffer, offset: number, byteLength: number) => + new DataView(buffer, offset, byteLength), + }, + ...[ + Int8Array, + Uint8ClampedArray, + Int16Array, + Uint16Array, + Int32Array, + Uint32Array, + BigInt64Array, + BigUint64Array, + Float32Array, + Float64Array, + ].map(Type => ({ + name: Type.name, + elementSize: Type.BYTES_PER_ELEMENT, + makeView: (buffer: ArrayBuffer, offset: number, byteLength: number) => + new Type(buffer, offset, byteLength / Type.BYTES_PER_ELEMENT), + })), + ]; + + for (let {name, elementSize, makeView} of cases) { + // Use a non-zero offset and extra trailing byte to verify only the view's visible byte range + // is serialized, not the whole backing buffer. + let byteLength = elementSize * 2; + let offset = elementSize; + let backing = new ArrayBuffer(offset + byteLength + 1); + let bytes = new Uint8Array(byteLength); + for (let i = 0; i < bytes.length; i++) bytes[i] = i + 1; + new Uint8Array(backing, offset, byteLength).set(bytes); + + let view = makeView(backing, offset, byteLength); + let serialized = serialize(view); + let parsed = JSON.parse(serialized) as [string, string, string]; + expect(parsed[0]).toBe("bytes"); + expect(parsed[2]).toBe(name); + + let deserialized = deserialize(serialized) as ArrayBufferView; + expect(Object.getPrototypeOf(deserialized)).toBe(Object.getPrototypeOf(view)); + expect(new Uint8Array( + deserialized.buffer, deserialized.byteOffset, deserialized.byteLength)).toStrictEqual(bytes); + } + }) + + it("throws for unknown bytes type markers", () => { + expect(() => deserialize('["bytes","SGVsbG8","invalidUint8Array"]')).toThrowError( + "Unknown bytes type marker: invalidUint8Array"); + expect(() => deserialize('["bytes","SGVsbG8",123]')).toThrowError( + "Unknown bytes type marker type: number"); + }) + it("preserves Invalid Date values through serialization", () => { let invalidDate = new Date(NaN); let serialized = serialize(invalidDate); diff --git a/protocol.md b/protocol.md index b1797954..b21bfa7d 100644 --- a/protocol.md +++ b/protocol.md @@ -168,9 +168,14 @@ The literal value `undefined`. The values Infinity, -Infinity, and NaN. -`["bytes", base64]` - -A `Uint8Array`, represented as a base64-encoded string. +`["bytes", base64]`, `["bytes", base64, type]` + +A byte container, represented as a base64-encoded string. If `type` is omitted, the receiver +should deserialize bytes as its default `Uint8Array` for backwards compatibility. Otherwise, +`type` preserves the byte container type across the wire. The supported `type` values are +`ArrayBuffer`, `DataView`, `Int8Array`, `Uint8ClampedArray`, +`Int16Array`, `Uint16Array`, `Int32Array`, `Uint32Array`, `BigInt64Array`, `BigUint64Array`, +`Float32Array`, and `Float64Array`. `["blob", type, readableExpression]` diff --git a/src/core.ts b/src/core.ts index 2ea587e6..d163a20a 100644 --- a/src/core.ts +++ b/src/core.ts @@ -95,6 +95,18 @@ export function typeForRpc(value: unknown): TypeForRpc { case Uint8Array.prototype: case BUFFER_PROTOTYPE: + case ArrayBuffer.prototype: + case DataView.prototype: + case Int8Array.prototype: + case Uint8ClampedArray.prototype: + case Int16Array.prototype: + case Uint16Array.prototype: + case Int32Array.prototype: + case Uint32Array.prototype: + case BigInt64Array.prototype: + case BigUint64Array.prototype: + case Float32Array.prototype: + case Float64Array.prototype: return "bytes"; case WritableStream.prototype: diff --git a/src/serialize.ts b/src/serialize.ts index 97074c0c..9bb0dc00 100644 --- a/src/serialize.ts +++ b/src/serialize.ts @@ -162,12 +162,59 @@ export class Devaluator { } case "bytes": { - let bytes = value as Uint8Array; - if (bytes.toBase64) { - return ["bytes", bytes.toBase64({omitPadding: true})]; + let alternateTypeName: string | undefined; + let bytes: Uint8Array | undefined; + switch (Object.getPrototypeOf(value)) { + case ArrayBuffer.prototype: + alternateTypeName = "ArrayBuffer"; + bytes = new Uint8Array(value as ArrayBuffer); + break; + case DataView.prototype: + alternateTypeName = "DataView"; + break; + case Int8Array.prototype: + alternateTypeName = "Int8Array"; + break; + case Uint8ClampedArray.prototype: + alternateTypeName = "Uint8ClampedArray"; + break; + case Int16Array.prototype: + alternateTypeName = "Int16Array"; + break; + case Uint16Array.prototype: + alternateTypeName = "Uint16Array"; + break; + case Int32Array.prototype: + alternateTypeName = "Int32Array"; + break; + case Uint32Array.prototype: + alternateTypeName = "Uint32Array"; + break; + case BigInt64Array.prototype: + alternateTypeName = "BigInt64Array"; + break; + case BigUint64Array.prototype: + alternateTypeName = "BigUint64Array"; + break; + case Float32Array.prototype: + alternateTypeName = "Float32Array"; + break; + case Float64Array.prototype: + alternateTypeName = "Float64Array"; + break; + default: + bytes = value as Uint8Array; + break; + } + if (bytes === undefined) { + let view = value as ArrayBufferView; + bytes = new Uint8Array(view.buffer, view.byteOffset, view.byteLength); } + let b64: string; - if (typeof Buffer !== "undefined") { + if (bytes.toBase64) { + b64 = bytes.toBase64({omitPadding: true}); + } else if (typeof Buffer !== "undefined") { let buf = bytes instanceof Buffer ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); b64 = buf.toString("base64"); @@ -178,7 +225,8 @@ export class Devaluator { } b64 = btoa(binary); } - return ["bytes", b64.replace(/=+$/, "")]; + b64 = b64.replace(/=+$/, ""); + return alternateTypeName === undefined ? ["bytes", b64] : ["bytes", b64, alternateTypeName]; } case "headers": @@ -601,19 +649,45 @@ export class Evaluator { break; case "bytes": { if (typeof value[1] == "string") { + let bytes: Uint8Array; if (typeof Buffer !== "undefined") { - return Buffer.from(value[1], "base64"); + bytes = Buffer.from(value[1], "base64"); } else if (Uint8Array.fromBase64) { - return Uint8Array.fromBase64(value[1]); + bytes = Uint8Array.fromBase64(value[1]); } else { let bs = atob(value[1]); let len = bs.length; - let bytes = new Uint8Array(len); + bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = bs.charCodeAt(i); } + } + if (value.length === 2) { return bytes; } + if (typeof value[2] !== "string") { + throw new TypeError(`Unknown bytes type marker type: ${typeof value[2]}`); + } + + let buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + switch (value[2]) { + case "ArrayBuffer": return buffer; + case "DataView": return new DataView(buffer); + case "Int8Array": return new Int8Array(buffer); + case "Uint8ClampedArray": return new Uint8ClampedArray(buffer); + case "Int16Array": return new Int16Array(buffer); + case "Uint16Array": return new Uint16Array(buffer); + case "Int32Array": return new Int32Array(buffer); + case "Uint32Array": return new Uint32Array(buffer); + case "BigInt64Array": return new BigInt64Array(buffer); + case "BigUint64Array": return new BigUint64Array(buffer); + case "Float32Array": return new Float32Array(buffer); + case "Float64Array": return new Float64Array(buffer); + default: { + let marker = value[2].slice(0, 64); + throw new TypeError(`Unknown bytes type marker: ${marker}`); + } + } } break; }