From 5dffdf8f79fe1544d0a2a896fbffea163d8213bc Mon Sep 17 00:00:00 2001 From: u9g Date: Sun, 6 Sep 2026 10:32:22 -0400 Subject: [PATCH 1/4] Add the hash datatype ["hash", { alg, type, body }] writes the value serialized as `body`, hashed with `alg`, as `type`; reading yields the digest. crc32 and crc32c are built in, anything else goes through node's crypto and is a Buffer. A CRC written into a signed type takes its two's complement. The compiled writer needs the size of the body before it can serialize it, which no writer could get at until now: WriteCompiler.callTypeSize and SizeOfCompiler.callTypeWrite generate code with the sibling compiler in the current scope and run it against that compiler's context. SizeOfCompiler now remembers fixed-size natives so hashes into a fixed-width type are sized without hashing. --- ProtoDef | 2 +- doc/compiler.md | 17 +++++++- src/compiler.js | 51 ++++++++++++++++++++++ src/datatypes/compiler-utils.js | 26 +++++++++++ src/datatypes/utils.js | 31 +++++++++++++- src/hash.js | 39 +++++++++++++++++ test/misc.js | 76 +++++++++++++++++++++++++++++++++ 7 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 src/hash.js diff --git a/ProtoDef b/ProtoDef index 8e07785..647e754 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit 8e07785e94626882fa3333184bb366e3f6625356 +Subproject commit 647e754b8147b40dd05e947801e00ef3f38f704f diff --git a/doc/compiler.md b/doc/compiler.md index a97a078..ac34893 100644 --- a/doc/compiler.md +++ b/doc/compiler.md @@ -228,4 +228,19 @@ compiledProto.setVariable('noArraySizeCheck', true); // Use it as if it were a normal ProtoDef const buffer = compiledProto.createPacketBuffer('mainType', result) const result = compiledProto.parsePacketBuffer('mainType', buffer) -``` \ No newline at end of file +``` +### Sizing inside a writer, writing inside a sizer + +A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code computing the size of `value` as `type`, and `SizeOfCompiler.callTypeWrite(value, type, offsetExpr)` returns code writing it into `buffer`. Both resolve field references against the current scope and run against the other compiler's context, so they are only available when the types are compiled through `ProtoDefCompiler`. The `hash` datatype is built on them: + +```javascript +Write: { + hash: ['parametrizable', (compiler, { alg, type, body }) => { + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += 'return ' + compiler.callType('hash', type) + return compiler.wrapCode(code) + }] +} +``` diff --git a/src/compiler.js b/src/compiler.js index 9c8f057..4547361 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -12,6 +12,8 @@ class ProtoDefCompiler { this.readCompiler = new ReadCompiler() this.writeCompiler = new WriteCompiler() this.sizeOfCompiler = new SizeOfCompiler() + this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler + this.sizeOfCompiler.writeCompiler = this.writeCompiler } addTypes (types) { @@ -62,6 +64,9 @@ class CompiledProtodef { this.sizeOfCtx = sizeOfCtx this.writeCtx = writeCtx this.readCtx = readCtx + // Code from callTypeSize / callTypeWrite runs against the other context + writeCtx.sizeOfCtx = sizeOfCtx + sizeOfCtx.writeCtx = writeCtx } read (buffer, cursor, type) { @@ -174,6 +179,24 @@ class Compiler { } } + /** + * Generates code with another compiler inside this compiler's scope, so that + * field references resolve to the same variables, and binds it to that + * compiler's context. Natives are reachable through the context as well. + */ + callTypeIn (other, ctxName, generate) { + if (!other) throw new Error(`${ctxName} is only available when compiling with ProtoDefCompiler`) + const scopeStack = other.scopeStack + other.scopeStack = this.scopeStack + try { + const code = generate(other) + if (!isNaN(code)) return code + return `((ctx, native) => ${code})(ctx.${ctxName}, ctx.${ctxName})` + } finally { + other.scopeStack = scopeStack + } + } + addTypesToCompile (types) { for (const [type, json] of Object.entries(types)) { // Replace native type, otherwise first in wins @@ -259,6 +282,7 @@ class Compiler { // Local variable to provide some context to eval() const native = this.native // eslint-disable-line const { PartialReadError } = require('./utils') // eslint-disable-line + const hashDigest = require('./hash').digest // eslint-disable-line return eval(code)() // eslint-disable-line } } @@ -361,11 +385,20 @@ class WriteCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, buffer, ${offsetExpr}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value}, buffer, ${offsetExpr})` } + + /** + * Code computing the size of `value` as `type`, for writers that need to + * serialize part of a value before they can write it + */ + callTypeSize (value, type, args = []) { + return this.callTypeIn(this.sizeOfCompiler, 'sizeOfCtx', compiler => compiler.callType(value, type, args)) + } } class SizeOfCompiler extends Compiler { constructor () { super() + this.constants = {} this.addTypes(conditionalDatatypes.SizeOf) this.addTypes(structuresDatatypes.SizeOf) @@ -390,12 +423,22 @@ class SizeOfCompiler extends Compiler { this.primitiveTypes[type] = `native.${type}` if (!isNaN(fn)) { this.native[type] = (value) => { return fn } + this.constants[type] = fn } else { this.native[type] = fn } this.types[type] = 'native' } + /** + * The size of `type` when it doesn't depend on the value, following + * aliases down to a fixed-size native; undefined otherwise + */ + constantSize (type) { + while (typeof type === 'string' && typeof this.types[type] === 'string' && this.types[type] !== 'native') type = this.types[type] + return this.constants[type] + } + compileType (type) { if (type instanceof Array) { if (this.parameterizableTypes[type[0]]) { return this.parameterizableTypes[type[0]](this, type[1]) } @@ -429,6 +472,14 @@ class SizeOfCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value})` } + + /** + * Code writing `value` as `type` into `buffer` at `offsetExpr`, for sizers + * whose result depends on the serialized form of a value + */ + callTypeWrite (value, type, offsetExpr = 'offset', args = []) { + return this.callTypeIn(this.writeCompiler, 'writeCtx', compiler => compiler.callType(value, type, offsetExpr, args)) + } } module.exports = { diff --git a/src/datatypes/compiler-utils.js b/src/datatypes/compiler-utils.js index d60eda5..e528e90 100644 --- a/src/datatypes/compiler-utils.js +++ b/src/datatypes/compiler-utils.js @@ -83,6 +83,9 @@ return { value, size } let code = 'const { value, size } = ' + compiler.callType(mapper.type) + '\n' code += 'return { value: ' + JSON.stringify(sanitizeMappings(mapper.mappings)) + '[value] || value, size }' return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { type }) => { + return compiler.wrapCode('return ' + compiler.callType(type)) }] }, @@ -163,6 +166,18 @@ return (ctx.${type})(val, buffer, offset) code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += 'try {\n' + code += ' return ' + compiler.callType('hash', type) + '\n' + code += '} catch (e) {\n' + code += ' if (!(e instanceof RangeError) || typeof hash !== "number") throw e\n' + code += ' return ' + compiler.callType('hash | 0', type) + '\n' + code += '}' + return compiler.wrapCode(code) }] }, @@ -217,6 +232,17 @@ return (ctx.${type})(val) code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + const constant = compiler.constantSize(type) + if (constant !== undefined) return String(constant) + const size = compiler.callType('hash', type) + if (!isNaN(size)) return size + let code = `const bodyBuffer = Buffer.alloc(${compiler.callType('value', body)})\n` + code += `;((buffer) => ${compiler.callTypeWrite('value', body, '0')})(bodyBuffer)\n` + code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += 'return ' + size + return compiler.wrapCode(code) }] } } diff --git a/src/datatypes/utils.js b/src/datatypes/utils.js index 2f1722b..ff380a2 100644 --- a/src/datatypes/utils.js +++ b/src/datatypes/utils.js @@ -1,4 +1,5 @@ -const { getCount, sendCount, calcCount, PartialReadError } = require('../utils') +const { getCount, sendCount, calcCount, getFieldInfo, PartialReadError } = require('../utils') +const { digest } = require('../hash') module.exports = { bool: [readBool, writeBool, 1, require('../../ProtoDef/schemas/utils.json').bool], @@ -9,6 +10,7 @@ module.exports = { bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags], cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring], mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper], + hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash], ...require('./varint') } @@ -280,3 +282,30 @@ function sizeOfBitflags (value, { type, flags, shift, big }, rootNode) { } return this.sizeOf(mappedValue, type, rootNode) } + +function readHash (buffer, offset, { type }, rootNode) { + return this.read(buffer, offset, type, rootNode) +} + +function hashOf (value, { alg, body }, rootNode) { + const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) + this.write(value, bodyBuffer, 0, body, rootNode) + return digest(alg, bodyBuffer) +} + +// A CRC is unsigned; a signed `type` takes its two's complement. +function writeHash (value, buffer, offset, typeArgs, rootNode) { + const hash = hashOf.call(this, value, typeArgs, rootNode) + try { + return this.write(hash, buffer, offset, typeArgs.type, rootNode) + } catch (e) { + if (!(e instanceof RangeError) || typeof hash !== 'number') throw e + return this.write(hash | 0, buffer, offset, typeArgs.type, rootNode) + } +} + +function sizeOfHash (value, typeArgs, rootNode) { + const functions = this.types[getFieldInfo(typeArgs.type).type] + if (functions && typeof functions[2] === 'number') return functions[2] + return this.sizeOf(hashOf.call(this, value, typeArgs, rootNode), typeArgs.type, rootNode) +} diff --git a/src/hash.js b/src/hash.js new file mode 100644 index 0000000..a1023d5 --- /dev/null +++ b/src/hash.js @@ -0,0 +1,39 @@ +const crypto = require('crypto') + +// Reflected table-driven CRC with all-ones init and final xor; `poly` is the +// reversed polynomial. +const tables = {} +function table (poly) { + if (!tables[poly]) { + const t = new Int32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >>> 1) : c >>> 1 + t[n] = c + } + tables[poly] = t + } + return tables[poly] +} + +function crc (poly, buffer) { + const t = table(poly) + let c = -1 + for (let i = 0; i < buffer.length; i++) c = t[(c ^ buffer[i]) & 0xff] ^ (c >>> 8) + return (c ^ -1) >>> 0 +} + +const algorithms = { + crc32: buffer => crc(0xEDB88320, buffer), + crc32c: buffer => crc(0x82F63B78, buffer) +} + +// CRC digests are unsigned integers; every other algorithm is delegated to +// node's crypto and yields a Buffer. +function digest (alg, buffer) { + const algorithm = algorithms[alg] + if (algorithm) return algorithm(buffer) + return crypto.createHash(alg).update(buffer).digest() +} + +module.exports = { digest, algorithms } diff --git a/test/misc.js b/test/misc.js index dab6359..d4a7829 100644 --- a/test/misc.js +++ b/test/misc.js @@ -25,3 +25,79 @@ describe('mapper', () => { }) } }) + +describe('hash', () => { + const { digest } = require('../src/hash') + const types = { + crc32: ['hash', { alg: 'crc32', type: 'u32', body: ['buffer', { count: 9 }] }], + crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }], + signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: ['buffer', { count: 9 }] }], + HashCode: 'i32', + asVarint: ['hash', { alg: 'crc32c', type: 'varint', body: ['buffer', { count: 9 }] }], + sha256: ['hash', { alg: 'sha256', type: ['buffer', { count: 32 }], body: ['buffer', { count: 9 }] }], + // A field of the enclosing container selects the body's type + tagged: ['container', [ + { name: 'kind', type: 'u8' }, + { name: 'hash', type: ['hash', { alg: 'crc32c', type: 'u32', body: ['switch', { compareTo: 'kind', fields: { 0: 'u8', 1: 'u16' } }] }] } + ]], + // A hash over a list of hashes + entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]], + list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }], + nested: ['hash', { alg: 'crc32c', type: 'lu32', body: 'list' }] + } + const proto = new ProtoDef() + proto.addTypes(types) + const compiler = new ProtoDefCompiler() + compiler.addTypesToCompile(types) + const compiled = compiler.compileProtoDefSync() + const check = Buffer.from('123456789') + const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b } + const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b } + + it('crc32 and crc32c match their check values', () => { + assert.strictEqual(digest('crc32', check), 0xCBF43926) + assert.strictEqual(digest('crc32c', check), 0xE3069283) + }) + + for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) { + describe(label, () => { + it('writes the hash of the serialized body', () => { + assert.deepStrictEqual(p.createPacketBuffer('crc32', check), u32(0xCBF43926)) + assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283)) + }) + it('reads the hash, not the value', () => { + assert.strictEqual(p.parsePacketBuffer('crc32c', u32(0xE3069283)).data, 0xE3069283) + }) + it('writes a signed type in two\'s complement', () => { + const buffer = p.createPacketBuffer('signed', check) + assert.deepStrictEqual(buffer, u32(0xE3069283)) + assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0) + }) + it('sizes a fixed-size type without hashing', () => { + assert.strictEqual(p.sizeOf(check, 'signed'), 4) + assert.strictEqual(p.sizeOf(check, 'sha256'), 32) + }) + it('sizes a variable-size type from the hash', () => { + const buffer = p.createPacketBuffer('asVarint', check) + assert.strictEqual(p.sizeOf(check, 'asVarint'), buffer.length) + assert.deepStrictEqual(buffer, p.createPacketBuffer('varint', 0xE3069283 | 0)) + }) + it('writes a crypto digest as a buffer', () => { + assert.deepStrictEqual(p.createPacketBuffer('sha256', check), + require('crypto').createHash('sha256').update(check).digest()) + }) + it('resolves body fields against the enclosing container', () => { + assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 1, hash: 300 }), + Buffer.concat([Buffer.from([1]), u32(digest('crc32c', Buffer.from([0x01, 0x2C])))])) + assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 0, hash: 44 }), + Buffer.concat([Buffer.from([0]), u32(digest('crc32c', Buffer.from([44])))])) + }) + it('nests hashes of hashes', () => { + const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }] + const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))]) + assert.deepStrictEqual(p.createPacketBuffer('list', value), list) + assert.deepStrictEqual(p.createPacketBuffer('nested', value), lu32(digest('crc32c', list))) + }) + }) + } +}) From 3d001d3344e464dc4d0f757e454017cbd6ed14f8 Mon Sep 17 00:00:00 2001 From: u9g Date: Sat, 12 Sep 2026 14:11:24 -0400 Subject: [PATCH 2/4] hash: narrow to crc32c, and drop what the open-ended alg cost With the spec defining an explicit list of algorithms, a digest's width is known before hashing: the compiled sizer is a constant and type has to be of constant size, checked when compiling. That was the only caller of SizeOfCompiler.callTypeWrite, so it and the sizeOf -> write back-reference are gone; a sizer never writes. The hashing code moves to src/datatypes/hash.js beside the interpreter functions, leaving no general-purpose hashing module in the lib, and no crypto import. The digest function is copied into the compiled context as ctx._crc32c rather than injected into compile()'s eval scope, so generated code is self-contained and an unknown algorithm is caught when compiling. A named body type is sized with a direct ctx.sizeOfCtx. call; only an anonymous one, which has no function to call, still has its sizer generated in place. --- ProtoDef | 2 +- doc/compiler.md | 11 ++++-- src/compiler.js | 22 +++++------ src/datatypes/compiler-utils.js | 29 +++++++++------ src/datatypes/hash.js | 65 +++++++++++++++++++++++++++++++++ src/datatypes/utils.js | 32 +--------------- src/hash.js | 39 -------------------- test/misc.js | 40 +++++++++++--------- 8 files changed, 124 insertions(+), 116 deletions(-) create mode 100644 src/datatypes/hash.js delete mode 100644 src/hash.js diff --git a/ProtoDef b/ProtoDef index 647e754..c6e9efe 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit 647e754b8147b40dd05e947801e00ef3f38f704f +Subproject commit c6e9efee15a278d8d34039756b84f8eb469b5256 diff --git a/doc/compiler.md b/doc/compiler.md index ac34893..e86c38e 100644 --- a/doc/compiler.md +++ b/doc/compiler.md @@ -229,18 +229,23 @@ compiledProto.setVariable('noArraySizeCheck', true); const buffer = compiledProto.createPacketBuffer('mainType', result) const result = compiledProto.parsePacketBuffer('mainType', buffer) ``` -### Sizing inside a writer, writing inside a sizer +### Sizing inside a writer -A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code computing the size of `value` as `type`, and `SizeOfCompiler.callTypeWrite(value, type, offsetExpr)` returns code writing it into `buffer`. Both resolve field references against the current scope and run against the other compiler's context, so they are only available when the types are compiled through `ProtoDefCompiler`. The `hash` datatype is built on them: +A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code computing the size of `value` as `type`. It is only available when the types are compiled through `ProtoDefCompiler`, since it calls into the sizeOf context. A named type is already a function there and is called directly; an anonymous type has none, so its sizer is generated in place, against the current scope so that field references resolve to the same variables. + +A datatype that needs a helper function in its generated code registers it as a context type, which copies the function's source into the compiled output, rather than reaching for something outside it. The `hash` datatype is built on both: ```javascript Write: { + _crc32c: ['context', crc32c], hash: ['parametrizable', (compiler, { alg, type, body }) => { let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` - code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += `const hash = ctx._${alg}(bodyBuffer)\n` code += 'return ' + compiler.callType('hash', type) return compiler.wrapCode(code) }] } ``` + +The context is shared with the protocol's own type names, and a context entry wins over a type of the same name, so a helper's name is underscored to keep it out of the way. diff --git a/src/compiler.js b/src/compiler.js index 4547361..ea223e1 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -13,7 +13,6 @@ class ProtoDefCompiler { this.writeCompiler = new WriteCompiler() this.sizeOfCompiler = new SizeOfCompiler() this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler - this.sizeOfCompiler.writeCompiler = this.writeCompiler } addTypes (types) { @@ -64,9 +63,8 @@ class CompiledProtodef { this.sizeOfCtx = sizeOfCtx this.writeCtx = writeCtx this.readCtx = readCtx - // Code from callTypeSize / callTypeWrite runs against the other context + // Code from callTypeSize runs against the sizeOf context writeCtx.sizeOfCtx = sizeOfCtx - sizeOfCtx.writeCtx = writeCtx } read (buffer, cursor, type) { @@ -282,7 +280,6 @@ class Compiler { // Local variable to provide some context to eval() const native = this.native // eslint-disable-line const { PartialReadError } = require('./utils') // eslint-disable-line - const hashDigest = require('./hash').digest // eslint-disable-line return eval(code)() // eslint-disable-line } } @@ -388,9 +385,16 @@ class WriteCompiler extends Compiler { /** * Code computing the size of `value` as `type`, for writers that need to - * serialize part of a value before they can write it + * serialize part of a value before they can write it. A named type is + * already a function in the sizeOf context and is called directly; an + * anonymous one has none, so its sizer is generated here instead. */ callTypeSize (value, type, args = []) { + if (!this.sizeOfCompiler) throw new Error('sizeOfCtx is only available when compiling with ProtoDefCompiler') + if (typeof type === 'string' && this.sizeOfCompiler.types[type] && this.sizeOfCompiler.types[type] !== 'native') { + const params = [value, ...args.map(name => this.getField(name))] + return `ctx.sizeOfCtx.${type}(${params.join(', ')})` + } return this.callTypeIn(this.sizeOfCompiler, 'sizeOfCtx', compiler => compiler.callType(value, type, args)) } } @@ -472,14 +476,6 @@ class SizeOfCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value})` } - - /** - * Code writing `value` as `type` into `buffer` at `offsetExpr`, for sizers - * whose result depends on the serialized form of a value - */ - callTypeWrite (value, type, offsetExpr = 'offset', args = []) { - return this.callTypeIn(this.writeCompiler, 'writeCtx', compiler => compiler.callType(value, type, offsetExpr, args)) - } } module.exports = { diff --git a/src/datatypes/compiler-utils.js b/src/datatypes/compiler-utils.js index e528e90..0b28e2d 100644 --- a/src/datatypes/compiler-utils.js +++ b/src/datatypes/compiler-utils.js @@ -1,3 +1,5 @@ +const { algorithms: hashAlgorithms } = require('./hash') + module.exports = { Read: { pstring: ['parametrizable', (compiler, string) => { @@ -90,6 +92,10 @@ return { value, size } }, Write: { + // A hash digest is taken in generated code, so the digest function is + // copied into the compiled context rather than reached for outside it. + // Underscored: the context is shared with the protocol's own type names. + _crc32c: ['context', hashAlgorithms.crc32c.digest], pstring: ['parametrizable', (compiler, string) => { let code = `const length = Buffer.byteLength(value, "${string.encoding || 'utf8'}")\n` if (string.countType) { @@ -168,13 +174,15 @@ return (ctx.${type})(val, buffer, offset) return compiler.wrapCode(code) }], hash: ['parametrizable', (compiler, { alg, type, body }) => { + if (!hashAlgorithms[alg]) throw new Error('Unknown hash algorithm: ' + alg) let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` - code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` + code += `const hash = ctx._${alg}(bodyBuffer)\n` + // A CRC is unsigned; a signed `type` takes its two's complement code += 'try {\n' code += ' return ' + compiler.callType('hash', type) + '\n' code += '} catch (e) {\n' - code += ' if (!(e instanceof RangeError) || typeof hash !== "number") throw e\n' + code += ' if (!(e instanceof RangeError)) throw e\n' code += ' return ' + compiler.callType('hash | 0', type) + '\n' code += '}' return compiler.wrapCode(code) @@ -233,16 +241,13 @@ return (ctx.${type})(val) code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) }], - hash: ['parametrizable', (compiler, { alg, type, body }) => { - const constant = compiler.constantSize(type) - if (constant !== undefined) return String(constant) - const size = compiler.callType('hash', type) - if (!isNaN(size)) return size - let code = `const bodyBuffer = Buffer.alloc(${compiler.callType('value', body)})\n` - code += `;((buffer) => ${compiler.callTypeWrite('value', body, '0')})(bodyBuffer)\n` - code += `const hash = hashDigest(${JSON.stringify(alg)}, bodyBuffer)\n` - code += 'return ' + size - return compiler.wrapCode(code) + // The digest has a fixed width, so a hash is sized without hashing: its + // size is the size of `type`, which the spec requires to be constant + hash: ['parametrizable', (compiler, { alg, type }) => { + const size = compiler.constantSize(type) + if (size === undefined) throw new Error('hash type must be of constant size, ' + JSON.stringify(type) + ' is not') + if (size < hashAlgorithms[alg].bytes) throw new Error('hash type is too small for a ' + alg + ' digest') + return String(size) }] } } diff --git a/src/datatypes/hash.js b/src/datatypes/hash.js new file mode 100644 index 0000000..6c66c08 --- /dev/null +++ b/src/datatypes/hash.js @@ -0,0 +1,65 @@ +const { getFieldInfo } = require('../utils') + +// CRC-32C (Castagnoli): reflected table-driven, all-ones init and final xor. +// The compiler copies this function into the code it generates, so it has to +// stand on its own: no imports, no module scope, its table cached on itself. +function crc32c (buffer) { + let table = crc32c.table + if (!table) { + table = crc32c.table = new Int32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0x82F63B78 ^ (c >>> 1) : c >>> 1 + table[n] = c + } + } + let c = -1 + for (let i = 0; i < buffer.length; i++) c = table[(c ^ buffer[i]) & 0xff] ^ (c >>> 8) + return (c ^ -1) >>> 0 +} + +// `alg` is an explicit list in the spec, so a protocol means the same thing in +// every implementation; adding an algorithm here is a spec change. The width of +// the digest is what lets a hash be sized without hashing. +const algorithms = { + crc32c: { bytes: 4, digest: crc32c } +} + +function digest (alg, buffer) { + const algorithm = algorithms[alg] + if (!algorithm) throw new Error('Unknown hash algorithm: ' + alg) + return algorithm.digest(buffer) +} + +function readHash (buffer, offset, { type }, rootNode) { + return this.read(buffer, offset, type, rootNode) +} + +// A CRC is unsigned; a signed `type` takes its two's complement. +function writeHash (value, buffer, offset, { alg, type, body }, rootNode) { + const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) + this.write(value, bodyBuffer, 0, body, rootNode) + const hash = digest(alg, bodyBuffer) + try { + return this.write(hash, buffer, offset, type, rootNode) + } catch (e) { + if (!(e instanceof RangeError)) throw e + return this.write(hash | 0, buffer, offset, type, rootNode) + } +} + +// The digest has a fixed width, so the size of a hash never depends on the +// value: `type` is required to be of constant size and is looked up as one. +function sizeOfHash (value, { alg, type }, rootNode) { + const functions = this.types[getFieldInfo(type).type] + const size = functions ? functions[2] : undefined + if (typeof size !== 'number') throw new Error('hash type must be of constant size, ' + JSON.stringify(type) + ' is not') + if (size < algorithms[alg].bytes) throw new Error('hash type is too small for a ' + alg + ' digest') + return size +} + +module.exports = { + digest, + algorithms, + hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash] +} diff --git a/src/datatypes/utils.js b/src/datatypes/utils.js index ff380a2..5dead75 100644 --- a/src/datatypes/utils.js +++ b/src/datatypes/utils.js @@ -1,5 +1,4 @@ -const { getCount, sendCount, calcCount, getFieldInfo, PartialReadError } = require('../utils') -const { digest } = require('../hash') +const { getCount, sendCount, calcCount, PartialReadError } = require('../utils') module.exports = { bool: [readBool, writeBool, 1, require('../../ProtoDef/schemas/utils.json').bool], @@ -10,7 +9,7 @@ module.exports = { bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags], cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring], mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper], - hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash], + hash: require('./hash').hash, ...require('./varint') } @@ -282,30 +281,3 @@ function sizeOfBitflags (value, { type, flags, shift, big }, rootNode) { } return this.sizeOf(mappedValue, type, rootNode) } - -function readHash (buffer, offset, { type }, rootNode) { - return this.read(buffer, offset, type, rootNode) -} - -function hashOf (value, { alg, body }, rootNode) { - const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) - this.write(value, bodyBuffer, 0, body, rootNode) - return digest(alg, bodyBuffer) -} - -// A CRC is unsigned; a signed `type` takes its two's complement. -function writeHash (value, buffer, offset, typeArgs, rootNode) { - const hash = hashOf.call(this, value, typeArgs, rootNode) - try { - return this.write(hash, buffer, offset, typeArgs.type, rootNode) - } catch (e) { - if (!(e instanceof RangeError) || typeof hash !== 'number') throw e - return this.write(hash | 0, buffer, offset, typeArgs.type, rootNode) - } -} - -function sizeOfHash (value, typeArgs, rootNode) { - const functions = this.types[getFieldInfo(typeArgs.type).type] - if (functions && typeof functions[2] === 'number') return functions[2] - return this.sizeOf(hashOf.call(this, value, typeArgs, rootNode), typeArgs.type, rootNode) -} diff --git a/src/hash.js b/src/hash.js deleted file mode 100644 index a1023d5..0000000 --- a/src/hash.js +++ /dev/null @@ -1,39 +0,0 @@ -const crypto = require('crypto') - -// Reflected table-driven CRC with all-ones init and final xor; `poly` is the -// reversed polynomial. -const tables = {} -function table (poly) { - if (!tables[poly]) { - const t = new Int32Array(256) - for (let n = 0; n < 256; n++) { - let c = n - for (let k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >>> 1) : c >>> 1 - t[n] = c - } - tables[poly] = t - } - return tables[poly] -} - -function crc (poly, buffer) { - const t = table(poly) - let c = -1 - for (let i = 0; i < buffer.length; i++) c = t[(c ^ buffer[i]) & 0xff] ^ (c >>> 8) - return (c ^ -1) >>> 0 -} - -const algorithms = { - crc32: buffer => crc(0xEDB88320, buffer), - crc32c: buffer => crc(0x82F63B78, buffer) -} - -// CRC digests are unsigned integers; every other algorithm is delegated to -// node's crypto and yields a Buffer. -function digest (alg, buffer) { - const algorithm = algorithms[alg] - if (algorithm) return algorithm(buffer) - return crypto.createHash(alg).update(buffer).digest() -} - -module.exports = { digest, algorithms } diff --git a/test/misc.js b/test/misc.js index d4a7829..a743d7f 100644 --- a/test/misc.js +++ b/test/misc.js @@ -27,14 +27,12 @@ describe('mapper', () => { }) describe('hash', () => { - const { digest } = require('../src/hash') + const { digest } = require('../src/datatypes/hash') + const varintHash = ['hash', { alg: 'crc32c', type: 'varint', body: ['buffer', { count: 9 }] }] const types = { - crc32: ['hash', { alg: 'crc32', type: 'u32', body: ['buffer', { count: 9 }] }], crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }], signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: ['buffer', { count: 9 }] }], HashCode: 'i32', - asVarint: ['hash', { alg: 'crc32c', type: 'varint', body: ['buffer', { count: 9 }] }], - sha256: ['hash', { alg: 'sha256', type: ['buffer', { count: 32 }], body: ['buffer', { count: 9 }] }], // A field of the enclosing container selects the body's type tagged: ['container', [ { name: 'kind', type: 'u8' }, @@ -54,15 +52,31 @@ describe('hash', () => { const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b } const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b } - it('crc32 and crc32c match their check values', () => { - assert.strictEqual(digest('crc32', check), 0xCBF43926) + it('crc32c matches its check value', () => { assert.strictEqual(digest('crc32c', check), 0xE3069283) }) + it('rejects an algorithm the spec does not define', () => { + assert.throws(() => digest('sha256', check), /Unknown hash algorithm/) + const log = console.log // the validator dumps the type it rejected + console.log = () => {} + try { + assert.throws(() => new ProtoDef().addTypes({ bad: ['hash', { alg: 'sha256', type: 'u32', body: 'u8' }] })) + } finally { + console.log = log + } + }) + + it('rejects a hash written as a variable-size type', () => { + assert.throws(() => proto.sizeOf(check, varintHash), /constant size/) + const c = new ProtoDefCompiler() + c.addTypesToCompile({ asVarint: varintHash }) + assert.throws(() => c.compileProtoDefSync(), /constant size/) + }) + for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) { describe(label, () => { it('writes the hash of the serialized body', () => { - assert.deepStrictEqual(p.createPacketBuffer('crc32', check), u32(0xCBF43926)) assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283)) }) it('reads the hash, not the value', () => { @@ -73,18 +87,8 @@ describe('hash', () => { assert.deepStrictEqual(buffer, u32(0xE3069283)) assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0) }) - it('sizes a fixed-size type without hashing', () => { + it('sizes without hashing', () => { assert.strictEqual(p.sizeOf(check, 'signed'), 4) - assert.strictEqual(p.sizeOf(check, 'sha256'), 32) - }) - it('sizes a variable-size type from the hash', () => { - const buffer = p.createPacketBuffer('asVarint', check) - assert.strictEqual(p.sizeOf(check, 'asVarint'), buffer.length) - assert.deepStrictEqual(buffer, p.createPacketBuffer('varint', 0xE3069283 | 0)) - }) - it('writes a crypto digest as a buffer', () => { - assert.deepStrictEqual(p.createPacketBuffer('sha256', check), - require('crypto').createHash('sha256').update(check).digest()) }) it('resolves body fields against the enclosing container', () => { assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 1, hash: 300 }), From b86c30b0a2cab57ca3e249ff1048bf77b04981d3 Mon Sep 17 00:00:00 2001 From: u9g Date: Sat, 12 Sep 2026 16:25:39 -0400 Subject: [PATCH 3/4] Point the submodule at the trimmed hash doc No schema change, so nothing here reads differently. --- ProtoDef | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProtoDef b/ProtoDef index c6e9efe..ab8287c 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit c6e9efee15a278d8d34039756b84f8eb469b5256 +Subproject commit ab8287cebdc03869494ac0aa48b269b51f7e91b7 From dde6ce755124e8f42b4fc1e53ce2d84c73acdb09 Mon Sep 17 00:00:00 2001 From: u9g Date: Sat, 12 Sep 2026 16:35:17 -0400 Subject: [PATCH 4/4] hash: require body to be a named type, and drop the cross-compiler codegen callTypeSize now emits a call to the sizer the sizeOf context already holds, ctx.sizeOfCtx.(value), and throws when the body is not a named type. That leaves nothing generating code with another compiler, so callTypeIn and its scope-stack swap are gone: what the write compiler borrows from the sizeOf compiler is one function call, resolved by name. The capability this gives up, a body whose type is selected by a field of the enclosing container, was reachable only from an inline type. A named type cannot reference a parent field at all -- getField throws while the type is being generated -- so hash was alone in being able to do it, and the scope sharing was the whole reason it could. The interpreter rejects an inline body too, rather than accepting schemas the compiler refuses. --- ProtoDef | 2 +- doc/compiler.md | 2 +- src/compiler.js | 33 +++++++-------------------------- src/datatypes/hash.js | 1 + test/misc.js | 26 ++++++++++++-------------- 5 files changed, 22 insertions(+), 42 deletions(-) diff --git a/ProtoDef b/ProtoDef index ab8287c..a2b1cd2 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit ab8287cebdc03869494ac0aa48b269b51f7e91b7 +Subproject commit a2b1cd2d84488012b7949fbe578d3781cc3329a4 diff --git a/doc/compiler.md b/doc/compiler.md index e86c38e..c946c05 100644 --- a/doc/compiler.md +++ b/doc/compiler.md @@ -231,7 +231,7 @@ const result = compiledProto.parsePacketBuffer('mainType', buffer) ``` ### Sizing inside a writer -A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code computing the size of `value` as `type`. It is only available when the types are compiled through `ProtoDefCompiler`, since it calls into the sizeOf context. A named type is already a function there and is called directly; an anonymous type has none, so its sizer is generated in place, against the current scope so that field references resolve to the same variables. +A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code calling the sizer for `type`, which the sizeOf context already holds since it is generated first. It is only available when the types are compiled through `ProtoDefCompiler`, and `type` has to be a named type, since an anonymous one has no function in that context to call. A datatype that needs a helper function in its generated code registers it as a context type, which copies the function's source into the compiled output, rather than reaching for something outside it. The `hash` datatype is built on both: diff --git a/src/compiler.js b/src/compiler.js index ea223e1..24c5aa1 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -177,24 +177,6 @@ class Compiler { } } - /** - * Generates code with another compiler inside this compiler's scope, so that - * field references resolve to the same variables, and binds it to that - * compiler's context. Natives are reachable through the context as well. - */ - callTypeIn (other, ctxName, generate) { - if (!other) throw new Error(`${ctxName} is only available when compiling with ProtoDefCompiler`) - const scopeStack = other.scopeStack - other.scopeStack = this.scopeStack - try { - const code = generate(other) - if (!isNaN(code)) return code - return `((ctx, native) => ${code})(ctx.${ctxName}, ctx.${ctxName})` - } finally { - other.scopeStack = scopeStack - } - } - addTypesToCompile (types) { for (const [type, json] of Object.entries(types)) { // Replace native type, otherwise first in wins @@ -385,17 +367,16 @@ class WriteCompiler extends Compiler { /** * Code computing the size of `value` as `type`, for writers that need to - * serialize part of a value before they can write it. A named type is - * already a function in the sizeOf context and is called directly; an - * anonymous one has none, so its sizer is generated here instead. + * serialize part of a value before they can write it. The sizer is a + * function in the sizeOf context, which is generated first, so `type` has + * to be a named one for there to be a function to call. */ - callTypeSize (value, type, args = []) { + callTypeSize (value, type) { if (!this.sizeOfCompiler) throw new Error('sizeOfCtx is only available when compiling with ProtoDefCompiler') - if (typeof type === 'string' && this.sizeOfCompiler.types[type] && this.sizeOfCompiler.types[type] !== 'native') { - const params = [value, ...args.map(name => this.getField(name))] - return `ctx.sizeOfCtx.${type}(${params.join(', ')})` + if (typeof type !== 'string' || !this.sizeOfCompiler.types[type]) { + throw new Error('cannot size ' + JSON.stringify(type) + ' from a writer, it is not a named type') } - return this.callTypeIn(this.sizeOfCompiler, 'sizeOfCtx', compiler => compiler.callType(value, type, args)) + return `ctx.sizeOfCtx.${type}(${value})` } } diff --git a/src/datatypes/hash.js b/src/datatypes/hash.js index 6c66c08..89622e3 100644 --- a/src/datatypes/hash.js +++ b/src/datatypes/hash.js @@ -37,6 +37,7 @@ function readHash (buffer, offset, { type }, rootNode) { // A CRC is unsigned; a signed `type` takes its two's complement. function writeHash (value, buffer, offset, { alg, type, body }, rootNode) { + if (typeof body !== 'string') throw new Error('hash body must be a named type, ' + JSON.stringify(body) + ' is not') const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) this.write(value, bodyBuffer, 0, body, rootNode) const hash = digest(alg, bodyBuffer) diff --git a/test/misc.js b/test/misc.js index a743d7f..c823bd0 100644 --- a/test/misc.js +++ b/test/misc.js @@ -28,16 +28,13 @@ describe('mapper', () => { describe('hash', () => { const { digest } = require('../src/datatypes/hash') - const varintHash = ['hash', { alg: 'crc32c', type: 'varint', body: ['buffer', { count: 9 }] }] + const varintHash = ['hash', { alg: 'crc32c', type: 'varint', body: 'Body' }] + const inlineBody = ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }] const types = { - crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }], - signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: ['buffer', { count: 9 }] }], + Body: ['buffer', { count: 9 }], + crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: 'Body' }], + signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: 'Body' }], HashCode: 'i32', - // A field of the enclosing container selects the body's type - tagged: ['container', [ - { name: 'kind', type: 'u8' }, - { name: 'hash', type: ['hash', { alg: 'crc32c', type: 'u32', body: ['switch', { compareTo: 'kind', fields: { 0: 'u8', 1: 'u16' } }] }] } - ]], // A hash over a list of hashes entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]], list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }], @@ -67,6 +64,13 @@ describe('hash', () => { } }) + it('rejects a body that is not a named type', () => { + assert.throws(() => proto.write(check, Buffer.alloc(4), 0, inlineBody), /named type/) + const c = new ProtoDefCompiler() + c.addTypesToCompile({ withInlineBody: inlineBody }) + assert.throws(() => c.compileProtoDefSync(), /named type/) + }) + it('rejects a hash written as a variable-size type', () => { assert.throws(() => proto.sizeOf(check, varintHash), /constant size/) const c = new ProtoDefCompiler() @@ -90,12 +94,6 @@ describe('hash', () => { it('sizes without hashing', () => { assert.strictEqual(p.sizeOf(check, 'signed'), 4) }) - it('resolves body fields against the enclosing container', () => { - assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 1, hash: 300 }), - Buffer.concat([Buffer.from([1]), u32(digest('crc32c', Buffer.from([0x01, 0x2C])))])) - assert.deepStrictEqual(p.createPacketBuffer('tagged', { kind: 0, hash: 44 }), - Buffer.concat([Buffer.from([0]), u32(digest('crc32c', Buffer.from([44])))])) - }) it('nests hashes of hashes', () => { const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }] const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))])