From 647e754b8147b40dd05e947801e00ef3f38f704f Mon Sep 17 00:00:00 2001 From: u9g Date: Sun, 6 Sep 2026 10:32:18 -0400 Subject: [PATCH 1/4] Add the hash datatype: a value written as the hash of its serialized form --- doc/datatypes.md | 1 + doc/datatypes/utils.md | 24 ++++++++++++++++++++++++ schemas/datatype.json | 3 ++- schemas/utils.json | 26 ++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/doc/datatypes.md b/doc/datatypes.md index 578ce05..73d3ddc 100644 --- a/doc/datatypes.md +++ b/doc/datatypes.md @@ -18,3 +18,4 @@ * * [bitfield](./datatypes/utils.md) * * [mapper](./datatypes/utils.md) * * [pstring](./datatypes/utils.md) +* * [hash](./datatypes/utils.md) diff --git a/doc/datatypes/utils.md b/doc/datatypes/utils.md index c64a3aa..2bafc8f 100644 --- a/doc/datatypes/utils.md +++ b/doc/datatypes/utils.md @@ -124,3 +124,27 @@ Example: A string length prefixed by a varint. ] ``` Example of value: `"my string"` + +### **hash** ({ alg: String, type: Type, body: Type }) +Arguments: +* alg : the hash algorithm: `crc32`, `crc32c`, or any digest the platform's crypto library provides (`md5`, `sha1`, `sha256`, ...) +* type : the type the hash is written and read as +* body : the type the value is serialized as before being hashed + +Represents a hash of a value instead of the value itself: writing serializes the value as `body`, hashes the bytes and writes the hash as `type`; reading reads `type` and yields the hash. The value is not recoverable from the buffer, so the read side sees only the hash. + +CRC hashes are unsigned integers. When `type` is signed (for example `i32`) the hash is written in two's complement, so it reads back the way a language with only signed integers would produce it. Other algorithms produce raw digest bytes, so `type` must then be a `buffer` of the digest's length. + +Example: a CRC32C of an item component, written as a signed int like Minecraft's `HashedSlot` carries it. +```json +[ + "hash", + { + "alg": "crc32c", + "type": "i32", + "body": "ItemComponent" + } +] +``` + +Example of value: `{ "id": 5, "level": 3 }` (whatever `body` accepts) / reads as `-486237565` diff --git a/schemas/datatype.json b/schemas/datatype.json index 66b37d8..94a1431 100644 --- a/schemas/datatype.json +++ b/schemas/datatype.json @@ -45,6 +45,7 @@ { "$ref": "buffer" }, { "$ref": "bitfield" }, { "$ref": "bitflags" }, - { "$ref": "mapper" } + { "$ref": "mapper" }, + { "$ref": "hash" } ] } diff --git a/schemas/utils.json b/schemas/utils.json index c9d48b5..99690c3 100644 --- a/schemas/utils.json +++ b/schemas/utils.json @@ -144,5 +144,31 @@ } ], "additionalItems": false + }, + "hash": { + "title": "hash", + "type": "array", + "items": [ + { + "enum": ["hash"] + }, + { + "type": "object", + "properties": { + "alg": { + "type": "string" + }, + "type": { + "$ref": "dataType" + }, + "body": { + "$ref": "dataType" + } + }, + "required": ["alg", "type", "body"], + "additionalProperties": false + } + ], + "additionalItems": false } } From c6e9efee15a278d8d34039756b84f8eb469b5256 Mon Sep 17 00:00:00 2001 From: u9g Date: Sat, 12 Sep 2026 14:11:15 -0400 Subject: [PATCH 2/4] hash: make alg an explicit list Defining the algorithms in the spec rather than deferring to whatever the implementation's platform exposes means a protocol using hash means the same thing everywhere, and it fixes the digest's width: crc32c is 4 bytes, so the size of a hash field is known without hashing anything. type is required to be of constant size and at least as wide as the digest, which a variable-length type such as varint is not. --- doc/datatypes/utils.md | 12 ++++++++++-- schemas/utils.json | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/datatypes/utils.md b/doc/datatypes/utils.md index 2bafc8f..3dda78d 100644 --- a/doc/datatypes/utils.md +++ b/doc/datatypes/utils.md @@ -127,13 +127,21 @@ Example of value: `"my string"` ### **hash** ({ alg: String, type: Type, body: Type }) Arguments: -* alg : the hash algorithm: `crc32`, `crc32c`, or any digest the platform's crypto library provides (`md5`, `sha1`, `sha256`, ...) +* alg : the hash algorithm, currently only `crc32c` * type : the type the hash is written and read as * body : the type the value is serialized as before being hashed Represents a hash of a value instead of the value itself: writing serializes the value as `body`, hashes the bytes and writes the hash as `type`; reading reads `type` and yields the hash. The value is not recoverable from the buffer, so the read side sees only the hash. -CRC hashes are unsigned integers. When `type` is signed (for example `i32`) the hash is written in two's complement, so it reads back the way a language with only signed integers would produce it. Other algorithms produce raw digest bytes, so `type` must then be a `buffer` of the digest's length. +`alg` is an explicit list rather than whatever hashes the implementation's platform happens to offer, so that a protocol using `hash` means the same thing in every implementation: + +| alg | digest | +| --- | --- | +| `crc32c` | a 4 byte unsigned integer (CRC-32C, the Castagnoli polynomial, reflected, all-ones init and final xor) | + +Because the digest's width is fixed by `alg`, the size of a `hash` field is known without hashing anything. `type` must therefore be a type of constant size, at least as wide as the digest; a variable-length `type` such as `varint` is an error. + +A CRC is unsigned. When `type` is signed (for example `i32`) the hash is written in two's complement, so it reads back the way a language with only signed integers would produce it. Example: a CRC32C of an item component, written as a signed int like Minecraft's `HashedSlot` carries it. ```json diff --git a/schemas/utils.json b/schemas/utils.json index 99690c3..8f18bb2 100644 --- a/schemas/utils.json +++ b/schemas/utils.json @@ -156,7 +156,7 @@ "type": "object", "properties": { "alg": { - "type": "string" + "enum": ["crc32c"] }, "type": { "$ref": "dataType" From ab8287cebdc03869494ac0aa48b269b51f7e91b7 Mon Sep 17 00:00:00 2001 From: u9g Date: Sat, 12 Sep 2026 16:25:27 -0400 Subject: [PATCH 3/4] hash: cut the doc down to what an implementer needs The section explained its own design where the neighbouring types state a rule and stop: why alg is an explicit list, why a fixed width means a hash can be sized without hashing, why two's complement reads back as it does. The rules those sentences surround stay -- the constraint on type moves into its argument bullet, and the CRC-32C parameters into alg's, since no other section in the file uses a table for one value. The example's value was unverifiable: ItemComponent is not defined here, and -486237565 is not the CRC32C of any reading of { id: 5, level: 3 }. The line now gives the bytes the body serializes to, so the digest can be checked against the section above it. --- doc/datatypes/utils.md | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/doc/datatypes/utils.md b/doc/datatypes/utils.md index 3dda78d..b4fe8a3 100644 --- a/doc/datatypes/utils.md +++ b/doc/datatypes/utils.md @@ -127,23 +127,13 @@ Example of value: `"my string"` ### **hash** ({ alg: String, type: Type, body: Type }) Arguments: -* alg : the hash algorithm, currently only `crc32c` -* type : the type the hash is written and read as +* alg : the hash algorithm, one of : `crc32c` (CRC-32C, the Castagnoli polynomial, reflected, all-ones init and final xor, a 4 byte digest) +* type : the type the hash is written and read as, of constant size and at least as wide as the digest * body : the type the value is serialized as before being hashed -Represents a hash of a value instead of the value itself: writing serializes the value as `body`, hashes the bytes and writes the hash as `type`; reading reads `type` and yields the hash. The value is not recoverable from the buffer, so the read side sees only the hash. +Represents a hash of a value instead of the value itself : writing serializes the value as `body`, hashes those bytes and writes the digest as `type`; reading reads `type` and yields the digest. When `type` is signed the digest is written in two's complement. -`alg` is an explicit list rather than whatever hashes the implementation's platform happens to offer, so that a protocol using `hash` means the same thing in every implementation: - -| alg | digest | -| --- | --- | -| `crc32c` | a 4 byte unsigned integer (CRC-32C, the Castagnoli polynomial, reflected, all-ones init and final xor) | - -Because the digest's width is fixed by `alg`, the size of a `hash` field is known without hashing anything. `type` must therefore be a type of constant size, at least as wide as the digest; a variable-length `type` such as `varint` is an error. - -A CRC is unsigned. When `type` is signed (for example `i32`) the hash is written in two's complement, so it reads back the way a language with only signed integers would produce it. - -Example: a CRC32C of an item component, written as a signed int like Minecraft's `HashedSlot` carries it. +Example: A CRC32C of an item component, written as a signed int. ```json [ "hash", @@ -155,4 +145,4 @@ Example: a CRC32C of an item component, written as a signed int like Minecraft's ] ``` -Example of value: `{ "id": 5, "level": 3 }` (whatever `body` accepts) / reads as `-486237565` +Example of value: `{ "id": 5, "level": 3 }`, serializing to `05 03` / reads as `-1088848499` From a2b1cd2d84488012b7949fbe578d3781cc3329a4 Mon Sep 17 00:00:00 2001 From: u9g Date: Sat, 12 Sep 2026 16:34:58 -0400 Subject: [PATCH 4/4] hash: require body to be a named type A compiled implementation sizes the body by calling the sizer generated for its type, and only a named type has one. An inline body would also be free to reference a field of whatever contains the hash, which nothing it could be called from is able to resolve. The type this exists for, an item component hashed into a HashedSlot, is a named type, so the restriction costs it nothing. --- doc/datatypes/utils.md | 2 +- schemas/utils.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/datatypes/utils.md b/doc/datatypes/utils.md index b4fe8a3..787a8c8 100644 --- a/doc/datatypes/utils.md +++ b/doc/datatypes/utils.md @@ -129,7 +129,7 @@ Example of value: `"my string"` Arguments: * alg : the hash algorithm, one of : `crc32c` (CRC-32C, the Castagnoli polynomial, reflected, all-ones init and final xor, a 4 byte digest) * type : the type the hash is written and read as, of constant size and at least as wide as the digest -* body : the type the value is serialized as before being hashed +* body : the name of the type the value is serialized as before being hashed Represents a hash of a value instead of the value itself : writing serializes the value as `body`, hashes those bytes and writes the digest as `type`; reading reads `type` and yields the digest. When `type` is signed the digest is written in two's complement. diff --git a/schemas/utils.json b/schemas/utils.json index 8f18bb2..5b70022 100644 --- a/schemas/utils.json +++ b/schemas/utils.json @@ -162,7 +162,7 @@ "$ref": "dataType" }, "body": { - "$ref": "dataType" + "type": "string" } }, "required": ["alg", "type", "body"],