From ac202d7519b2183d7c39be5a5738514507bdb3bb Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Fri, 7 Aug 2026 17:55:28 +0200 Subject: [PATCH 1/4] feat(specs,test-types): add EIP-8297 binary tree state provider --- .../base_types/composite_types.py | 3 + .../cli/pytest_commands/fill.py | 1 + .../pytest_ini_files/pytest-fill.ini | 1 + .../test_types/account_types.py | 3 + .../test_types/tests/test_alloc_prestate.py | 71 +- pyproject.toml | 1 + src/ethereum/exceptions.py | 19 + src/ethereum/partitioned_binary_tree.py | 1150 ++++++++ src/ethereum/state.py | 13 +- src/ethereum/state_mpt.py | 16 +- src/ethereum/state_pbt.py | 398 +++ .../evm_tools/t8n/result.py | 13 +- tests/binary_trie/__init__.py | 1 + tests/binary_trie/incremental_trie.py | 204 ++ tests/binary_trie/test_block_execution.py | 303 ++ tests/binary_trie/test_differential_mpt.py | 591 ++++ tests/binary_trie/test_embedding.py | 1193 ++++++++ tests/binary_trie/test_state_pbt.py | 2509 +++++++++++++++++ tests/binary_trie/test_trie.py | 1039 +++++++ uv.lock | 78 + vulture_whitelist.py | 21 + 21 files changed, 7622 insertions(+), 6 deletions(-) create mode 100644 src/ethereum/partitioned_binary_tree.py create mode 100644 src/ethereum/state_pbt.py create mode 100644 tests/binary_trie/__init__.py create mode 100644 tests/binary_trie/incremental_trie.py create mode 100644 tests/binary_trie/test_block_execution.py create mode 100644 tests/binary_trie/test_differential_mpt.py create mode 100644 tests/binary_trie/test_embedding.py create mode 100644 tests/binary_trie/test_state_pbt.py create mode 100644 tests/binary_trie/test_trie.py diff --git a/packages/testing/src/execution_testing/base_types/composite_types.py b/packages/testing/src/execution_testing/base_types/composite_types.py index b25ff1dcec1..742761a16fb 100644 --- a/packages/testing/src/execution_testing/base_types/composite_types.py +++ b/packages/testing/src/execution_testing/base_types/composite_types.py @@ -578,6 +578,9 @@ class StateCommitment(Enum): MPT = auto() """Merkle-Patricia trie.""" + PBT = auto() + """EIP-8297 partitioned binary tree.""" + class AccessList(CamelModel, RLPSerializable): """Access List for transactions.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/fill.py b/packages/testing/src/execution_testing/cli/pytest_commands/fill.py index fbd513b823d..a849911ee52 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/fill.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/fill.py @@ -142,6 +142,7 @@ def _add_default_ignores(self, args: List[str]) -> List[str]: default_ignores = [ "tests/evm_tools", "tests/json_loader", + "tests/binary_trie", "tests/fixtures", ] diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini index e627de46c9a..c2144be7170 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini +++ b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini @@ -20,6 +20,7 @@ addopts = --dist loadgroup --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/ --ignore tests/json_loader + --ignore tests/binary_trie --ignore tests/evm_tools # these customizations require the pytest-custom-report plugin report_passed_verbose = FILLED diff --git a/packages/testing/src/execution_testing/test_types/account_types.py b/packages/testing/src/execution_testing/test_types/account_types.py index 29454e84db9..1d96a505a86 100644 --- a/packages/testing/src/execution_testing/test_types/account_types.py +++ b/packages/testing/src/execution_testing/test_types/account_types.py @@ -18,6 +18,7 @@ import ethereum.state as spec_state import ethereum.state_mpt as spec_state_mpt +import ethereum.state_pbt as spec_state_pbt from ethereum.crypto.hash import Hash32 from ethereum.crypto.hash import keccak256 as spec_keccak256 from ethereum_types.bytes import Bytes, Bytes20 @@ -381,6 +382,8 @@ def _state_module(self) -> ModuleType: ) if self._state_commitment is StateCommitment.MPT: return spec_state_mpt + if self._state_commitment is StateCommitment.PBT: + return spec_state_pbt raise NotImplementedError("State commitment type not yet implemented.") def _materialize_state(self) -> spec_state.PreState: diff --git a/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py b/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py index 57b558882f3..2d1bda44442 100644 --- a/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py +++ b/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py @@ -17,12 +17,13 @@ import ethereum.state as spec_state import ethereum.state_mpt as spec_state_mpt +import ethereum.state_pbt as spec_state_pbt import pytest from ethereum.crypto.hash import keccak256 from ethereum_types.bytes import Bytes20, Bytes32 from ethereum_types.numeric import U256, Uint -from execution_testing.base_types import Account, StateCommitment +from execution_testing.base_types import Account, Hash, StateCommitment from execution_testing.test_types import Alloc from execution_testing.test_types.account_types import _Phase @@ -287,3 +288,71 @@ def test_apply_diff_round_trip_matches_independent_post_state() -> None: account_changes={}, storage_changes={}, code_changes={} ) ) + + +def _pbt_alloc(commitment: StateCommitment) -> Alloc: + """ + Build a small, deterministic two-account allocation committed + through `commitment`. + + A fresh `Alloc` is returned on every call because a state + commitment, once migrated onto an instance, sticks to it -- a + shared module-level allocation could not be reused across + assertions that expect different commitment schemes. + """ + alloc = Alloc.model_validate( + { + 0xA: { + "balance": 1000, + "nonce": 2, + "code": "0x00", + "storage": {"0x01": "0x02"}, + }, + 0xB: {"balance": 5, "nonce": 0, "code": "0x"}, + } + ) + alloc.migrate_state_commitment(commitment) + return alloc + + +def test_pbt_state_root_matches_state_pbt() -> None: + """ + An alloc committed through `StateCommitment.PBT` returns a + root that differs from the plain MPT `state_root()` and matches the + root computed directly through `ethereum.state_pbt` for the same + accounts. + """ + mpt_root = _pbt_alloc(StateCommitment.MPT).state_root() + + pbt_root = _pbt_alloc(StateCommitment.PBT).state_root() + assert pbt_root != mpt_root + + # Build the same accounts directly through `ethereum.state_pbt`, + # mirroring `Alloc._materialize_state`. + state = spec_state_pbt.State() + alloc = _pbt_alloc(StateCommitment.PBT) + for address, account in alloc.root.items(): + assert account is not None + addr = Bytes20(address) + code = bytes(account.code) if account.code else b"" + code_hash = spec_state_pbt.store_code(state, code) + spec_state_pbt.set_account( + state, + addr, + spec_state.Account( + nonce=Uint(int(account.nonce)), + balance=U256(int(account.balance)), + code_hash=code_hash, + ), + ) + for key, value in account.storage.root.items(): + value_int = int(value) + if value_int == 0: + continue + spec_state_pbt.set_storage( + state, + addr, + Bytes32(int(key).to_bytes(32, "big")), + U256(value_int), + ) + assert Hash(spec_state_pbt.state_root(state)) == pbt_root diff --git a/pyproject.toml b/pyproject.toml index 23a1eff6527..07702cfd41c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "ethereum-rlp>=0.1.6,<0.2", "cryptography>=45.0.1,<46", "platformdirs>=4.2,<5", + "blake3>=1.0,<2", "libcst>=1.8,<2", ] diff --git a/src/ethereum/exceptions.py b/src/ethereum/exceptions.py index dafb73b64fc..ac8536e15af 100644 --- a/src/ethereum/exceptions.py +++ b/src/ethereum/exceptions.py @@ -73,3 +73,22 @@ class NonceOverflowError(InvalidTransaction): """ Thrown when a transaction's nonce is greater than `2**64 - 2`. """ + + +class BalanceOverflowError(InvalidBlock): + """ + Thrown when an account's balance is too large to fit the sixteen-byte + balance field of the binary tree's basic data leaf. + """ + + +class UnknownCodeHashError(EthereumException): + """ + Thrown when a code hash has no bytecode stored for it in the state's + code store. + + Indicates a malformed pre-state rather than an invalid block, so this + is deliberately not an [`InvalidBlock`]. + + [`InvalidBlock`]: ref:ethereum.exceptions.InvalidBlock + """ diff --git a/src/ethereum/partitioned_binary_tree.py b/src/ethereum/partitioned_binary_tree.py new file mode 100644 index 00000000000..90d905ad706 --- /dev/null +++ b/src/ethereum/partitioned_binary_tree.py @@ -0,0 +1,1150 @@ +""" +The [EIP-8297] Partitioned Binary Tree: a single authenticated +key/value tree holding all of Ethereum state. + +The raw tree is a compressed binary radix trie mapping +variable-length keys to 32-byte values, committing to its entire +contents with one root hash. Keys are consumed bit by bit, most +significant bit first, and must be prefix-free; see [`Key`]. The +mapping of keys to values is exposed through [`BinaryTrie`], and the +[`root`] function reduces a tree to its 32-byte commitment. The hash +function follows the EIP's reference implementation (BLAKE3). + +The rest of the module defines the _embedding_: how accounts, +storage slots, and code chunks are assigned keys and packed into +values, merging the account and storage tries of the Merkle-Patricia +design into this one tree. State is written through +[`embed_account`] and [`embed_storage_slot`] and removed through +[`remove_account`], [`remove_storage_slot`][rss], and +[`remove_code_chunks`], all built on the raw tree operations. + +The first byte of every key is a **zone** identifier labeling the +category of state the key holds: account headers live in +[`ACCOUNT_ZONE`], content-addressed code in [`CODE_ZONE`], and +overflow storage in [`STORAGE_ZONE`]. Keys are variable length, but +every key of a zone has the same length, keeping keys prefix-free as +the tree requires. + +A key's **stem** is every byte except its final sub-index byte. Keys +sharing a stem form one group of up to [`STEM_SUBTREE_WIDTH`] +co-located values, all reachable through the same branch of the +tree. This keeps data that is accessed together cheap to prove: an +account's header stem holds its basic data, its code hash or its +delegation, and its first storage slots, so one proof path covers +them all. + +Code is not keyed by account at all: every chunk lives in +[`CODE_ZONE`], content-addressed by code hash, so contracts with +identical bytecode share their chunk leaves. Overflow storage and +code are co-located at coarser granularity: aligned groups of up to +[`STEM_SUBTREE_WIDTH`] consecutive slots or chunks share a stem, so +neighboring values are still proved through one shared path rather +than one path each. + +[EIP-8297]: https://eips.ethereum.org/EIPS/eip-8297 +[`Key`]: ref:ethereum.partitioned_binary_tree.Key +[`BinaryTrie`]: ref:ethereum.partitioned_binary_tree.BinaryTrie +[`root`]: ref:ethereum.partitioned_binary_tree.root +[`ACCOUNT_ZONE`]: ref:ethereum.partitioned_binary_tree.ACCOUNT_ZONE +[`CODE_ZONE`]: ref:ethereum.partitioned_binary_tree.CODE_ZONE +[`STORAGE_ZONE`]: ref:ethereum.partitioned_binary_tree.STORAGE_ZONE +[`STEM_SUBTREE_WIDTH`]: ref:ethereum.partitioned_binary_tree.STEM_SUBTREE_WIDTH +[`embed_account`]: ref:ethereum.partitioned_binary_tree.embed_account +[`embed_storage_slot`]: ref:ethereum.partitioned_binary_tree.embed_storage_slot +[`remove_account`]: ref:ethereum.partitioned_binary_tree.remove_account +[rss]: ref:ethereum.partitioned_binary_tree.remove_storage_slot +[`remove_code_chunks`]: ref:ethereum.partitioned_binary_tree.remove_code_chunks +""" + +import copy +from dataclasses import dataclass, field +from typing import Dict, List, Mapping, Optional, Union, final + +from blake3 import blake3 +from ethereum_types.bytes import Bytes, Bytes20, Bytes32 +from ethereum_types.frozen import slotted_freezable +from ethereum_types.numeric import U8, U32, U64, U256, Uint + +from ethereum.crypto.hash import Hash32, keccak256 +from ethereum.exceptions import BalanceOverflowError +from ethereum.utils.byte import left_pad_zero_bytes, right_pad_zero_bytes + +EMPTY_TRIE_ROOT = Hash32(b"\x00" * 32) +""" +Root hash of an empty binary tree, defined as 32 zero bytes. + +This is a sentinel value rather than a hash output: no input is +expected to hash to all zeroes, so it cannot collide with the +commitment of a non-empty tree. +""" + + +def blake3_hash(data: Bytes) -> Hash32: + """ + Hash `data` with the tree's hash function. + """ + return Hash32(blake3(data).digest()) + + +def bytes_to_bit_list(data: Bytes) -> Bytes: + """ + Expand each input byte into eight bits, most significant bit first. + """ + return Bytes( + bytearray( + (byte >> (7 - offset)) & 1 for byte in data for offset in range(8) + ) + ) + + +Key = Bytes +""" +A tree key is any non-empty byte string, consumed bit by bit, MSB first. + +Keys must be **prefix-free**, so no key may be a prefix of another. A +key is its path and a [`LeafNode`] ends a path, so a longer key +could never pass through the position a shorter key terminates at. + +[`LeafNode`]: ref:ethereum.partitioned_binary_tree.LeafNode +""" + +MAX_KEY_LENGTH = Uint(8192) +""" +Longest key the tree accepts, in bytes. + +The bound is derived from the prefix encoding algorithm: a branch prefix can +approach the full bit length of the keys sharing it, and +[`encode_bit_prefix`] stores the bit count in two bytes, so keys +longer than this could produce a prefix the encoding cannot +represent. That bound is the worst case and longer keys often +encode fine, but enforcing it on every key in [`trie_set`] keeps +the limit a stated contract instead of a data-dependent failure +during merkleization. + +[`encode_bit_prefix`]: ref:ethereum.partitioned_binary_tree.encode_bit_prefix +[`trie_set`]: ref:ethereum.partitioned_binary_tree.trie_set +""" + +LEAF_NODE_TAG = Bytes(b"\x00") +""" +First byte of every [`LeafNode`] hash preimage. + +This is needed so that two different nodes can never share a +preimage (since their first byte will always differ). + +See [`merkleize`] for usage. + +[`LeafNode`]: ref:ethereum.partitioned_binary_tree.LeafNode +[`merkleize`]: ref:ethereum.partitioned_binary_tree.merkleize +""" + +BRANCH_NODE_TAG = Bytes(b"\x01") +""" +First byte of every [`BranchNode`] hash preimage. + +[`BranchNode`]: ref:ethereum.partitioned_binary_tree.BranchNode +""" + + +@final +@slotted_freezable +@dataclass +class LeafNode: + """ + Terminal node holding a single key's value. + + Note: the complete key is committed, not just the bits below the + leaf's position, so a leaf's meaning never depends on the path + taken to reach it. + """ + + key: Key + """ + The complete key whose value this leaf holds. + """ + + value: Bytes32 + """ + The 32-byte value stored under [`key`]. + + [`key`]: ref:ethereum.partitioned_binary_tree.LeafNode.key + """ + + +@final +@slotted_freezable +@dataclass +class BranchNode: + """ + Binary branch splitting on a single bit, carrying the run of bits + every key below it shares beyond the bits consumed above it. + + This inlines the Merkle Patricia Trie's extension-node concept + into the branch itself. + """ + + prefix: Bytes + """ + The compressed run of bits shared by every key below this branch, + one bit per byte, in consumption order. + This is empty when the keys diverge immediately. + + Like the MPT, the run is relative: it holds only the bits between the + parent's split point and this branch's split bit, never the path + from the root, which is reconstructed by the walk down. + """ + + left: "BinaryNode" + """ + Subtree of keys whose bit after [`prefix`] is `0`. + + [`prefix`]: ref:ethereum.partitioned_binary_tree.BranchNode.prefix + """ + + right: "BinaryNode" + """ + Subtree of keys whose bit after [`prefix`] is `1`. + + [`prefix`]: ref:ethereum.partitioned_binary_tree.BranchNode.prefix + """ + + +BinaryNode = Union[BranchNode, LeafNode] +""" +Either of the node types making up a non-empty binary tree. +""" + + +@final +@dataclass +class BinaryTrie: + """ + Mapping of variable-length keys to 32-byte values with a single + root hash that cryptographically commits to the mapping. + + Only the key/value pairs are stored; [`root`] rebuilds the node + structure and rehashes it from scratch on every call, which makes + the canonical compressed form automatic rather than a rule the + caller must maintain. + + A production client would instead keep the tree's nodes in + memory between calls and recompute only the hashes along the + path to a changed key; this reference implementation rebuilds + everything each time for readability. + + [`root`]: ref:ethereum.partitioned_binary_tree.root + """ + + _data: Dict[Key, Bytes32] = field(default_factory=dict) + + +def copy_trie(trie: BinaryTrie) -> BinaryTrie: + """ + Create a copy of `trie`. + + Keys and values are immutable, so the contents are shared between + the original and the copy. + """ + return BinaryTrie(copy.copy(trie._data)) + + +def trie_set(trie: BinaryTrie, key: Key, value: Optional[Bytes32]) -> None: + """ + Insert or update `key` in `trie` with the given `value`; setting + `None` removes the key, and removing an absent key does nothing. + + `None` can mark absence because it lies outside the value space: + every 32-byte value, including all zeroes, is a legitimate leaf, + so no stored value could play the role the Merkle Patricia + Trie's default value does. The same convention marks deleted + accounts in [`BlockDiff`], and mirrors [`trie_get`], which + returns `None` for absent keys. + + Since [`root`] rebuilds the node structure from the surviving + entries, a removal needs no node surgery here: branches held + open by the removed key simply never form, and the trie commits + as if the key had never been inserted. + + The caller must keep keys prefix-free; see [`Key`]. + + [`Key`]: ref:ethereum.partitioned_binary_tree.Key + [`root`]: ref:ethereum.partitioned_binary_tree.root + [`BlockDiff`]: ref:ethereum.state.BlockDiff + """ + assert ( + len(key) >= 1 + ) # Reject the empty key since it is a prefix of every other key + assert Uint(len(key)) <= MAX_KEY_LENGTH + if value is None: + trie._data.pop(key, None) + return + # `Bytes32` already rejects other lengths; asserted anyway, as in + # `root`, to keep the EIP's explicit validation visible. + assert len(value) == 32 + trie._data[key] = value + + +def trie_get(trie: BinaryTrie, key: Key) -> Optional[Bytes32]: + """ + Look up `key` in `trie`, returning `None` if absent. + """ + return trie._data.get(key) + + +def remove_subtree(trie: BinaryTrie, prefix: Bytes) -> None: + """ + Remove every key of `trie` beginning with `prefix`; a prefix + matching nothing does nothing. + + Keys are consumed most significant bit first, so the keys sharing + a `prefix` are exactly the keys of one subtree, and this removes + that subtree whole. Callers reach for it when the set of keys to + remove is known by where it sits in the tree rather than by + enumeration; see [`remove_account`], which drops an account's + unbounded storage without being told which slots it holds. + + A production client would unlink the subtree's node from its + parent and be done, which is why this is a tree operation and not + a loop of removals; the scan here follows [`BinaryTrie`] storing + only key/value pairs. + + [`remove_account`]: ref:ethereum.partitioned_binary_tree.remove_account + [`BinaryTrie`]: ref:ethereum.partitioned_binary_tree.BinaryTrie + """ + for key in [key for key in trie._data if key.startswith(prefix)]: + del trie._data[key] + + +def encode_bit_prefix(prefix: Bytes) -> Bytes: + """ + Encode a branch prefix: a two-byte big-endian bit + count followed by the bits packed most significant bit first, + zero padded to a byte boundary. + + The explicit bit count keeps the encoding injective. Without it, two + prefixes differing only by trailing zero bits would pack to the + same bytes and two different trees could share a root. + + Two bytes are enough because a prefix cannot outgrow the bit length + of the keys sharing it, and [`trie_set`] bounds every key at + [`MAX_KEY_LENGTH`]. + + [`trie_set`]: ref:ethereum.partitioned_binary_tree.trie_set + [`MAX_KEY_LENGTH`]: ref:ethereum.partitioned_binary_tree.MAX_KEY_LENGTH + """ + assert len(prefix) < 2**16 + packed = bytearray((len(prefix) + 7) // 8) + for bit_index, bit in enumerate(prefix): + packed[bit_index // 8] |= bit << (7 - bit_index % 8) + return Bytes(len(prefix).to_bytes(2, "big") + bytes(packed)) + + +def merkleize(node: BinaryNode) -> Hash32: + """ + Compute the hash committing to `node` and everything below it. + """ + if isinstance(node, LeafNode): + return blake3_hash(LEAF_NODE_TAG + node.key + node.value) + return blake3_hash( + BRANCH_NODE_TAG + + encode_bit_prefix(node.prefix) + + merkleize(node.left) + + merkleize(node.right) + ) + + +def binarize(entries: Mapping[Key, Bytes32], depth: Uint) -> BinaryNode: + """ + Build the canonical node structure for `entries`, whose keys all + share their first `depth` bits. `entries` must not be empty. + + A single entry becomes a [`LeafNode`] immediately. Multiple + entries become a [`BranchNode`] carrying the run of bits they + share beyond `depth` and splitting on the first bit where they + differ. + + [`LeafNode`]: ref:ethereum.partitioned_binary_tree.LeafNode + [`BranchNode`]: ref:ethereum.partitioned_binary_tree.BranchNode + """ + assert len(entries) > 0 + if len(entries) == 1: + ((key, value),) = entries.items() + return LeafNode(key, value) + + bit_lists = {key: bytes_to_bit_list(key) for key in entries} + + split = depth + while True: + # A key running out of bits while still grouped with others + # would be a prefix of theirs; see `Key`. + for bit_list in bit_lists.values(): + assert split < Uint(len(bit_list)) + distinct_bits_at_split = { + bit_list[split] for bit_list in bit_lists.values() + } + if len(distinct_bits_at_split) > 1: + break + split += Uint(1) + + left = { + key: value + for key, value in entries.items() + if bit_lists[key][split] == 0 + } + right = { + key: value + for key, value in entries.items() + if bit_lists[key][split] == 1 + } + shared_bits = next(iter(bit_lists.values())) + return BranchNode( + Bytes(shared_bits[depth:split]), + binarize(left, split + Uint(1)), + binarize(right, split + Uint(1)), + ) + + +def root(trie: BinaryTrie) -> Hash32: + """ + Compute the root hash of `trie`. + + An empty trie commits to [`EMPTY_TRIE_ROOT`]; any other trie + commits to the hash of its canonical node structure. + + Every entry is validated before hashing, as in the EIP's + `state_root`: computing the root rejects out-of-range keys and + values that are not 32 bytes, even though [`trie_set`] already + enforced both at write time. Prefix-freeness is enforced during + the walk itself, in [`binarize`]. + + [`EMPTY_TRIE_ROOT`]: ref:ethereum.partitioned_binary_tree.EMPTY_TRIE_ROOT + [`trie_set`]: ref:ethereum.partitioned_binary_tree.trie_set + [`binarize`]: ref:ethereum.partitioned_binary_tree.binarize + """ + for key, value in trie._data.items(): + assert len(key) >= 1 + assert Uint(len(key)) <= MAX_KEY_LENGTH + assert len(value) == 32 + if len(trie._data) == 0: + return EMPTY_TRIE_ROOT + return merkleize(binarize(trie._data, Uint(0))) + + +Zone = U8 +""" +One-byte identifier labeling the category of state a key holds, +prepended as the first byte of every key. + +Zones are the partitions of the Partitioned Binary Tree: because the +tree consumes key bits most significant first, every zone owns its +own region of the key space. +Defined zones are [`ACCOUNT_ZONE`], [`CODE_ZONE`], and +[`STORAGE_ZONE`]; the remaining values are reserved for future +state categories. + +[`ACCOUNT_ZONE`]: ref:ethereum.partitioned_binary_tree.ACCOUNT_ZONE +[`CODE_ZONE`]: ref:ethereum.partitioned_binary_tree.CODE_ZONE +[`STORAGE_ZONE`]: ref:ethereum.partitioned_binary_tree.STORAGE_ZONE +""" + +Address32 = Bytes32 +""" +32-byte address used to key the tree. + +Legacy 20-byte addresses are converted by [`address20_to_address32`]. + +[`address20_to_address32`]: ref:ethereum.partitioned_binary_tree.address20_to_address32 +""" # noqa: E501 + +BASIC_DATA_LEAF_KEY = Uint(0) +""" +Sub-index of the account header leaf packing version, code size, +nonce, and balance. +""" + +BASIC_DATA_VERSION = Uint(0) +""" +Version of the basic data leaf layout, packed as the leaf's first +byte by [`encode_basic_data`]. A future change to the layout bumps +the version so readers can tell the encodings apart. + +[`encode_basic_data`]: ref:ethereum.partitioned_binary_tree.encode_basic_data +""" + +CODE_HASH_LEAF_KEY = Uint(1) +""" +Sub-index of the account header leaf holding the code hash. + +An account that is delegated holds no such leaf; its code is its +delegation indicator, kept at [`DELEGATION_LEAF_KEY`] instead. Every +account that exists holds exactly one of the two. + +[`DELEGATION_LEAF_KEY`]: ref:ethereum.partitioned_binary_tree.DELEGATION_LEAF_KEY +""" # noqa: E501 + +DELEGATION_LEAF_KEY = Uint(2) +""" +Sub-index of the account header leaf holding a delegation indicator. + +The leaf determines both the code and its hash: a code read takes +the leading `code_size` bytes of the value, and `EXTCODEHASH` hashes +them. Holding it in the header rather than as content-addressed code +keeps the indicator private to one account, so replacing or clearing +a delegation touches no leaf another account shares; see +[`embed_account`]. + +[`embed_account`]: ref:ethereum.partitioned_binary_tree.embed_account +""" + +DELEGATION_MARKER = Bytes(b"\xef\x01\x00") +""" +Leading bytes marking an account's code as a delegation indicator. + +Defined here rather than imported so this module stays independent +of any fork, as [`EMPTY_CODE_HASH`] is. + +[`EMPTY_CODE_HASH`]: ref:ethereum.partitioned_binary_tree.EMPTY_CODE_HASH +""" + +DELEGATION_CODE_LENGTH = Uint(23) +""" +Length of a delegation indicator: the marker and a 20-byte address. +""" + +EMPTY_CODE_HASH = keccak256(b"") +""" +Code hash for accounts without code. + +The code hash leaf is written on account creation, EOAs included, +and holds the Keccak hash of empty bytecode: `EXTCODEHASH` of a +codeless account, existing or newly created, must keep returning +this value. + +`code_hash` is an EVM-observable value stored in a leaf, not a tree +commitment, so it stays Keccak even though the tree hashes with +[`blake3_hash`]. + +[`blake3_hash`]: ref:ethereum.partitioned_binary_tree.blake3_hash +""" + +HEADER_STORAGE_OFFSET = Uint(64) +""" +Sub-index of storage slot `0` within the account header stem. Slots +`0` through `63` live in the header. +""" + +HEADER_STORAGE_SLOTS = Uint(64) +""" +Number of storage slots co-located in the account header stem: +slots `0` through `HEADER_STORAGE_SLOTS - 1` live there, at +sub-indices counted from [`HEADER_STORAGE_OFFSET`], and every later +slot lives in [`STORAGE_ZONE`]; see +[`get_tree_key_for_storage_slot`]. + +[`HEADER_STORAGE_OFFSET`]: ref:ethereum.partitioned_binary_tree.HEADER_STORAGE_OFFSET +[`STORAGE_ZONE`]: ref:ethereum.partitioned_binary_tree.STORAGE_ZONE +[`get_tree_key_for_storage_slot`]: ref:ethereum.partitioned_binary_tree.get_tree_key_for_storage_slot +""" # noqa: E501 + +STEM_SUBTREE_WIDTH = Uint(256) +""" +Maximum number of values grouped under a single stem: the size of +the sub-index byte's space. + +The EIP requires `HEADER_STORAGE_OFFSET + HEADER_STORAGE_SLOTS <= +STEM_SUBTREE_WIDTH` as an invariant; the header storage sweep and +the storage key split assume it. +""" + +ACCOUNT_ZONE = Zone(0) +""" +Zone byte of account header stems. +""" + +CODE_ZONE = Zone(1) +""" +Zone byte of content-addressed code stems. + +Code chunk keys derive from the code hash rather than from any +account, so contracts with identical bytecode share their chunk +leaves; see [`get_tree_key_for_code_chunk`]. + +[`get_tree_key_for_code_chunk`]: ref:ethereum.partitioned_binary_tree.get_tree_key_for_code_chunk +""" # noqa: E501 + +STORAGE_ZONE = Zone(255) +""" +Zone byte of overflow storage stems. + +Storage sits at the far end of the zone byte, leaving zones `2` +through `254` reserved for future state categories. + +Note: Because keys are variable length, a zone's one-byte label +says nothing about its capacity so every zone's key space is +unbounded behind its prefix. +""" + +ACCOUNT_KEY_LENGTH = Uint(34) +""" +Length of every account zone key: the zone byte, a full address +digest, and the sub-index byte. +""" + +CODE_KEY_LENGTH = Uint(34) +""" +Length of every code zone key: the zone byte, a full digest of the +code hash and group index, and the sub-index byte. +""" + +STORAGE_KEY_LENGTH = Uint(66) +""" +Length of every storage zone key: the zone byte, two full digests +binding the account and its group index, and the sub-index byte. +""" + +PUSH_OFFSET = Uint(95) +""" +Opcode value one below `PUSH1`, so `PUSH_OFFSET + n` is the opcode +pushing `n` bytes. +""" + +PUSH1 = PUSH_OFFSET + Uint(1) +""" +Opcode of the smallest push instruction. +""" + +PUSH32 = PUSH_OFFSET + Uint(32) +""" +Opcode of the largest push instruction. +""" + + +def address20_to_address32(address: Bytes20) -> Address32: + """ + Convert a legacy 20-byte address by prepending 12 zero bytes. + + The embedding keys the tree by 32-byte addresses so that a future + address-space extension needs no re-keying. + """ + return Address32(left_pad_zero_bytes(address, 32)) + + +def key_hash(data: Bytes) -> Hash32: + """ + Hash `data` for use in tree key derivation. + + Key derivation reuses [`blake3_hash`], the tree's merkleization + hash. + + [`blake3_hash`]: ref:ethereum.partitioned_binary_tree.blake3_hash + """ + return blake3_hash(data) + + +def get_tree_key(zone: Zone, tree_position: Bytes, sub_index: U8) -> Key: + """ + Build a key from its three parts: the `zone` byte, the + hash-derived `tree_position`, and the final `sub_index` byte. + """ + return Key(bytes([int(zone)]) + tree_position + bytes([int(sub_index)])) + + +def get_tree_key_for_header(address: Address32, sub_index: Uint) -> Key: + """ + Compute the key of the account header leaf at `sub_index`. + + The header stem is in [`ACCOUNT_ZONE`] and is keyed by the address + alone, so each account has exactly one header stem. The header is + not one key: it is up to [`STEM_SUBTREE_WIDTH`] separate leaves + sharing that stem, and `sub_index` selects which one; basic + data, code hash, delegation, or an early storage slot. The + embedding derives no header key outside those sub-indices, so + the rest of the stem's space is unallocated and reserved for + future header fields. + + [`ACCOUNT_ZONE`]: ref:ethereum.partitioned_binary_tree.ACCOUNT_ZONE + [`STEM_SUBTREE_WIDTH`]: ref:ethereum.partitioned_binary_tree.STEM_SUBTREE_WIDTH + """ # noqa: E501 + key = get_tree_key(ACCOUNT_ZONE, key_hash(address), U8(sub_index)) + assert len(key) == int(ACCOUNT_KEY_LENGTH) + return key + + +def account_header_stem(address: Address32) -> Bytes: + """ + Compute the stem shared by every leaf of an account's header: + its basic data, its code hash or its delegation, and its first + storage slots. + + Every key under this prefix belongs to `address` and no key of + `address`'s header sits outside it, so the prefix is the account + header as one addressable region; see [`remove_account`]. + + [`remove_account`]: ref:ethereum.partitioned_binary_tree.remove_account + """ + return Bytes(bytes([int(ACCOUNT_ZONE)]) + key_hash(address)) + + +def account_storage_prefix(address: Address32) -> Bytes: + """ + Compute the prefix covering every overflow storage leaf of an + account, across all of its storage groups. + + Unlike [`account_header_stem`] this spans many stems: it is the + outer digest of [`storage_tree_position`], deliberately shared by + an account's whole overflow storage so that storage forms one + contiguous key range rather than locations scattered across the + tree. + + The range is unbounded, since an account may hold slots at any + of `2**256` positions, so it can only be addressed as a prefix, + never enumerated key by key. + + [`account_header_stem`]: ref:ethereum.partitioned_binary_tree.account_header_stem + [`storage_tree_position`]: ref:ethereum.partitioned_binary_tree.storage_tree_position + """ # noqa: E501 + return Bytes(bytes([int(STORAGE_ZONE)]) + key_hash(address)) + + +def get_tree_key_for_basic_data(address: Address32) -> Key: + """ + Compute the key of the account's basic data leaf. + """ + return get_tree_key_for_header(address, BASIC_DATA_LEAF_KEY) + + +def get_tree_key_for_code_hash(address: Address32) -> Key: + """ + Compute the key of the account's code hash leaf. + """ + return get_tree_key_for_header(address, CODE_HASH_LEAF_KEY) + + +def get_tree_key_for_delegation(address: Address32) -> Key: + """ + Compute the key of the account's delegation leaf. + """ + return get_tree_key_for_header(address, DELEGATION_LEAF_KEY) + + +def is_delegation(code: Bytes) -> bool: + """ + Check whether `code` is a delegation indicator. + + Deployed code may not begin with the marker's first byte, so an + account holds an indicator only by delegating; the classification + is a function of the code alone, never of its hash, which an + attacker could otherwise grind to have a contract read as + delegated. + """ + return ( + Uint(len(code)) == DELEGATION_CODE_LENGTH + and code[: len(DELEGATION_MARKER)] == DELEGATION_MARKER + ) + + +def encode_delegation(code: Bytes) -> Bytes32: + """ + Pack a delegation indicator into the 32-byte value stored at + [`DELEGATION_LEAF_KEY`]. + + The indicator occupies the leading bytes and the remainder is + zero. This is not the chunk encoding: a chunk reserves its first + byte for a push-data count, which an indicator, never being + executed as code, does not carry. + + [`DELEGATION_LEAF_KEY`]: ref:ethereum.partitioned_binary_tree.DELEGATION_LEAF_KEY + """ # noqa: E501 + return Bytes32(right_pad_zero_bytes(code, 32)) + + +def storage_tree_position(address: Address32, tree_index: U256) -> Bytes: + """ + Build the hash-derived position of an account's overflow storage + group at `tree_index`. + + The position carries two full digests: + + - `key_hash(address)` gathers all of an account's overflow + storage under one subtree, which future expiry and sync + schemes could use as their unit of work: a contract's whole + storage is one contiguous key range that can be expired or + served as a single subtree, rather than locations scattered + across the whole tree. + - `key_hash(address ‖ tree_index)` spreads the account's groups + within that subtree. + + Both digests depend on the address, so storage keys that an + attacker grinds to sit close together under one contract cannot + be reused against a different contract. + + `key_hash(address)` is the same digest [`get_tree_key_for_header`] + uses for the account's header stem; the two never collide because + they sit in different zones, differing in the key's first byte. + + [`get_tree_key_for_header`]: ref:ethereum.partitioned_binary_tree.get_tree_key_for_header + """ # noqa: E501 + prefix = key_hash(address) + suffix = key_hash(address + tree_index.to_be_bytes32()) + return Bytes(prefix + suffix) + + +def get_tree_key_for_storage_slot( + address: Address32, storage_key: U256 +) -> Key: + """ + Compute the key of a storage slot. + + The first [`HEADER_STORAGE_SLOTS`] slots live in the account + header stem, co-located with the account's basic data; all other + slots live in the storage zone. + + This leaves group `0` (`tree_index == 0`) short; its + storage-zone leaves are only sub-indices `64`-`255`, 192 slots + rather than the full 256 every later group has. + + [`HEADER_STORAGE_SLOTS`]: ref:ethereum.partitioned_binary_tree.HEADER_STORAGE_SLOTS + """ # noqa: E501 + if storage_key < U256(HEADER_STORAGE_SLOTS): + return get_tree_key_for_header( + address, HEADER_STORAGE_OFFSET + Uint(storage_key) + ) + tree_index = storage_key // U256(STEM_SUBTREE_WIDTH) + sub_index = storage_key % U256(STEM_SUBTREE_WIDTH) + key = get_tree_key( + STORAGE_ZONE, + storage_tree_position(address, tree_index), + U8(sub_index), + ) + assert len(key) == int(STORAGE_KEY_LENGTH) + return key + + +def get_tree_key_for_code_chunk(code_hash: Hash32, chunk_id: Uint) -> Key: + """ + Compute the key of a code chunk, which lives in [`CODE_ZONE`]. + + No address takes part: the key is content-addressed by + `code_hash`, so every account running the same bytecode shares + the leaf. That sharing is why chunks outlive the accounts + referencing them; see [`remove_code_chunks`]. + + An aligned range of [`STEM_SUBTREE_WIDTH`] chunks sharing one + `tree_index` is a **code group**: its chunks share a stem and + differ only in the sub-index byte. + + [`CODE_ZONE`]: ref:ethereum.partitioned_binary_tree.CODE_ZONE + [`STEM_SUBTREE_WIDTH`]: ref:ethereum.partitioned_binary_tree.STEM_SUBTREE_WIDTH + [`remove_code_chunks`]: ref:ethereum.partitioned_binary_tree.remove_code_chunks + """ # noqa: E501 + tree_index = chunk_id // STEM_SUBTREE_WIDTH + sub_index = chunk_id % STEM_SUBTREE_WIDTH + key = get_tree_key( + CODE_ZONE, + key_hash(code_hash + tree_index.to_be_bytes32()), + U8(sub_index), + ) + assert len(key) == int(CODE_KEY_LENGTH) + return key + + +def chunkify_code(code: Bytes) -> List[Bytes32]: + """ + Split `code` into the 32-byte chunks stored in the tree. + + Chunk `i` holds the `i`-th 31-byte slice of the code in bytes `1` + through `31`, preceded by one byte counting how many of the + slice's leading bytes are data of a push instruction that began in + an earlier chunk. + + The count lets a chunk be interpreted without + its predecessors and is capped at `31`, the chunk payload size. + """ + if len(code) % 31 != 0: + pad_amount = 31 - (len(code) % 31) + code = Bytes(right_pad_zero_bytes(code, len(code) + pad_amount)) + + # Number of push-data bytes remaining at each position, counting + # the position itself; `0` marks executable bytes. The extra 32 + # entries let the largest push record data past the end of the + # code. + remaining_push_data = [0] * (len(code) + 32) + position = 0 + while position < len(code): + opcode = Uint(code[position]) + if PUSH1 <= opcode <= PUSH32: + push_data_bytes = int(opcode - PUSH_OFFSET) + else: + push_data_bytes = 0 + position += 1 + for offset in range(push_data_bytes): + remaining_push_data[position + offset] = push_data_bytes - offset + position += push_data_bytes + + return [ + Bytes32( + bytes([min(remaining_push_data[start], 31)]) + + code[start : start + 31] + ) + for start in range(0, len(code), 31) + ] + + +def encode_basic_data(code_size: U32, nonce: U64, balance: U256) -> Bytes32: + """ + Pack an account's basic data into the 32-byte value stored at + [`BASIC_DATA_LEAF_KEY`]. + + The fields are packed big-endian, consistent with every other + encoding in the embedding: + + - one version byte, currently zero + - three reserved zero bytes + - four bytes of code size + - eight bytes of nonce + - sixteen bytes of balance + + The code size and nonce parameters are typed at their field + widths; the nonce cannot exceed eight bytes by [EIP-2681]. + Balances are protocol-level `U256` values, so the parameter + keeps that type; a balance too large for the sixteen-byte + field cannot be committed and raises [`BalanceOverflowError`], + invalidating the block whose state would hold it. + + The four-byte `code_size` at offset four matches EIP-8297; it is + one byte wider than EIP-7864's three-byte field at offset five, + from which this layout descends. + + [`BASIC_DATA_LEAF_KEY`]: ref:ethereum.partitioned_binary_tree.BASIC_DATA_LEAF_KEY + [`BalanceOverflowError`]: ref:ethereum.exceptions.BalanceOverflowError + [EIP-2681]: https://eips.ethereum.org/EIPS/eip-2681 + """ # noqa: E501 + if balance >= U256(2) ** U256(128): # U128 doesn't exist + raise BalanceOverflowError( + f"balance {balance} does not fit the sixteen-byte " + f"basic data balance field" + ) + return Bytes32( + bytes([int(BASIC_DATA_VERSION)]) + # Reserved bytes: headroom for future header fields. + + b"\x00" * 3 + + code_size.to_be_bytes4() + + nonce.to_be_bytes8() + + int(balance).to_bytes(16, "big") + ) + + +ZERO_VALUE = Bytes32(b"\x00" * 32) +""" +The value that [`state_write`] resolves to a deletion rather +than an insertion. + +[`state_write`]: ref:ethereum.partitioned_binary_tree.state_write +""" + + +def state_write(trie: BinaryTrie, key: Key, value: Bytes32) -> None: + """ + Write `value` at `key`, resolving 32 zero bytes to a deletion + rather than an insertion. + + The tree itself has no value meaning absence: every 32-byte + value is storable, and only a key's presence distinguishes it + from an absent one. Collapsing zero onto absence is the state + model's choice, made here so that state written through this + module cannot commit to a zero-valued leaf, and so an absent key + and a zero one are the same state with the same root. + + Reads recover the collapsed value: an absent key reads back as + the zero it stood for, whether that is an empty storage slot or + a code chunk of 31 zero bytes. + """ + trie_set(trie, key, None if value == ZERO_VALUE else value) + + +def embed_account( + trie: BinaryTrie, + address32: Address32, + nonce: U64, + balance: U256, + code_hash: Hash32, + code: Bytes, +) -> None: + """ + Write an account's leaves into `trie`: packed basic data, then + either a delegation leaf or a code hash leaf and one leaf per + chunk of `code`. + + Being delegated and holding contract code are exclusive, so an + account holds exactly one of the two leaves and the other is + removed here. Writing over an existing account updates its leaves + in place and is told nothing of what the account was before, so + both removals are unconditional: an account that has just + delegated still carries the code hash leaf it held a moment ago, + and one that has just cleared its delegation still carries the + delegation leaf. + + Chunk leaves are content-addressed, so accounts sharing bytecode + write the same leaves with the same values, and a re-embedding + is idempotent. Leaves of a previous, different code are not + touched here: content addressing keeps them out of this code's + key set, and reclaiming them is a reference check against the + resulting state; see [`remove_code_chunks`]. + + Every leaf goes through [`state_write`], so any of them encoding + to 32 zero bytes is left absent and reads back as the zero it + stood for. Two cases reach that: + + - A chunk of 31 zero bytes, as in a run of `STOP` or a + zero-filled data region. Chunk presence therefore does not + delimit the code; its length is `code_size`. + - The basic data of an account with zero nonce, zero balance and + no code, since the version byte and the reserved bytes are + zero too. Such an account is still distinguished from an + absent one by the one header leaf it always holds. + + [`remove_code_chunks`]: ref:ethereum.partitioned_binary_tree.remove_code_chunks + [`state_write`]: ref:ethereum.partitioned_binary_tree.state_write + """ # noqa: E501 + state_write( + trie, + get_tree_key_for_basic_data(address32), + encode_basic_data( + code_size=U32(len(code)), + nonce=nonce, + balance=balance, + ), + ) + if is_delegation(code): + state_write( + trie, + get_tree_key_for_delegation(address32), + encode_delegation(code), + ) + trie_set(trie, get_tree_key_for_code_hash(address32), None) + return + + trie_set(trie, get_tree_key_for_delegation(address32), None) + state_write( + trie, + get_tree_key_for_code_hash(address32), + Bytes32(code_hash), + ) + for chunk_id, chunk in enumerate(chunkify_code(code)): + state_write( + trie, + get_tree_key_for_code_chunk(code_hash, Uint(chunk_id)), + chunk, + ) + + +def embed_storage_slot( + trie: BinaryTrie, + address32: Address32, + storage_key: U256, + value: Bytes32, +) -> None: + """ + Write one storage slot's leaf into `trie`, in the account header + stem or the account's overflow storage subtree as the slot + number dictates. + + Writing zero removes the slot's leaf, per [`state_write`], so a + slot cleared to zero is indistinguishable from one never + written. + + [`state_write`]: ref:ethereum.partitioned_binary_tree.state_write + """ + state_write( + trie, get_tree_key_for_storage_slot(address32, storage_key), value + ) + + +def remove_account(trie: BinaryTrie, address32: Address32) -> None: + """ + Remove an account from `trie` entirely: its basic data, its code + hash or delegation, and every storage slot it holds. + + An account owns exactly two regions of the key space, both fixed + by its address: its [`account_header_stem`] and its + [`account_storage_prefix`]. Removing an account is removing those + two subtrees, so what has to go is read off the address rather + than out of a list of the account's slots, which the caller may + not have, and which for storage is unbounded anyway. + + Code chunks are the one thing an account holds that it does not + own: they live in [`CODE_ZONE`], content-addressed, and are + shared with every other account running the same bytecode, so + they outlive the account that referenced them and are not removed + here. Dropping them takes a reference check against the resulting + state; see [`remove_code_chunks`]. + + Removing an absent account does nothing. + + [`account_header_stem`]: ref:ethereum.partitioned_binary_tree.account_header_stem + [`account_storage_prefix`]: ref:ethereum.partitioned_binary_tree.account_storage_prefix + [`CODE_ZONE`]: ref:ethereum.partitioned_binary_tree.CODE_ZONE + [`remove_code_chunks`]: ref:ethereum.partitioned_binary_tree.remove_code_chunks + """ # noqa: E501 + remove_subtree(trie, account_header_stem(address32)) + remove_subtree(trie, account_storage_prefix(address32)) + + +def remove_code_chunks( + trie: BinaryTrie, code_hash: Hash32, code: Bytes +) -> None: + """ + Remove the [`CODE_ZONE`] leaves of `code` from `trie`. + + These leaves are content-addressed, so they belong to the + bytecode rather than to any account holding it. They may be + removed only once no account in the resulting state has + `code_hash`, which the caller establishes; removing them while an + account still runs that code would take its bytecode with it. + + The sweep covers every chunk of `code` without consulting the + tree: chunks encoding to 32 zero bytes were never in it (see + [`state_write`]) and removing an absent chunk does nothing. + + [`CODE_ZONE`]: ref:ethereum.partitioned_binary_tree.CODE_ZONE + [`state_write`]: ref:ethereum.partitioned_binary_tree.state_write + """ # noqa: E501 + for chunk_id in range(len(chunkify_code(code))): + trie_set( + trie, + get_tree_key_for_code_chunk(code_hash, Uint(chunk_id)), + None, + ) + + +def remove_all_storage(trie: BinaryTrie, address32: Address32) -> None: + """ + Remove every storage slot leaf of an account from `trie`, leaving + the account itself and its code in place. + + An account's storage straddles the two regions its address fixes: + the header slots sit in the header stem beside the basic data + and the code hash or delegation that must survive, so the header + is swept one slot sub-index at a time, while the overflow + storage subtree goes whole. As in [`remove_account`], no list of + the account's slots is needed. + + [`remove_account`]: ref:ethereum.partitioned_binary_tree.remove_account + """ + for sub_index in range( + HEADER_STORAGE_OFFSET, HEADER_STORAGE_OFFSET + HEADER_STORAGE_SLOTS + ): + trie_set( + trie, get_tree_key_for_header(address32, Uint(sub_index)), None + ) + remove_subtree(trie, account_storage_prefix(address32)) + + +def remove_storage_slot( + trie: BinaryTrie, address32: Address32, storage_key: U256 +) -> None: + """ + Remove one storage slot's leaf from `trie`; removing an absent + slot does nothing. + """ + trie_set(trie, get_tree_key_for_storage_slot(address32, storage_key), None) diff --git a/src/ethereum/state.py b/src/ethereum/state.py index e7903318c83..9f9e98471fa 100644 --- a/src/ethereum/state.py +++ b/src/ethereum/state.py @@ -115,7 +115,11 @@ def get_code(self, code_hash: Hash32) -> Bytes: """ Get the bytecode for a given code hash. - Return ``b""`` for ``EMPTY_CODE_HASH``. + Return ``b""`` for ``EMPTY_CODE_HASH``. A code hash with no + stored bytecode is a malformed pre-state; providers raise + [`UnknownCodeHashError`] rather than a raw lookup error. + + [`UnknownCodeHashError`]: ref:ethereum.exceptions.UnknownCodeHashError """ ... @@ -140,6 +144,13 @@ def compute_state_root(self, block_diff: BlockDiff) -> Root: new bytecode is not yet in the provider's code store when the root is computed. + A commitment whose encoding bounds a field more tightly than + the protocol types -- such as the binary tree's sixteen-byte + balance field -- raises [`InvalidBlock`] when the diffed state + cannot be committed. + Return the new state root. + + [`InvalidBlock`]: ref:ethereum.exceptions.InvalidBlock """ ... diff --git a/src/ethereum/state_mpt.py b/src/ethereum/state_mpt.py index dbdd7a718e0..299a22cc083 100644 --- a/src/ethereum/state_mpt.py +++ b/src/ethereum/state_mpt.py @@ -18,6 +18,7 @@ from ethereum_types.numeric import U256 from ethereum.crypto.hash import Hash32, keccak256 +from ethereum.exceptions import UnknownCodeHashError from ethereum.merkle_patricia_trie import ( EMPTY_TRIE_ROOT, Trie, @@ -50,11 +51,20 @@ def get_code(self, code_hash: Hash32) -> Bytes: """ Get the bytecode for a given code hash. - Return ``b""`` for ``EMPTY_CODE_HASH``. - """ + Return ``b""`` for ``EMPTY_CODE_HASH``. Any other hash with no + stored bytecode raises [`UnknownCodeHashError`]: an account + referencing such a hash is a malformed pre-state. + + [`UnknownCodeHashError`]: ref:ethereum.exceptions.UnknownCodeHashError + """ # noqa: E501 if code_hash == EMPTY_CODE_HASH: return b"" - return self._code_store[code_hash] + code = self._code_store.get(code_hash) + if code is None: + raise UnknownCodeHashError( + f"no bytecode stored for code hash 0x{code_hash.hex()}" + ) + return code def get_account_optional(self, address: Address) -> Optional[Account]: """ diff --git a/src/ethereum/state_pbt.py b/src/ethereum/state_pbt.py new file mode 100644 index 00000000000..7ddfca65d14 --- /dev/null +++ b/src/ethereum/state_pbt.py @@ -0,0 +1,398 @@ +""" +Binary-tree-backed implementation of the shared state model. + +The [`State`] class here is the [EIP-8297] counterpart of +[`ethereum.state_mpt`]: an in-memory implementation of the +[`PreState`] protocol whose state roots are binary tree commitments. +Accounts and storage live in plain mappings; [`embed_flat_state`] +maps them to tree keys and values and [`compute_state_root`] commits +the result, both through [`ethereum.partitioned_binary_tree`]. + +This provider makes one deliberate simplification: no transition +machinery. On mainnet the EIP's tree would start empty beside a +frozen Merkle Patricia Trie; here all state is in the tree from the +start, which keeps the commitment testable in isolation. + +Zero means absent, as [EIP-8297] requires and +[`ethereum.state_mpt`] independently does: a write of 32 zero bytes +resolves to a deletion, so no leaf holds zero and an absent key +reads back as the zero it stood for. The tree itself can represent +both, which is why the collapse is the state model's to make; see +[`state_write`][psw]. + +[EIP-8297]: https://eips.ethereum.org/EIPS/eip-8297 +[`State`]: ref:ethereum.state_pbt.State +[`PreState`]: ref:ethereum.state.PreState +[`embed_flat_state`]: ref:ethereum.state_pbt.embed_flat_state +[`compute_state_root`]: ref:ethereum.state_pbt.State.compute_state_root +[`ethereum.state_mpt`]: ref:ethereum.state_mpt +[`ethereum.partitioned_binary_tree`]: ref:ethereum.partitioned_binary_tree +[psw]: ref:ethereum.partitioned_binary_tree.state_write +""" + +from dataclasses import dataclass, field +from typing import Callable, Dict, Mapping, Optional, final + +from ethereum_types.bytes import Bytes, Bytes32 +from ethereum_types.numeric import U64, U256 + +from ethereum.crypto.hash import Hash32, keccak256 +from ethereum.exceptions import UnknownCodeHashError +from ethereum.partitioned_binary_tree import ( + BinaryTrie, + address20_to_address32, + embed_account, + embed_storage_slot, + is_delegation, + remove_account, + remove_all_storage, + remove_code_chunks, + remove_storage_slot, +) +from ethereum.partitioned_binary_tree import root as binary_tree_root +from ethereum.state import EMPTY_CODE_HASH, Account, Address, BlockDiff, Root + + +def embed_flat_state( + accounts: Mapping[Address, Account], + storages: Mapping[Address, Mapping[Bytes32, U256]], + get_code: Callable[[Hash32], Bytes], +) -> BinaryTrie: + """ + Embed a flat snapshot of state into a fresh binary tree. + + `accounts` maps addresses to their account records, `storages` + maps addresses to their storage slots keyed by the raw 32-byte + slot key, and `get_code` resolves a code hash to its bytecode. + Every account contributes its basic data and code hash leaves, + one leaf per code chunk, and one leaf per storage slot it has in + `storages`. Chunk leaves are content-addressed, so accounts with + identical bytecode write the same leaves with the same values; + the repetition is idempotent and the embedding is independent of + account order. + + Addresses appearing in `storages` but not in `accounts` are + ignored: storage belongs to an account, so slots without one + have no place in the tree. [`_apply_diff_to_trie`] holds to the + same rule when it reaches the tree incrementally. + + [`_apply_diff_to_trie`]: ref:ethereum.state_pbt._apply_diff_to_trie + """ + trie = BinaryTrie() + + for address, account in accounts.items(): + embed_account( + trie, + address20_to_address32(address), + U64(account.nonce), + account.balance, + account.code_hash, + get_code(account.code_hash), + ) + + for address, slots in storages.items(): + if address not in accounts: + continue + address32 = address20_to_address32(address) + for key, value in slots.items(): + embed_storage_slot( + trie, + address32, + U256.from_be_bytes(key), + value.to_be_bytes32(), + ) + + return trie + + +@final +@dataclass +class State: + """ + Contains all information that is preserved between transactions. + """ + + _accounts: Dict[Address, Account] = field(default_factory=dict) + _storage: Dict[Address, Dict[Bytes32, U256]] = field(default_factory=dict) + _code_store: Dict[Hash32, Bytes] = field( + default_factory=dict, compare=False + ) + + def get_code(self, code_hash: Hash32) -> Bytes: + """ + Get the bytecode for a given code hash. + + Return ``b""`` for ``EMPTY_CODE_HASH``. Any other hash with no + stored bytecode raises [`UnknownCodeHashError`]: an account + referencing such a hash is a malformed pre-state. + + [`UnknownCodeHashError`]: ref:ethereum.exceptions.UnknownCodeHashError + """ + if code_hash == EMPTY_CODE_HASH: + return b"" + code = self._code_store.get(code_hash) + if code is None: + raise UnknownCodeHashError( + f"no bytecode stored for code hash 0x{code_hash.hex()}" + ) + return code + + def get_account_optional(self, address: Address) -> Optional[Account]: + """ + Get the account at an address. + + Return ``None`` if there is no account at the address. + """ + return self._accounts.get(address) + + def get_storage(self, address: Address, key: Bytes32) -> U256: + """ + Get a storage value. + + Return ``U256(0)`` if the key has not been set. + """ + return self._storage.get(address, {}).get(key, U256(0)) + + def account_has_storage(self, address: Address) -> bool: + """ + Check whether an account has any storage. + + Only needed for EIP-7610. The Merkle Patricia Trie answered + this from an account's ``storage_root``; the binary tree has + no such node, so the answer is whether any slot leaf of the + address exists; which is what an entry here means, since + storage without an account never reaches the tree. + """ + return address in self._storage + + def compute_state_root(self, block_diff: BlockDiff) -> Root: + """ + Compute the state root after applying `block_diff` to the + pre-state. The pre-state itself is not modified: its + embedding is built fresh and the diff is applied to that + tree, via [`_apply_diff_to_trie`], as explicit insertions, + updates, and deletions. + + The diff's ``code_changes`` are needed here, unlike in the + Merkle Patricia Trie: code chunk leaves commit the code + itself, not just its hash, and newly deployed code is not yet + in the code store when the root is computed. The same lookup + serves removals, resolving the bytecode whose chunks a + deletion or code change sweeps away. + + [`_apply_diff_to_trie`]: ref:ethereum.state_pbt._apply_diff_to_trie + """ + trie = embed_flat_state(self._accounts, self._storage, self.get_code) + _apply_diff_to_trie(trie, self, block_diff) + return binary_tree_root(trie) + + +def _apply_diff_to_trie( + trie: BinaryTrie, pre_state: State, diff: BlockDiff +) -> None: + """ + Apply `diff` to `trie`, the embedding of `pre_state`, as tree + operations: writes become insertions or in-place updates, and + removals become deletions. This function decides *what* changed; + the embedding's operations decide which keys that touches. + + Account and storage removals are addressed rather than + enumerated: an account owns a known region of the key space, so + deleting it or wiping its storage is a tree operation on that + region, and the diff never has to say which slots the account + held. Code is the exception on both counts: its chunks sit in a + shared region no account owns, so removal enumerates them from + the bytecode, and it may happen at all only after + `code_hash_survives` has scanned every account the diff does not + touch to establish that nothing remaining still runs the code. + Beyond that scan, the pre-state supplies what the diff leaves + implicit: an account's previous code, and whether an address had + an account at all. + + Only a deleted account reaches that scan. A live account cannot + replace non-empty code: deployment requires an empty code hash, + an indicator may not be deployed as code, and a delegation is + not code at all, being a header leaf its account replaces on its + own. So a code change needs no reclamation, and re-embedding the + account is the whole of it. + + Storage belongs to an account, so an address the diff leaves + without one owns no slot leaves, exactly as in + [`embed_flat_state`]: a deleted account's slots go with it, and + a write to an address the same diff deletes never reaches the + tree. + + [`embed_flat_state`]: ref:ethereum.state_pbt.embed_flat_state + """ + + def code_for(code_hash: Hash32) -> Bytes: + if code_hash in diff.code_changes: + return diff.code_changes[code_hash] + return pre_state.get_code(code_hash) + + def has_account(address: Address) -> bool: + if address in diff.account_changes: + return diff.account_changes[address] is not None + return address in pre_state._accounts + + def code_hash_survives(code_hash: Hash32) -> bool: + """ + Whether any account in the resulting state has `code_hash`. + """ + for account in diff.account_changes.values(): + if account is not None and account.code_hash == code_hash: + return True + return any( + account.code_hash == code_hash + for address, account in pre_state._accounts.items() + if address not in diff.account_changes + ) + + def drop_unreferenced_code(pre_account: Optional[Account]) -> None: + """ + Remove a deleted account's code chunks once nothing in the + resulting state runs that code. + + The chunks are content-addressed and possibly shared, so they + go only when `code_hash_survives` finds no remaining holder. + A delegation indicator is exempt: it lives in its account's + header, so deleting the account takes it and there is no + shared leaf to reclaim. + """ + if pre_account is None: + return + code_hash = pre_account.code_hash + if code_hash == EMPTY_CODE_HASH: + return + if is_delegation(code_for(code_hash)): + return + if code_hash_survives(code_hash): + return + remove_code_chunks(trie, code_hash, code_for(code_hash)) + + for address in diff.storage_clears: + remove_all_storage(trie, address20_to_address32(address)) + + for address, account in diff.account_changes.items(): + address32 = address20_to_address32(address) + pre_account = pre_state._accounts.get(address) + if account is None: + remove_account(trie, address32) + drop_unreferenced_code(pre_account) + continue + embed_account( + trie, + address32, + U64(account.nonce), + account.balance, + account.code_hash, + code_for(account.code_hash), + ) + + for address, slots in diff.storage_changes.items(): + if not has_account(address): + continue + address32 = address20_to_address32(address) + for key, value in slots.items(): + if value == U256(0): + remove_storage_slot(trie, address32, U256.from_be_bytes(key)) + else: + embed_storage_slot( + trie, + address32, + U256.from_be_bytes(key), + value.to_be_bytes32(), + ) + + +def apply_changes_to_state(state: State, diff: BlockDiff) -> None: + """ + Apply block-level diff to the ``State`` for the next block. + + Storage belongs to an account, so a write to an address the diff + leaves without one is dropped rather than kept as storage no + account owns. That keeps [`account_has_storage`] answering as the + tree does, since such slots never reach the tree; see + [`_apply_diff_to_trie`]. + + [`account_has_storage`]: ref:ethereum.state_pbt.State.account_has_storage + [`_apply_diff_to_trie`]: ref:ethereum.state_pbt._apply_diff_to_trie + """ + for address in diff.storage_clears: + state._storage.pop(address, None) + + for address, account in diff.account_changes.items(): + if account is None: + state._accounts.pop(address, None) + state._storage.pop(address, None) + else: + state._accounts[address] = account + + for address, slots in diff.storage_changes.items(): + if address not in state._accounts: + continue + slot_values = state._storage.setdefault(address, {}) + for key, value in slots.items(): + if value == U256(0): + slot_values.pop(key, None) + else: + slot_values[key] = value + if slot_values == {}: + del state._storage[address] + + state._code_store.update(diff.code_changes) + + +def store_code(state: State, code: Bytes) -> Hash32: + """ + Store bytecode in ``State``. + """ + code_hash = keccak256(code) + if code_hash != EMPTY_CODE_HASH: + state._code_store[code_hash] = code + return code_hash + + +def set_account( + state: State, + address: Address, + account: Optional[Account], +) -> None: + """ + Set an account in a ``State``. + + Setting to ``None`` deletes the account. + """ + if account is None: + state._accounts.pop(address, None) + else: + state._accounts[address] = account + + +def set_storage( + state: State, + address: Address, + key: Bytes32, + value: U256, +) -> None: + """ + Set a storage value in a ``State``. + + Setting to ``U256(0)`` deletes the key. + """ + assert address in state._accounts + + slot_values = state._storage.setdefault(address, {}) + if value == U256(0): + slot_values.pop(key, None) + else: + slot_values[key] = value + if slot_values == {}: + del state._storage[address] + + +def state_root(state: State) -> Root: + """ + Compute the state root of the current state. + """ + return state.compute_state_root(BlockDiff()) diff --git a/src/ethereum_spec_tools/evm_tools/t8n/result.py b/src/ethereum_spec_tools/evm_tools/t8n/result.py index 5bab2af3b75..008bafa56e8 100644 --- a/src/ethereum_spec_tools/evm_tools/t8n/result.py +++ b/src/ethereum_spec_tools/evm_tools/t8n/result.py @@ -11,7 +11,9 @@ from ethereum_rlp import rlp from ethereum.crypto.hash import keccak256 +from ethereum.exceptions import InvalidBlock from ethereum.merkle_patricia_trie import root, trie_get +from ethereum.state import BlockDiff if TYPE_CHECKING: from execution_testing.client_clis.cli_types import ( @@ -79,7 +81,16 @@ def build_result( from execution_testing.client_clis.cli_types import Result as TestingResult diff = t8n.fork.extract_block_diff(t8n._block_state) - state_root = t8n.alloc.compute_state_root(diff) + try: + state_root = t8n.alloc.compute_state_root(diff) + except InvalidBlock as e: + # The active commitment cannot encode the resulting state (see + # `PreState.compute_state_root`). Such a block is rejected and + # leaves the chain state unchanged, so report the pre-state + # root. + if block_exception is None: + block_exception = f"{e}" + state_root = t8n.alloc.compute_state_root(BlockDiff()) arguments: Dict[str, Any] = { "state_root": state_root, diff --git a/tests/binary_trie/__init__.py b/tests/binary_trie/__init__.py new file mode 100644 index 00000000000..01fc2527d8f --- /dev/null +++ b/tests/binary_trie/__init__.py @@ -0,0 +1 @@ +"""Tests for the binary trie.""" diff --git a/tests/binary_trie/incremental_trie.py b/tests/binary_trie/incremental_trie.py new file mode 100644 index 00000000000..3b4440439df --- /dev/null +++ b/tests/binary_trie/incremental_trie.py @@ -0,0 +1,204 @@ +""" +Independent, insertion-based (incremental) implementation of the +binary radix trie, used by `test_trie.py` to cross-check +`ethereum.partitioned_binary_tree`'s rebuild-from-scratch implementation. +""" + +from typing import List, Optional + +from blake3 import blake3 + + +def _bits(data: bytes) -> List[int]: + return [(byte >> (7 - i)) & 1 for byte in data for i in range(8)] + + +def _pack_padded(bits: List[int]) -> bytes: + packed = bytearray((len(bits) + 7) // 8) + for i, bit in enumerate(bits): + packed[i // 8] |= bit << (7 - i % 8) + return bytes(packed) + + +def _leaf_hash(key: bytes, value: bytes) -> bytes: + return blake3(b"\x00" + key + value).digest() + + +def _branch_hash(prefix: List[int], left: bytes, right: bytes) -> bytes: + return blake3( + b"\x01" + + len(prefix).to_bytes(2, "big") + + _pack_padded(prefix) + + left + + right + ).digest() + + +class IncrementalRadixTree: + """ + Insertion-based compressed binary radix tree, used only to + cross-check `ethereum.partitioned_binary_tree`. + + Follows the standard descend/split insertion algorithm and hashes + with independently written tagged rules, so agreement with the + rebuild-from-scratch spec implementation also checks that both + produce the same canonical structure. + """ + + class Leaf: + """ + Terminal node of the incremental implementation. + """ + + def __init__(self, key: bytes, value: bytes) -> None: + self.key = key + self.value = value + + class Branch: + """ + Prefix-carrying binary branch of the incremental implementation. + """ + + def __init__( + self, prefix: List[int], left: object, right: object + ) -> None: + self.prefix = prefix + self.left = left + self.right = right + + def __init__(self) -> None: + self.root: Optional[object] = None + + def insert(self, key: bytes, value: bytes) -> None: + """ + Insert `key` and `value`, splitting nodes as needed. + """ + assert len(value) == 32 + if self.root is None: + self.root = self.Leaf(key, value) + return + self.root = self._insert(self.root, _bits(key), key, value, 0) + + def _insert( # type: ignore[no-untyped-def] + self, node, bits, key, value, depth + ): + if isinstance(node, self.Leaf): + if node.key == key: + node.value = value + return node + other_bits = _bits(node.key) + run = 0 + while True: + position = depth + run + assert position < len(bits) and position < len(other_bits) + if bits[position] != other_bits[position]: + break + run += 1 + prefix = bits[depth : depth + run] + leaf = self.Leaf(key, value) + if bits[depth + run] == 0: + return self.Branch(prefix, leaf, node) + return self.Branch(prefix, node, leaf) + + matched = 0 + while matched < len(node.prefix): + position = depth + matched + assert position < len(bits) + if bits[position] != node.prefix[matched]: + break + matched += 1 + if matched == len(node.prefix): + split = depth + matched + assert split < len(bits) + if bits[split] == 0: + node.left = self._insert( + node.left, bits, key, value, split + 1 + ) + else: + node.right = self._insert( + node.right, bits, key, value, split + 1 + ) + return node + # The key diverges inside the prefix: the surviving branch + # keeps the bits after the divergence, and a new branch takes + # the bits before it. + survivor = self.Branch( + node.prefix[matched + 1 :], node.left, node.right + ) + leaf = self.Leaf(key, value) + if bits[depth + matched] == 0: + return self.Branch(node.prefix[:matched], leaf, survivor) + return self.Branch(node.prefix[:matched], survivor, leaf) + + def delete(self, key: bytes) -> None: + """ + Remove `key` if present, collapsing the branch it leaves + behind; deleting an absent key does nothing. + """ + if self.root is None: + return + if isinstance(self.root, self.Leaf): + if self.root.key == key: + self.root = None + return + self.root = self._delete(self.root, _bits(key), key, 0) + + def _delete( # type: ignore[no-untyped-def] + self, node, bits, key, depth + ): + assert isinstance(node, self.Branch) + matched = 0 + while matched < len(node.prefix): + position = depth + matched + if position >= len(bits) or bits[position] != node.prefix[matched]: + return node # The key is not in this subtree. + matched += 1 + split = depth + matched + if split >= len(bits): + return node # The key ends at the split; not present. + + take_left = bits[split] == 0 + child = node.left if take_left else node.right + if isinstance(child, self.Branch): + replacement = self._delete(child, bits, key, split + 1) + if take_left: + node.left = replacement + else: + node.right = replacement + return node + + if child.key != key: + return node + + # The child leaf is the deletion target: this branch now has a + # single subtree, so the sibling takes its place. A leaf + # sibling moves up unchanged (it commits its full key); a + # branch sibling absorbs this branch's prefix and the split + # bit that selected it into its own prefix. + sibling = node.right if take_left else node.left + if isinstance(sibling, self.Leaf): + return sibling + assert isinstance(sibling, self.Branch) + sibling_bit = 1 if take_left else 0 + return self.Branch( + node.prefix + [sibling_bit] + sibling.prefix, + sibling.left, + sibling.right, + ) + + def merkelize(self) -> bytes: + """ + Compute the root hash of the reference tree. + """ + if self.root is None: + return b"\x00" * 32 + + def _hash(node: object) -> bytes: + if isinstance(node, self.Leaf): + return _leaf_hash(node.key, node.value) + assert isinstance(node, self.Branch) + return _branch_hash( + node.prefix, _hash(node.left), _hash(node.right) + ) + + return _hash(self.root) diff --git a/tests/binary_trie/test_block_execution.py b/tests/binary_trie/test_block_execution.py new file mode 100644 index 00000000000..c4d20eee419 --- /dev/null +++ b/tests/binary_trie/test_block_execution.py @@ -0,0 +1,303 @@ +""" +Unit test proving the PBT-mode block-execution path rejects a +block whose header claims a `state_root` that does not match the +tree-computed root. + +Fixture tests only *record* this expectation: fill-time +verification is skipped whenever `rlp_modifier` is set +(`BlockchainTest.generate_block_data`), and no client consumes +PBT-mode fixtures either, so nothing exercises the check end to +end today. This test drives it directly, through +`ethereum.forks.amsterdam.fork.execute_block`, whose +`block_state_root != block.header.state_root` comparison is what +raises `InvalidBlock`. + +Building a self-consistent block by hand hits the chicken-and-egg +problem every block builder faces: `execute_block` only *validates* a +header against outputs it (re)computes, it never *returns* the +correct header. `_build_valid_block_one` resolves this the way a real +block builder would: it runs the block body once (`apply_body`, on a +from-scratch `BlockEnvironment`) to learn the real outputs, using the +same helper functions `execute_block` itself calls right after -- +none of which is the comparison under test here -- and packages a +header from the results. +`test_execute_block_rejects_a_tampered_state_root` then asserts that +block executes cleanly (the control) before tampering with its +`state_root` in isolation, keeping `execute_block` itself, unmodified, +as the thing that raises `InvalidBlock`. + +The block itself is deliberately minimal: no transactions, no +withdrawals, and a one-byte `STOP` stub deployed at every system +contract address `apply_body` unconditionally calls -- just enough for +`process_checked_system_transaction`'s "contract has code" precondition +to pass. A block this empty produces an empty `BlockDiff`, so its +`state_root` is simply the pre-state's own, already-known root. +""" + +from dataclasses import replace +from typing import Tuple + +import pytest +from ethereum_rlp import rlp +from ethereum_types.bytes import Bytes, Bytes8, Bytes32 +from ethereum_types.numeric import U64, U256, Uint + +from ethereum.crypto.hash import Hash32, keccak256 +from ethereum.exceptions import InvalidBlock +from ethereum.forks.amsterdam.block_access_lists import ( + BlockAccessListBuilder, + hash_block_access_list, +) +from ethereum.forks.amsterdam.blocks import Block, Header +from ethereum.forks.amsterdam.bloom import logs_bloom +from ethereum.forks.amsterdam.fork import ( + BEACON_ROOTS_ADDRESS, + BUILDER_DEPOSIT_CONTRACT_ADDRESS, + BUILDER_EXIT_CONTRACT_ADDRESS, + CONSOLIDATION_REQUEST_PREDEPLOY_ADDRESS, + EMPTY_OMMER_HASH, + HISTORY_STORAGE_ADDRESS, + WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, + ChainContext, + apply_body, + calculate_base_fee_per_gas, + execute_block, +) +from ethereum.forks.amsterdam.fork_types import Bloom +from ethereum.forks.amsterdam.requests import compute_requests_hash +from ethereum.forks.amsterdam.state_tracker import ( + BlockState, + extract_block_diff, +) +from ethereum.forks.amsterdam.vm import BlockEnvironment +from ethereum.forks.amsterdam.vm.gas import calculate_excess_blob_gas +from ethereum.merkle_patricia_trie import root as mpt_root +from ethereum.state import Account, Address +from ethereum.state_pbt import State, set_account, state_root, store_code + +STUB_CODE = Bytes(b"\x00") +""" +Minimal, always-successful contract body: a single `STOP`. Long enough +(one byte) to satisfy `process_checked_system_transaction`'s +"contract has code" precondition, and simple enough to halt every call +made to it with no state effect. +""" + +SYSTEM_CONTRACT_ADDRESSES = ( + BEACON_ROOTS_ADDRESS, + HISTORY_STORAGE_ADDRESS, + WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, + CONSOLIDATION_REQUEST_PREDEPLOY_ADDRESS, + BUILDER_DEPOSIT_CONTRACT_ADDRESS, + BUILDER_EXIT_CONTRACT_ADDRESS, +) +""" +Every system contract address `apply_body` unconditionally calls +during block processing, checked or not. +""" + + +def _stubbed_pre_state() -> State: + """ + Build a `State` with `STUB_CODE` deployed at every system contract + address `apply_body` unconditionally calls. + + `process_checked_system_transaction` raises `InvalidBlock` outright + if the withdrawal, consolidation, or either builder-request + contract has no code; without this, every call to `execute_block` + below would raise for that reason, regardless of `state_root`. + """ + state = State() + for address in SYSTEM_CONTRACT_ADDRESSES: + code_hash = store_code(state, STUB_CODE) + set_account( + state, + address, + Account(nonce=Uint(0), balance=U256(0), code_hash=code_hash), + ) + return state + + +def _synthetic_parent_header() -> Header: + """ + Build the arbitrary "block 0" header that block one extends. + + Only the fields `validate_header` actually reads off the parent + (`gas_limit`, `gas_used`, `base_fee_per_gas`, `timestamp`, + `number`, `excess_blob_gas`, `blob_gas_used`, and its own RLP + encoding, for the child's `parent_hash`) matter here; every other + field is an unused placeholder. + """ + return Header( + parent_hash=Hash32(b"\x00" * 32), + ommers_hash=EMPTY_OMMER_HASH, + coinbase=Address(b"\x00" * 20), + state_root=Hash32(b"\x00" * 32), + transactions_root=Hash32(b"\x00" * 32), + receipt_root=Hash32(b"\x00" * 32), + bloom=Bloom(b"\x00" * 256), + difficulty=Uint(0), + number=Uint(0), + gas_limit=Uint(30_000_000), + gas_used=Uint(0), + timestamp=U256(1_000), + extra_data=Bytes(b""), + prev_randao=Bytes32(b"\x00" * 32), + nonce=Bytes8(b"\x00" * 8), + base_fee_per_gas=Uint(1_000_000_000), + withdrawals_root=Hash32(b"\x00" * 32), + blob_gas_used=U64(0), + excess_blob_gas=U64(0), + parent_beacon_block_root=Hash32(b"\x00" * 32), + requests_hash=Hash32(b"\x00" * 32), + block_access_list_hash=Hash32(b"\x00" * 32), + slot_number=U64(0), + ) + + +def _build_valid_block_one( + pre_state: State, +) -> Tuple[Block, ChainContext]: + """ + Build block one and the `ChainContext` it extends, with every + header field set to what `execute_block` will itself recompute and + check it against. + + Runs the real block body (`apply_body`, empty transactions and + withdrawals) once against a from-scratch `BlockEnvironment` to + learn the outputs, exactly as `execute_block` does internally + right up to (but not including) its validation `if`s -- see the + module docstring for why this bootstrap is necessary and does not + weaken the test. + """ + parent_header = _synthetic_parent_header() + parent_hash = keccak256(rlp.encode(parent_header)) + chain_context = ChainContext( + chain_id=U64(1), + block_hashes=[parent_hash], + parent_header=parent_header, + ) + + gas_limit = parent_header.gas_limit + base_fee_per_gas = calculate_base_fee_per_gas( + gas_limit, + parent_header.gas_limit, + parent_header.gas_used, + parent_header.base_fee_per_gas, + ) + excess_blob_gas = calculate_excess_blob_gas(parent_header) + + # Every field below is either read straight off the parent (the + # ones just computed above) or this test's own arbitrary, but + # self-consistent, choice. `state_root`, `transactions_root`, + # `receipt_root`, `bloom`, `gas_used`, `withdrawals_root`, + # `requests_hash`, and `block_access_list_hash` are placeholders + # here, overwritten below from the real outputs of running the + # block body. + header_shell = Header( + parent_hash=parent_hash, + ommers_hash=EMPTY_OMMER_HASH, + coinbase=Address(b"\xbb" * 20), + state_root=Hash32(b"\x00" * 32), + transactions_root=Hash32(b"\x00" * 32), + receipt_root=Hash32(b"\x00" * 32), + bloom=Bloom(b"\x00" * 256), + difficulty=Uint(0), + number=parent_header.number + Uint(1), + gas_limit=gas_limit, + gas_used=Uint(0), + timestamp=parent_header.timestamp + U256(12), + extra_data=Bytes(b""), + prev_randao=Bytes32(b"\x00" * 32), + nonce=Bytes8(b"\x00" * 8), + base_fee_per_gas=base_fee_per_gas, + withdrawals_root=Hash32(b"\x00" * 32), + blob_gas_used=U64(0), + excess_blob_gas=excess_blob_gas, + parent_beacon_block_root=Hash32(b"\x00" * 32), + requests_hash=Hash32(b"\x00" * 32), + block_access_list_hash=Hash32(b"\x00" * 32), + slot_number=U64(1), + ) + + block_state = BlockState(pre_state=pre_state) + block_env = BlockEnvironment( + chain_id=chain_context.chain_id, + state=block_state, + block_gas_limit=header_shell.gas_limit, + block_hashes=chain_context.block_hashes, + coinbase=header_shell.coinbase, + number=header_shell.number, + base_fee_per_gas=header_shell.base_fee_per_gas, + time=header_shell.timestamp, + prev_randao=header_shell.prev_randao, + excess_blob_gas=header_shell.excess_blob_gas, + parent_beacon_block_root=header_shell.parent_beacon_block_root, + block_access_list_builder=BlockAccessListBuilder(), + slot_number=header_shell.slot_number, + ) + block_output = apply_body( + block_env=block_env, transactions=(), withdrawals=() + ) + block_diff = extract_block_diff(block_state) + + header = replace( + header_shell, + state_root=pre_state.compute_state_root(block_diff), + transactions_root=mpt_root(block_output.transactions_trie), + receipt_root=mpt_root(block_output.receipts_trie), + bloom=logs_bloom(block_output.block_logs), + withdrawals_root=mpt_root(block_output.withdrawals_trie), + requests_hash=Hash32(compute_requests_hash(block_output.requests)), + block_access_list_hash=hash_block_access_list( + block_output.block_access_list + ), + gas_used=max( + block_output.block_gas_used, block_output.block_state_gas_used + ), + ) + block = Block(header=header, transactions=(), ommers=(), withdrawals=()) + return block, chain_context + + +def test_execute_block_rejects_a_tampered_state_root() -> None: + """ + `execute_block` accepts a correctly built block one and rejects an + otherwise-identical copy whose header's `state_root` has one byte + flipped. + + The control call (the real, computed root) must succeed first -- + proving the block built above is genuinely valid, not merely "some + header that happens to raise" -- so the second call's + `InvalidBlock` is attributable to the tampered `state_root` + specifically. Both calls go through `execute_block` unmodified; + its own `block_state_root != block.header.state_root` comparison + in `fork.py` is what raises. + """ + pre_state = _stubbed_pre_state() + block, chain_context = _build_valid_block_one(pre_state) + + # This block writes no state (no transactions, no withdrawals, and + # the system-contract stubs are pure STOPs), so its state_root is + # simply the pre-state's own, unchanged, root. + assert block.header.state_root == state_root(pre_state) + + # `execute_block` annotates `pre_state` with the MPT provider, but + # its body only touches the commitment-neutral `PreState` protocol + # surface, which the PBT provider implements too. + execute_block(block, pre_state, chain_context) # type: ignore[arg-type] + + original_root = block.header.state_root + tampered_root = Hash32( + bytes([original_root[0] ^ 0xFF]) + original_root[1:] + ) + tampered_block = replace( + block, header=replace(block.header, state_root=tampered_root) + ) + + with pytest.raises(InvalidBlock): + execute_block( + tampered_block, + pre_state, # type: ignore[arg-type] + chain_context, + ) diff --git a/tests/binary_trie/test_differential_mpt.py b/tests/binary_trie/test_differential_mpt.py new file mode 100644 index 00000000000..d79a83ed835 --- /dev/null +++ b/tests/binary_trie/test_differential_mpt.py @@ -0,0 +1,591 @@ +""" +Differential tests between `ethereum.state_mpt` and `ethereum.state_pbt`. + +Both modules implement the same `PreState` protocol (`ethereum.state`), +and on the `binary_tree` fork `ethereum.state_pbt` is meant to be a +pure commitment-scheme swap for `ethereum.state_mpt`: every +provider-level observable (accounts, storage, code, +`account_has_storage`) should agree given identical inputs. + +The first group of tests directs specific `BlockDiff`s at both +providers to pin the storage a deleted account leaves behind +(detailed on each test), which the two answer differently: PBT drops +it, MPT keeps it, visible through EIP-7610's `account_has_storage` +gate on `CREATE2`. This is not a bug in either provider. EIP-8297 +fixes `account_has_storage` for the tree, to whether a slot leaf of +the address exists, and `state_pbt` follows it; `state_mpt` answers +from its own storage tries, exactly as it does with no binary tree in +the picture. The second group applies random sequences of 5-8 diffs +to both providers and checks observable equivalence after every diff, +except at addresses known to carry that divergence, tracked via a +`divergent` set. + +Everything else should agree. EIP-8297's "Zero values and deletion" +section requires a write of 32 zero bytes to resolve to a deletion, +which is what both providers do and what the MPT did already, so the +zero-write equalities below are conformance rather than pinned +behavior. `test_trie.py::test_zero_value_is_not_absence` is not in +tension with that: the raw `BinaryTrie` does keep a zero-valued leaf, +and collapsing zero onto absence is the state model's job. +""" + +import random +from typing import Dict, List, Optional, Set, Tuple + +import pytest +from ethereum_types.bytes import Bytes, Bytes20, Bytes32 +from ethereum_types.numeric import U256, Uint + +from ethereum.crypto.hash import Hash32, keccak256 +from ethereum.exceptions import UnknownCodeHashError +from ethereum.merkle_patricia_trie import ( + EMPTY_TRIE_ROOT as MPT_EMPTY_TRIE_ROOT, +) +from ethereum.partitioned_binary_tree import ( + EMPTY_TRIE_ROOT as PBT_EMPTY_TRIE_ROOT, +) +from ethereum.partitioned_binary_tree import ( + address20_to_address32, + get_tree_key_for_code_hash, + get_tree_key_for_delegation, + is_delegation, +) +from ethereum.state import ( + EMPTY_ACCOUNT, + EMPTY_CODE_HASH, + Account, + Address, + BlockDiff, +) +from ethereum.state_mpt import State as MptState +from ethereum.state_mpt import ( + apply_changes_to_state as mpt_apply_changes_to_state, +) +from ethereum.state_mpt import set_account as mpt_set_account +from ethereum.state_mpt import set_storage as mpt_set_storage +from ethereum.state_mpt import state_root as mpt_state_root +from ethereum.state_mpt import store_code as mpt_store_code +from ethereum.state_pbt import State as PbtState +from ethereum.state_pbt import ( + apply_changes_to_state as pbt_apply_changes_to_state, +) +from ethereum.state_pbt import embed_flat_state +from ethereum.state_pbt import set_account as pbt_set_account +from ethereum.state_pbt import set_storage as pbt_set_storage +from ethereum.state_pbt import state_root as pbt_state_root +from ethereum.state_pbt import store_code as pbt_store_code + +ADDRESS_X = Bytes20(b"\xaa" * 20) +STORAGE_KEY = Bytes32(U256(1).to_be_bytes32()) +STORAGE_VALUE = U256(7) +STORAGE_KEY_2 = Bytes32(U256(2).to_be_bytes32()) +STORAGE_VALUE_2 = U256(9) + +CODELESS_ACCOUNT = Account( + nonce=Uint(1), balance=U256(1000), code_hash=EMPTY_CODE_HASH +) + +NEW_CODE = Bytes(b"\x60\x00\x60\x00\x00") +NEW_CODE_HASH = keccak256(NEW_CODE) + +# Fixed 12-address universe and probe-key set for the randomized +# differential test below; small and reused across diffs so deletes +# and recreates of the same address collide often. +RANDOM_ADDRESSES = [Bytes20(bytes([i]) * 20) for i in range(1, 13)] +WRITABLE_KEYS = [Bytes32(bytes([i]) * 32) for i in range(1, 6)] +NEVER_WRITTEN_KEY = Bytes32(b"\xff" * 32) +PROBE_KEYS = WRITABLE_KEYS + [NEVER_WRITTEN_KEY] + + +def test_account_delete_diverges_on_account_has_storage() -> None: + """ + Deleting an account leaves its storage trie intact under MPT but + pops it under PBT. + + Both providers start identical: account `X` with code-less storage + `{STORAGE_KEY: STORAGE_VALUE}`. A single diff deletes `X` and + touches no storage. `state_mpt.apply_changes_to_state` writes + `account_changes` into `_main_trie` without touching + `_storage_tries`, so the storage trie survives; PBT's + `state_pbt.apply_changes_to_state` pops `_storage[address]` in the + same branch that pops the account. + + This is the EIP-7610-visible divergence: right after this diff, a + `CREATE2` at `X` would be rejected under MPT (`account_has_storage` + still `True`) but allowed under PBT (`False`). Open EIP-8297 + consensus question, not a verdict on which provider is right. + """ + mpt_state = MptState() + pbt_state = PbtState() + mpt_set_account(mpt_state, ADDRESS_X, CODELESS_ACCOUNT) + pbt_set_account(pbt_state, ADDRESS_X, CODELESS_ACCOUNT) + mpt_set_storage(mpt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + pbt_set_storage(pbt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + + diff = BlockDiff(account_changes={ADDRESS_X: None}) + mpt_apply_changes_to_state(mpt_state, diff) + pbt_apply_changes_to_state(pbt_state, diff) + + assert mpt_state.get_account_optional(ADDRESS_X) is None + assert pbt_state.get_account_optional(ADDRESS_X) is None + assert mpt_state.account_has_storage(ADDRESS_X) is True + assert pbt_state.account_has_storage(ADDRESS_X) is False + + +def test_delete_then_recreate_resurrects_storage_only_under_mpt() -> None: + """ + Recreating a deleted account resurrects its old storage value + under MPT but starts empty under PBT. + + Continues the sequence pinned by + `test_account_delete_diverges_on_account_has_storage`: after `X` + is deleted while holding storage, a second diff recreates it as a + fresh `EMPTY_ACCOUNT`. MPT's orphaned storage trie is untouched by + either diff, so the pre-delete value reappears with no write ever + setting it; PBT's `_storage` was popped on delete and nothing + refills it, so the slot reads back as never written. + """ + mpt_state = MptState() + pbt_state = PbtState() + mpt_set_account(mpt_state, ADDRESS_X, CODELESS_ACCOUNT) + pbt_set_account(pbt_state, ADDRESS_X, CODELESS_ACCOUNT) + mpt_set_storage(mpt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + pbt_set_storage(pbt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + + delete_diff = BlockDiff(account_changes={ADDRESS_X: None}) + mpt_apply_changes_to_state(mpt_state, delete_diff) + pbt_apply_changes_to_state(pbt_state, delete_diff) + + recreate_diff = BlockDiff(account_changes={ADDRESS_X: EMPTY_ACCOUNT}) + mpt_apply_changes_to_state(mpt_state, recreate_diff) + pbt_apply_changes_to_state(pbt_state, recreate_diff) + + assert mpt_state.get_storage(ADDRESS_X, STORAGE_KEY) == STORAGE_VALUE + assert pbt_state.get_storage(ADDRESS_X, STORAGE_KEY) == U256(0) + + +def test_account_delete_with_same_diff_storage_writes() -> None: + """ + A single diff that both deletes an account and writes its storage + leaves an MPT trie holding the old key alongside the new one, and + a PBT state holding neither. + + `X` starts holding `{STORAGE_KEY: STORAGE_VALUE}`. One diff sets + `account_changes={X: None}` and, in the same diff, + `storage_changes={X: {STORAGE_KEY_2: STORAGE_VALUE_2}}`. + `state_mpt.apply_changes_to_state` applies the two against + separate containers, so the write lands in the trie that still + holds `STORAGE_KEY`. `state_pbt.apply_changes_to_state` pops + `_storage[X]` with the account and then drops the write too, + since storage belongs to an account and `X` no longer has one. + + Both providers still agree the account is gone. They disagree on + `account_has_storage`: PBT answers as [EIP-8297] requires, from + whether any slot leaf of the address exists, and none does. Same + divergence family as + `test_account_delete_diverges_on_account_has_storage`, surfacing + within one diff instead of across two. + + [EIP-8297]: https://eips.ethereum.org/EIPS/eip-8297 + """ + mpt_state = MptState() + pbt_state = PbtState() + mpt_set_account(mpt_state, ADDRESS_X, CODELESS_ACCOUNT) + pbt_set_account(pbt_state, ADDRESS_X, CODELESS_ACCOUNT) + mpt_set_storage(mpt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + pbt_set_storage(pbt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + + diff = BlockDiff( + account_changes={ADDRESS_X: None}, + storage_changes={ADDRESS_X: {STORAGE_KEY_2: STORAGE_VALUE_2}}, + ) + mpt_apply_changes_to_state(mpt_state, diff) + pbt_apply_changes_to_state(pbt_state, diff) + + assert mpt_state.get_account_optional(ADDRESS_X) is None + assert pbt_state.get_account_optional(ADDRESS_X) is None + + # The freshly written key: MPT keeps it against an address with + # no account, PBT drops it. + key_2 = STORAGE_KEY_2 + assert mpt_state.get_storage(ADDRESS_X, key_2) == STORAGE_VALUE_2 + assert pbt_state.get_storage(ADDRESS_X, key_2) == U256(0) + + # The pre-existing key: MPT resurrects it, PBT does not. + assert mpt_state.get_storage(ADDRESS_X, STORAGE_KEY) == STORAGE_VALUE + assert pbt_state.get_storage(ADDRESS_X, STORAGE_KEY) == U256(0) + + assert mpt_state.account_has_storage(ADDRESS_X) is True + assert pbt_state.account_has_storage(ADDRESS_X) is False + + +def test_all_zero_storage_changes_matches_never_written() -> None: + """ + Writing only zeros to slots an account never held reads back + identically to never having written them in both providers, and + PBT additionally commits to the same root either way. MPT is + checked only via `get_storage`/`account_has_storage`; its root is + never computed in this test. + + The equality is what EIP-8297 requires of the tree, and what the + MPT gave already: a write of zero is a deletion, so writing zero + to an absent slot is a no-op. + """ + diff = BlockDiff( + storage_changes={ + ADDRESS_X: {STORAGE_KEY: U256(0), STORAGE_KEY_2: U256(0)} + } + ) + + mpt_state = MptState() + mpt_set_account(mpt_state, ADDRESS_X, CODELESS_ACCOUNT) + mpt_apply_changes_to_state(mpt_state, diff) + + pbt_state = PbtState() + pbt_set_account(pbt_state, ADDRESS_X, CODELESS_ACCOUNT) + never_written_root = pbt_state_root(pbt_state) + pbt_apply_changes_to_state(pbt_state, diff) + + assert mpt_state.get_storage(ADDRESS_X, STORAGE_KEY) == U256(0) + assert mpt_state.get_storage(ADDRESS_X, STORAGE_KEY_2) == U256(0) + assert pbt_state.get_storage(ADDRESS_X, STORAGE_KEY) == U256(0) + assert pbt_state.get_storage(ADDRESS_X, STORAGE_KEY_2) == U256(0) + assert mpt_state.account_has_storage( + ADDRESS_X + ) == pbt_state.account_has_storage(ADDRESS_X) + assert pbt_state_root(pbt_state) == never_written_root + + +def test_code_changes_only_diff() -> None: + """ + A diff carrying only `code_changes` leaves every observable + account and storage slot unchanged in both providers, and the new + bytecode becomes retrievable by hash in both. + """ + mpt_state = MptState() + pbt_state = PbtState() + mpt_set_account(mpt_state, ADDRESS_X, CODELESS_ACCOUNT) + pbt_set_account(pbt_state, ADDRESS_X, CODELESS_ACCOUNT) + mpt_set_storage(mpt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + pbt_set_storage(pbt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + + diff = BlockDiff(code_changes={NEW_CODE_HASH: NEW_CODE}) + mpt_apply_changes_to_state(mpt_state, diff) + pbt_apply_changes_to_state(pbt_state, diff) + + assert mpt_state.get_account_optional(ADDRESS_X) == CODELESS_ACCOUNT + assert pbt_state.get_account_optional(ADDRESS_X) == CODELESS_ACCOUNT + assert mpt_state.get_storage(ADDRESS_X, STORAGE_KEY) == STORAGE_VALUE + assert pbt_state.get_storage(ADDRESS_X, STORAGE_KEY) == STORAGE_VALUE + + assert mpt_state.get_code(NEW_CODE_HASH) == NEW_CODE + assert pbt_state.get_code(NEW_CODE_HASH) == NEW_CODE + + +def test_zero_write_to_existing_slot_deletes_in_both() -> None: + """ + Zeroing an account's only storage slot deletes it identically in + both providers, agreeing that the account no longer has storage. + + Unlike the delete-account divergence pinned above, no account + deletion is involved: both providers' storage-changes loops + discard their now-empty container for the address once its one + key is zeroed. + + Zeroing the last slot is a deletion under EIP-8297 as it is + under the MPT. + """ + diff = BlockDiff(storage_changes={ADDRESS_X: {STORAGE_KEY: U256(0)}}) + + mpt_state = MptState() + mpt_set_account(mpt_state, ADDRESS_X, CODELESS_ACCOUNT) + mpt_set_storage(mpt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + mpt_apply_changes_to_state(mpt_state, diff) + + never_had_key_state = PbtState() + pbt_set_account(never_had_key_state, ADDRESS_X, CODELESS_ACCOUNT) + never_had_key_root = pbt_state_root(never_had_key_state) + + pbt_state = PbtState() + pbt_set_account(pbt_state, ADDRESS_X, CODELESS_ACCOUNT) + pbt_set_storage(pbt_state, ADDRESS_X, STORAGE_KEY, STORAGE_VALUE) + pbt_apply_changes_to_state(pbt_state, diff) + + assert mpt_state.get_storage(ADDRESS_X, STORAGE_KEY) == U256(0) + assert pbt_state.get_storage(ADDRESS_X, STORAGE_KEY) == U256(0) + assert mpt_state.account_has_storage(ADDRESS_X) is False + assert pbt_state.account_has_storage(ADDRESS_X) is False + assert pbt_state_root(pbt_state) == never_had_key_root + + +def _random_account(rng: random.Random, code_hash: Hash32) -> Account: + """ + Build an `Account` with `code_hash`, a random nonce below `2**64`, + and a random balance below `2**128`. + + The cap is not cosmetic: `encode_basic_data` rejects balances + past its sixteen-byte field with `BalanceOverflowError`, so + `2**128` or more would invalidate root computation rather than + merely being unrealistic. + """ + return Account( + nonce=Uint(rng.randrange(0, 2**64)), + balance=U256(rng.randrange(0, 2**128)), + code_hash=code_hash, + ) + + +def _random_new_code(rng: random.Random) -> Tuple[Hash32, Bytes]: + """ + Generate a random 1-39-byte code blob and return it with its + keccak hash. + """ + length = rng.randrange(1, 40) + code = Bytes(bytes(rng.randrange(0, 256) for _ in range(length))) + return keccak256(code), code + + +def _random_storage_slots(rng: random.Random) -> Dict[Bytes32, U256]: + """ + Build a random mix of zero and non-zero writes over a random + subset of `WRITABLE_KEYS`. + """ + slots: Dict[Bytes32, U256] = {} + for key in rng.sample(WRITABLE_KEYS, rng.randint(1, len(WRITABLE_KEYS))): + slots[key] = ( + U256(0) if rng.random() < 0.3 else U256(rng.randrange(1, 2**64)) + ) + return slots + + +def _build_random_initial_state( + rng: random.Random, +) -> Tuple[MptState, PbtState, List[Tuple[Hash32, Bytes]]]: + """ + Build identical initial states over 5-10 of `RANDOM_ADDRESSES`, + mixing EOAs and contracts with code and storage. + + Return the two providers, plus the pool of every code hash and + blob deployed so far, seeded on both providers via `store_code`. + """ + mpt_state = MptState() + pbt_state = PbtState() + code_pool: List[Tuple[Hash32, Bytes]] = [] + + initial_count = rng.randint(5, 10) + for address in rng.sample(RANDOM_ADDRESSES, initial_count): + code_hash = EMPTY_CODE_HASH + if rng.random() < 0.5: + code_hash, code = _random_new_code(rng) + mpt_store_code(mpt_state, code) + pbt_store_code(pbt_state, code) + code_pool.append((code_hash, code)) + + account = _random_account(rng, code_hash) + mpt_set_account(mpt_state, address, account) + pbt_set_account(pbt_state, address, account) + + if code_hash != EMPTY_CODE_HASH: + for key, value in _random_storage_slots(rng).items(): + mpt_set_storage(mpt_state, address, key, value) + pbt_set_storage(pbt_state, address, key, value) + + return mpt_state, pbt_state, code_pool + + +def _random_block_diff( + rng: random.Random, code_pool: List[Tuple[Hash32, Bytes]] +) -> BlockDiff: + """ + Build one random `BlockDiff` mixing account create/modify, account + delete, storage writes (including zeros), and new code. + + Appends any newly introduced bytecode to `code_pool` in place so + later diffs may deploy an account that references it. + """ + code_changes: Dict[Hash32, Bytes] = {} + if rng.random() < 0.4: + new_code_hash, new_code = _random_new_code(rng) + code_changes[new_code_hash] = new_code + code_pool.append((new_code_hash, new_code)) + + account_changes: Dict[Address, Optional[Account]] = {} + storage_changes: Dict[Address, Dict[Bytes32, U256]] = {} + + touch_count = rng.randint(2, 6) + for address in rng.sample(RANDOM_ADDRESSES, touch_count): + if rng.random() < 0.25: + account_changes[address] = None + else: + chosen_hash, _ = ( + rng.choice(code_pool) + if code_pool + else (EMPTY_CODE_HASH, Bytes(b"")) + ) + account_changes[address] = _random_account(rng, chosen_hash) + + if rng.random() < 0.7: + storage_changes[address] = _random_storage_slots(rng) + + return BlockDiff( + account_changes=account_changes, + storage_changes=storage_changes, + code_changes=code_changes, + ) + + +def _mark_divergent( + mpt_state: MptState, + pbt_state: PbtState, + diff: BlockDiff, + divergent: Set[Address], +) -> None: + """ + Add every address `diff` deletes whose storage the two providers + may then disagree about, before `diff` is applied. + + Two families, both descended from + `test_account_delete_diverges_on_account_has_storage`, and both + rooted in MPT keeping a deleted account's storage where PBT does + not: + + - The address already holds storage in either provider. MPT + leaves the storage trie in place; PBT pops it with the account. + - The same diff writes storage to the address. MPT records those + writes against an address with no account; PBT drops them, + since storage without an account never reaches the tree. + + Either way the providers may disagree about that address's + storage for the rest of the run. + """ + for address, account in diff.account_changes.items(): + if account is not None: + continue + has_storage = mpt_state.account_has_storage( + address + ) or pbt_state.account_has_storage(address) + if has_storage or address in diff.storage_changes: + divergent.add(address) + + +def _assert_equivalent( + mpt_state: MptState, + pbt_state: PbtState, + divergent: Set[Address], + code_pool: List[Tuple[Hash32, Bytes]], +) -> None: + """ + Assert both providers agree on every observable over the full + random address universe, skipping storage-related checks for + `divergent` addresses, and check every code hash ever stored. + + Also recomputes each provider's own root over the live, randomly + built state: PBT's root is computed twice (determinism) and, + whenever an account survives, asserted to differ from the + empty-tree sentinel; MPT gets the same two checks. Neither check + alone would catch a leaf silently dropped inside + `embed_flat_state` -- the root would still be deterministic and + non-empty -- so this also re-embeds the live PBT state directly + (bypassing `state_root`'s wrapper) and spot-checks every surviving + account's code-hash leaf against ground truth. The two providers' + roots are never compared against each other: they commit to + different schemes, so there is nothing for such a comparison to + mean. + """ + for address in RANDOM_ADDRESSES: + assert mpt_state.get_account_optional( + address + ) == pbt_state.get_account_optional(address) + + if address in divergent: + continue + + assert mpt_state.account_has_storage( + address + ) == pbt_state.account_has_storage(address) + for key in PROBE_KEYS: + assert mpt_state.get_storage( + address, key + ) == pbt_state.get_storage(address, key) + + assert mpt_state.get_code(EMPTY_CODE_HASH) == b"" + assert pbt_state.get_code(EMPTY_CODE_HASH) == b"" + for code_hash, code in code_pool: + assert mpt_state.get_code(code_hash) == code + assert pbt_state.get_code(code_hash) == code + + any_account_survives = any( + pbt_state.get_account_optional(address) is not None + for address in RANDOM_ADDRESSES + ) + + pbt_root_first = pbt_state_root(pbt_state) + pbt_root_second = pbt_state_root(pbt_state) + assert pbt_root_first == pbt_root_second + if any_account_survives: + assert pbt_root_first != PBT_EMPTY_TRIE_ROOT + + mpt_root_first = mpt_state_root(mpt_state) + mpt_root_second = mpt_state_root(mpt_state) + assert mpt_root_first == mpt_root_second + if any_account_survives: + assert mpt_root_first != MPT_EMPTY_TRIE_ROOT + + # Ground-truth spot-check for the case the checks above would miss + # (see docstring): a leaf `embed_flat_state` silently drops. + # Every account holds exactly one of the code hash and delegation + # leaves, so which one is present is itself part of the check. + embedded = embed_flat_state( + pbt_state._accounts, pbt_state._storage, pbt_state.get_code + ) + for address in RANDOM_ADDRESSES: + account = pbt_state.get_account_optional(address) + if account is None: + continue + address32 = address20_to_address32(address) + code_hash_leaf = embedded._data.get( + get_tree_key_for_code_hash(address32) + ) + delegation_leaf = embedded._data.get( + get_tree_key_for_delegation(address32) + ) + if is_delegation(pbt_state.get_code(account.code_hash)): + assert code_hash_leaf is None + assert delegation_leaf is not None + else: + assert code_hash_leaf == account.code_hash + assert delegation_leaf is None + + +@pytest.mark.parametrize("seed", [8297, 7610, 20260727]) +def test_random_diff_sequences_keep_providers_equivalent(seed: int) -> None: + """ + Random sequences of block diffs keep both providers equivalent at + every step, except at addresses known to carry the delete-while- + holding-storage divergence pinned by the directed tests above. + """ + rng = random.Random(seed) + mpt_state, pbt_state, code_pool = _build_random_initial_state(rng) + + divergent: Set[Address] = set() + + for _ in range(rng.randint(5, 8)): + diff = _random_block_diff(rng, code_pool) + _mark_divergent(mpt_state, pbt_state, diff, divergent) + + mpt_apply_changes_to_state(mpt_state, diff) + pbt_apply_changes_to_state(pbt_state, diff) + + _assert_equivalent(mpt_state, pbt_state, divergent, code_pool) + + +def test_get_code_contract_is_identical_across_providers() -> None: + """ + Both providers implement the `PreState.get_code` error contract + the same way: empty code for `EMPTY_CODE_HASH`, and the typed + `UnknownCodeHashError` for any other hash with no stored bytecode. + """ + for state in (MptState(), PbtState()): + assert state.get_code(EMPTY_CODE_HASH) == b"" + with pytest.raises(UnknownCodeHashError): + state.get_code(keccak256(b"never stored")) diff --git a/tests/binary_trie/test_embedding.py b/tests/binary_trie/test_embedding.py new file mode 100644 index 00000000000..8c9d2e7187c --- /dev/null +++ b/tests/binary_trie/test_embedding.py @@ -0,0 +1,1193 @@ +""" +Tests for the embedding of state into the binary tree. +""" + +from typing import List + +import pytest +from blake3 import blake3 +from ethereum_types.bytes import Bytes, Bytes20, Bytes32 +from ethereum_types.numeric import U8, U32, U64, U256, Uint + +from ethereum.crypto.hash import keccak256 +from ethereum.exceptions import BalanceOverflowError, InvalidBlock +from ethereum.partitioned_binary_tree import ( + EMPTY_CODE_HASH, + Address32, + BinaryTrie, + Zone, + address20_to_address32, + chunkify_code, + embed_account, + embed_storage_slot, + encode_basic_data, + encode_delegation, + get_tree_key, + get_tree_key_for_basic_data, + get_tree_key_for_code_chunk, + get_tree_key_for_code_hash, + get_tree_key_for_delegation, + get_tree_key_for_header, + get_tree_key_for_storage_slot, + is_delegation, + key_hash, + remove_account, + remove_all_storage, + remove_code_chunks, + remove_storage_slot, + root, +) +from ethereum.state import EMPTY_CODE_HASH as MPT_STATE_EMPTY_CODE_HASH + +ADDRESS = Address32(b"\x00" * 12 + b"\xaa" * 20) + +CODE_HASH = Bytes32(blake3(b"some code").digest()) + + +def _header_stem(address: Address32) -> bytes: + """ + Build `0x00 || H(address)`, the 33-byte account header stem, from + scratch. + """ + return bytes([0]) + blake3(bytes(address)).digest() + + +def _storage_overflow_key( + address: Address32, tree_index_bytes: bytes, sub_index: int +) -> bytes: + """ + Build `0xFF || H(A) || H(A || tree_index_bytes) || sub_index`, + the 66-byte overflow storage key, from scratch. + """ + prefix = blake3(bytes(address)).digest() + suffix = blake3(bytes(address) + tree_index_bytes).digest() + return bytes([255]) + prefix + suffix + bytes([sub_index]) + + +def _code_chunk_key( + code_hash: Bytes32, tree_index_bytes: bytes, sub_index: int +) -> bytes: + """ + Build `0x01 || H(C || tree_index_bytes) || sub_index`, the + 34-byte content-addressed code-chunk key, from scratch. + """ + digest = blake3(bytes(code_hash) + tree_index_bytes).digest() + return bytes([1]) + digest + bytes([sub_index]) + + +def test_address20_to_address32_prepends_zeros() -> None: + """ + Legacy addresses convert by prepending 12 zero bytes. + """ + address = Bytes20(b"\xaa" * 20) + expected = Address32(b"\x00" * 12 + b"\xaa" * 20) + assert address20_to_address32(address) == expected + + +def test_empty_code_hash_is_keccak_of_empty() -> None: + """ + The empty-code leaf value is the classic Keccak empty-code hash, + and agrees with the shared MPT state module's definition. + """ + assert EMPTY_CODE_HASH == bytes.fromhex( + "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + ) + assert EMPTY_CODE_HASH == MPT_STATE_EMPTY_CODE_HASH + + +def test_key_hash_is_blake3() -> None: + """ + Key derivation hashes with BLAKE3. + """ + assert key_hash(ADDRESS) == blake3(bytes(ADDRESS)).digest() + + +def test_get_tree_key_concatenates_its_three_parts() -> None: + """ + A key is the zone byte, the whole hash-derived position, and the + sub-index byte. + """ + digest = blake3(b"digest").digest() + for zone in (0, 1, 2, 254, 255): + key = get_tree_key(Zone(zone), digest, U8(7)) + assert len(key) == 34 + assert key == bytes([zone]) + digest + b"\x07" + + +def test_header_sub_index_wider_than_one_byte_is_rejected() -> None: + """ + A header sub-index that does not fit the key's final byte fails + at the narrowing to one byte inside the derivation. + """ + with pytest.raises(OverflowError): + get_tree_key_for_header(ADDRESS, Uint(256)) + + +def test_header_key_vectors() -> None: + """ + Header keys are `0x00 || H(A)` plus the leaf's sub-index, + 34 bytes in total. + """ + stem = _header_stem(ADDRESS) + + assert get_tree_key_for_basic_data(ADDRESS) == stem + b"\x00" + assert get_tree_key_for_code_hash(ADDRESS) == stem + b"\x01" + + +def test_delegation_key_vector() -> None: + """ + EIP key vector: an account's delegation leaf is its header stem + at sub-index `0x02`, 34 bytes in total. + + The sub-index is what distinguishes a delegation from a code + hash, so the two keys must differ; a discriminator that instead + read the leaf's leading bytes would let an attacker grind code + whose hash begins with the marker and have the contract read as + delegated. + """ + stem = _header_stem(ADDRESS) + + key = get_tree_key_for_delegation(ADDRESS) + + assert key == stem + b"\x02" + assert len(key) == 34 + assert key != get_tree_key_for_code_hash(ADDRESS) + + +def test_delegation_leaf_value_layout() -> None: + """ + EIP value vector: the leaf holds the indicator followed by nine + zero bytes. + + The target is a distinct byte per position, so a reversal or an + off-by-one slice shows. The nine trailing zeros are what set this + apart from the chunk encoding, which reserves a leading byte for + its push-data count and so pads with eight: the two encodings are + the same length and share no byte position. + """ + target = bytes(range(1, 21)) + designator = Bytes(b"\xef\x01\x00" + target) + + value = encode_delegation(designator) + + assert len(value) == 32 + assert value[:3] == b"\xef\x01\x00" + assert value[3:23] == target + assert value[23:] == b"\x00" * 9 + assert value != chunkify_code(designator)[0] + + +def test_storage_slot_in_header_vector() -> None: + """ + EIP sub-index vector: storage slot 5 lives in the header at + sub-index 0x45. + """ + key = get_tree_key_for_storage_slot(ADDRESS, U256(5)) + assert key == _header_stem(ADDRESS) + bytes([0x45]) + + +def test_storage_slot_overflow_vector() -> None: + """ + Slot 1000 maps to tree index 3, sub-index 0xE8, with the 65-byte + stem `0xFF || H(A) || H(A || 3)`. + """ + prefix = blake3(bytes(ADDRESS)).digest() + suffix = blake3(bytes(ADDRESS) + (3).to_bytes(32, "big")).digest() + stem = bytes([255]) + prefix + suffix + + key = get_tree_key_for_storage_slot(ADDRESS, U256(1000)) + assert key == stem + bytes([0xE8]) + + +def test_storage_slot_boundary_is_64() -> None: + """ + Slot 63 is the last header slot and slot 64 the first overflow + slot, landing at tree index 0, sub-index 64. + """ + assert get_tree_key_for_storage_slot(ADDRESS, U256(63)) == _header_stem( + ADDRESS + ) + bytes([127]) + + overflow_stem = ( + bytes([255]) + + blake3(bytes(ADDRESS)).digest() + + blake3(bytes(ADDRESS) + (0).to_bytes(32, "big")).digest() + ) + assert get_tree_key_for_storage_slot( + ADDRESS, U256(64) + ) == overflow_stem + bytes([64]) + + +@pytest.mark.parametrize( + "slot", + [ + pytest.param(0, id="slot-0-header-first"), + pytest.param(1, id="slot-1-header"), + pytest.param(62, id="slot-62-header-last-but-one"), + pytest.param(65, id="slot-65-overflow-group-0"), + pytest.param(255, id="slot-255-group-0-last"), + pytest.param(256, id="slot-256-group-1-first"), + pytest.param(257, id="slot-257-group-1"), + pytest.param(511, id="slot-511-group-1-last"), + pytest.param(512, id="slot-512-group-2-first"), + pytest.param(2**32, id="slot-2-32"), + pytest.param(2**256 - 1, id="slot-max-u256"), + ], +) +def test_storage_slot_key_matrix(slot: int) -> None: + """ + A matrix of storage slots rebuilds each expected key from + scratch, header form below 64 and overflow form at and above it. + + The overflow cases cross every group rollover in range: 255 is + group 0's last sub-index, 256 opens group 1 at sub-index 0, 511 + is group 1's last sub-index, and 512 opens group 2. + """ + if slot < 64: + expected = _header_stem(ADDRESS) + bytes([64 + slot]) + expected_length = 34 + else: + tree_index = slot // 256 + sub_index = slot % 256 + expected = _storage_overflow_key( + ADDRESS, tree_index.to_bytes(32, "big"), sub_index + ) + expected_length = 66 + + key = get_tree_key_for_storage_slot(ADDRESS, U256(slot)) + + assert key == expected + assert len(key) == expected_length + + +def test_storage_group_zero_never_uses_low_sub_indices() -> None: + """ + Group 0 of the storage zone is short. + + Slots 0 through 63 stay in the account header, so group 0's + overflow leaves cover only sub-indices 64 through 255 -- 192 + slots rather than the 256 every later group has. Every overflow + key derived from a group-0 slot must therefore end in a byte of + at least 64. + """ + group_zero_slots = 0 + for slot in range(64, 1001): + if slot // 256 != 0: + continue + group_zero_slots += 1 + key = get_tree_key_for_storage_slot(ADDRESS, U256(slot)) + assert key[-1] >= 64, ( + f"slot {slot}: group-0 overflow key ends in {key[-1]}, " + "below the 64 floor" + ) + # Vacuity control: `key[-1] >= 64` above is this loop's only + # assertion (there is no set-equality assertion anywhere in this + # test to imply it), so it is also the loop's sole guard against + # running zero times. If a future edit narrows `range(64, 1001)` + # or the `continue` guard so the body never executes, this line + # is what still catches the test passing vacuously. + assert group_zero_slots == 192 + + +def test_storage_tree_index_is_a_32_byte_big_endian_suffix() -> None: + """ + The overflow position's group half hashes the tree index as a + full 32-byte big-endian integer, not some narrower width. + + Encoding the same tree index over only 8 bytes changes the hash + input and therefore the key, so a future narrowing regression + would be caught here. + """ + slot = 256 * 5 + tree_index = slot // 256 + sub_index = slot % 256 + assert tree_index == 5 + + key = get_tree_key_for_storage_slot(ADDRESS, U256(slot)) + wide_key = _storage_overflow_key( + ADDRESS, tree_index.to_bytes(32, "big"), sub_index + ) + narrow_key = _storage_overflow_key( + ADDRESS, tree_index.to_bytes(8, "big"), sub_index + ) + + assert key == wide_key + assert key != narrow_key + + +def test_code_chunk_vector() -> None: + """ + EIP key vector: code chunk 5 lives in the code zone at sub-index + 0x05, with the 33-byte stem `0x01 || H(C || 0)`. + """ + code_hash = Bytes32(blake3(b"some code").digest()) + + key = get_tree_key_for_code_chunk(code_hash, Uint(5)) + assert key == _code_chunk_key(code_hash, (0).to_bytes(32, "big"), 0x05) + + +def test_code_chunk_second_group_vector() -> None: + """ + EIP key vector: chunk 300 lands in code group 1 at sub-index + 0x2C, with the 33-byte stem `0x01 || H(C || 1)`. + """ + code_hash = Bytes32(blake3(b"some code").digest()) + + key = get_tree_key_for_code_chunk(code_hash, Uint(300)) + assert key == _code_chunk_key(code_hash, (1).to_bytes(32, "big"), 0x2C) + + +def test_code_keys_are_content_addressed() -> None: + """ + Chunk keys depend only on the code hash and chunk id: no address + takes part in the derivation, so accounts sharing bytecode share + every chunk key, and distinct bytecodes share none, their stems + diverging at the hash. + """ + code_hash = Bytes32(blake3(b"shared bytecode").digest()) + other_hash = Bytes32(blake3(b"different bytecode").digest()) + + for chunk_id in (0, 5, 200): + ours = get_tree_key_for_code_chunk(code_hash, Uint(chunk_id)) + theirs = get_tree_key_for_code_chunk(other_hash, Uint(chunk_id)) + assert ours != theirs + assert ours[:-1] != theirs[:-1], "stems must differ, not just keys" + + +@pytest.mark.parametrize( + "chunk_id", + [ + pytest.param(0, id="chunk-0-group-0-first"), + pytest.param(1, id="chunk-1-group-0"), + pytest.param(255, id="chunk-255-group-0-last"), + pytest.param(256, id="chunk-256-group-1-first"), + pytest.param(257, id="chunk-257-group-1"), + pytest.param(511, id="chunk-511-group-1-last"), + pytest.param(512, id="chunk-512-group-2-first"), + ], +) +def test_code_chunk_key_matrix(chunk_id: int) -> None: + """ + A matrix of code chunk ids rebuilds each expected key from + scratch: every chunk is content-addressed in the code zone, and + the stem changes exactly where `chunk_id // 256` does. The + rollover itself is pinned separately by + `test_code_group_rollover_changes_the_stem`. + """ + tree_index = chunk_id // 256 + sub_index = chunk_id % 256 + expected = _code_chunk_key( + CODE_HASH, tree_index.to_bytes(32, "big"), sub_index + ) + + key = get_tree_key_for_code_chunk(CODE_HASH, Uint(chunk_id)) + + assert key == expected + assert len(key) == 34 + assert key[0] == 1 + + +def test_code_group_rollover_changes_the_stem() -> None: + """ + Crossing a code-group boundary changes the key's stem, not just + its sub-index: chunk 255 sits at group 0's last sub-index and + chunk 256 opens group 1 under a fresh hash-derived stem, and + likewise at 511 -> 512. Kept out of the key matrix so narrowing + the matrix's parameters can never silently retire this check. + """ + for last_of_group, first_of_next in ((255, 256), (511, 512)): + last_key = get_tree_key_for_code_chunk(CODE_HASH, Uint(last_of_group)) + next_key = get_tree_key_for_code_chunk(CODE_HASH, Uint(first_of_next)) + assert last_key[-1] == 255 + assert next_key[-1] == 0 + assert last_key[:-1] != next_key[:-1], "rollover must change stem" + + +def test_max_code_size_chunk_keys() -> None: + """ + `MAX_CODE_SIZE` (0x10000 = 65536 bytes, defined in + `ethereum.forks.amsterdam.vm.interpreter`) chunkifies into + `ceil(65536 / 31) == 2115` chunks. + + The last chunk, id 2114, lands in the code zone at group 8, + sub-index 66; the group/sub-index are computed here from the EIP + formula, with the concrete numbers also asserted so the + arithmetic stays pinned. + """ + max_code_size = 0x10000 + chunk_count = (max_code_size + 30) // 31 + assert chunk_count == 2115 + + last_chunk_id = chunk_count - 1 + tree_index = last_chunk_id // 256 + sub_index = last_chunk_id % 256 + assert (tree_index, sub_index) == (8, 66) + + key = get_tree_key_for_code_chunk(CODE_HASH, Uint(last_chunk_id)) + expected = _code_chunk_key( + CODE_HASH, tree_index.to_bytes(32, "big"), sub_index + ) + + assert key == expected + assert len(key) == 34 + + +def test_key_derivations_assert_their_own_key_length( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + Every key-deriving function asserts the length of the key it + builds before returning it (EIP-8297, "Tree embedding": + "Implementations MUST assert the length of every key they + construct"). + + Monkeypatching `key_hash` to return a too-short digest breaks + that invariant for whichever derivation calls it, so each of the + three length asserts in `embedding.py` -- + `get_tree_key_for_header`'s `ACCOUNT_KEY_LENGTH`, + `get_tree_key_for_storage_slot`'s `STORAGE_KEY_LENGTH`, and + `get_tree_key_for_code_chunk`'s `CODE_KEY_LENGTH` -- fires + instead of silently returning a malformed key. The storage slot + is chosen in the storage zone's range, so the call reaches that + function's own assert rather than delegating to + `get_tree_key_for_header`'s; every code chunk id reaches the + code derivation directly. + """ + monkeypatch.setattr( + "ethereum.partitioned_binary_tree.key_hash", + lambda _data: b"\x00" * 16, + ) + + with pytest.raises(AssertionError): + get_tree_key_for_header(ADDRESS, Uint(0)) + + with pytest.raises(AssertionError): + get_tree_key_for_storage_slot(ADDRESS, U256(1000)) + + with pytest.raises(AssertionError): + get_tree_key_for_code_chunk(CODE_HASH, Uint(300)) + + +def test_chunkify_empty_code() -> None: + """ + Empty code produces no chunks. + """ + assert chunkify_code(Bytes(b"")) == [] + + +def test_chunkify_code_without_pushes_pads_to_31_bytes() -> None: + """ + Code shorter than a chunk is zero-padded to 31 bytes. + """ + code = Bytes(b"\x01\x02\x03") # ADD MUL SUB + + chunks = chunkify_code(code) + + # The leading byte is the push-data offset count (zero here), not + # padding; only the trailing zeros pad the code to 31 bytes. + assert chunks == [Bytes32(b"\x00" + code + b"\x00" * 28)] + + +def test_chunkify_code_eip_example() -> None: + """ + EIP example: push data spanning a chunk boundary is recorded in + the second chunk's leading byte. + """ + # `...PUSH4 99 98 | 97 96 PUSH1 128 MSTORE...` where `|` begins a + # new chunk; the second chunk records that its first 2 bytes are + # push data. + push4 = 0x63 + push1 = 0x60 + mstore = 0x52 + code = Bytes( + b"\x00" * 28 + bytes([push4, 99, 98, 97, 96, push1, 128, mstore]) + ) + + chunks = chunkify_code(code) + + assert len(chunks) == 2 + assert chunks[0] == Bytes32(b"\x00" * 29 + bytes([push4, 99, 98])) + assert chunks[1] == Bytes32( + bytes([2, 97, 96, push1, 128, mstore]) + b"\x00" * 26 + ) + + +def test_chunkify_code_caps_leading_push_data_count_at_31() -> None: + """ + A chunk consisting entirely of push data reports 31 leading push + data bytes, the chunk-payload maximum, rather than 32. + """ + push32 = 0x7F + push_data = bytes(range(1, 33)) + code = Bytes(b"\x00" * 30 + bytes([push32]) + push_data) + + chunks = chunkify_code(code) + + assert len(chunks) == 3 + assert chunks[0] == Bytes32(b"\x00" * 31 + bytes([push32])) + assert chunks[1] == Bytes32(bytes([31]) + push_data[:31]) + assert chunks[2] == Bytes32(bytes([1]) + push_data[31:] + b"\x00" * 30) + + +def test_chunkify_code_push_data_truncated_by_end_of_code() -> None: + """ + A push instruction with its data cut off by the end of the code + still chunks cleanly. + """ + push32 = 0x7F + code = Bytes(bytes([push32])) + + chunks = chunkify_code(code) + + assert chunks == [Bytes32(b"\x00" + bytes([push32]) + b"\x00" * 30)] + + +def _reference_chunkify(code: bytes) -> List[bytes]: + """ + Chunkify `code` with an independent reimplementation. + + A direct transcription of EIP-8297's own `chunkify_code` + pseudocode, using the EIP text's own variable names rather than + `chunkify_code`'s, so the two implementations are genuinely + independent, not a copy-paste. + """ + push_offset = 95 + push1 = push_offset + 1 + push32 = push_offset + 32 + + if len(code) % 31 != 0: + code = code + b"\x00" * (31 - (len(code) % 31)) + bytes_to_exec_data = [0] * (len(code) + 32) + pos = 0 + while pos < len(code): + if push1 <= code[pos] <= push32: + pushdata_bytes = code[pos] - push_offset + else: + pushdata_bytes = 0 + pos += 1 + for x in range(pushdata_bytes): + bytes_to_exec_data[pos + x] = pushdata_bytes - x + pos += pushdata_bytes + return [ + bytes([min(bytes_to_exec_data[pos], 31)]) + code[pos : pos + 31] + for pos in range(0, len(code), 31) + ] + + +def test_chunkify_push0_is_not_push_data() -> None: + """ + `PUSH0` (0x5F) carries no push data. + + `PUSH_OFFSET` (95) sits one below `PUSH1` (96), so the `PUSH1` + through `PUSH32` range (96 through 127) deliberately excludes + `PUSH0`; a run of `PUSH0` bytes therefore chunkifies as plain + non-push code, every chunk's leading byte staying 0. + """ + code = Bytes(b"\x5f" * 40) + + chunks = chunkify_code(code) + + assert len(chunks) == 2 + for chunk in chunks: + assert chunk[0] == 0 + + +def test_chunkify_push_data_overhanging_into_padding() -> None: + """ + Padding happens before the push-data scan, so a push instruction + truncated by the end of the code can have its declared data + overhang into the padding. + + `PUSH32` demands 32 data bytes but this 32-byte code (opcode plus + 31 data bytes) supplies only 31, so the scan -- which runs over + the already-padded buffer -- counts the first zero padding byte + as the push's 32nd data byte too. + """ + push32 = 0x7F + data = bytes(range(1, 32)) # 31 bytes: 1, 2, ..., 31 + code = Bytes(bytes([push32]) + data) + assert len(code) == 32 + + chunks = chunkify_code(code) + + assert len(chunks) == 2 + # The PUSH32 opcode itself is not push data. + assert chunks[0] == Bytes32(bytes([0, push32]) + data[:30]) + # Byte 31 (last real data byte) and byte 32 (first padding byte) + # both count as carried-over push data. + assert chunks[1] == Bytes32(bytes([2]) + bytes([31]) + b"\x00" * 30) + + +def test_chunkify_push_ending_exactly_at_chunk_boundary() -> None: + """ + A push whose data ends exactly on a chunk's last payload byte + leaves no push data carried into the next chunk. + + The `PUSH1` at position 29 has its one data byte at position 30, + chunk 0's last payload byte, so chunk 1 opens with a plain opcode + byte and no phantom extra chunk appears. + """ + push1 = 0x60 + code = Bytes(b"\x00" * 29 + bytes([push1, 0xAB]) + b"\x00" * 31) + assert len(code) == 62 + + chunks = chunkify_code(code) + + assert len(chunks) == 2 + assert chunks[0] == Bytes32( + bytes([0]) + b"\x00" * 29 + bytes([push1, 0xAB]) + ) + assert chunks[1] == Bytes32(bytes([0]) + b"\x00" * 31) + + +@pytest.mark.parametrize( + "length", + [ + pytest.param(30, id="length-30-needs-padding"), + pytest.param(31, id="length-31-exact-multiple"), + pytest.param(32, id="length-32-needs-padding"), + pytest.param(62, id="length-62-exact-multiple"), + pytest.param(93, id="length-93-exact-multiple"), + ], +) +def test_chunkify_code_length_multiples_need_no_padding( + length: int, +) -> None: + """ + Chunk count always follows `ceil(len(code) / 31)`. + + When the code length is itself a multiple of 31 (31, 62, and 93 + bytes here), no padding byte is introduced: the final chunk's + 31-byte payload is entirely original code. Filling the code with + a non-zero repeating byte (0xAB, never a push opcode) makes any + padding zero bytes stand out. + """ + fill = 0xAB + code = Bytes(bytes([fill]) * length) + + chunks = chunkify_code(code) + + expected_chunk_count = (length + 30) // 31 + assert len(chunks) == expected_chunk_count + + if length % 31 == 0: + last_chunk = chunks[-1] + assert last_chunk[0] == 0 + assert last_chunk[1:] == bytes([fill]) * 31 + + +def test_chunkify_all_push32_code_matches_reference_scanner() -> None: + """ + `chunkify_code` agrees with an independent reimplementation of + the EIP pseudocode for code built entirely of `PUSH32` + instructions. + + 31 repeats of a 33-byte `PUSH32 || 32 data bytes` instruction is + 1023 bytes, exercising a full alignment cycle between the + 33-byte instruction and the 31-byte chunk. Chunk 32 additionally + gets a hand-derived pin independent of the reference scanner, so + a bug shared by both scanners would still be caught. + """ + instruction = bytes([0x7F]) + bytes(range(1, 33)) + assert len(instruction) == 33 + code = Bytes(instruction * 31) + assert len(code) == 1023 + + chunks = chunkify_code(code) + + assert chunks == [Bytes32(c) for c in _reference_chunkify(bytes(code))] + assert len(chunks) == 33 + assert chunks[32] == Bytes32(bytes([31]) + bytes(range(2, 33))) + + +def test_chunkify_push_data_containing_push_opcodes() -> None: + """ + Bytes that fall inside a push instruction's data are never + reinterpreted as fresh opcodes, even when their value (0x60 + through 0x7F) would otherwise mean `PUSH1` through `PUSH32`. + + Each `PUSH2` here carries `0x7F` and `0x60` as its two data + bytes; the scanner must skip both wholesale rather than restart a + push count on them. Chunk 2 additionally gets a hand-derived pin + independent of the reference scanner: a scanner that wrongly + restarted counting on a push-opcode-valued data byte would read + 31 there instead of 1. + """ + push2 = 0x61 + code = Bytes((bytes([push2, 0x7F, 0x60]) + b"\x00") * 16) + + chunks = chunkify_code(code) + + assert chunks == [Bytes32(c) for c in _reference_chunkify(bytes(code))] + assert len(chunks) == 3 + assert chunks[2] == Bytes32(bytes([1, 0x60]) + b"\x00" * 30) + + +def test_chunkify_consecutive_pushes_across_boundary() -> None: + """ + Back-to-back pushes straddling a chunk edge chunkify consistently + with the reference scanner. + + A `PUSH4` positioned so its fourth data byte falls exactly on + chunk 1's first byte is immediately followed by a `PUSH1` and a + `PUSH3`, both entirely inside chunk 1; chunk 1 opens with exactly + one carried-over push-data byte. + """ + push4, push1, push3 = 0x63, 0x60, 0x62 + code = Bytes( + b"\x00" * 27 # positions 0-26 + + bytes([push4, 1, 2, 3, 4]) # opcode 27, data 28-31 + + bytes([push1, 0xAA]) # opcode 32, data 33 + + bytes([push3, 0xBB, 0xCC, 0xDD]) # opcode 34, data 35-37 + + b"\x00" * 24 # positions 38-61 + ) + assert len(code) == 62 + + chunks = chunkify_code(code) + + assert chunks == [Bytes32(c) for c in _reference_chunkify(bytes(code))] + assert chunks[1][0] == 1 + + +CODE_HASHING_TO_THE_DELEGATION_MARKER = Bytes( + bytes.fromhex("0000000000000000000000000000000000000000637401") +) +""" +Deployable 23-byte code whose Keccak hash begins `0xef0100`. + +Found by grinding roughly 2**24 candidates offline, the work the +EIP names when rejecting a discriminator that reads a leaf's leading +bytes. It opens with `STOP`, so [EIP-3541] permits its deployment. + +[EIP-3541]: https://eips.ethereum.org/EIPS/eip-3541 +""" + + +@pytest.mark.parametrize( + "code, delegated", + [ + pytest.param( + Bytes(b"\xef\x01\x00" + b"\x11" * 20), True, id="indicator" + ), + pytest.param( + Bytes(b"\xef\x01\x00" + b"\x11" * 20 + b"\x00"), + False, + id="one-byte-too-long", + ), + pytest.param( + Bytes(b"\xef\x01\x00" + b"\x11" * 19), + False, + id="one-byte-too-short", + ), + pytest.param( + Bytes(b"\xef\x01\x01" + b"\x11" * 20), False, id="wrong-marker" + ), + pytest.param(Bytes(b"\x01" * 23), False, id="right-length-no-marker"), + pytest.param(Bytes(b""), False, id="no-code"), + pytest.param( + CODE_HASHING_TO_THE_DELEGATION_MARKER, + False, + id="hash-begins-with-the-marker", + ), + ], +) +def test_delegation_is_classified_by_code_never_by_hash( + code: Bytes, delegated: bool +) -> None: + """ + An indicator is the marker and the exact length, both read from + the code itself. + + The last case is the one the EIP's rationale turns on: its code + hash begins with the marker, which a discriminator reading the + code hash leaf's leading bytes would take for a delegation to an + attacker-chosen address. Classifying by the code keeps it a + contract, and putting the delegation at its own sub-index means + the leaf that answers the question is the one whose presence is + the answer. + """ + assert is_delegation(code) is delegated + if code == CODE_HASHING_TO_THE_DELEGATION_MARKER: + assert keccak256(code)[:3] == b"\xef\x01\x00" + + +def test_embed_account_reads_the_code_not_the_code_hash() -> None: + """ + Which leaf an account gets is decided by its code alone. + + `embed_account` is handed both the code and its hash, so this + passes a hash that contradicts the code in each direction: a + contract whose hash begins with the marker still gets a code hash + leaf and a chunk, and an indicator still gets a delegation leaf + even when handed an ordinary hash. Only an implementation that + consults the hash can tell these apart, and that is the design + the EIP rejects. + """ + contract = CODE_HASHING_TO_THE_DELEGATION_MARKER + contract_hash = keccak256(contract) + designator = Bytes(b"\xef\x01\x00" + b"\x11" * 20) + + trie = BinaryTrie() + embed_account(trie, ADDRESS, U64(1), U256(0), contract_hash, contract) + + stem = _header_stem(ADDRESS) + assert stem + b"\x01" in trie._data + assert stem + b"\x02" not in trie._data + assert get_tree_key_for_code_chunk(contract_hash, Uint(0)) in trie._data + + other = BinaryTrie() + embed_account( + other, ADDRESS, U64(1), U256(0), keccak256(designator), designator + ) + + assert stem + b"\x02" in other._data + assert stem + b"\x01" not in other._data + assert not any(key[0] == 1 for key in other._data) + + +def test_remove_account_takes_the_delegation_leaf() -> None: + """ + Deleting a delegated account removes its delegation leaf with the + rest of its header. + + The sweep is a subtree removal over the header stem, so it covers + every sub-index by construction. What this guards is a rewrite + into an enumeration of the sub-indices in use -- the shape + `remove_all_storage` already has -- which would be correct for + the code hash leaf and silently orphan the delegation one. + """ + designator = Bytes(b"\xef\x01\x00" + b"\x11" * 20) + + trie = BinaryTrie() + empty = root(trie) + embed_account( + trie, ADDRESS, U64(1), U256(5), keccak256(designator), designator + ) + assert _header_stem(ADDRESS) + b"\x02" in trie._data + + remove_account(trie, ADDRESS) + + assert trie._data == {} + assert root(trie) == empty + + +def test_chunkify_designator_shaped_code_still_chunks() -> None: + """ + `chunkify_code` has no notion of delegation: given designator + bytes it chunks them like any other code. + + No account reaches this, since a delegated account's indicator + goes to its header leaf and is never chunked. What the case pins + is that the two encodings of the same 23 bytes stay distinct -- + the chunk spends its leading byte on a push-data count and pads + with eight zeros, where the leaf pads with nine. + """ + designator = bytes([0xEF, 0x01, 0x00]) + b"\xcc" * 20 + assert len(designator) == 23 + code = Bytes(designator) + + chunks = chunkify_code(code) + + assert len(chunks) == 1 + assert chunks[0] == Bytes32(bytes([0]) + designator + b"\x00" * 8) + assert chunks[0] != encode_delegation(code) + + +def test_encode_basic_data_layout() -> None: + """ + Basic data packs version, code size, nonce, and balance at the + offsets given by the EIP, and the all-zero leaf -- a freshly + created, codeless, nonce-0, balance-0 account -- packs to 32 zero + bytes. + """ + code_size_hex = "11223344" + nonce_hex = "5566778899aabbcc" + balance_hex = "0123456789abcdef0123456789abcdef" + + value = encode_basic_data( + code_size=U32(int(code_size_hex, 16)), + nonce=U64(int(nonce_hex, 16)), + balance=U256(int(balance_hex, 16)), + ) + + assert len(value) == 32 + assert value[0] == 0 # version + assert value[1:4] == b"\x00" * 3 # reserved + assert value[4:8] == bytes.fromhex(code_size_hex) + assert value[8:16] == bytes.fromhex(nonce_hex) + assert value[16:32] == bytes.fromhex(balance_hex) + + # The all-zero leaf can't distinguish WHERE code_size sits (every + # offset reads zero either way), so it doesn't pin the + # offset-4/offset-5 EIP-7864 divergence noted on + # `encode_basic_data`; `test_encode_basic_data_maximum_fields` is + # what pins that. + all_zero = encode_basic_data( + code_size=U32(0), nonce=U64(0), balance=U256(0) + ) + assert all_zero == Bytes32(b"\x00" * 32) + + +def test_encode_basic_data_rejects_balance_past_sixteen_bytes() -> None: + """ + A balance that does not fit the sixteen-byte field raises + `BalanceOverflowError` -- an `InvalidBlock` -- rather than being + silently truncated by `to_bytes`. + """ + assert issubclass(BalanceOverflowError, InvalidBlock) + with pytest.raises(BalanceOverflowError): + encode_basic_data( + code_size=U32(0), + nonce=U64(0), + balance=U256(2) ** U256(128), + ) + + +OTHER_ADDRESS = Address32(b"\x00" * 12 + b"\xbb" * 20) + + +def test_remove_account_restores_prior_root() -> None: + """ + Removing a bare account deletes exactly its two header leaves, + restoring the commitment to what it was before the account was + embedded. + """ + trie = BinaryTrie() + embed_account( + trie, OTHER_ADDRESS, U64(1), U256(5), EMPTY_CODE_HASH, Bytes(b"") + ) + before = root(trie) + + embed_account(trie, ADDRESS, U64(2), U256(9), EMPTY_CODE_HASH, Bytes(b"")) + assert root(trie) != before + remove_account(trie, ADDRESS) + assert root(trie) == before + + +def test_remove_account_takes_header_and_storage_with_it() -> None: + """ + An account owns its header stem and its overflow storage + subtree, so removing it undoes its basic data, code hash, and + storage on both sides of the header boundary, without being told + which slots it held. Its code chunks are content-addressed, not + owned, and stay behind until `remove_code_chunks` drops them. + """ + code = Bytes(b"\x01" * 40) # two chunks, both in the code zone + code_hash = Bytes32(b"\x22" * 32) + + trie = BinaryTrie() + embed_account( + trie, OTHER_ADDRESS, U64(1), U256(5), EMPTY_CODE_HASH, Bytes(b"") + ) + before = root(trie) + + embed_account(trie, ADDRESS, U64(2), U256(9), code_hash, code) + for slot in (U256(0), U256(63), U256(64), U256(1000), U256(2**200)): + embed_storage_slot(trie, ADDRESS, slot, Bytes32(b"\x07" * 32)) + assert root(trie) != before + + remove_account(trie, ADDRESS) + assert root(trie) != before, "content-addressed chunks stay behind" + remove_code_chunks(trie, code_hash, code) + assert root(trie) == before + + +def test_remove_account_never_reaches_the_code_zone() -> None: + """ + The sweep covers the account and storage zones only, so + content-addressed chunks -- a neighbour's or the removed + account's own -- are untouched by a removal beside them. + """ + long_code = Bytes(b"\x01" * 4000) # 130 chunks, one code group + long_hash = Bytes32(b"\x33" * 32) + short_code = Bytes(b"\x02" * 40) # two chunks + short_hash = Bytes32(b"\x22" * 32) + + trie = BinaryTrie() + embed_account(trie, OTHER_ADDRESS, U64(1), U256(5), long_hash, long_code) + before = root(trie) + + embed_account(trie, ADDRESS, U64(2), U256(9), short_hash, short_code) + remove_account(trie, ADDRESS) + + for chunk_id in (Uint(0), Uint(129)): + assert get_tree_key_for_code_chunk(long_hash, chunk_id) in trie._data + for chunk_id in (Uint(0), Uint(1)): + assert get_tree_key_for_code_chunk(short_hash, chunk_id) in trie._data + + remove_code_chunks(trie, short_hash, short_code) + assert root(trie) == before + + +def test_remove_account_leaves_code_for_the_caller() -> None: + """ + Removing an account never takes content-addressed chunks with + it: whether they may go depends on the resulting state, which + the embedding cannot see. They are dropped separately, once the + caller has established nothing else runs the code. + """ + code = Bytes(b"\x01" * 4000) # 130 chunks + code_hash = Bytes32(b"\x22" * 32) + + trie = BinaryTrie() + empty = root(trie) + embed_account(trie, ADDRESS, U64(1), U256(5), code_hash, code) + + remove_account(trie, ADDRESS) + + residue = [ + get_tree_key_for_code_chunk(code_hash, Uint(chunk_id)) + for chunk_id in range(130) + ] + assert sorted(trie._data) == sorted(residue) + + remove_code_chunks(trie, code_hash, code) + assert root(trie) == empty + + +def test_remove_code_chunks_spares_the_header() -> None: + """ + Dropping a code's shared leaves removes exactly the code zone's + keys for it: a holder's basic data and code hash leaves are in + the account zone and survive untouched. + """ + code = Bytes(b"\x01" * 4000) # 130 chunks + code_hash = Bytes32(b"\x22" * 32) + + trie = BinaryTrie() + embed_account(trie, ADDRESS, U64(1), U256(5), code_hash, code) + + remove_code_chunks(trie, code_hash, code) + + assert all(key[0] != 1 for key in trie._data) + assert get_tree_key_for_basic_data(ADDRESS) in trie._data + assert trie._data[get_tree_key_for_code_hash(ADDRESS)] == code_hash + + +def test_all_zero_basic_data_is_absent_from_the_tree() -> None: + """ + Zero resolves to absence over the whole value space, basic data + included: an account with zero nonce, zero balance and no code + packs to 32 zero bytes, since the version and reserved bytes are + zero too. Its code hash leaf still distinguishes it from an + account that is not there at all. + """ + trie = BinaryTrie() + embed_account(trie, ADDRESS, U64(0), U256(0), EMPTY_CODE_HASH, Bytes(b"")) + + assert encode_basic_data( + code_size=U32(0), nonce=U64(0), balance=U256(0) + ) == Bytes32(b"\x00" * 32) + assert get_tree_key_for_basic_data(ADDRESS) not in trie._data + assert trie._data[get_tree_key_for_code_hash(ADDRESS)] == EMPTY_CODE_HASH + + +def test_emptying_basic_data_removes_its_leaf() -> None: + """ + The rule applies to updates as well as fresh writes: an account + drained to zero nonce and balance loses the leaf rather than + keeping a zero-valued one, landing on the commitment of a state + where it was always zero. + """ + fresh = BinaryTrie() + embed_account(fresh, ADDRESS, U64(0), U256(0), EMPTY_CODE_HASH, Bytes(b"")) + + drained = BinaryTrie() + embed_account( + drained, ADDRESS, U64(3), U256(99), EMPTY_CODE_HASH, Bytes(b"") + ) + assert root(drained) != root(fresh) + + embed_account( + drained, ADDRESS, U64(0), U256(0), EMPTY_CODE_HASH, Bytes(b"") + ) + assert root(drained) == root(fresh) + + +def test_zero_code_chunks_are_absent_from_the_tree() -> None: + """ + A chunk of 31 zero bytes encodes to 32 zero bytes and is left + absent like any other zero value, so chunk presence does not + delimit the code. + """ + code = Bytes(b"\x00" * 62) # two chunks, both entirely zero + code_hash = Bytes32(b"\x22" * 32) + + assert chunkify_code(code) == [Bytes32(b"\x00" * 32)] * 2 + + trie = BinaryTrie() + embed_account(trie, ADDRESS, U64(1), U256(5), code_hash, code) + + for chunk_id in (Uint(0), Uint(1)): + key = get_tree_key_for_code_chunk(code_hash, chunk_id) + assert key not in trie._data + # The account is still distinguished from one with no code. + assert trie._data[get_tree_key_for_code_hash(ADDRESS)] == code_hash + + +def test_remove_all_storage_keeps_the_account_and_its_code() -> None: + """ + A storage wipe straddles the header boundary: slots `0`-`63` + share the header stem with the basic data and code hash that + must survive, while the rest live in the overflow subtree. The + account's code chunks sit in the code zone, out of the sweep's + reach entirely. + """ + code = Bytes(b"\x01" * 40) + code_hash = Bytes32(b"\x22" * 32) + + trie = BinaryTrie() + embed_account(trie, ADDRESS, U64(1), U256(9), code_hash, code) + before = root(trie) + + for slot in (U256(0), U256(63), U256(64), U256(1000), U256(2**200)): + embed_storage_slot(trie, ADDRESS, slot, Bytes32(b"\x07" * 32)) + assert root(trie) != before + + remove_all_storage(trie, ADDRESS) + assert root(trie) == before + + +def test_embed_and_remove_storage_slot_roundtrip() -> None: + """ + A header slot and an overflow slot each embed as one leaf, and + removing them restores the prior commitment. + """ + trie = BinaryTrie() + embed_account(trie, ADDRESS, U64(1), U256(1), EMPTY_CODE_HASH, Bytes(b"")) + before = root(trie) + + embed_storage_slot(trie, ADDRESS, U256(1), Bytes32(b"\x07" * 32)) + embed_storage_slot(trie, ADDRESS, U256(1000), Bytes32(b"\x09" * 32)) + assert root(trie) != before + + remove_storage_slot(trie, ADDRESS, U256(1)) + remove_storage_slot(trie, ADDRESS, U256(1000)) + assert root(trie) == before + + +def test_encode_basic_data_maximum_fields() -> None: + """ + Every field at its type's maximum packs into the expected 32 + bytes. + + `code_size` fills all four of its bytes at offset 4 -- one byte + wider than EIP-7864's three-byte field at offset 5, as + `encode_basic_data`'s note records -- `nonce` fills eight bytes + and `balance` sixteen. + """ + value = encode_basic_data( + code_size=U32(2**32 - 1), + nonce=U64(2**64 - 1), + balance=U256(2**128 - 1), + ) + + expected = ( + bytes([0]) # version + + b"\x00" * 3 # reserved + + b"\xff" * 4 # code_size + + b"\xff" * 8 # nonce + + b"\xff" * 16 # balance + ) + assert len(expected) == 32 + assert value == Bytes32(expected) diff --git a/tests/binary_trie/test_state_pbt.py b/tests/binary_trie/test_state_pbt.py new file mode 100644 index 00000000000..f8b42064859 --- /dev/null +++ b/tests/binary_trie/test_state_pbt.py @@ -0,0 +1,2509 @@ +""" +Tests for `ethereum.state_pbt`. + +The first group covers `embed_flat_state`: each kind of state +(account fields, code chunks, storage slots) must land in the tree +as exactly the expected leaves, with expected keys and values built +by hand from the derivation functions. + +The second group covers the provider: `compute_state_root` applies a +block diff (deletions, zero-writes, freshly deployed code) and embeds +the result, checked either against a post-state built directly in the +MPT-backed container or against a known invariant (the empty root, a +zero-write matching a never-written slot, and so on). + +Later groups pin exact key sets, the BASIC_DATA leaf's byte layout, +and further provider semantics: `storage_clears` ordering, +account-delete/storage-orphan interactions, the asymmetry between +`set_account` and the diff path, pre-state immutability, sequential +diffs, and the storage sub-index boundaries. A final group follows +the content-addressed code lifecycle: chunk-leaf sharing, the +survives-check on deletion, removal across code groups, and the +delegation leaves that are deliberately outside all of it. Key sets +are rebuilt +from raw `blake3` and literal zone/sub-index bytes, never by calling +the derivation functions under test, so a wrong key that still +produces the right leaf count -- a swapped zone byte, an off-by-one +sub-index -- is still caught; a leaf count alone would miss it. + +EIP-8297's "Zero values and deletion" section now requires what this +module's `State` does: a write of 32 zero bytes resolves to a +deletion, and deleting an account removes its header leaves and its +storage leaves. The roots asserted below are conformance, not merely +pinned current behavior. + +Two things the EIP settles that are easy to misread as bugs. Zero +means absent over the whole value space, so a chunk of 31 zero bytes +and the basic data of an account with zero nonce, zero balance and +no code are all left out of the tree; presence of a leaf is not what +makes an account or a code chunk exist. +`test_trie.py::test_zero_value_is_not_absence` still holds and is +not in tension with that: the raw `BinaryTrie` does keep a +zero-valued leaf, and collapsing zero onto absence is the state +model's job, done in `embedding.state_write`. + +Where this provider parts company with `state_mpt` is storage owned +by no account. The EIP fixes `account_has_storage` to whether a slot +leaf of the address exists, so a write to an address the same diff +deletes is dropped here; `state_mpt` keeps it. See +`test_differential_mpt.py::test_account_delete_diverges_on_account_has_storage` +for the EIP-7610-visible consequence, which the EIP resolves for the +tree and leaves open for the Merkle Patricia Trie. +""" # noqa: E501 + +import random +from typing import Dict, Optional, Tuple + +import pytest +from blake3 import blake3 +from ethereum_types.bytes import Bytes, Bytes20, Bytes32 +from ethereum_types.numeric import U32, U64, U256, Uint + +from ethereum.crypto.hash import keccak256 +from ethereum.exceptions import ( + BalanceOverflowError, + InvalidBlock, + UnknownCodeHashError, +) +from ethereum.partitioned_binary_tree import ( + EMPTY_TRIE_ROOT, + HEADER_STORAGE_OFFSET, + HEADER_STORAGE_SLOTS, + BinaryTrie, + address20_to_address32, + chunkify_code, + encode_basic_data, + get_tree_key_for_basic_data, + get_tree_key_for_code_chunk, + get_tree_key_for_code_hash, + get_tree_key_for_storage_slot, + root, + trie_set, +) +from ethereum.state import EMPTY_CODE_HASH, Account, BlockDiff +from ethereum.state_mpt import State as MptState +from ethereum.state_mpt import set_account as mpt_set_account +from ethereum.state_mpt import set_storage as mpt_set_storage +from ethereum.state_mpt import store_code as mpt_store_code +from ethereum.state_pbt import ( + State, + _apply_diff_to_trie, + apply_changes_to_state, + embed_flat_state, + set_account, + set_storage, + state_root, + store_code, +) + +ADDRESS_A = Bytes20(b"\xaa" * 20) +ADDRESS_B = Bytes20(b"\xbb" * 20) +ADDRESS_C = Bytes20(b"\xcc" * 20) + +# EIP-7702 delegation designators: the one code a live account can +# replace, and never code in the tree -- each is a single leaf in its +# own account's header stem, private to that account. +DELEGATION_A = Bytes(b"\xef\x01\x00" + b"\x11" * 20) +DELEGATION_B = Bytes(b"\xef\x01\x00" + b"\x22" * 20) + + +def embed_state(state: MptState) -> BinaryTrie: + """ + Embed every account, code chunk, and storage slot of the + MPT-backed `state` into a fresh `BinaryTrie`. + """ + accounts = { + address: account + for address, account in state._main_trie._data.items() + if account is not None + } + storages = { + address: dict(trie._data) + for address, trie in state._storage_tries.items() + } + return embed_flat_state(accounts, storages, state.get_code) + + +def test_empty_state_embeds_to_empty_root() -> None: + """ + A state with no accounts embeds to the empty tree. + """ + assert root(embed_state(MptState())) == EMPTY_TRIE_ROOT + + +def test_eoa_embeds_basic_data_and_code_hash_leaves() -> None: + """ + An EOA produces exactly its two header leaves: packed basic data + and the empty-code hash. No chunk or storage leaves appear. + """ + state = MptState() + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(5), balance=U256(1000), code_hash=EMPTY_CODE_HASH), + ) + + embedded = embed_state(state) + + address32 = address20_to_address32(ADDRESS_A) + expected = BinaryTrie() + trie_set( + expected, + get_tree_key_for_basic_data(address32), + encode_basic_data(code_size=U32(0), nonce=U64(5), balance=U256(1000)), + ) + trie_set( + expected, + get_tree_key_for_code_hash(address32), + Bytes32(EMPTY_CODE_HASH), + ) + assert len(embedded._data) == 2 + assert root(embedded) == root(expected) + + +def test_contract_embeds_chunks_and_storage_slots() -> None: + """ + A contract account produces its header leaves, one leaf per code + chunk, and one leaf per non-zero storage slot, header slot and + overflow slot alike. + """ + code = Bytes(b"\x01" * 40) # two chunks, in the code zone + state = MptState() + code_hash = mpt_store_code(state, code) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + mpt_set_storage( + state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7) + ) + mpt_set_storage( + state, ADDRESS_A, Bytes32(U256(100).to_be_bytes32()), U256(9) + ) + + embedded = embed_state(state) + + address32 = address20_to_address32(ADDRESS_A) + expected = BinaryTrie() + trie_set( + expected, + get_tree_key_for_basic_data(address32), + encode_basic_data(code_size=U32(40), nonce=U64(1), balance=U256(0)), + ) + trie_set( + expected, + get_tree_key_for_code_hash(address32), + Bytes32(code_hash), + ) + for chunk_id, chunk in enumerate(chunkify_code(code)): + trie_set( + expected, + get_tree_key_for_code_chunk(code_hash, Uint(chunk_id)), + chunk, + ) + trie_set( + expected, + get_tree_key_for_storage_slot(address32, U256(1)), + U256(7).to_be_bytes32(), + ) + trie_set( + expected, + get_tree_key_for_storage_slot(address32, U256(100)), + U256(9).to_be_bytes32(), + ) + assert len(embedded._data) == 6 + assert root(embedded) == root(expected) + + +def test_identical_bytecode_shares_chunk_leaves() -> None: + """ + Two contracts with the same bytecode share every chunk leaf: the + embedded tree holds one content-addressed copy of each chunk, + plus per-account header leaves. + """ + code = Bytes(b"\x01" * 4000) # 130 chunks + state = MptState() + code_hash = mpt_store_code(state, code) + for address in (ADDRESS_A, ADDRESS_B): + mpt_set_account( + state, + address, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + + embedded = embed_state(state) + + assert len(chunkify_code(code)) == 130 + # Per account: basic data and code hash. The 130 chunks are + # content-addressed and stored once, shared by both accounts. + assert len(embedded._data) == 2 * 2 + 130 + + +def test_empty_provider_commits_to_empty_root() -> None: + """ + A provider with no accounts commits to the empty tree. + """ + assert state_root(State()) == EMPTY_TRIE_ROOT + + +def test_empty_diff_root_matches_direct_embedding() -> None: + """ + With no changes, the root is the embedding of the pre-state. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(100), code_hash=EMPTY_CODE_HASH), + ) + set_storage(state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7)) + + mpt_state = MptState() + mpt_set_account( + mpt_state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(100), code_hash=EMPTY_CODE_HASH), + ) + mpt_set_storage( + mpt_state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7) + ) + + assert state_root(state) == root(embed_state(mpt_state)) + + +def test_diff_root_matches_directly_built_post_state() -> None: + """ + A diff deploying code, touching storage, zeroing a slot, and + deleting an account produces the same root as building the + post-state directly in the MPT container and embedding it. + + The zeroed slot is deleted and the deleted account's storage + goes with it, both as EIP-8297 requires, so the two ways of + reaching the post state agree. + """ + code = Bytes(b"\x01" * 40) + code_hash = keccak256(code) + + pre_state = State() + set_account( + pre_state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(100), code_hash=EMPTY_CODE_HASH), + ) + set_storage( + pre_state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7) + ) + set_account( + pre_state, + ADDRESS_B, + Account(nonce=Uint(9), balance=U256(5), code_hash=EMPTY_CODE_HASH), + ) + + # The block: account A becomes a contract (deployed code, new + # storage written, old slot zeroed) and account B is deleted. + post_account_a = Account( + nonce=Uint(2), balance=U256(50), code_hash=code_hash + ) + computed = pre_state.compute_state_root( + BlockDiff( + account_changes={ADDRESS_A: post_account_a, ADDRESS_B: None}, + storage_changes={ + ADDRESS_A: { + Bytes32(U256(1).to_be_bytes32()): U256(0), + Bytes32(U256(2).to_be_bytes32()): U256(11), + } + }, + code_changes={code_hash: code}, + ) + ) + + post_state = MptState() + assert mpt_store_code(post_state, code) == code_hash + mpt_set_account(post_state, ADDRESS_A, post_account_a) + mpt_set_storage( + post_state, ADDRESS_A, Bytes32(U256(2).to_be_bytes32()), U256(11) + ) + + assert computed == root(embed_state(post_state)) + + +def test_deleting_the_only_account_empties_the_tree() -> None: + """ + Deleting the last account, bare as deletable accounts must be, + leaves the empty tree commitment. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + + computed = state.compute_state_root( + BlockDiff(account_changes={ADDRESS_A: None}) + ) + + assert computed == EMPTY_TRIE_ROOT + + +def test_zero_write_matches_never_written() -> None: + """ + Writing a slot to zero commits identically to never having + written it: zero resolves to a deletion, as EIP-8297 requires + and as the MPT state semantics already had it. + """ + + def fresh() -> State: + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + return state + + written_then_zeroed = fresh() + set_storage(written_then_zeroed, ADDRESS_A, Bytes32(b"\x05" * 32), U256(9)) + set_storage(written_then_zeroed, ADDRESS_A, Bytes32(b"\x05" * 32), U256(0)) + + assert state_root(written_then_zeroed) == state_root(fresh()) + + # And via a block diff rather than direct writes. + with_slot = fresh() + set_storage(with_slot, ADDRESS_A, Bytes32(b"\x05" * 32), U256(9)) + zeroed_by_diff = with_slot.compute_state_root( + BlockDiff( + storage_changes={ADDRESS_A: {Bytes32(b"\x05" * 32): U256(0)}} + ) + ) + assert zeroed_by_diff == state_root(fresh()) + + +def _clone(state: State) -> State: + """Deep-copy a provider state's three mappings.""" + return State( + _accounts=dict(state._accounts), + _storage={ + address: dict(slots) for address, slots in state._storage.items() + }, + _code_store=dict(state._code_store), + ) + + +def _flat_oracle_root(pre: State, diff: BlockDiff) -> bytes: + """ + Compute the post-root the pre-incremental way: apply the diff to + a copy of the flat state and re-embed everything from scratch. + """ + post = _clone(pre) + apply_changes_to_state(post, diff) + return bytes(state_root(post)) + + +def test_storage_clear_deletes_every_slot_leaf() -> None: + """ + Clearing an account's storage removes its header-stem slot + leaves and its overflow-subtree slot leaves, leaving the root of + a state where the storage was never written. + """ + + def account_only() -> State: + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + return state + + pre = account_only() + set_storage(pre, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7)) + set_storage(pre, ADDRESS_A, Bytes32(U256(100).to_be_bytes32()), U256(9)) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, BlockDiff(storage_clears={ADDRESS_A})) + + assert root(trie) == state_root(account_only()) + + +def test_delegation_change_replaces_the_header_leaf() -> None: + """ + Re-delegating overwrites the account's delegation leaf in place; + un-delegating removes it and restores the code hash leaf. Both + leave the root of a state that only ever held the final code, so + a stale leaf of either kind -- the delegation an un-delegating + account has left behind, or the code hash a delegating account + has replaced -- moves the root and fails here. + """ + for new_code in (DELEGATION_B, Bytes(b"")): + pre = State() + old_hash = store_code(pre, DELEGATION_A) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=old_hash), + ) + + new_hash = keccak256(new_code) + post_account = Account( + nonce=Uint(2), balance=U256(1), code_hash=new_hash + ) + diff = BlockDiff( + account_changes={ADDRESS_A: post_account}, + code_changes={new_hash: new_code} if new_code else {}, + ) + + fresh = State() + assert store_code(fresh, new_code) == new_hash + set_account(fresh, ADDRESS_A, post_account) + + assert pre.compute_state_root(diff) == state_root(fresh), ( + f"new code {new_code.hex()}" + ) + + +def test_deleting_a_sole_holder_removes_its_short_code() -> None: + """ + Short code is content-addressed like any other: deleting the only + holder drops its chunk leaves, resolved from the bytecode rather + than from any per-account key range, and the state commits to the + empty root. + """ + pre = State() + code_hash = store_code(pre, Bytes(b"\x01" * 40)) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + assert bytes(pre.compute_state_root(diff)) == _flat_oracle_root(pre, diff) + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +def test_deleting_the_last_holder_removes_its_code() -> None: + """ + Content-addressed chunks may go once no account in the resulting + state has their code hash. With the only holder deleted, nothing + references the code and its shared leaves go with it. + """ + pre = State() + code_hash = store_code(pre, Bytes(b"\x01" * 4000)) # 130 chunks + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + assert bytes(pre.compute_state_root(diff)) == _flat_oracle_root(pre, diff) + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +def test_deleting_one_holder_keeps_shared_code() -> None: + """ + A second account still running the bytecode keeps its chunks + alive: they are removed only if no account in the resulting state + has the code hash. + """ + code = Bytes(b"\x01" * 4000) # 130 chunks + pre = State() + code_hash = store_code(pre, code) + for address in (ADDRESS_A, ADDRESS_B): + set_account( + pre, + address, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + post = pre.compute_state_root(diff) + + assert bytes(post) == _flat_oracle_root(pre, diff) + + survivor = State() + assert store_code(survivor, code) == code_hash + set_account( + survivor, + ADDRESS_B, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + assert post == state_root(survivor) + + +def test_deleting_an_account_removes_its_storage_leaves() -> None: + """ + An account can hold storage without holding code: genesis + allocates such accounts directly, and no bytecode is needed to + keep slots that were allocated rather than written. Touching one + empties it under EIP-161, and the deletion emits no storage diff + of its own, so the deletion must drop the slot leaves the account + still owns in the pre-state tree. + """ + pre = State() + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(0), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + # A header-stem slot and an overflow-subtree slot. + set_storage(pre, ADDRESS_A, Bytes32(U256(3).to_be_bytes32()), U256(4)) + set_storage(pre, ADDRESS_A, Bytes32(U256(300).to_be_bytes32()), U256(5)) + set_account( + pre, + ADDRESS_B, + Account(nonce=Uint(1), balance=U256(9), code_hash=EMPTY_CODE_HASH), + ) + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + assert bytes(pre.compute_state_root(diff)) == _flat_oracle_root(pre, diff) + + +def test_deleting_a_cleared_account_removes_its_storage_leaves() -> None: + """ + A pre-EIP-6780 `SELFDESTRUCT` wipes storage and removes the + account in one diff. The wipe is recorded against the tree, not + against the pre-state, so the deletion cannot tell that the + storage is already gone; removing the leaves twice must still + land on the post-state root. + """ + pre = State() + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + set_storage(pre, ADDRESS_A, Bytes32(U256(3).to_be_bytes32()), U256(4)) + + diff = BlockDiff( + storage_clears={ADDRESS_A}, account_changes={ADDRESS_A: None} + ) + + assert bytes(pre.compute_state_root(diff)) == _flat_oracle_root(pre, diff) + + +def test_storage_written_to_a_deleted_account_is_not_embedded() -> None: + """ + Storage belongs to an account: [`embed_flat_state`] ignores slots + whose address has no account, so a write landing on an address + the same block deletes must not reach the tree either. + + [`embed_flat_state`]: ref:ethereum.state_pbt.embed_flat_state + """ + pre = State() + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + + diff = BlockDiff( + account_changes={ADDRESS_A: None}, + storage_changes={ + ADDRESS_A: {Bytes32(U256(3).to_be_bytes32()): U256(7)} + }, + ) + + assert bytes(pre.compute_state_root(diff)) == _flat_oracle_root(pre, diff) + + +def test_delegating_an_account_reclaims_nothing() -> None: + """ + A live account replacing its code reclaims no chunk leaf: only + deleting an account can, so the change is a re-embedding and + nothing else. + + Delegation is the one such replacement a live account can make. + An indicator is not code, so nothing referenced a chunk in the + first place: the account's whole code footprint is the header + leaf that replaces its code hash, and the code zone stays empty + throughout. + """ + pre = State() + delegation_hash = store_code(pre, DELEGATION_A) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + + diff = BlockDiff( + account_changes={ + ADDRESS_A: Account( + nonce=Uint(2), balance=U256(1), code_hash=delegation_hash + ) + }, + code_changes={delegation_hash: DELEGATION_A}, + ) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + assert not any(key[0] == 1 for key in trie._data) + + fresh = State() + assert store_code(fresh, DELEGATION_A) == delegation_hash + set_account( + fresh, + ADDRESS_A, + Account(nonce=Uint(2), balance=U256(1), code_hash=delegation_hash), + ) + assert pre.compute_state_root(diff) == state_root(fresh) + + +def test_random_diffs_match_flat_application_and_rebuild() -> None: + """ + Randomized protocol-shaped diffs, covering deletions of accounts + with and without allocated storage, delegation churn, contract + storage writes and zeroes, and fresh deployments, produce the + same root through incremental trie application as through + applying the diff to the flat state and re-embedding everything. + """ + rng = random.Random(8297) + long_code = Bytes(bytes(range(256)) * 16) # 133 chunks, varied bytes + + for trial in range(10): + pre = State() + long_hash = store_code(pre, long_code) + delegation_hashes = [ + store_code(pre, code) for code in (DELEGATION_A, DELEGATION_B) + ] + + # EOAs: empty code or a pre-existing delegation. Some hold + # allocated storage, which no bytecode is needed to keep and + # which a deletion must take with it. + eoas = [Bytes20(rng.randbytes(20)) for _ in range(6)] + for address in eoas: + set_account( + pre, + address, + Account( + nonce=Uint(rng.randrange(1, 5)), + balance=U256(rng.randrange(1, 10**9)), + code_hash=rng.choice( + [EMPTY_CODE_HASH, EMPTY_CODE_HASH] + delegation_hashes + ), + ), + ) + if rng.random() < 0.4: + for _ in range(rng.randrange(1, 3)): + slot = rng.choice( + [rng.randrange(0, 64), rng.randrange(64, 10**9)] + ) + set_storage( + pre, + address, + Bytes32(U256(slot).to_be_bytes32()), + U256(rng.randrange(1, 100)), + ) + + # Contracts: immutable code, mutable storage. + contracts = [Bytes20(rng.randbytes(20)) for _ in range(3)] + for address in contracts: + set_account( + pre, + address, + Account( + nonce=Uint(1), + balance=U256(rng.randrange(1, 10**9)), + code_hash=long_hash, + ), + ) + for _ in range(rng.randrange(1, 4)): + # Header slots (0-63) and overflow slots alike. + slot = rng.choice( + [rng.randrange(0, 64), rng.randrange(64, 10**9)] + ) + set_storage( + pre, + address, + Bytes32(U256(slot).to_be_bytes32()), + U256(rng.randrange(1, 100)), + ) + + account_changes: Dict[Bytes20, Optional[Account]] = {} + storage_changes: Dict[Bytes20, Dict[Bytes32, U256]] = {} + for address in eoas: + roll = rng.random() + bare = pre._accounts[address].code_hash == EMPTY_CODE_HASH + if roll < 0.25 and bare: + # EIP-6780-style same-transaction deletion. + account_changes[address] = None + elif roll < 0.6: + # Delegate, re-delegate, or un-delegate. + account_changes[address] = Account( + nonce=Uint(rng.randrange(1, 5)), + balance=U256(rng.randrange(1, 10**9)), + code_hash=rng.choice( + [EMPTY_CODE_HASH] + delegation_hashes + ), + ) + for address in contracts: + if rng.random() < 0.5: + account_changes[address] = Account( + nonce=Uint(1), + balance=U256(rng.randrange(1, 10**9)), + code_hash=long_hash, + ) + if rng.random() < 0.7: + storage_changes[address] = { + Bytes32(U256(1).to_be_bytes32()): U256(rng.choice([0, 7])), + Bytes32(U256(100).to_be_bytes32()): U256( + rng.choice([0, 9]) + ), + } + + fresh_code = Bytes(rng.randbytes(200)) + fresh_hash = keccak256(fresh_code) + created = Bytes20(rng.randbytes(20)) + account_changes[created] = Account( + nonce=Uint(1), balance=U256(5), code_hash=fresh_hash + ) + storage_changes[created] = {Bytes32(U256(2).to_be_bytes32()): U256(9)} + + diff = BlockDiff( + account_changes=account_changes, + storage_changes=storage_changes, + code_changes={fresh_hash: fresh_code}, + ) + + assert bytes(pre.compute_state_root(diff)) == _flat_oracle_root( + pre, diff + ), f"trial {trial}" + + +def test_header_root_matches_the_advanced_chain_state() -> None: + """ + The fork commits a block's header root with + [`compute_state_root`] but advances the chain with + [`apply_changes_to_state`]. The two are separate + implementations, one walking the tree and the other the flat + mappings, so every block's header root must equal the root of + the state the chain + carries into the next block, or the second block of a chain is + built on a state the first block never committed to. + + [`compute_state_root`]: ref:ethereum.state_pbt.State.compute_state_root + [`apply_changes_to_state`]: ref:ethereum.state_pbt.apply_changes_to_state + """ # noqa: E501 + chain = State() + delegation_hash = store_code(chain, DELEGATION_A) + set_account( + chain, + ADDRESS_A, + Account(nonce=Uint(0), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + set_storage(chain, ADDRESS_A, Bytes32(U256(3).to_be_bytes32()), U256(4)) + set_storage(chain, ADDRESS_A, Bytes32(U256(300).to_be_bytes32()), U256(5)) + set_account( + chain, + ADDRESS_B, + Account(nonce=Uint(1), balance=U256(9), code_hash=EMPTY_CODE_HASH), + ) + + deployed = Bytes(b"\x60\x01" * 100) + deployed_hash = keccak256(deployed) + + blocks = [ + # Touching an empty account that still holds storage deletes + # it under EIP-161, storage leaves and all. + BlockDiff(account_changes={ADDRESS_A: None}), + # Delegating an existing EOA swaps its code hash leaf for a + # delegation leaf. + BlockDiff( + account_changes={ + ADDRESS_B: Account( + nonce=Uint(2), balance=U256(9), code_hash=delegation_hash + ) + } + ), + # A fresh deployment, with storage, on the delegated account's + # neighbour, plus the delegation being revoked. + BlockDiff( + account_changes={ + ADDRESS_B: Account( + nonce=Uint(3), balance=U256(9), code_hash=EMPTY_CODE_HASH + ), + ADDRESS_A: Account( + nonce=Uint(1), balance=U256(2), code_hash=deployed_hash + ), + }, + storage_changes={ + ADDRESS_A: {Bytes32(U256(7).to_be_bytes32()): U256(8)} + }, + code_changes={deployed_hash: deployed}, + ), + ] + + for number, diff in enumerate(blocks): + header_root = chain.compute_state_root(diff) + apply_changes_to_state(chain, diff) + assert header_root == state_root(chain), f"block {number}" + + +def test_store_code_round_trips() -> None: + """ + Stored bytecode is retrievable by its hash, and the empty code + hash resolves to empty bytes without storage. + """ + state = State() + code = Bytes(b"\x60\x00") + code_hash = store_code(state, code) + + assert state.get_code(code_hash) == code + assert state.get_code(EMPTY_CODE_HASH) == b"" + + +def _account_header_stem(address32: bytes) -> bytes: + """ + Build an account's 33-byte header stem from scratch. + + The stem is `0x00 || blake3(address32)`: the account zone byte + followed by the address digest, computed here independently of + `get_tree_key_for_header`. + """ + return bytes([0]) + blake3(address32).digest() + + +def _storage_overflow_stem(address32: bytes, tree_index: int) -> bytes: + """ + Build a 65-byte overflow storage stem from scratch. + + The stem is `0xff || blake3(address32) || + blake3(address32 || tree_index)`, computed here independently of + `get_tree_key_for_storage_slot`. + """ + prefix = blake3(address32).digest() + suffix = blake3(address32 + tree_index.to_bytes(32, "big")).digest() + return bytes([255]) + prefix + suffix + + +def _code_zone_stem(code_hash: bytes, tree_index: int) -> bytes: + """ + Build a 33-byte code zone stem from scratch. + + The stem is `0x01 || blake3(code_hash || tree_index)`, computed + here independently of `get_tree_key_for_code_chunk`. + """ + digest = blake3(code_hash + tree_index.to_bytes(32, "big")).digest() + return bytes([1]) + digest + + +def test_embedded_key_set_for_a_crafted_contract() -> None: + """ + One contract, crafted so its code and storage exercise every + sub-index boundary the embedding defines, embeds to an exact, + independently rebuilt key set. + + Code spanning 129 chunks (`31 * 129 = 3999` bytes) fills + code-zone sub-indices 0-128 of group 0; storage at slots 63, 64, + and 256 puts one slot in the header and two in the storage zone, + each its own overflow group. + """ + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + code = Bytes(b"\x01" * (31 * 129)) + state = MptState() + code_hash = mpt_store_code(state, code) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + for slot, value in ((63, 1), (64, 2), (256, 3)): + mpt_set_storage( + state, + ADDRESS_A, + Bytes32(U256(slot).to_be_bytes32()), + U256(value), + ) + + embedded = embed_state(state) + + header_stem = _account_header_stem(address32) + expected_keys = { + header_stem + bytes([0]), # basic data + header_stem + bytes([1]), # code hash + header_stem + bytes([127]), # storage slot 63 + } + expected_keys |= { + _code_zone_stem(code_hash, 0) + bytes([chunk_id]) + for chunk_id in range(129) + } + expected_keys.add(_storage_overflow_stem(address32, 0) + bytes([64])) + expected_keys.add(_storage_overflow_stem(address32, 1) + bytes([0])) + + assert set(embedded._data.keys()) == expected_keys + assert all(len(key) in (34, 66) for key in expected_keys) + + +def test_embedded_keys_never_use_a_reserved_zone_byte() -> None: + """ + Every key's first byte is one of the three zones this embedding + defines -- `ACCOUNT_ZONE` (0x00), `CODE_ZONE` (0x01), or + `STORAGE_ZONE` (0xFF) -- never one of the `0x02`-`0xFE` zone bytes + EIP-8297 reserves for future state categories ("New categories + MUST be allocated from `0x02`-`0xFE` and MUST keep their keys + mutually prefix-free"). + + Reuses `test_embedded_key_set_for_a_crafted_contract`'s crafted + contract, so the embedded state populates every leaf category: + header leaves, header and overflow storage, and content-addressed + code. + """ + code = Bytes(b"\x01" * (31 * 129)) + state = MptState() + code_hash = mpt_store_code(state, code) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + for slot, value in ((63, 1), (64, 2), (256, 3)): + mpt_set_storage( + state, + ADDRESS_A, + Bytes32(U256(slot).to_be_bytes32()), + U256(value), + ) + + embedded = embed_state(state) + + allowed_zone_bytes = {0x00, 0x01, 0xFF} + for key in embedded._data: + assert key[0] in allowed_zone_bytes, ( + f"key {key.hex()} uses zone byte {key[0]:#04x}, outside " + "the three zones this embedding defines" + ) + + +def test_embedded_state_root_is_pinned() -> None: + """ + The same crafted state as `test_embedded_key_set_for_a_crafted_contract` + commits to a hardcoded root hash. + + A deliberate change-detector for the hash function, node tags, + prefix encoding, and the embedding built on top of them -- same + spirit as `test_trie.py::test_fixed_trie_root_is_pinned`. Every + other root assertion in this module compares two roots the code + itself computed, so a systematic but deterministic bug in the hash + function or merkleization would move both sides identically and + pass unnoticed there; only a value hardcoded from a known-good run + catches that. To regenerate after a deliberate, reviewed change: + print `root(embedded).hex()` for this same state and paste the new + value below. + """ + code = Bytes(b"\x01" * (31 * 129)) + state = MptState() + code_hash = mpt_store_code(state, code) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + for slot, value in ((63, 1), (64, 2), (256, 3)): + mpt_set_storage( + state, + ADDRESS_A, + Bytes32(U256(slot).to_be_bytes32()), + U256(value), + ) + + embedded = embed_state(state) + + assert root(embedded) == bytes.fromhex( + "84d204064e6f2d3f8862bf399d9c1d7eb46a47d041930beec3c1d1dd124e6bc8" + ) + + +def test_embedded_key_set_for_maximum_header_occupancy() -> None: + """ + An account with 64 header storage slots fills every header + sub-index this embedding can ever populate, and embeds to exactly + that key set plus its code's content-addressed leaves. + + Storage slots 0-63 fill header sub-indices 64-127; together with + basic data (0) and the code hash (1) that is the whole allocated + header range, the maximum-occupancy case the EIP's + `HEADER_STORAGE_OFFSET + HEADER_STORAGE_SLOTS <= + STEM_SUBTREE_WIDTH` invariant exists to protect. + + Most of the 256 sub-indices are unreachable by any account: + 3-63 sit unassigned between the delegation leaf (2) and the + first header storage slot (64), reserved for future header + fields, and 128-255 -- the old code range -- are unallocated + since code moved wholly into the code zone. This account holds + contract code, so its expected header set is + `{0, 1} | set(range(64, 128))`; a delegated account would hold + 2 in place of 1, the two being exclusive. + """ + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + code = Bytes(b"\x01" * (31 * 128)) # 128 chunks + state = MptState() + code_hash = mpt_store_code(state, code) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + for slot in range(64): + mpt_set_storage( + state, + ADDRESS_A, + Bytes32(U256(slot).to_be_bytes32()), + U256(slot + 1), + ) + + embedded = embed_state(state) + + header_stem = _account_header_stem(address32) + header_keys = { + key for key in embedded._data if key.startswith(header_stem) + } + header_sub_indices = {key[-1] for key in header_keys} + + assert header_sub_indices == {0, 1} | set(range(64, 128)) + code_zone_keys = {key for key in embedded._data if key[0] == 1} + assert code_zone_keys == { + _code_zone_stem(code_hash, 0) + bytes([chunk_id]) + for chunk_id in range(128) + } + assert embedded._data.keys() == header_keys | code_zone_keys, ( + "maximum header occupancy plus its code must produce no other key" + ) + + +def test_identical_code_shares_every_chunk_key() -> None: + """ + Two accounts with identical 129-chunk code produce one shared set + of content-addressed chunk keys, and no chunk key anywhere else. + + Pins this by KEY, not leaf count: the code zone must hold exactly + the 129 keys of the shared bytecode's group 0, and each account's + header stem must carry nothing beyond its basic data and code + hash. + """ + code = Bytes(b"\x01" * (31 * 129)) + state = MptState() + code_hash = mpt_store_code(state, code) + for address in (ADDRESS_A, ADDRESS_B): + mpt_set_account( + state, + address, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + + embedded = embed_state(state) + + stem_a = _account_header_stem(b"\x00" * 12 + bytes(ADDRESS_A)) + stem_b = _account_header_stem(b"\x00" * 12 + bytes(ADDRESS_B)) + for stem in (stem_a, stem_b): + header_sub_indices = { + key[-1] for key in embedded._data if key.startswith(stem) + } + assert header_sub_indices == {0, 1} + + code_zone_keys = {key for key in embedded._data if key[0] == 1} + assert code_zone_keys == { + _code_zone_stem(code_hash, 0) + bytes([chunk_id]) + for chunk_id in range(129) + } + + +_NON_PUSH_BYTES = [b for b in range(256) if not (0x60 <= b <= 0x7F)] +""" +The 224 byte values outside the `PUSH1`..`PUSH32` opcode range. +""" + + +def _code_chunk_filler_byte(chunk_id: int) -> int: + """ + Map a chunk index to a filler byte never in the `PUSH1`..`PUSH32` + range (0x60-0x7F), so a chunk of repeats carries no push data. + + Cycles through the non-push byte values, so any two chunks fewer + than `len(_NON_PUSH_BYTES)` apart -- in particular any + neighbours -- get distinct filler bytes. + """ + return _NON_PUSH_BYTES[chunk_id % len(_NON_PUSH_BYTES)] + + +def _distinct_chunk_code(chunk_count: int, *, salt: int = 1) -> Bytes: + """ + Build code of `chunk_count` chunks, chunk `i` filled with 31 + repeats of `_code_chunk_filler_byte(salt + i)`. + + The default salt starts past filler byte zero, so every chunk of + a code shorter than `len(_NON_PUSH_BYTES) - salt` chunks is + nonzero and present in the tree; a longer code wraps back through + zero and gains one absent chunk per cycle, a hole its callers + rely on removal skipping. Two codes built with salts fewer than + `len(_NON_PUSH_BYTES)` apart share no chunk value across their + overlapping indices. + """ + return Bytes( + b"".join( + bytes([_code_chunk_filler_byte(salt + i)]) * 31 + for i in range(chunk_count) + ) + ) + + +def test_chunk_values_are_distinct_across_the_code_group_boundary() -> None: + r""" + Chunks 254-257, each filled with its own distinct byte, get their + exact 32-byte values pinned by rebuilt key: group 0's last two + chunks (254, 255) and group 1's first two (256, 257) -- covering + the one boundary in EIP-8297 where a code key's stem changes, the + `tree_index` advancing while the sub-index wraps to zero. + + Every other coverage of this boundary in this suite builds code as + `b"\\x01" * N`, making neighbouring chunks identical and + interchangeable: swapping two chunks' stored values inside + `embed_flat_state` would pass every one of those tests, key-set + assertions included. Filling chunk `i` with + `_code_chunk_filler_byte(i)` keeps every filler byte outside the + `PUSH1`..`PUSH32` range, so each chunk's expected value is + trivially `0x00` followed by 31 repeats of its own filler byte. + """ + chunk_count = 258 # chunks 0..257: 31 * 258 = 7998, well under + # MAX_CODE_SIZE, and comfortably covers chunks 254-257. + code = Bytes( + b"".join( + bytes([_code_chunk_filler_byte(i)]) * 31 + for i in range(chunk_count) + ) + ) + assert len(code) == 31 * chunk_count == 7998 + + state = State() + code_hash = store_code(state, code) + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + + trie = embed_flat_state(state._accounts, state._storage, state.get_code) + + group_0_stem = _code_zone_stem(code_hash, 0) + group_1_stem = _code_zone_stem(code_hash, 1) + + def expected_chunk(chunk_id: int) -> Bytes32: + filler = bytes([_code_chunk_filler_byte(chunk_id)]) + return Bytes32(bytes([0]) + filler * 31) + + # Chunks 0-255 share group 0's stem at sub-index chunk_id; chunk + # 256 is the first of group 1, at sub-index 0; 257 is the next. + assert trie._data[group_0_stem + bytes([254])] == expected_chunk(254) + assert trie._data[group_0_stem + bytes([255])] == expected_chunk(255) + assert trie._data[group_1_stem + bytes([0])] == expected_chunk(256) + assert trie._data[group_1_stem + bytes([1])] == expected_chunk(257) + + +def test_short_identical_code_shares_both_chunk_leaves() -> None: + """ + Two accounts with the same 2-chunk code -- the common case under + content addressing -- embed to exactly six leaves: two header + pairs and one shared copy of each chunk, with the chunk values + pinned per key. + + The two chunks carry distinct bytes, so a wrong chunk landing in + the right key set -- swapped values, a copy under a wrong stem -- + cannot cancel out. + """ + code = _distinct_chunk_code(2) + state = MptState() + code_hash = mpt_store_code(state, code) + for address in (ADDRESS_A, ADDRESS_B): + mpt_set_account( + state, + address, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + + embedded = embed_state(state) + + stem_a = _account_header_stem(b"\x00" * 12 + bytes(ADDRESS_A)) + stem_b = _account_header_stem(b"\x00" * 12 + bytes(ADDRESS_B)) + code_stem = _code_zone_stem(code_hash, 0) + assert set(embedded._data.keys()) == { + stem_a + bytes([0]), + stem_a + bytes([1]), + stem_b + bytes([0]), + stem_b + bytes([1]), + code_stem + bytes([0]), + code_stem + bytes([1]), + } + for chunk_id, chunk in enumerate(chunkify_code(code)): + assert embedded._data[code_stem + bytes([chunk_id])] == chunk + + +@pytest.mark.parametrize( + "survivor_placement", + [ + pytest.param("pre_state", id="survivor_untouched_in_pre_state"), + pytest.param("diff_first", id="survivor_listed_before_the_loser"), + pytest.param("diff_last", id="survivor_listed_after_the_loser"), + ], +) +def test_deleting_a_holder_keeps_chunks_a_survivor_still_holds( + survivor_placement: str, +) -> None: + """ + Account A, holding code H, is deleted while account B still + holds H: H's chunks must survive, and the root must equal a fresh + embed of the post state. + + `code_hash_survives` has two arms -- the diff's own values and + the untouched pre-state -- and a survivor can satisfy either, so + the diff arm is exercised beside the pre-state one. Within the + diff, the survivor's position matters more than it looks: listed + after the loser, `embed_account` re-writes whatever a broken + removal took, so only the survivor-first ordering detects an + implementation that skips the diff arm entirely. + """ + code = _distinct_chunk_code(3) + + pre = State() + code_hash = store_code(pre, code) + for address in (ADDRESS_A, ADDRESS_B): + set_account( + pre, + address, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + survivor = Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash) + touched = Account(nonce=Uint(1), balance=U256(2), code_hash=code_hash) + account_changes: Dict[Bytes20, Optional[Account]] = {} + if survivor_placement == "diff_first": + survivor = touched + account_changes[ADDRESS_B] = survivor + account_changes[ADDRESS_A] = None + if survivor_placement == "diff_last": + survivor = touched + account_changes[ADDRESS_B] = survivor + diff = BlockDiff(account_changes=account_changes) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + stem = _code_zone_stem(code_hash, 0) + for chunk_id in range(3): + assert stem + bytes([chunk_id]) in trie._data + + fresh = State() + assert store_code(fresh, code) == code_hash + set_account(fresh, ADDRESS_B, survivor) + assert pre.compute_state_root(diff) == state_root(fresh) + + +def test_deleting_the_last_holder_drops_every_group() -> None: + """ + The deleted account was the only holder of a code spanning two + code groups (257 chunks): the sweep must remove group 1's chunks + as well as group 0's, emptying the tree. + + Every legacy-size code in this suite fits inside group 0, so a + removal bug scoped to `tree_index == 0` -- sweeping sub-indices + without ever advancing the group -- is caught only here. + """ + # The filler cycle passes zero at chunk 223, leaving that chunk + # absent -- a hole the removal sweep must treat as a no-op. + code = _distinct_chunk_code(257) + + pre = State() + code_hash = store_code(pre, code) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + assert not any(key[0] == 1 for key in trie._data) + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +@pytest.mark.parametrize( + "code, storage_slots, delegated", + [ + pytest.param(Bytes(b""), (), False, id="codeless_eoa"), + pytest.param(_distinct_chunk_code(3), (), False, id="contract"), + pytest.param(DELEGATION_A, (), True, id="delegated_eoa"), + pytest.param( + DELEGATION_A, (0, 63, 64), True, id="delegated_with_storage" + ), + ], +) +def test_every_account_holds_exactly_one_of_the_two_leaves( + code: Bytes, storage_slots: Tuple[int, ...], delegated: bool +) -> None: + """ + Being delegated and holding contract code are exclusive, so an + account holds the code hash leaf or the delegation leaf and never + both, whatever else it carries. + + The shapes are parametrized because a single one cannot tell the + rule from an implementation that hardcodes an answer: suppress + the code hash leaf everywhere and only the codeless and contract + cases fail; write both leaves and only the delegated ones do. + """ + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + + state = State() + code_hash = store_code(state, code) + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + for slot in storage_slots: + set_storage( + state, ADDRESS_A, Bytes32(U256(slot).to_be_bytes32()), U256(7) + ) + + trie = embed_flat_state(state._accounts, state._storage, state.get_code) + + held = { + sub_index + for sub_index in (1, 2) + if header_stem + bytes([sub_index]) in trie._data + } + assert held == ({2} if delegated else {1}) + + +def test_delegating_replaces_the_code_hash_leaf() -> None: + """ + An account that delegates gains a delegation leaf and loses the + code hash leaf it held a moment earlier. + + The incremental path re-embeds over a trie that still carries the + pre-state leaf, so an implementation that writes the delegation + leaf without removing the code hash leaf leaves the account + holding both. The key set is asserted on both the incremental + trie and a fresh embedding, since a root comparison alone would + move together with the bug: both sides embed through the same + function. + """ + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + + pre = State() + delegation_hash = store_code(pre, DELEGATION_A) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + assert ( + header_stem + bytes([1]) + in embed_flat_state(pre._accounts, pre._storage, pre.get_code)._data + ) + + delegated = Account( + nonce=Uint(2), balance=U256(1), code_hash=delegation_hash + ) + diff = BlockDiff( + account_changes={ADDRESS_A: delegated}, + code_changes={delegation_hash: DELEGATION_A}, + ) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + assert header_stem + bytes([1]) not in trie._data + assert trie._data[header_stem + bytes([2])] == Bytes32( + bytes(DELEGATION_A) + b"\x00" * 9 + ) + + fresh = State() + assert store_code(fresh, DELEGATION_A) == delegation_hash + set_account(fresh, ADDRESS_A, delegated) + fresh_trie = embed_flat_state( + fresh._accounts, fresh._storage, fresh.get_code + ) + assert header_stem + bytes([1]) not in fresh_trie._data + assert pre.compute_state_root(diff) == state_root(fresh) + + +def test_undelegating_restores_the_empty_code_hash_leaf() -> None: + """ + An authorization to the zero address clears the delegation, + replacing the leaf with a code hash leaf holding the hash of + empty bytecode and zeroing `code_size`. + + The restored leaf must be present, not merely absent-as-zero: the + empty-code hash is what distinguishes an account that exists from + one that does not, and an implementation that removed the + delegation leaf without writing it back would erase the account. + """ + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + empty_code_hash = bytes.fromhex( + "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + ) + + pre = State() + delegation_hash = store_code(pre, DELEGATION_A) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=delegation_hash), + ) + + cleared = Account( + nonce=Uint(2), balance=U256(1), code_hash=EMPTY_CODE_HASH + ) + diff = BlockDiff(account_changes={ADDRESS_A: cleared}) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + assert header_stem + bytes([2]) not in trie._data + assert trie._data[header_stem + bytes([1])] == empty_code_hash + assert trie._data[header_stem + bytes([0])][4:8] == b"\x00" * 4 + + fresh = State() + set_account(fresh, ADDRESS_A, cleared) + assert pre.compute_state_root(diff) == state_root(fresh) + + +def test_authorities_to_one_target_hold_separate_delegation_leaves() -> None: + """ + Two EOAs delegating to the same target hold one delegation leaf + each, under their own header stems, carrying byte-identical + values. Nothing is shared, so one authority re-delegating leaves + the other's leaf untouched, and deleting one leaves the other's + standing. + + This is the property the header placement exists for: while a + designator was content-addressed, both authorities named one + leaf, and clearing a delegation could only be resolved by asking + whether any account anywhere still delegated to that target. Both + authorities use the *same* target here, since with different + targets even a content-addressed implementation would produce two + leaves and pass. + """ + stem_a = _account_header_stem(b"\x00" * 12 + bytes(ADDRESS_A)) + stem_b = _account_header_stem(b"\x00" * 12 + bytes(ADDRESS_B)) + delegation_key_a = stem_a + bytes([2]) + delegation_key_b = stem_b + bytes([2]) + + pre = State() + hash_a = store_code(pre, DELEGATION_A) + for address in (ADDRESS_A, ADDRESS_B): + set_account( + pre, + address, + Account(nonce=Uint(1), balance=U256(1), code_hash=hash_a), + ) + + initial = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + assert not any(key[0] == 1 for key in initial._data), ( + "a delegation must reach no code-zone leaf" + ) + assert initial._data[delegation_key_a] == Bytes32( + bytes(DELEGATION_A) + b"\x00" * 9 + ) + assert initial._data[delegation_key_b] == initial._data[delegation_key_a] + + hash_b = keccak256(DELEGATION_B) + step_1 = BlockDiff( + account_changes={ + ADDRESS_A: Account( + nonce=Uint(2), balance=U256(1), code_hash=hash_b + ) + }, + code_changes={hash_b: DELEGATION_B}, + ) + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, step_1) + assert trie._data[delegation_key_a] == Bytes32( + bytes(DELEGATION_B) + b"\x00" * 9 + ) + assert trie._data[delegation_key_b] == Bytes32( + bytes(DELEGATION_A) + b"\x00" * 9 + ), "the other authority's leaf must not move" + + step_1_post = State() + assert store_code(step_1_post, DELEGATION_A) == hash_a + assert store_code(step_1_post, DELEGATION_B) == hash_b + set_account( + step_1_post, + ADDRESS_A, + Account(nonce=Uint(2), balance=U256(1), code_hash=hash_b), + ) + set_account( + step_1_post, + ADDRESS_B, + Account(nonce=Uint(1), balance=U256(1), code_hash=hash_a), + ) + assert pre.compute_state_root(step_1) == state_root(step_1_post) + + apply_changes_to_state(pre, step_1) + step_2 = BlockDiff(account_changes={ADDRESS_A: None}) + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, step_2) + assert delegation_key_a not in trie._data + assert trie._data[delegation_key_b] == Bytes32( + bytes(DELEGATION_A) + b"\x00" * 9 + ), "deleting one authority must not disturb the other" + + +def test_shared_code_survives_until_the_last_holder_is_gone() -> None: + """ + Three accounts share one bytecode. Deleting them one block at a + time keeps the chunk leaves through the first two deletions -- + pinned by key after each step -- and the third deletion takes + them with it, returning the tree to empty. + """ + code = _distinct_chunk_code(3) + addresses = (ADDRESS_A, ADDRESS_B, ADDRESS_C) + + pre = State() + code_hash = store_code(pre, code) + for address in addresses: + set_account( + pre, + address, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + stem = _code_zone_stem(code_hash, 0) + chunk_keys = {stem + bytes([chunk_id]) for chunk_id in range(3)} + + for deletions_so_far, address in enumerate(addresses): + diff = BlockDiff(account_changes={address: None}) + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + present = chunk_keys & set(trie._data.keys()) + if deletions_so_far < 2: + assert present == chunk_keys, ( + f"deletion {deletions_so_far + 1} of 3 must keep the " + "shared chunks" + ) + else: + assert present == set() + assert root(trie) == EMPTY_TRIE_ROOT + apply_changes_to_state(pre, diff) + + +def test_two_holders_deleted_in_one_block_drop_the_code_once() -> None: + """ + Both remaining holders of a bytecode go in the same block: the + first removal drops the shared chunks and the second finds them + already gone, a no-op rather than an error, leaving the empty + root. + """ + code = _distinct_chunk_code(2) + + pre = State() + code_hash = store_code(pre, code) + for address in (ADDRESS_A, ADDRESS_B): + set_account( + pre, + address, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + diff = BlockDiff(account_changes={ADDRESS_A: None, ADDRESS_B: None}) + + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +def test_contract_deleted_then_recreated_with_different_code() -> None: + """ + A sole holder's deletion drops its chunks in one block; the next + block re-creates the address with different code through its own + diff. Each block's root is computed incrementally and must match + a fresh embed of that block's post state -- the drop and the + re-add land in separate tries, rebuilt from the advanced flat + state, so nothing of the old code may linger. + """ + old_code = _distinct_chunk_code(3) + new_code = _distinct_chunk_code(2, salt=40) + + pre = State() + old_hash = store_code(pre, old_code) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=old_hash), + ) + + delete = BlockDiff(account_changes={ADDRESS_A: None}) + assert pre.compute_state_root(delete) == EMPTY_TRIE_ROOT + apply_changes_to_state(pre, delete) + + new_hash = keccak256(new_code) + recreated = Account(nonce=Uint(1), balance=U256(2), code_hash=new_hash) + recreate = BlockDiff( + account_changes={ADDRESS_A: recreated}, + code_changes={new_hash: new_code}, + ) + + fresh = State() + assert store_code(fresh, new_code) == new_hash + set_account(fresh, ADDRESS_A, recreated) + assert pre.compute_state_root(recreate) == state_root(fresh) + + +def test_delegation_and_storage_writes_share_a_diff() -> None: + """ + One diff both delegates an account -- replacing its code hash + leaf with a delegation leaf -- and writes its storage. The two + live in one header stem at sub-indices the other never touches, + and the storage loop runs after the account loop; the + combination must land on the root of a fresh embed of the post + state. + """ + slot = Bytes32(U256(2).to_be_bytes32()) + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + + pre = State() + delegation_hash = store_code(pre, DELEGATION_A) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + set_storage(pre, ADDRESS_A, slot, U256(5)) + + delegated = Account( + nonce=Uint(2), balance=U256(1), code_hash=delegation_hash + ) + diff = BlockDiff( + account_changes={ADDRESS_A: delegated}, + storage_changes={ADDRESS_A: {slot: U256(9)}}, + code_changes={delegation_hash: DELEGATION_A}, + ) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + _apply_diff_to_trie(trie, pre, diff) + header_sub_indices = { + key[-1] for key in trie._data if key.startswith(header_stem) + } + assert header_sub_indices == {0, 2, 64 + 2} + + fresh = State() + assert store_code(fresh, DELEGATION_A) == delegation_hash + set_account(fresh, ADDRESS_A, delegated) + set_storage(fresh, ADDRESS_A, slot, U256(9)) + assert pre.compute_state_root(diff) == state_root(fresh) + + +def test_group_exact_code_fills_group_zero_and_nothing_more() -> None: + """ + Code of exactly 256 chunks fills code group 0 to its last + sub-index and derives no key in group 1: the group boundary is + exclusive on the right, `chunk_id // 256`. + """ + code = Bytes(b"\x01" * (31 * 256)) # 256 chunks, none zero + + pre = State() + code_hash = store_code(pre, code) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + + group_0_stem = _code_zone_stem(code_hash, 0) + group_1_stem = _code_zone_stem(code_hash, 1) + code_zone_keys = {key for key in trie._data if key[0] == 1} + assert code_zone_keys == { + group_0_stem + bytes([sub_index]) for sub_index in range(256) + } + assert not any(key.startswith(group_1_stem) for key in trie._data) + + +def test_change_to_an_unresolvable_code_hash_is_a_pre_state_error() -> None: + """ + A diff that points an account at a code hash resolvable neither + from its `code_changes` nor from the store fails with + `UnknownCodeHashError` when the root is computed: malformed + input, deliberately not an invalid block. + """ + pre = State() + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH), + ) + phantom = keccak256(b"never stored") + diff = BlockDiff( + account_changes={ + ADDRESS_A: Account( + nonce=Uint(1), balance=U256(1), code_hash=phantom + ) + }, + ) + + with pytest.raises(UnknownCodeHashError): + pre.compute_state_root(diff) + assert not issubclass(UnknownCodeHashError, InvalidBlock) + + +def test_absent_chunk_in_a_later_group_does_not_stall_removal() -> None: + """ + A multi-group code with an all-zero chunk in group 1 has a hole + where that leaf would sit. Deleting the last holder must remove + every present chunk on both sides of the hole, leaving the empty + root. + """ + chunk_count = 258 + mutable = bytearray(b"\x01" * (31 * chunk_count)) + mutable[31 * 256 : 31 * 257] = b"\x00" * 31 # chunk 256, group 1 + code = Bytes(bytes(mutable)) + + pre = State() + code_hash = store_code(pre, code) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + group_1_stem = _code_zone_stem(code_hash, 1) + assert group_1_stem + bytes([0]) not in trie._data # the hole + assert group_1_stem + bytes([1]) in trie._data + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +def test_push_data_continuation_chunk_of_zero_bytes_is_present() -> None: + """ + Zero bytes that continue PUSHDATA from an earlier chunk do not + qualify as a zero chunk -- the EIP's "Code" section is explicit + that byte 0 then records the continuation -- so a chunk whose 31 + code bytes are all zero is absent only when its leading byte is + zero too. + + Here `PUSH32` ends chunk 0 and its data fills chunk 1 with 31 + zero bytes, so chunk 1 encodes to `0x1f` followed by zeros -- not + the zero value -- and its leaf must be in the tree. An + implementation keying absence off the 31-byte code slice alone + would drop it and corrupt the committed code. Deleting the last + holder must still take it away with the rest. + """ + # PUSH32 at position 30: data occupies positions 31..62, so chunk + # 1's code bytes are 31 zero bytes of push data. + code = Bytes(b"\x01" * 30 + b"\x7f" + b"\x00" * 31) + assert len(code) == 62 # two chunks + + pre = State() + code_hash = store_code(pre, code) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + trie = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + stem = _code_zone_stem(code_hash, 0) + continuation_key = stem + bytes([1]) + assert continuation_key in trie._data + assert trie._data[continuation_key] == Bytes32(bytes([31]) + b"\x00" * 31) + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +def test_account_has_storage_matches_the_embedded_leaf_set() -> None: + """ + EIP-8297 defines non-empty storage by leaf existence: a leaf at + one of the header's storage sub-indices or anywhere in the + address's storage bucket. The provider answers from its flat + map, which stays equivalent only through upkeep on every + mutation path; this pins the equivalence itself, over states + reached through `set_*` calls and through applied diffs. + + The one deliberate exception is storage orphaned by + `set_account(..., None)`, whose flat/leaf divergence + `test_set_account_none_leaves_storage_while_the_diff_path_clears_it` + pins separately. + """ + + def storage_leaf_exists(state: State, address: Bytes20) -> bool: + trie = embed_flat_state( + state._accounts, state._storage, state.get_code + ) + address32 = b"\x00" * 12 + bytes(address) + header_stem = _account_header_stem(address32) + bucket_prefix = bytes([0xFF]) + blake3(address32).digest() + first = int(HEADER_STORAGE_OFFSET) + last = int(HEADER_STORAGE_OFFSET + HEADER_STORAGE_SLOTS) - 1 + return any( + (key.startswith(header_stem) and first <= key[-1] <= last) + or key.startswith(bucket_prefix) + for key in trie._data + ) + + key_header = Bytes32(U256(3).to_be_bytes32()) + key_bucket = Bytes32(U256(1000).to_be_bytes32()) + account = Account( + nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH + ) + + def fresh(*slots: Bytes32) -> State: + state = State() + set_account(state, ADDRESS_A, account) + for slot in slots: + set_storage(state, ADDRESS_A, slot, U256(7)) + return state + + no_storage = fresh() + header_only = fresh(key_header) + bucket_only = fresh(key_bucket) + zeroed = fresh(key_header) + apply_changes_to_state( + zeroed, BlockDiff(storage_changes={ADDRESS_A: {key_header: U256(0)}}) + ) + deleted = fresh(key_header, key_bucket) + apply_changes_to_state( + deleted, BlockDiff(account_changes={ADDRESS_A: None}) + ) + cleared = fresh(key_header, key_bucket) + apply_changes_to_state(cleared, BlockDiff(storage_clears={ADDRESS_A})) + + def delegated(*slots: Bytes32) -> State: + state = State() + delegation_hash = store_code(state, DELEGATION_A) + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=delegation_hash), + ) + for slot in slots: + set_storage(state, ADDRESS_A, slot, U256(7)) + return state + + expectations = [ + (no_storage, False), + (header_only, True), + (bucket_only, True), + (zeroed, False), + (deleted, False), + (cleared, False), + # The delegation leaf sits below the storage sub-indices, so + # holding one is not holding storage. + (delegated(), False), + (delegated(key_header), True), + (delegated(key_bucket), True), + ] + for state, expected in expectations: + assert state.account_has_storage(ADDRESS_A) is expected + assert storage_leaf_exists(state, ADDRESS_A) is expected + + +def test_deleting_an_unknown_address_is_a_no_op() -> None: + """ + A diff may delete an address the pre-state never held. Removing + the absent account's regions is a no-op, and the code check is + skipped outright -- there is no previous account to read a code + hash from -- leaving the root exactly where the pre-state's was. + """ + pre = State() + code_hash = store_code(pre, _distinct_chunk_code(2)) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + before = state_root(pre) + + diff = BlockDiff(account_changes={ADDRESS_B: None}) + + assert pre.compute_state_root(diff) == before + + +def test_removing_code_with_an_absent_first_chunk_leaves_nothing() -> None: + """ + A code whose first chunk is 31 zero bytes has no leaf at chunk 0 + -- zero collapses to absence -- so a presence probe on any fixed + chunk could call the code leafless and leak the rest. Deleting + the last holder must remove every later chunk regardless, + leaving the empty root. + """ + code = Bytes(b"\x00" * 31 + bytes(_distinct_chunk_code(2))) + + pre = State() + code_hash = store_code(pre, code) + set_account( + pre, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(1), code_hash=code_hash), + ) + + stem = _code_zone_stem(code_hash, 0) + before = embed_flat_state(pre._accounts, pre._storage, pre.get_code) + assert stem + bytes([0]) not in before._data + assert stem + bytes([1]) in before._data + assert stem + bytes([2]) in before._data + + diff = BlockDiff(account_changes={ADDRESS_A: None}) + + assert pre.compute_state_root(diff) == EMPTY_TRIE_ROOT + + +def test_basic_data_leaf_bytes_carry_code_size_nonce_and_balance() -> None: + """ + The BASIC_DATA leaf's 32 bytes pack the resolved code's length, + not any field stored on `Account` itself. + + `Account` has no `code_size` field, so this proves the provider + derives it from `get_code(code_hash)` at embed time, not from + anything cached. + """ + code = Bytes(b"\x01" * 40) + state = MptState() + code_hash = mpt_store_code(state, code) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(7), balance=U256(12345), code_hash=code_hash), + ) + + embedded = embed_state(state) + + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + key = _account_header_stem(address32) + bytes([0]) + leaf = embedded._data[key] + + assert leaf[0:1] == b"\x00" # version + assert leaf[1:4] == b"\x00" * 3 # reserved + assert leaf[4:8] == len(code).to_bytes(4, "big") # code_size + assert leaf[8:16] == (7).to_bytes(8, "big") # nonce + assert leaf[16:32] == (12345).to_bytes(16, "big") # balance + + +def test_balance_at_or_above_the_sixteen_byte_field_rejects_at_root_time() -> ( + None +): + """ + A balance of exactly `2**128` is rejected when the root is + computed, not when it is set. + + `Account.balance` is a full `U256` and `set_account` stays + unbounded, so the sixteen-byte field EIP-8297 packs it into is + enforced at commitment time, as `BalanceOverflowError` -- an + `InvalidBlock` -- rather than a raw `AssertionError`. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account( + nonce=Uint(0), + balance=U256(2) ** U256(128), + code_hash=EMPTY_CODE_HASH, + ), + ) + + with pytest.raises(BalanceOverflowError): + state_root(state) + + +def test_block_diff_minting_over_cap_balance_rejects_the_block() -> None: + """ + A diff that raises an account's balance to `2**128` makes + `compute_state_root` raise `BalanceOverflowError`: the block + minting the balance is invalid, and the pre-state, whose balance + still fits the field, embeds cleanly afterwards. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account( + nonce=Uint(0), + balance=U256(2) ** U256(128) - U256(1), + code_hash=EMPTY_CODE_HASH, + ), + ) + diff = BlockDiff( + account_changes={ + ADDRESS_A: Account( + nonce=Uint(0), + balance=U256(2) ** U256(128), + code_hash=EMPTY_CODE_HASH, + ) + } + ) + + with pytest.raises(BalanceOverflowError): + state.compute_state_root(diff) + + assert state.compute_state_root(BlockDiff()) == state_root(state) + + +def test_empty_code_contract_embeds_like_an_eoa() -> None: + """ + A contract account whose code is explicitly `b""`, stored through + `store_code` rather than merely defaulted, embeds identically to + an EOA: exactly the two header leaves and no code chunk leaves. + + Unlike `test_eoa_embeds_basic_data_and_code_hash_leaves` (which + never calls `store_code`), this pins that routing empty bytes + through the store still resolves to `EMPTY_CODE_HASH`. + """ + state = State() + code_hash = store_code(state, Bytes(b"")) + assert code_hash == EMPTY_CODE_HASH + + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(3), balance=U256(42), code_hash=code_hash), + ) + + trie = embed_flat_state(state._accounts, state._storage, state.get_code) + + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + basic_data_key = header_stem + bytes([0]) + code_hash_key = header_stem + bytes([1]) + + assert set(trie._data.keys()) == {basic_data_key, code_hash_key} + assert trie._data[code_hash_key] == EMPTY_CODE_HASH + + +def test_delegation_designator_account_embedding() -> None: + """ + An EIP-7702 delegation designator (`0xef0100` followed by a + 20-byte address, 23 bytes total) embeds as one header leaf beside + the account's basic data: no code hash leaf, since the delegation + leaf determines the code and its hash, and no code-zone leaf, + since the indicator is not code. + + The value is the designator followed by nine zero bytes. That is + deliberately not the chunk encoding, which spends its first byte + on a push-data count and so pads with eight; an implementation + reusing `chunkify_code` here writes a plausible 32 bytes that are + wrong in every position. + """ + designator = Bytes(b"\xef\x01\x00" + bytes(ADDRESS_B)) + assert len(designator) == 23 + + state = MptState() + code_hash = mpt_store_code(state, designator) + mpt_set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=code_hash), + ) + + embedded = embed_state(state) + + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + basic_data_key = header_stem + bytes([0]) + delegation_key = header_stem + bytes([2]) + + assert set(embedded._data.keys()) == {basic_data_key, delegation_key} + assert embedded._data[delegation_key] == Bytes32( + bytes(designator) + b"\x00" * 9 + ) + assert _code_zone_stem(code_hash, 0) + bytes([0]) not in embedded._data, ( + "the designator must reach no code-zone leaf" + ) + + basic_data = embedded._data[basic_data_key] + assert basic_data[4:8] == (23).to_bytes(4, "big") # code_size + + +def test_get_code_raises_for_an_unknown_code_hash() -> None: + """ + An account whose `code_hash` was never stored raises + `UnknownCodeHashError` when the root is computed, because + `embed_flat_state` resolves every account's code through + `get_code` to size it. + + Only `EMPTY_CODE_HASH` needs no store entry. An unstored hash is + a malformed pre-state, not an invalid block, so the error is an + `EthereumException` but deliberately not an `InvalidBlock`. + """ + assert not issubclass(UnknownCodeHashError, InvalidBlock) + + state = State() + unknown_hash = keccak256(b"never stored") + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=unknown_hash), + ) + + assert state.get_code(EMPTY_CODE_HASH) == b"" + with pytest.raises(UnknownCodeHashError): + state.get_code(unknown_hash) + with pytest.raises(UnknownCodeHashError): + state_root(state) + + +def test_storage_clears_removes_slots_before_other_changes() -> None: + """ + `storage_clears` empties an address's storage before + `storage_changes` is applied: a pre-existing slot is cleared like + any other, while a brand-new key written by the same diff still + lands with its new value. + + `binary_tree`'s own `extract_block_diff` (copied from Amsterdam, + post-EIP-6780) never sets this field, so the branch is reachable + only from this unit test today; pinning it keeps the provider + honest in case a pre-Cancun-style tracker is ever wired up to it. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + key_1 = Bytes32(U256(1).to_be_bytes32()) + key_2 = Bytes32(U256(2).to_be_bytes32()) + set_storage(state, ADDRESS_A, key_1, U256(7)) + + apply_changes_to_state( + state, + BlockDiff( + storage_clears={ADDRESS_A}, + storage_changes={ADDRESS_A: {key_2: U256(9)}}, + ), + ) + + assert state.get_storage(ADDRESS_A, key_1) == U256(0) + assert state.get_storage(ADDRESS_A, key_2) == U256(9) + assert state._storage[ADDRESS_A] == {key_2: U256(9)} + + +def test_account_deletion_also_drops_its_storage() -> None: + """ + Deleting an account through a diff also pops its storage: + `_storage` no longer has an entry for the address, so + `account_has_storage` reads back `False`. + + `test_differential_mpt.py`'s + `test_account_delete_diverges_on_account_has_storage` pins the + contrasting MPT behavior, where storage survives an account + delete; this test only pins PBT's own side of that divergence. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + set_storage(state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7)) + assert state.account_has_storage(ADDRESS_A) is True + + apply_changes_to_state(state, BlockDiff(account_changes={ADDRESS_A: None})) + + assert ADDRESS_A not in state._storage + assert state.account_has_storage(ADDRESS_A) is False + + +def test_storage_written_for_a_deleted_account_is_dropped() -> None: + """ + A diff that deletes an account and writes to its storage in the + same step leaves no storage behind at all. + + Storage belongs to an account, so a write to an address the diff + leaves without one is dropped rather than kept as an orphan no + account owns. That keeps `account_has_storage` answering as + [EIP-8297] requires, from whether any slot leaf of the address + exists: `embed_flat_state` would skip such an orphan anyway, so + reporting storage for it would claim leaves the tree does not + have. + + [EIP-8297]: https://eips.ethereum.org/EIPS/eip-8297 + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + set_account( + state, + ADDRESS_B, + Account(nonce=Uint(2), balance=U256(5), code_hash=EMPTY_CODE_HASH), + ) + set_storage(state, ADDRESS_B, Bytes32(U256(9).to_be_bytes32()), U256(3)) + + key = Bytes32(U256(1).to_be_bytes32()) + apply_changes_to_state( + state, + BlockDiff( + account_changes={ADDRESS_A: None}, + storage_changes={ADDRESS_A: {key: U256(7)}}, + ), + ) + + assert state.get_account_optional(ADDRESS_A) is None + assert state.account_has_storage(ADDRESS_A) is False + assert state.get_storage(ADDRESS_A, key) == U256(0) + + without_a = State() + set_account( + without_a, + ADDRESS_B, + Account(nonce=Uint(2), balance=U256(5), code_hash=EMPTY_CODE_HASH), + ) + set_storage( + without_a, ADDRESS_B, Bytes32(U256(9).to_be_bytes32()), U256(3) + ) + + assert state_root(state) == state_root(without_a) + + +def test_all_zero_storage_change_drops_the_address_entry() -> None: + """ + A diff writing only zeros to slots an account already holds + empties `_storage[address]` down to nothing, and the address key + itself is dropped, not left mapped to an empty dict. + + `account_has_storage` reads back `False` and the root matches a + state that was never written to, which is the answer EIP-8297 + fixes for the tree: no slot leaf of the address survives. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + key_1 = Bytes32(U256(1).to_be_bytes32()) + key_2 = Bytes32(U256(2).to_be_bytes32()) + set_storage(state, ADDRESS_A, key_1, U256(7)) + set_storage(state, ADDRESS_A, key_2, U256(9)) + assert state.account_has_storage(ADDRESS_A) is True + + apply_changes_to_state( + state, + BlockDiff( + storage_changes={ADDRESS_A: {key_1: U256(0), key_2: U256(0)}} + ), + ) + + assert ADDRESS_A not in state._storage + assert state.account_has_storage(ADDRESS_A) is False + + never_written = State() + set_account( + never_written, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + assert state_root(state) == state_root(never_written) + + +def test_set_account_none_leaves_storage_while_the_diff_path_clears_it() -> ( + None +): + """ + `set_account(state, A, None)` pops the account but never touches + its storage, while the diff path pops both together. + + Two identically set-up states, one deleted through `set_account` + and the other through `apply_changes_to_state`, diverge on + `account_has_storage` depending only on which route deleted the + account. + + Rooting both states shows the divergence stops at the flat map: + the embedding skips storage whose address has no account, so the + orphaned slots never reach the tree and both states commit to + the empty root. + """ + key = Bytes32(U256(1).to_be_bytes32()) + + via_set_account = State() + set_account( + via_set_account, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + set_storage(via_set_account, ADDRESS_A, key, U256(7)) + set_account(via_set_account, ADDRESS_A, None) + + via_diff = State() + set_account( + via_diff, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + set_storage(via_diff, ADDRESS_A, key, U256(7)) + apply_changes_to_state( + via_diff, BlockDiff(account_changes={ADDRESS_A: None}) + ) + + assert via_set_account.get_account_optional(ADDRESS_A) is None + assert via_diff.get_account_optional(ADDRESS_A) is None + assert via_set_account.account_has_storage(ADDRESS_A) is True + assert via_diff.account_has_storage(ADDRESS_A) is False + assert state_root(via_set_account) == EMPTY_TRIE_ROOT + assert state_root(via_diff) == EMPTY_TRIE_ROOT + + +def test_set_storage_requires_an_existing_account() -> None: + """ + `set_storage` asserts the account already exists; it is not a + valid way to create storage for an address with no account. + """ + state = State() + with pytest.raises(AssertionError): + set_storage( + state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7) + ) + + +def test_compute_state_root_leaves_the_pre_state_untouched() -> None: + """ + `compute_state_root` applies the diff to a copy: calling it twice + with a non-trivial diff returns the same root both times, and the + pre-state's accounts, storage, and code store are unchanged. + """ + code = Bytes(b"\x01" * 40) + code_hash = keccak256(code) + + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(100), code_hash=EMPTY_CODE_HASH), + ) + set_storage(state, ADDRESS_A, Bytes32(U256(1).to_be_bytes32()), U256(7)) + + accounts_before = dict(state._accounts) + storage_before = { + address: dict(slots) for address, slots in state._storage.items() + } + code_store_before = dict(state._code_store) + + diff = BlockDiff( + account_changes={ + ADDRESS_A: Account( + nonce=Uint(2), balance=U256(50), code_hash=code_hash + ), + ADDRESS_B: Account( + nonce=Uint(1), balance=U256(1), code_hash=EMPTY_CODE_HASH + ), + }, + storage_changes={ + ADDRESS_A: {Bytes32(U256(1).to_be_bytes32()): U256(0)} + }, + code_changes={code_hash: code}, + ) + + first_root = state.compute_state_root(diff) + second_root = state.compute_state_root(diff) + + assert first_root == second_root + assert state._accounts == accounts_before + assert state._storage == storage_before + assert state._code_store == code_store_before + + +def test_sequential_block_diffs_evolve_the_root() -> None: + """ + Three sequential diffs applied to the same live `State` via + `apply_changes_to_state`: writing a slot changes the root, + writing a second slot changes it again, and zeroing the second + slot returns the root to exactly what it was after the first + write. + + Zero-means-absent composes across separate blocks, not just + within one diff. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + key_1 = Bytes32(U256(1).to_be_bytes32()) + key_2 = Bytes32(U256(2).to_be_bytes32()) + root_empty = state_root(state) + + apply_changes_to_state( + state, BlockDiff(storage_changes={ADDRESS_A: {key_1: U256(7)}}) + ) + root_after_first_write = state_root(state) + assert root_after_first_write != root_empty + + apply_changes_to_state( + state, BlockDiff(storage_changes={ADDRESS_A: {key_2: U256(9)}}) + ) + root_after_second_write = state_root(state) + assert root_after_second_write != root_after_first_write + + apply_changes_to_state( + state, BlockDiff(storage_changes={ADDRESS_A: {key_2: U256(0)}}) + ) + assert state_root(state) == root_after_first_write + + +def test_storage_boundary_slots_through_the_provider() -> None: + """ + Slots 0, 63, 64, 255, 256, and `2**256 - 1`, set on one account + through `set_storage`, land on whichever of the header (0, 63) + or overflow (64, 255, 256, `2**256 - 1`) forms the embedding + defines for that slot, holding the exact 32-byte value + `set_storage` wrote there -- not merely a key that exists. + + Rebuilt here from raw `blake3` and literal zone/sub-index bytes, + not `get_tree_key_for_storage_slot`. Each slot gets its own + distinct value (`index + 1`), so a swap between any two of the six + leaves is detectable: every other boundary-focused test in this + module asserts key sets only, so mutating all header (or all + overflow) leaf values inside `embed_flat_state` would otherwise go + uncaught here. + """ + state = State() + set_account( + state, + ADDRESS_A, + Account(nonce=Uint(1), balance=U256(0), code_hash=EMPTY_CODE_HASH), + ) + slots = (0, 63, 64, 255, 256, 2**256 - 1) + values = {slot: U256(index + 1) for index, slot in enumerate(slots)} + for slot, value in values.items(): + set_storage( + state, ADDRESS_A, Bytes32(U256(slot).to_be_bytes32()), value + ) + + trie = embed_flat_state(state._accounts, state._storage, state.get_code) + + address32 = b"\x00" * 12 + bytes(ADDRESS_A) + header_stem = _account_header_stem(address32) + storage_keys = { + 0: header_stem + bytes([64]), + 63: header_stem + bytes([127]), + 64: _storage_overflow_stem(address32, 0) + bytes([64]), + 255: _storage_overflow_stem(address32, 0) + bytes([255]), + 256: _storage_overflow_stem(address32, 1) + bytes([0]), + 2**256 - 1: _storage_overflow_stem(address32, 2**248 - 1) + + bytes([255]), + } + expected_keys = { + header_stem + bytes([0]), # basic data + header_stem + bytes([1]), # code hash + *storage_keys.values(), + } + + assert set(trie._data.keys()) == expected_keys + for slot, key in storage_keys.items(): + assert trie._data[key] == values[slot].to_be_bytes32(), ( + f"slot {slot}: unexpected leaf value" + ) diff --git a/tests/binary_trie/test_trie.py b/tests/binary_trie/test_trie.py new file mode 100644 index 00000000000..9519e41d2c7 --- /dev/null +++ b/tests/binary_trie/test_trie.py @@ -0,0 +1,1039 @@ +""" +Tests for the raw binary tree structure. +""" + +import random +import sys +from typing import Dict, Set + +import pytest +from blake3 import blake3 +from ethereum_types.bytes import Bytes, Bytes32 +from ethereum_types.numeric import Uint + +from ethereum.partitioned_binary_tree import ( + BRANCH_NODE_TAG, + EMPTY_TRIE_ROOT, + LEAF_NODE_TAG, + BinaryNode, + BinaryTrie, + BranchNode, + LeafNode, + binarize, + bytes_to_bit_list, + copy_trie, + encode_bit_prefix, + remove_subtree, + root, + trie_get, + trie_set, +) + +from .incremental_trie import ( + IncrementalRadixTree, + _bits, + _branch_hash, + _leaf_hash, +) + + +def test_bytes_to_bit_list_is_msb_first() -> None: + """ + Bytes expand to bits most significant bit first. + """ + assert bytes_to_bit_list(Bytes(b"\x80")) == Bytes( + bytes([1, 0, 0, 0, 0, 0, 0, 0]) + ) + assert bytes_to_bit_list(Bytes(b"\x01")) == Bytes( + bytes([0, 0, 0, 0, 0, 0, 0, 1]) + ) + assert bytes_to_bit_list(Bytes(b"\xa5")) == Bytes( + bytes([1, 0, 1, 0, 0, 1, 0, 1]) + ) + + +def test_encode_bit_prefix_layout() -> None: + """ + A prefix encodes as a two-byte big-endian bit count followed by + the bits packed most significant bit first, zero padded to a byte + boundary. + """ + assert encode_bit_prefix(Bytes(b"")) == b"\x00\x00" + assert encode_bit_prefix(Bytes(bytes([1, 0, 1]))) == b"\x00\x03\xa0" + # Nine bits cross a byte boundary into a zero-padded second byte. + assert encode_bit_prefix(Bytes(bytes([1] * 9))) == b"\x00\x09\xff\x80" + + +def test_encode_bit_prefix_rejects_unrepresentable_counts() -> None: + """ + A prefix whose bit count does not fit the two-byte field is + rejected. + """ + with pytest.raises(AssertionError): + encode_bit_prefix(Bytes(bytes(2**16))) + + +def test_encode_bit_prefix_counts_trailing_zero_bits() -> None: + """ + Prefixes differing only by trailing zero bits pack to the same + bytes; the explicit count is what keeps their encodings and + subsequently their commitments distinct. + """ + shorter = Bytes(bytes([0, 1, 1, 0])) + longer = Bytes(bytes([0, 1, 1, 0, 0])) + + assert encode_bit_prefix(shorter)[2:] == encode_bit_prefix(longer)[2:] + assert encode_bit_prefix(shorter) != encode_bit_prefix(longer) + + +def test_empty_trie_root_is_all_zeros() -> None: + """ + An empty trie commits to 32 zero bytes. + """ + trie: BinaryTrie = BinaryTrie(_data={}) + assert root(trie) == b"\x00" * 32 + assert EMPTY_TRIE_ROOT == b"\x00" * 32 + + +def test_trie_set_and_get() -> None: + """ + Values can be stored, retrieved, and overwritten. + """ + trie = BinaryTrie() + key = Bytes32(b"\x01" * 32) + value = Bytes32(b"\x02" * 32) + + assert trie_get(trie, key) is None + trie_set(trie, key, value) + assert trie_get(trie, key) == value + + replacement = Bytes32(b"\x03" * 32) + trie_set(trie, key, replacement) + assert trie_get(trie, key) == replacement + + +def test_trie_set_rejects_malformed_inputs() -> None: + """ + Empty keys, keys past the maximum length, and values that are not + 32 bytes are rejected. + + The EIP's `state_root` asserts these bounds when the root is + computed; `trie_set` enforcing them at write time is strictly + earlier, and `root` re-checks every entry regardless, so a + malformed entry can neither enter the map nor reach a + commitment. + """ + trie = BinaryTrie() + with pytest.raises(AssertionError): + trie_set(trie, Bytes(b""), Bytes32(b"\x01" * 32)) + with pytest.raises(AssertionError): + trie_set(trie, Bytes(b"\x01" * 8193), Bytes32(b"\x01" * 32)) + with pytest.raises(AssertionError): + trie_set( + trie, + Bytes(b"\x01"), + Bytes(b"\x02" * 31), # type: ignore[arg-type] + ) + + +def test_copy_trie_is_independent() -> None: + """ + Mutating a copy leaves the original untouched, and vice versa. + """ + key = Bytes32(b"\x01" * 32) + other_key = Bytes32(b"\x02" * 32) + value = Bytes32(b"\x03" * 32) + + original = BinaryTrie() + trie_set(original, key, value) + + duplicate = copy_trie(original) + assert trie_get(duplicate, key) == value + + trie_set(duplicate, other_key, value) + assert trie_get(original, other_key) is None + assert root(duplicate) != root(original) + + +def test_single_key_is_a_leaf_at_the_root() -> None: + """ + A trie with one key commits to a single leaf carrying its full + key. + """ + key = Bytes(b"\x00" + b"\x42" * 32 + b"\x07") + value = Bytes32(b"\x11" * 32) + + trie = BinaryTrie() + trie_set(trie, key, value) + + assert root(trie) == _leaf_hash(key, value) + + +def test_keys_sharing_a_stem_split_under_one_branch() -> None: + """ + Two keys sharing a 33-byte stem diverge in their final byte's + first bit: a single branch carrying the whole stem as its prefix, + over two leaves. + """ + stem = b"\x00" + b"\x42" * 32 + low_key = Bytes(stem + b"\x00") + high_key = Bytes(stem + b"\xff") + low_value = Bytes32(b"\x01" * 32) + high_value = Bytes32(b"\x02" * 32) + + trie = BinaryTrie() + trie_set(trie, low_key, low_value) + trie_set(trie, high_key, high_value) + + assert root(trie) == _branch_hash( + _bits(stem), + _leaf_hash(low_key, low_value), + _leaf_hash(high_key, high_value), + ) + + +def test_first_bit_divergence_has_empty_prefix() -> None: + """ + Keys differing in their first bit branch at the root with an + empty prefix. + """ + zero_key = Bytes(b"\x00" * 34) + one_key = Bytes(b"\xff" * 66) + value = Bytes32(b"\x33" * 32) + + trie = BinaryTrie() + trie_set(trie, zero_key, value) + trie_set(trie, one_key, value) + + assert root(trie) == _branch_hash( + [], _leaf_hash(zero_key, value), _leaf_hash(one_key, value) + ) + + +def test_canonical_form_example() -> None: + """ + Three keys sharing a stem, with sub-indices 0, 1, and 128: a + branch carrying the stem as its prefix, splitting on the first + sub-index bit; below it, a branch carrying the next six shared + bits over the two low leaves, and the high leaf directly on the + other side. + """ + stem = b"\xff" + b"\xab" * 32 + key_0 = Bytes(stem + b"\x00") + key_1 = Bytes(stem + b"\x01") + key_128 = Bytes(stem + b"\x80") + value = Bytes32(b"\x44" * 32) + + trie = BinaryTrie() + for key in (key_0, key_1, key_128): + trie_set(trie, key, value) + + low_side = _branch_hash( + [0] * 6, + _leaf_hash(key_0, value), + _leaf_hash(key_1, value), + ) + assert root(trie) == _branch_hash( + _bits(stem), low_side, _leaf_hash(key_128, value) + ) + + +def test_binarize_builds_relative_prefixes_and_full_key_leaves() -> None: + """ + Branch prefixes are relative (only the bits shared beyond the + parent's split point), while leaves commit their complete keys + wherever they sit in the tree. + """ + stem = b"\xff" + b"\xab" * 32 + key_0 = Bytes(stem + b"\x00") + key_1 = Bytes(stem + b"\x01") + key_128 = Bytes(stem + b"\x80") + value = Bytes32(b"\x44" * 32) + + top = binarize({key_0: value, key_1: value, key_128: value}, Uint(0)) + + assert isinstance(top, BranchNode) + assert top.prefix == bytes_to_bit_list(Bytes(stem)) + + low = top.left + assert isinstance(low, BranchNode) + # Relative to the split above it, not 271 bits from the root. + assert low.prefix == Bytes(bytes([0] * 6)) + assert isinstance(low.left, LeafNode) + assert isinstance(low.right, LeafNode) + assert low.left.key == key_0 + assert low.right.key == key_1 + + high = top.right + assert isinstance(high, LeafNode) + assert high.key == key_128 + + +def test_zero_value_is_not_absence() -> None: + """ + Storing 32 zero bytes commits differently from storing nothing. + """ + key = Bytes32(b"\x07" * 31 + b"\x00") + + trie = BinaryTrie() + trie_set(trie, key, Bytes32(b"\x00" * 32)) + + assert root(trie) != EMPTY_TRIE_ROOT + + +def test_prefix_key_violation_is_rejected() -> None: + """ + A key that is a prefix of another key makes the tree ill-defined, + and computing the root fails the prefix-freeness assertion. + """ + trie = BinaryTrie() + trie_set(trie, Bytes(b"\xaa" * 34), Bytes32(b"\x01" * 32)) + trie_set(trie, Bytes(b"\xaa" * 34 + b"\xbb" * 32), Bytes32(b"\x02" * 32)) + + with pytest.raises(AssertionError): + root(trie) + + +def test_prefix_key_violation_in_mid_byte_group_is_rejected() -> None: + """ + A prefix-violating pair is rejected even when it is isolated only + by a split inside a byte, so the scan that runs the shorter key + out of bits starts at a bit position that is not byte-aligned. + + All three keys share bit 0 and diverge at bit 1, splitting + `0x00` off and leaving the offending pair grouped at depth 2. + """ + trie = BinaryTrie() + trie_set(trie, Bytes(b"\x50"), Bytes32(b"\x01" * 32)) + trie_set(trie, Bytes(b"\x50\xff"), Bytes32(b"\x02" * 32)) + trie_set(trie, Bytes(b"\x00"), Bytes32(b"\x03" * 32)) + + with pytest.raises(AssertionError): + root(trie) + + +def random_entries(rng: random.Random) -> Dict[bytes, bytes]: + """ + Generate a random key/value set mixing three key shapes: fully + random keys, keys sharing an existing 31-byte prefix, and keys + sharing a shorter prefix (forcing splits at every depth). + """ + entries: Dict[bytes, bytes] = {} + for _ in range(rng.randrange(1, 40)): + key = rng.randbytes(32) + entries[key] = rng.randbytes(32) + + # Same first 31 bytes, different final byte: long shared + # prefixes carried by one branch. + for _ in range(rng.randrange(0, 3)): + entries[key[:31] + rng.randbytes(1)] = rng.randbytes(32) + + # Same first 1, 7, or 30 bytes: splits 8, 56, or 240 bits + # deep. + for prefix_length in (1, 7, 30): + if rng.random() < 0.2: + cousin = ( + key[:prefix_length] + + rng.randbytes(31 - prefix_length) + + rng.randbytes(1) + ) + entries[cousin] = rng.randbytes(32) + return entries + + +def test_root_matches_reference_implementation() -> None: + """ + Randomized key/value sets produce the same root in the spec-style + rebuild and the insertion-based reference. + """ + rng = random.Random(8297) + + for trial in range(20): + entries = random_entries(rng) + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, Bytes(key), Bytes32(value)) + + assert root(trie) == reference.merkelize(), f"trial {trial}" + + +def test_root_matches_reference_with_variable_length_keys() -> None: + """ + Keys shaped like the embedding's (34-byte account and code keys, + 66-byte storage keys) produce the same root in both + implementations when mixed in one tree. + """ + rng = random.Random(11832) + + for trial in range(10): + entries: Dict[bytes, bytes] = {} + for _ in range(rng.randrange(1, 30)): + if rng.random() < 0.5: + # Account or code zone: 34-byte keys. + prefix = bytes([rng.choice((0, 1))]) + rng.randbytes(32) + else: + # Storage zone: 66-byte keys. + prefix = b"\xff" + rng.randbytes(64) + for _ in range(rng.randrange(1, 4)): + entries[prefix + rng.randbytes(1)] = rng.randbytes(32) + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, Bytes(key), Bytes32(value)) + + assert root(trie) == reference.merkelize(), f"trial {trial}" + + +def test_root_is_insertion_order_independent() -> None: + """ + The root depends only on the contents, not insertion order. + """ + rng = random.Random(1234) + entries = [ + (Bytes32(rng.randbytes(32)), Bytes32(rng.randbytes(32))) + for _ in range(16) + ] + + forward = BinaryTrie() + for key, value in entries: + trie_set(forward, key, value) + + backward = BinaryTrie() + for key, value in reversed(entries): + trie_set(backward, key, value) + + assert root(forward) == root(backward) + + +def test_reference_roots_are_insertion_order_independent() -> None: + """ + The insertion-based reference converges to the same canonical + structure whatever order keys arrive in. + + The rebuild-based spec is order-independent trivially; for the + incremental reference it is the canonicity property that splits + happening in different sequences must produce one structure. + """ + rng = random.Random(3102) + + for trial in range(10): + entries = list(random_entries(rng).items()) + + trie = BinaryTrie() + for key, value in entries: + trie_set(trie, Bytes(key), Bytes32(value)) + expected = root(trie) + + for _ in range(3): + rng.shuffle(entries) + reference = IncrementalRadixTree() + for key, value in entries: + reference.insert(key, value) + assert reference.merkelize() == expected, f"trial {trial}" + + +def test_setting_none_removes_key_and_restores_prior_root() -> None: + """ + Setting a key to `None` removes it from the mapping and returns + the commitment to what it was before the key was inserted. + """ + trie = BinaryTrie() + kept_key = Bytes32(b"\x01" * 32) + doomed_key = Bytes32(b"\x02" * 32) + value = Bytes32(b"\x03" * 32) + + trie_set(trie, kept_key, value) + before = root(trie) + + trie_set(trie, doomed_key, value) + assert root(trie) != before + trie_set(trie, doomed_key, None) + + assert trie_get(trie, doomed_key) is None + assert trie_get(trie, kept_key) == value + assert root(trie) == before + + +def test_setting_none_for_absent_key_is_a_no_op() -> None: + """ + Setting `None` for a key that is not in the trie leaves the + commitment unchanged, including on an empty trie. + """ + absent = Bytes32(b"\x0a" * 32) + + trie = BinaryTrie() + trie_set(trie, absent, None) + assert root(trie) == EMPTY_TRIE_ROOT + + trie_set(trie, Bytes32(b"\x0b" * 32), Bytes32(b"\x01" * 32)) + before = root(trie) + trie_set(trie, absent, None) + assert root(trie) == before + + +def test_delete_collapses_branches_to_canonical_form() -> None: + """ + Deleting keys from the three-key canonical example collapses the + branches they held open: the commitment equals that of a trie + that never held the deleted key, in both implementations. + + Deleting `key_1` collapses a two-leaf branch to its surviving + leaf; deleting `key_128` merges the top branch's prefix, its + split bit, and the low branch's prefix into one run. + """ + stem = b"\xff" + b"\xab" * 32 + key_0 = Bytes(stem + b"\x00") + key_1 = Bytes(stem + b"\x01") + key_128 = Bytes(stem + b"\x80") + value = Bytes32(b"\x44" * 32) + + for doomed in (key_1, key_128): + trie = BinaryTrie() + reference = IncrementalRadixTree() + for key in (key_0, key_1, key_128): + trie_set(trie, key, value) + reference.insert(key, value) + trie_set(trie, doomed, None) + reference.delete(doomed) + + fresh = BinaryTrie() + for key in (key_0, key_1, key_128): + if key != doomed: + trie_set(fresh, key, value) + + assert root(trie) == root(fresh), f"deleting {doomed.hex()}" + assert reference.merkelize() == root(fresh), f"deleting {doomed.hex()}" + + +def test_delete_matches_reference_and_rebuild() -> None: + """ + Deleting a random subset of keys leaves both implementations at + the root of a trie that only ever held the survivors. + """ + rng = random.Random(4242) + + for trial in range(20): + entries = random_entries(rng) + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, Bytes(key), Bytes32(value)) + + doomed = {key for key in entries if rng.random() < 0.5} + for key in doomed: + reference.delete(key) + trie_set(trie, Bytes(key), None) + + survivors = BinaryTrie() + for key, value in entries.items(): + if key not in doomed: + trie_set(survivors, Bytes(key), Bytes32(value)) + + expected = root(survivors) + assert root(trie) == expected, f"trial {trial}" + assert reference.merkelize() == expected, f"trial {trial}" + + +def test_deleting_every_key_recommits_to_the_empty_root() -> None: + """ + Removing every key, in an order unrelated to insertion, brings + both implementations back to the empty-trie commitment. + """ + rng = random.Random(97) + entries = random_entries(rng) + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, Bytes(key), Bytes32(value)) + + keys = list(entries) + rng.shuffle(keys) + for key in keys: + reference.delete(key) + trie_set(trie, Bytes(key), None) + + assert root(trie) == EMPTY_TRIE_ROOT + assert reference.merkelize() == EMPTY_TRIE_ROOT + + +def test_delete_then_reinsert_roundtrips() -> None: + """ + Deleting a key and reinserting it with the same value restores + the original commitment in both implementations. + """ + rng = random.Random(515) + entries = random_entries(rng) + doomed, doomed_value = next(iter(entries.items())) + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, Bytes(key), Bytes32(value)) + before = root(trie) + assert reference.merkelize() == before + + reference.delete(doomed) + trie_set(trie, Bytes(doomed), None) + assert root(trie) != before + + reference.insert(doomed, doomed_value) + trie_set(trie, Bytes(doomed), Bytes32(doomed_value)) + assert root(trie) == before + assert reference.merkelize() == before + + +def test_overwriting_a_value_recommits_to_the_final_value() -> None: + """ + Overwriting a key's value changes the root and matches a trie + that only ever held the final value; the insertion-based + reference reaches the same root through its equal-key path. + """ + stem = b"\x00" + b"\x42" * 32 + key = Bytes(stem + b"\x07") + neighbour = Bytes(stem + b"\x08") + first = Bytes32(b"\x01" * 32) + second = Bytes32(b"\x02" * 32) + + overwritten = BinaryTrie() + trie_set(overwritten, key, first) + trie_set(overwritten, neighbour, first) + old_root = root(overwritten) + trie_set(overwritten, key, second) + + fresh = BinaryTrie() + trie_set(fresh, key, second) + trie_set(fresh, neighbour, first) + + reference = IncrementalRadixTree() + reference.insert(key, first) + reference.insert(neighbour, first) + reference.insert(key, second) + + assert root(overwritten) != old_root + assert root(overwritten) == root(fresh) + assert reference.merkelize() == root(fresh) + + +def test_remove_subtree_removes_exactly_the_matching_keys() -> None: + """ + Removing a subtree drops every key under its prefix and nothing + else, leaving the commitment of a trie those keys never entered. + Keys that merely share a shorter prefix survive. + """ + doomed = [Bytes(b"\xff\xaa" + bytes([i]) + b"\x00" * 4) for i in range(5)] + spared = [ + Bytes(b"\xff\xab\x00" + b"\x00" * 4), # diverges in byte 1 + Bytes(b"\x00\xaa\x00" + b"\x00" * 4), # diverges in byte 0 + ] + value = Bytes32(b"\x07" * 32) + + survivors = BinaryTrie() + for key in spared: + trie_set(survivors, key, value) + + trie = copy_trie(survivors) + for key in doomed: + trie_set(trie, key, value) + assert root(trie) != root(survivors) + + remove_subtree(trie, Bytes(b"\xff\xaa")) + + assert trie._data == survivors._data + assert root(trie) == root(survivors) + + +def test_remove_subtree_of_an_absent_prefix_does_nothing() -> None: + """ + A prefix matching no key leaves the trie untouched. + """ + trie = BinaryTrie() + trie_set(trie, Bytes(b"\x01\x02\x03"), Bytes32(b"\x07" * 32)) + before = root(trie) + + remove_subtree(trie, Bytes(b"\x09")) + + assert root(trie) == before + + +def test_leaf_preimage_golden_vector() -> None: + """ + A single-leaf trie's root is the direct BLAKE3 hash of the leaf + tag, key, and value. + + Reconstructed here by calling `blake3` directly, independent of + `merkleize`, `blake3_hash`, and the incremental reference, to pin + the documented leaf preimage layout on its own. + """ + key = Bytes(b"\x00" + b"\xab" * 33) + value = Bytes32(b"\x99" * 32) + + trie = BinaryTrie() + trie_set(trie, key, value) + + assert root(trie) == blake3(b"\x00" + key + value).digest() + + +def test_branch_preimage_golden_vector() -> None: + """ + A two-leaf branch's root is the direct BLAKE3 hash of the branch + tag, a hand-packed prefix encoding, and the two leaf hashes. + + The 33-byte shared stem is exactly 264 bits, a byte boundary, so + its most-significant-bit-first packing is the stem's own bytes; + only the 2-byte big-endian bit count (0x0108) is prepended by + hand, without calling `encode_bit_prefix`. + """ + stem = b"\x11" * 33 + low_key = Bytes(stem + b"\x00") + high_key = Bytes(stem + b"\x80") + low_value = Bytes32(b"\x22" * 32) + high_value = Bytes32(b"\x33" * 32) + + trie = BinaryTrie() + trie_set(trie, low_key, low_value) + trie_set(trie, high_key, high_value) + + prefix_encoding = b"\x01\x08" + stem + leaf_lo = blake3(b"\x00" + low_key + low_value).digest() + leaf_hi = blake3(b"\x00" + high_key + high_value).digest() + expected = blake3(b"\x01" + prefix_encoding + leaf_lo + leaf_hi).digest() + + assert root(trie) == expected + + +def test_fixed_trie_root_is_pinned() -> None: + """ + A small fixed trie spanning the embedding's three key shapes + commits to a hardcoded root hash. + + A deliberate change-detector for the hash function, node tags, + and prefix encoding: the EIP's hash choice is not yet final, and + this test is meant to fail loudly the moment any of them change. + To regenerate after a deliberate, reviewed change: print + `root(trie).hex()` for this same trie and paste the new value + below. + """ + key_a = Bytes(b"\x00" + b"\x11" * 33) # 34-byte key, 0x00 zone + key_b = Bytes(b"\x01" + b"\x22" * 33) # 34-byte key, 0x01 zone + key_c = Bytes(b"\xff" + b"\x33" * 65) # 66-byte key, 0xff zone + + trie = BinaryTrie() + trie_set(trie, key_a, Bytes32(b"\xaa" * 32)) + trie_set(trie, key_b, Bytes32(b"\xbb" * 32)) + trie_set(trie, key_c, Bytes32(b"\xcc" * 32)) + + assert root(trie) == bytes.fromhex( + "580244b78611bbadd6b4b743bb973a6e4d7bcce6458a39450fc51d552437ec5a" + ) + + +def test_leaf_and_branch_tags_are_domain_separated() -> None: + """ + Leaf and branch nodes hash under different, fixed tag bytes, so + the same payload can never collide between the two node types. + """ + assert LEAF_NODE_TAG == b"\x00" + assert BRANCH_NODE_TAG == b"\x01" + + payload = b"\x99" * 40 + assert ( + blake3(LEAF_NODE_TAG + payload).digest() + != blake3(BRANCH_NODE_TAG + payload).digest() + ) + + +def test_max_length_keys_diverging_at_last_bit() -> None: + """ + Two maximum-length (8192-byte) keys differing only in their final + bit are accepted and pin the branch preimage at the deepest + prefix the encoding can represent. + """ + low_key = Bytes(b"\xab" * 8191 + b"\xfe") + high_key = Bytes(b"\xab" * 8191 + b"\xff") + low_value = Bytes32(b"\x01" * 32) + high_value = Bytes32(b"\x02" * 32) + + trie = BinaryTrie() + trie_set(trie, low_key, low_value) + trie_set(trie, high_key, high_value) + + shared_bits = bytes_to_bit_list(low_key)[:-1] + assert len(shared_bits) == 65535 + + expected = blake3( + b"\x01" + + encode_bit_prefix(Bytes(shared_bits)) + + _leaf_hash(low_key, low_value) + + _leaf_hash(high_key, high_value) + ).digest() + assert root(trie) == expected + + # Exact-boundary accept side of the two-byte count field; the + # 65536 reject already exists in + # test_encode_bit_prefix_rejects_unrepresentable_counts. + assert encode_bit_prefix(Bytes(bytes(2**16 - 1)))[:2] == b"\xff\xff" + + +def test_trie_set_rejects_zero_and_thirty_three_byte_values() -> None: + """ + Values shorter or longer than 32 bytes are rejected. + """ + trie = BinaryTrie() + with pytest.raises(AssertionError): + trie_set( + trie, + Bytes(b"\x01"), + Bytes(b""), # type: ignore[arg-type] + ) + with pytest.raises(AssertionError): + trie_set( + trie, + Bytes(b"\x02"), + Bytes(b"\x03" * 33), # type: ignore[arg-type] + ) + + +def test_root_is_idempotent_and_does_not_mutate() -> None: + """ + Computing the root does not mutate the trie's stored entries, and + repeated calls return the same value. + """ + rng = random.Random(42) + trie = BinaryTrie() + for _ in range(5): + trie_set(trie, Bytes(rng.randbytes(32)), Bytes32(rng.randbytes(32))) + + snapshot = dict(trie._data) + first = root(trie) + second = root(trie) + + assert first == second + assert trie._data == snapshot + + +def test_prefix_violation_only_fails_at_root_time() -> None: + """ + A prefix-violating pair of keys is accepted and stays readable + through ordinary `trie_set`/`trie_get` calls; prefix-freeness is + only enforced lazily, when `root` walks the tree. + + That is exactly where EIP-8297 places the rejection: "Computing + the root rejects keys that violate either constraint", and + `state_root` in the spec's pseudocode is where the checks run. + So this pins conformance, not a divergence: a pair of writes the + spec's tree could never commit to stays observable through the + map interface but can produce no root. + """ + prefix_key = Bytes(b"\x50" * 34) + extended_key = Bytes(b"\x50" * 34 + b"\x60") + prefix_value = Bytes32(b"\x01" * 32) + extended_value = Bytes32(b"\x02" * 32) + + trie = BinaryTrie() + trie_set(trie, prefix_key, prefix_value) + trie_set(trie, extended_key, extended_value) + + assert trie_get(trie, prefix_key) == prefix_value + assert trie_get(trie, extended_key) == extended_value + + with pytest.raises(AssertionError): + root(trie) + + +def assert_canonical_structure( + node: BinaryNode, keys: Set[Bytes], depth: Uint +) -> None: + """ + Check that `node` is the canonical `binarize` encoding of `keys`, + whose members all share their first `depth` bits. + + Recomputes each branch's split independently from `keys`, rather + than trusting the node's own claimed prefix: both subtrees must + be non-empty (what makes the prefix maximal), the prefix must be + exactly the run all keys share from `depth`, and every leaf's key + must be one of `keys`, on the path its own bits take. + """ + if len(keys) == 1: + assert isinstance(node, LeafNode) + assert node.key in keys + return + + assert isinstance(node, BranchNode) + bit_lists = {key: bytes_to_bit_list(key) for key in keys} + + for offset, prefix_bit in enumerate(node.prefix): + position = depth + Uint(offset) + for bits in bit_lists.values(): + assert bits[position] == prefix_bit + + split = depth + Uint(len(node.prefix)) + left_keys = {key for key in keys if bit_lists[key][split] == 0} + right_keys = {key for key in keys if bit_lists[key][split] == 1} + + assert len(left_keys) > 0 + assert len(right_keys) > 0 + assert isinstance(node.left, (BranchNode, LeafNode)) + assert isinstance(node.right, (BranchNode, LeafNode)) + + assert_canonical_structure(node.left, left_keys, split + Uint(1)) + assert_canonical_structure(node.right, right_keys, split + Uint(1)) + + +def test_canonical_structure_holds_for_fixed_trie() -> None: + """ + The fixed trie from `test_fixed_trie_root_is_pinned` binarizes + into a canonical structure. + """ + entries = { + Bytes(b"\x00" + b"\x11" * 33): Bytes32(b"\xaa" * 32), + Bytes(b"\x01" + b"\x22" * 33): Bytes32(b"\xbb" * 32), + Bytes(b"\xff" + b"\x33" * 65): Bytes32(b"\xcc" * 32), + } + + top = binarize(entries, Uint(0)) + + assert_canonical_structure(top, set(entries.keys()), Uint(0)) + + +def test_canonical_structure_holds_for_random_corpora() -> None: + """ + `binarize` produces a canonical structure over many random + key/value corpora, not merely the same root as the reference. + """ + rng = random.Random(90210) + + for trial in range(10): + entries = { + Bytes(key): Bytes32(value) + for key, value in random_entries(rng).items() + } + + top = binarize(entries, Uint(0)) + try: + assert_canonical_structure(top, set(entries.keys()), Uint(0)) + except AssertionError as exc: + raise AssertionError(f"trial {trial}") from exc + + +def _thermometer_key(ones: int, total_bits: int) -> bytes: + """ + Build the `total_bits`-bit key whose first `ones` bits are one + and the remainder are zero, most significant bit first. + """ + value = (2**ones - 1) << (total_bits - ones) + return value.to_bytes(total_bits // 8, "big") + + +def test_deep_thermometer_chain_matches_reference() -> None: + """ + A "thermometer" key set, one 66-byte key per possible run length + of leading one-bits, forces the deepest branch chain equal-length + keys can produce -- 528 levels, each splitting off the next key + one level below the last. The rebuild-based spec and the + insertion-based reference agree on its root. + """ + total_bits = 8 * 66 + entries = { + Bytes(_thermometer_key(ones, total_bits)): Bytes32( + bytes([ones % 256]) * 32 + ) + for ones in range(total_bits + 1) + } + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, key, value) + + assert root(trie) == reference.merkelize() + + +def test_deep_chain_past_recursion_limit_is_an_implementation_limit() -> None: + """ + A branch chain deeper than Python's recursion limit overflows + `root`'s recursive walk with `RecursionError`, even though + nothing about the chain violates the tree's rules. + + The limit is lowered here only to make the failure cheap to + trigger. At the interpreter's default limit (raised to 12288 by + `ethereum/__init__.py`), the same failure is reachable with legal + maximum-length (8192-byte) keys, whose shared-prefix chains can + run up to 65535 branches deep; this is therefore a limit of this + reference implementation, not a rule the specification imposes. + """ + total_bits = 8 * 34 + entries = { + Bytes(_thermometer_key(ones, total_bits)): Bytes32(b"\x01" * 32) + for ones in range(260) + } + + trie = BinaryTrie() + for key, value in entries.items(): + trie_set(trie, key, value) + + old_limit = sys.getrecursionlimit() + # 260 branches is already deep enough at this lowered limit; see + # the docstring for why the same failure applies to legal inputs + # at the default limit. + sys.setrecursionlimit(200) + try: + with pytest.raises(RecursionError): + root(trie) + finally: + sys.setrecursionlimit(old_limit) + + +def _large_variable_length_entries( + rng: random.Random, count: int +) -> Dict[bytes, bytes]: + """ + Generate `count` entries mixing 34-byte account/code-zone keys + and 66-byte storage-zone keys, plus a handful of 200-byte keys in + a fourth zone. + + Every zone uses a fixed, disjoint leading byte, so no key can + ever be a prefix of one from another zone regardless of the + random bytes that follow. + """ + entries: Dict[bytes, bytes] = {} + long_key_count = min(8, count // 20) + for _ in range(long_key_count): + entries[b"\x02" + rng.randbytes(199)] = rng.randbytes(32) + while len(entries) < count: + if rng.random() < 0.5: + key = bytes([rng.choice((0, 1))]) + rng.randbytes(33) + else: + key = b"\xff" + rng.randbytes(65) + entries[key] = rng.randbytes(32) + return entries + + +@pytest.mark.parametrize("seed", [20260727, 5551212, 918273645]) +def test_larger_random_corpora_match_reference(seed: int) -> None: + """ + Larger, more varied corpora of 300-800 entries spanning three key + lengths still agree between the rebuild-based spec and the + insertion-based reference. + """ + rng = random.Random(seed) + count = rng.randrange(300, 801) + entries = _large_variable_length_entries(rng, count) + + reference = IncrementalRadixTree() + trie = BinaryTrie() + for key, value in entries.items(): + reference.insert(key, value) + trie_set(trie, Bytes(key), Bytes32(value)) + + assert root(trie) == reference.merkelize() diff --git a/uv.lock b/uv.lock index c00273eacbf..fc75a9712ce 100644 --- a/uv.lock +++ b/uv.lock @@ -113,6 +113,82 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/fe/3aed5d0be4d404d12d36ab97e2f1791424d9ca39c2f754a6285d59a3b01d/beautifulsoup4-4.14.2-py3-none-any.whl", hash = "sha256:5ef6fa3a8cbece8488d66985560f97ed091e22bbc4e9c2338508a9d5de6d4515", size = 106392, upload-time = "2025-09-29T10:05:43.771Z" }, ] +[[package]] +name = "blake3" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/6a/4cc5a9dd40fd8a6d283fd3761e5f59c490109571ef8e3c73245417e5a305/blake3-1.0.9.tar.gz", hash = "sha256:5fa374fa5070ca084368776c19b420157eb0f2d3f091343d6bc59189929d62e2", size = 116872, upload-time = "2026-06-22T18:02:25.366Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/12/aa8d72228b6ff61c675bd6f55ab138a91d71499c8a707cc9fb2052f1d2b5/blake3-1.0.9-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f169519c7ef25ef2c446b05e2f08e7e59fae312d569f98a3134b38d4caf7abd4", size = 346253, upload-time = "2026-06-22T18:00:15.537Z" }, + { url = "https://files.pythonhosted.org/packages/72/3a/820d2f729dfe152d5ebde16390f808c762dce3f21fb764ab033803ff2b1a/blake3-1.0.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5e1f21b49492d01fa5a02084894c491ab9e7a1867fced107f7126c80d067c94", size = 335497, upload-time = "2026-06-22T18:00:16.942Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d6/d5462ec19a7f3d084fe327e08618fa107799ee708df04b3a2d620bd62816/blake3-1.0.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ee96daaa850700fd342a811fa10a8780fd2e8464a71b83a1779c7b6becd3dd5", size = 377621, upload-time = "2026-06-22T18:00:18.389Z" }, + { url = "https://files.pythonhosted.org/packages/92/98/dbc433f2a45be1b2344a6035d4212dfb6e6eb45046ad15103ead9c82d491/blake3-1.0.9-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:09deb024cd75cb200e7f647cd038800e6edc8f190c8188e0c69ec1c2b920e125", size = 377495, upload-time = "2026-06-22T18:00:20.067Z" }, + { url = "https://files.pythonhosted.org/packages/e0/3d/c7a699fb60d8ed31f3f28e6aec7658d29e45ec89e7054906b3040ce3ee65/blake3-1.0.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c99afb0459c82dd13e456b6b68d45c4768b539ca998dacd3ed726f1e75e91dc", size = 451158, upload-time = "2026-06-22T18:00:21.459Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a1/0b1b0dbf2dd772483e372237bb65385602b019e24b67424b1fc9e5447837/blake3-1.0.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28528d1f29e6f3d45faf3482e1197e5e175730eef38bdc74e56ee11b68e0ad0d", size = 491988, upload-time = "2026-06-22T18:00:22.984Z" }, + { url = "https://files.pythonhosted.org/packages/ee/d1/ed319477f6d263a4f6b7e9aa465b06be5235a854923edbc9ea09508b6638/blake3-1.0.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65c0c20014df687694af5ccf0cec3bdb194511da8ebd50c30b0fd55c83fa4fd5", size = 386848, upload-time = "2026-06-22T18:00:24.319Z" }, + { url = "https://files.pythonhosted.org/packages/80/3e/a4cfb269f3e0955598b415a7843c358c4f79e826e3c9118dc9fb1f101ee6/blake3-1.0.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:964b642631a3c8fe117b3439c8ae64a9a0981af9444e409656d1f1e464bfa125", size = 387842, upload-time = "2026-06-22T18:00:25.589Z" }, + { url = "https://files.pythonhosted.org/packages/59/0e/d4ee3d89eece42f86eb46663aa42702000516b7ffbc53f60b918efe95b57/blake3-1.0.9-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2fd000708662b04be211a22c1095b65fe399d7276e9f3bb2fd1ef8aacc545791", size = 384317, upload-time = "2026-06-22T18:00:26.891Z" }, + { url = "https://files.pythonhosted.org/packages/3a/aa/317106349d10de3b51332ad1e761f4864ebe887854396b75975304dcfbd1/blake3-1.0.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:82ecade6ac425fdfc39a4371d6d9232fd6e5c28748fd8d3489016ead17407014", size = 553005, upload-time = "2026-06-22T18:00:28.246Z" }, + { url = "https://files.pythonhosted.org/packages/39/cc/7fbce61a0b24bda1aac99da674bd74ac2b687b61db071c888ffdb30cb47a/blake3-1.0.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b4102ba86b86c992a931b4a88c58a632d6097461e14a1e63ebd2ecb98ff0898f", size = 595086, upload-time = "2026-06-22T18:00:29.96Z" }, + { url = "https://files.pythonhosted.org/packages/e6/91/6ddc7a8b582a0871f23d6db722f4950a8918096d5fa10f9f0f992c2aea39/blake3-1.0.9-cp311-cp311-win32.whl", hash = "sha256:2f4ce45da903f3d0a7e342fa70c7cce9c10cef6b529eadb4d6213be0ab0eaf84", size = 231230, upload-time = "2026-06-22T18:00:31.247Z" }, + { url = "https://files.pythonhosted.org/packages/23/68/ea698e6df48eeb417671544cfbb18c60f863cb689306cc52f19666dd98f8/blake3-1.0.9-cp311-cp311-win_amd64.whl", hash = "sha256:d819457dccfd82fe34684ec99e36725f747bd5761a0e17f537387fb31d121193", size = 220622, upload-time = "2026-06-22T18:00:32.495Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/9bdf8345c70993aaef635398f52edfb915d6e8ad2c000c801204e387c456/blake3-1.0.9-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a70c20542d5e7960983a0ff32999049a2b0e5ef1f22dbbbdfb51cf04828a4156", size = 344587, upload-time = "2026-06-22T18:00:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/36/9d/be8b1f7f85b12bb45a0fade6ca7bdbf83a507d23d0b6141ba29fe69c8cea/blake3-1.0.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:72cdecf088a9d25e6ec79948a578995649b0dbee407e7a46c543a9ecc0f6f281", size = 328864, upload-time = "2026-06-22T18:00:35.59Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/66580635d744c826671fd219938caffb16281a26f62c4f856695d4233677/blake3-1.0.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42fa57bf462285ef16400601b0fd32214c248ba92505bbb94b1221ab9af5a092", size = 373795, upload-time = "2026-06-22T18:00:36.887Z" }, + { url = "https://files.pythonhosted.org/packages/b1/79/b5b17d3004bb81a5732c0b176c812703d200ed8c652b3b7713b9633bbe10/blake3-1.0.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b25ccde5a64be070f20e5c7a81da70292db40b164b6c77588cbd6230856badbb", size = 374183, upload-time = "2026-06-22T18:00:38.205Z" }, + { url = "https://files.pythonhosted.org/packages/3c/63/0d209c44b2041bbe130ced12a23c92dd995fbfe5bce7ee77fffea16f5cb0/blake3-1.0.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a800b87433955f37691b5f361ad29c7dd3ee089c9cd109adc5aea8e24bc4c1f", size = 446783, upload-time = "2026-06-22T18:00:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/c5/51/efd1f9b8a9d3e9a0e235f3ced99a738529a1019fe78b3988e29d9c2fbba6/blake3-1.0.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6879739e7904b9c42afbedbcc2e8c36cebe140fb3fc3f5c492993579cf5cd516", size = 487369, upload-time = "2026-06-22T18:00:40.875Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3f/a8dcaea9e0b26e419a540ca0cd6203c9fbb505e85b02b03c5a59bf9e6a45/blake3-1.0.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6edeb3d49a24c307995899b70dd47aa901d0e9ad51d2f8a79aba4f074f32d8c5", size = 383845, upload-time = "2026-06-22T18:00:42.251Z" }, + { url = "https://files.pythonhosted.org/packages/f6/10/e9907f5b86410d5071982aaf05d149ca4d4fd8acab7e77eebbc9a333c7b4/blake3-1.0.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd56a7a972c4185070f7042ccc20166927eec3c0f98b8405f375d007b604a0b", size = 383851, upload-time = "2026-06-22T18:00:43.715Z" }, + { url = "https://files.pythonhosted.org/packages/34/cf/c7863a185550706a9624f6aa7b6d46470aaed0bb46a827c5cda2a7d03151/blake3-1.0.9-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:a288664d08dee154cc496e06e62517fc9e655ecec12b0d7db538d244ac79edf1", size = 380067, upload-time = "2026-06-22T18:00:45.249Z" }, + { url = "https://files.pythonhosted.org/packages/54/0a/e7af679c719368b400c9ba9c3460072aac2ba077ddbd4bc806fef28cda03/blake3-1.0.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:91db52a809b68b5bebe7c413ddcd230e1f759398e7fa7a873104595a4fa648b6", size = 549471, upload-time = "2026-06-22T18:00:46.793Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3c/37c1dd3539b7bd9b6d2eef019802aacdb4a3d48ab484b140603bbf9c5b5a/blake3-1.0.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfaa671b07eb73883162ca940442193868358b0b904cfa266e4b74131ce966da", size = 591396, upload-time = "2026-06-22T18:00:48.122Z" }, + { url = "https://files.pythonhosted.org/packages/ae/55/4f0a23b72795292e74084834130900ea778c0583004519c86698dfffe1a5/blake3-1.0.9-cp312-cp312-win32.whl", hash = "sha256:ae47c3d5729ff89baa6ddf6de47fcfcc915985d39eb1bfcd6db653331f3c6fcc", size = 229271, upload-time = "2026-06-22T18:00:49.377Z" }, + { url = "https://files.pythonhosted.org/packages/12/91/7db93e4689f0f145bcb954dc62936e5f5090548a9fa20c6bbebfaeaa648a/blake3-1.0.9-cp312-cp312-win_amd64.whl", hash = "sha256:15566065ff90ab3da46ec0be1417406f00507af902b6fb0fbc6563e77f02fc42", size = 218220, upload-time = "2026-06-22T18:00:50.659Z" }, + { url = "https://files.pythonhosted.org/packages/41/1b/95b473d649f5322e69674622a307ffdb4f0b63adb0a0adcbc5cb8a8833c2/blake3-1.0.9-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:69ff5aebc7650954443aa701feff2028d7c7ea5b5e18ee265f15e2104e892328", size = 343869, upload-time = "2026-06-22T18:00:51.936Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9d/adec22c719d8451af1dc9e624bf5907008ef1e0afa51aa69fd1e8c91e60e/blake3-1.0.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0cdfeff65488089ef86f7587c76055ff72b28d28d10e427b547f5711477c376d", size = 328482, upload-time = "2026-06-22T18:00:53.39Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/0a6967ff9a6ae182419a681aed54f7338b34a1f71372e90f787a2afa42e6/blake3-1.0.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:766f1555cbe614f14f399c2fbec0983568d20edb36837ba04040807eb9e1a609", size = 373616, upload-time = "2026-06-22T18:00:54.701Z" }, + { url = "https://files.pythonhosted.org/packages/1c/51/5d4e198bf3ae902c6697ad6ec77d7210736ad8f680980e8b648dcfcd09a0/blake3-1.0.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:128a62136c9a39c7cb9fdaa5fb38471f2418853da7f5a89f31495735d0ba6f2c", size = 374149, upload-time = "2026-06-22T18:00:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/7e/62/d3c7c364925b3f10828e5137376f3947f112c32188e899b42f09c2fde98a/blake3-1.0.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1ea0bf17b184b03444007646d902207d2b4d4f3e91a0cac3836552d83db74b9", size = 446151, upload-time = "2026-06-22T18:00:57.378Z" }, + { url = "https://files.pythonhosted.org/packages/b1/01/55b89389c5036c9d24b1d762d6265e91552e10b76a3c99fece3c4a7a4783/blake3-1.0.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73a48f7e9f0e047f51a445d9b0361ab1907bdc72b6857815a84dacd2e59556f8", size = 487256, upload-time = "2026-06-22T18:00:58.763Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7a/a21b52253292ad3e4df63ea4a01ce11d3ee8f4a8a8d80eaf0c7ce92a62bd/blake3-1.0.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b27550ada40f839aca64c66127940e4318bb6ef3e291890ef913017f6f637448", size = 383977, upload-time = "2026-06-22T18:01:00.192Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/fe7188201a29ee9b042616c786a98afd864d537ca96198e64c3fe4ff13a9/blake3-1.0.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66c84dbc2a31eda88b55bbf5c5b711037bf0698eba0fd1faf06bdaf313c39048", size = 383615, upload-time = "2026-06-22T18:01:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/22/08/f6a213b950e30fe9ef7d7fc061ec388e66ed62643570226882e6f7136ea3/blake3-1.0.9-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:dab59b324aa65c09e937d6c43de5de85ec9581627f4e79dcc9806d85b54a1c34", size = 380288, upload-time = "2026-06-22T18:01:03.025Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/b171e47c1b835483bcf1545ebc289458165f8dc0f5c7f74a9176d7e9af03/blake3-1.0.9-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:eca281fedcbe5c56655bd5a4176e6036eddbbe57df96114a03838fce08b1e0ca", size = 549122, upload-time = "2026-06-22T18:01:04.486Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/7bf71c2c85a0951e406971f151435e0751716907e3924c6c48a2d6dae0db/blake3-1.0.9-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3cbe7f190164896dc3908e920716ee66bc31d40f1a0fb603ed59ac53290fb9cf", size = 591183, upload-time = "2026-06-22T18:01:06.259Z" }, + { url = "https://files.pythonhosted.org/packages/20/85/34c3ea03cc90b2516628494ab3e0a98aec4ca8b04d037840ccd390e480ca/blake3-1.0.9-cp313-cp313-win32.whl", hash = "sha256:508ccaf8f9377cc47e6026c2897fdc37de61faeb1420dc023b6379cc2474eb65", size = 229053, upload-time = "2026-06-22T18:01:07.638Z" }, + { url = "https://files.pythonhosted.org/packages/db/2e/f09e8ed426f360aa2005206466ceab2f707486eb5d9db7051dbcbae056d1/blake3-1.0.9-cp313-cp313-win_amd64.whl", hash = "sha256:caded2806d2cbeed638c5e2517ed8b2a94165b3452fda35e72896142d22070e0", size = 217589, upload-time = "2026-06-22T18:01:08.922Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4b/b2dd7c25378a3b5de30ed908d38e6427bc4c644c0c12e8359361abd3a9ca/blake3-1.0.9-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:ab0c030cf6644c30e786b0e785bde4e4596013ae9ea6ce9877e39d52383e25d7", size = 345406, upload-time = "2026-06-22T18:01:10.311Z" }, + { url = "https://files.pythonhosted.org/packages/c6/dc/c0dab2963ddf04a4a938363f61716f9b75de6d3a9bc4a89e78f0854d4d31/blake3-1.0.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:83b4a2336105af3800f7e17ac4b943f293a3927a2d66a6308d50dba944a6953e", size = 330077, upload-time = "2026-06-22T18:01:11.926Z" }, + { url = "https://files.pythonhosted.org/packages/20/f1/d03950a86d105a6332a8c422cb87658a7d247e214f1ea8f29ed09ff04e00/blake3-1.0.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95fc3545f80901b0dcd0508d16bc40f15ae39556709fa6cf86675f742d4f3c9c", size = 375147, upload-time = "2026-06-22T18:01:13.198Z" }, + { url = "https://files.pythonhosted.org/packages/10/75/711b1842e0a90aaad6a1c9a9022e90aa16206ac1f224516118bc24482532/blake3-1.0.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1bd981dc318c05375c3160a99df493b7cc4c83fffa1a34d14b18a071b47b262b", size = 373711, upload-time = "2026-06-22T18:01:14.606Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a0/f512799d1d0c0b4718fa6f0e99ccbe108e98bac7bf82c200803a62b57876/blake3-1.0.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:689a7e4069de681d9c5d9445b8b6473ee880ad04d7960a6789c60bd788980250", size = 446993, upload-time = "2026-06-22T18:01:15.924Z" }, + { url = "https://files.pythonhosted.org/packages/60/fb/6636ae8a46fc3352694188f5a5a325567782bc88fd1823b0b67be2c92184/blake3-1.0.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8adb0b0032e53919ee95b3d4f911448d3268316c28cd7df232ff2a1e7c9a4ba4", size = 488478, upload-time = "2026-06-22T18:01:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c5/a2b3c086f7e37c9db6017dc2890a76ad2a729e4a554896e855e511811e6b/blake3-1.0.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32bd4521ec2d477627ad93eb70f9ac4d01e12d1489024159bcaeff79466332f6", size = 384900, upload-time = "2026-06-22T18:01:18.802Z" }, + { url = "https://files.pythonhosted.org/packages/e1/b8/1298806dd6c464a6f807df24c9640ad3bf27ee54ff4de82b2b5a823a8aba/blake3-1.0.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65d77eb05331495485048f6804f53885b192b998acb7e6fe1487d941bf08435", size = 384333, upload-time = "2026-06-22T18:01:20.35Z" }, + { url = "https://files.pythonhosted.org/packages/3c/cc/0c29d9404155adfd6db716e9765d36ea6cbed287060759f5d764f0d9d99e/blake3-1.0.9-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:ca7dfe8fb197ff8a3f5c915424183ccd52a99e8afb12680f51b2e1f4c9c6c97f", size = 381142, upload-time = "2026-06-22T18:01:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/d6/91/9af20d563f0ced71e08a60fc0ee534146da4e265710ed6792d5d799f4c0f/blake3-1.0.9-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:f5c9d57f0dcb92243b6ae575c3065793edc9df9008d0ebd98d8245cdeb7c3f84", size = 550587, upload-time = "2026-06-22T18:01:23.381Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fa/06f46fc0aa486b799d776f9a80ed0b3605e2be1570cf48007860948aa5d9/blake3-1.0.9-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:172d44245a19dfec08ab771c1b7a506b97783163cdc65f559fe020007e403c99", size = 591888, upload-time = "2026-06-22T18:01:24.805Z" }, + { url = "https://files.pythonhosted.org/packages/50/68/d6198f4069a7c4a184ed854df45b82cc3e2d4b0be476b2a3ee65ad2344cf/blake3-1.0.9-cp314-cp314-win32.whl", hash = "sha256:249e5964fa9e768924bc7cc3d4efe75a425bb5dd3fb7671c3eda8eeddfa50591", size = 229410, upload-time = "2026-06-22T18:01:26.24Z" }, + { url = "https://files.pythonhosted.org/packages/63/ab/f29af72a8312b3827b50e55491f1bf9ae2347591de5c47365c5cbd2525a9/blake3-1.0.9-cp314-cp314-win_amd64.whl", hash = "sha256:0aba416bb2e3ef0c65e74d5eba21062483c714cd78e7e303c9d03c547fc7d015", size = 218526, upload-time = "2026-06-22T18:01:27.779Z" }, + { url = "https://files.pythonhosted.org/packages/47/7e/d932fe437ccf656cfba77abc466fb3d1a0ce3c31df92e760d9e4c34932b4/blake3-1.0.9-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5b35abe24a66a7b3db423eb4f8668ed7be1a362aa9c0024ab6483ec0b2c16058", size = 345049, upload-time = "2026-06-22T18:01:29.228Z" }, + { url = "https://files.pythonhosted.org/packages/55/1e/d92fb284fcacf86f5d1083e29d0a8c834b60432786928915238d9760f514/blake3-1.0.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bbdff61e049297ef3180867ce1f079cea7e5b372fd76953c3183da5b8124206", size = 329367, upload-time = "2026-06-22T18:01:30.566Z" }, + { url = "https://files.pythonhosted.org/packages/9d/da/e25fa75d5bfea4527fc21024dde86a9376db798e469a084741968299f215/blake3-1.0.9-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a69fcedf06785bb81d4d3d39f95ee65dbaf2cb246e174cfc9ff64d027f7551", size = 374203, upload-time = "2026-06-22T18:01:31.998Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4d/0224916202b773dfdf08dcbe4ed1ad1018d4ddcd4df7a7e2978d28f89b74/blake3-1.0.9-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5d5bf0f68cd77108a942c95db98e960d9c3d5643b95172f783822ce22667759", size = 373713, upload-time = "2026-06-22T18:01:33.387Z" }, + { url = "https://files.pythonhosted.org/packages/b8/e5/4ba968831b7afaec431c588c826cef76a96d6d6976188ed07d932072e673/blake3-1.0.9-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9767f16199b99aa022b61ff825ac4dbd39864bf637ae712605a2ce1f8b6a55e0", size = 446574, upload-time = "2026-06-22T18:01:34.687Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f5/08a9099c7177f282d2563abe4f7cc626c636642f7979cf58f2ab7ded2096/blake3-1.0.9-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4865a8cfb2b3d7c0baf5267f2fa6816a3384e836cd1bd0caf359f406cb1e8fba", size = 487232, upload-time = "2026-06-22T18:01:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/9392bf1ebc81b5b09ce58b94613fa2d37308e825ff2dc7b54d00ee622c77/blake3-1.0.9-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42609e4adc4b2d7423137f2cb35135bca598b925c5af09d2bc0a2c368b25aeb1", size = 384751, upload-time = "2026-06-22T18:01:37.512Z" }, + { url = "https://files.pythonhosted.org/packages/84/fc/b6e9aef02ca14ef62fa47783b9eeeb5b2d3f73fdf698d8bb94c36f5dd69f/blake3-1.0.9-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7f648fa425138452d1e585ac625c7aefddb946d9765906c4c12d564a1523cd8", size = 384546, upload-time = "2026-06-22T18:01:38.868Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cb/452e92dba9402b36a953aa8b9b06253445ccce43dcd0bcf521c5e3c3e15d/blake3-1.0.9-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:9cef6d4d07a7de0c44f5ba17f6383d55276d9efc8d601f75113538fcaa35008b", size = 380596, upload-time = "2026-06-22T18:01:40.412Z" }, + { url = "https://files.pythonhosted.org/packages/b2/01/7a84a7e10c5d14e6ed8a4403bd7f64c1e01f8ebabea0d6fe5f093b894cbd/blake3-1.0.9-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:28404301de485e9546365d01b30f65eaa835520c4211d6ef61242975b6722b60", size = 550032, upload-time = "2026-06-22T18:01:41.955Z" }, + { url = "https://files.pythonhosted.org/packages/58/7d/7aea0222f59cf84044ec52e2bfdaa0e3c355d221292b0ea1b722cf1edd6c/blake3-1.0.9-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:8a99f896e7718050ed033a888245098aab3d6a5338f91cc9450c563b53f90ad5", size = 592244, upload-time = "2026-06-22T18:01:43.426Z" }, + { url = "https://files.pythonhosted.org/packages/c0/e5/b44c230108745ff9c70c7bbafe22563772bc0c22322a8d15c10455f6ca02/blake3-1.0.9-cp314-cp314t-win32.whl", hash = "sha256:021309d760b390706fecf13498f9a25aa8f689bbb65a0896029b8fa223aae18b", size = 229481, upload-time = "2026-06-22T18:01:45.307Z" }, + { url = "https://files.pythonhosted.org/packages/ec/a6/ac03f37dc9aeebf398d42089720648b3bc8438e733d3e522196c5d12ab39/blake3-1.0.9-cp314-cp314t-win_amd64.whl", hash = "sha256:5ea0c60dd9c1e3d05610606579e4bf80f562854c46ed55f9ee8545e18987a480", size = 217979, upload-time = "2026-06-22T18:01:46.629Z" }, +] + [[package]] name = "bracex" version = "2.6" @@ -839,6 +915,7 @@ wheels = [ name = "ethereum-execution" source = { editable = "." } dependencies = [ + { name = "blake3" }, { name = "cryptography" }, { name = "ethereum-rlp" }, { name = "ethereum-types" }, @@ -951,6 +1028,7 @@ test = [ [package.metadata] requires-dist = [ + { name = "blake3", specifier = ">=1.0,<2" }, { name = "cryptography", specifier = ">=45.0.1,<46" }, { name = "ethash", marker = "extra == 'optimized'", specifier = ">=1.1.0,<2" }, { name = "ethereum-rlp", specifier = ">=0.1.6,<0.2" }, diff --git a/vulture_whitelist.py b/vulture_whitelist.py index b68fbccedab..da70fe7fe84 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -11,6 +11,16 @@ from ethereum.ethash import * from ethereum.fork_criteria import Unscheduled +from ethereum.partitioned_binary_tree import ( + EMPTY_CODE_HASH, + address20_to_address32, + chunkify_code, + encode_basic_data, + get_tree_key_for_basic_data, + get_tree_key_for_code_chunk, + get_tree_key_for_code_hash, + get_tree_key_for_storage_slot, +) from ethereum.trace import EvmTracer from ethereum.utils.hexadecimal import hex_to_bytes256 from ethereum_optimized.state_db import State @@ -38,6 +48,17 @@ StringReplaceCommand, ) +# src/ethereum/binary_trie/embedding.py - EIP-8297 public API, exercised +# via tests +EMPTY_CODE_HASH +address20_to_address32 +chunkify_code +encode_basic_data +get_tree_key_for_basic_data +get_tree_key_for_code_chunk +get_tree_key_for_code_hash +get_tree_key_for_storage_slot + # src/ethereum/utils/hexadecimal.py hex_to_bytes256 From 0cd49119c937db40bad9077909da71760891718e Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Fri, 7 Aug 2026 17:55:28 +0200 Subject: [PATCH 2/4] feat(test-forks): add --state-trie flag to override state commitment --- .../pytest_commands/plugins/forks/forks.py | 29 +++++ .../forks/tests/test_state_trie_option.py | 52 +++++++++ .../client_clis/tests/test_transition_tool.py | 59 ++++++++++ .../fixtures/tests/test_pre_alloc_groups.py | 103 +++++++++++++++++- .../src/execution_testing/forks/base_fork.py | 21 +++- .../forks/tests/test_forks.py | 36 +++++- 6 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_state_trie_option.py diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py index df04e8d4a14..764f4df8502 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py @@ -26,6 +26,7 @@ from _pytest.mark.structures import ParameterSet from pytest import Mark, Metafunc, StashKey +from execution_testing.base_types import StateCommitment from execution_testing.client_clis import TransitionTool from execution_testing.forks import ( ALL_FORKS, @@ -40,6 +41,7 @@ get_transition_forks, transition_fork_to, ) +from execution_testing.forks.base_fork import BaseFork from execution_testing.logging import ( get_logger, ) @@ -50,6 +52,11 @@ # (see `get_unsupported_forks`). unsupported_forks_key: StashKey[FrozenSet[Fork | TransitionFork]] = StashKey() +STATE_TRIE_CHOICES: Dict[str, StateCommitment] = { + "mpt": StateCommitment.MPT, + "pbt": StateCommitment.PBT, +} + def pytest_addoption(parser: pytest.Parser) -> None: """Add command-line options to pytest.""" @@ -84,6 +91,20 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="", help="Fill tests until and including the specified fork.", ) + fork_group.addoption( + "--state-trie", + action="store", + dest="state_trie", + default=None, + type=str.lower, + choices=list(STATE_TRIE_CHOICES), + help=( + "Override the state-commitment scheme used to compute state " + "roots for every fork: 'mpt' for the Merkle-Patricia trie, " + "'pbt' for the partitioned binary tree. By default, each " + "fork defines its own scheme." + ), + ) @dataclass(kw_only=True) @@ -569,6 +590,9 @@ def get_fork_option( forks_until = get_fork_option(config, "forks_until", "--until") show_fork_help = config.getoption("show_fork_help") + if state_trie := config.getoption("state_trie"): + BaseFork.set_state_commitment_override(STATE_TRIE_CHOICES[state_trie]) + dev_forks_help = textwrap.dedent( "To run tests for a fork under active development, it must be " "specified explicitly via --until=FORK.\n" @@ -628,6 +652,11 @@ def get_fork_option( ) +def pytest_unconfigure() -> None: + """Reset global fork state derived from command-line options.""" + BaseFork.set_state_commitment_override(None) + + def get_unsupported_forks( config: pytest.Config, ) -> FrozenSet[Fork | TransitionFork]: diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_state_trie_option.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_state_trie_option.py new file mode 100644 index 00000000000..4331dae0337 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_state_trie_option.py @@ -0,0 +1,52 @@ +""" +Test the `--state-trie` option's session wiring: the override is set +at configure time and restored at unconfigure, so nested in-process +pytest sessions cannot leak the scheme into later sessions in the +same process. +""" + +import pytest + +from execution_testing.forks.base_fork import BaseFork + +FORKS_PLUGIN = "execution_testing.cli.pytest_commands.plugins.forks.forks" + + +@pytest.mark.parametrize( + "options, expected", + [ + pytest.param((), "MPT", id="default_is_mpt"), + pytest.param(("--state-trie", "pbt"), "PBT", id="pbt_override"), + pytest.param(("--state-trie", "mpt"), "MPT", id="explicit_mpt"), + ], +) +def test_state_trie_option_sets_and_resets_the_override( + pytester: pytest.Pytester, options: tuple, expected: str +) -> None: + """ + An inner session observes the commitment the option selects, and + the override is `None` again once the session ends. + """ + pytester.makepyfile( + f""" + from execution_testing.base_types import StateCommitment + from execution_testing.forks import Amsterdam + + def test_commitment(): + assert ( + Amsterdam.state_commitment() + is StateCommitment.{expected} + ) + """ + ) + + result = pytester.runpytest( + "-p", FORKS_PLUGIN, "--fork", "Amsterdam", *options + ) + + leaked = BaseFork._state_commitment_override + # Reset regardless, so a regression cannot poison sibling tests + # running later in this process. + BaseFork.set_state_commitment_override(None) + result.assert_outcomes(passed=1) + assert leaked is None diff --git a/packages/testing/src/execution_testing/client_clis/tests/test_transition_tool.py b/packages/testing/src/execution_testing/client_clis/tests/test_transition_tool.py index 97ff0ee83cc..f4520c7241d 100644 --- a/packages/testing/src/execution_testing/client_clis/tests/test_transition_tool.py +++ b/packages/testing/src/execution_testing/client_clis/tests/test_transition_tool.py @@ -8,6 +8,11 @@ import ijson # type: ignore[import-untyped] import pytest +from ethereum.merkle_patricia_trie import Trie +from ethereum.state import EMPTY_CODE_HASH, Account, Address, BlockDiff +from ethereum_spec_tools.evm_tools.t8n.result import build_result +from ethereum_types.bytes import Bytes20 +from ethereum_types.numeric import U256, Uint from execution_testing.base_types import StateCommitment from execution_testing.client_clis import ( @@ -486,3 +491,57 @@ def test_opcode_count_accumulation() -> None: tool.reset_opcode_count() assert tool.opcode_count == OpcodeCount({}) assert tool.opcode_count_per_block == [] + + +def test_build_result_reports_uncommittable_state_as_rejected() -> None: + """ + `build_result` catches `InvalidBlock` from `compute_state_root` -- + state the active commitment cannot encode, such as a balance past + the binary tree's sixteen-byte field -- and reports the block as + rejected with the pre-state root rather than crashing. + """ + from types import SimpleNamespace + + alloc = Alloc.model_validate( + {0xA: {"balance": 1, "nonce": 0, "code": "0x"}} + ) + alloc.migrate_state_commitment(StateCommitment.PBT) + pre_state_root = alloc.state_root() + + overflow_diff = BlockDiff( + account_changes={ + Address(Bytes20(b"\x00" * 19 + b"\x0a")): Account( + nonce=Uint(0), + balance=U256(2) ** U256(128), + code_hash=EMPTY_CODE_HASH, + ) + }, + ) + t8n_stub = SimpleNamespace( + fork=SimpleNamespace( + extract_block_diff=lambda _block_state: overflow_diff, + logs_bloom=lambda _logs: b"\x00" * 256, + ), + _block_state=None, + alloc=alloc, + exception_mapper=None, + ) + block_output = SimpleNamespace( + transactions_trie=Trie(secured=False, default=None), + receipts_trie=Trie(secured=False, default=None), + block_logs=(), + receipt_keys=(), + block_gas_used=Uint(0), + ) + + result = build_result( + t8n_stub, # type: ignore[arg-type] + block_env=SimpleNamespace(), + block_output=block_output, + block_exception=None, + rejected_transactions=[], + ) + + assert result.state_root == pre_state_root + assert result.block_exception is not None + assert "sixteen-byte" in str(result.block_exception) diff --git a/packages/testing/src/execution_testing/fixtures/tests/test_pre_alloc_groups.py b/packages/testing/src/execution_testing/fixtures/tests/test_pre_alloc_groups.py index f2bb4a4c797..6a4db3cd8ef 100644 --- a/packages/testing/src/execution_testing/fixtures/tests/test_pre_alloc_groups.py +++ b/packages/testing/src/execution_testing/fixtures/tests/test_pre_alloc_groups.py @@ -2,20 +2,22 @@ import json from pathlib import Path -from typing import Dict +from typing import Dict, Iterator import pytest -from execution_testing.base_types import Account, Address +from execution_testing.base_types import Account, Address, StateCommitment from execution_testing.fixtures.pre_alloc_groups import ( TEST_GROUP_INDEX_FILE, GroupIndexEntry, + PreAllocGroup, PreAllocGroupBuilder, pack_pre_alloc_groups, packed_group_hash_for_test, read_test_group_index, ) -from execution_testing.forks import Fork, Osaka, Prague +from execution_testing.forks import Amsterdam, Fork, Osaka, Prague +from execution_testing.forks.base_fork import BaseFork from execution_testing.test_types import Alloc, Environment @@ -526,3 +528,98 @@ def test_pack_isolates_disagreeing_shared_address(tmp_path: Path) -> None: "tests/b.py::test_b", "tests/c.py::test_c", ] + + +def _commitment_pre( + commitment: StateCommitment | None = None, +) -> Alloc: + """ + Build a small, deterministic pre-allocation for genesis tests. + + A fresh `Alloc` is returned on every call because a state + commitment, once migrated onto an instance (whether explicitly via + `commitment` or by a builder seeding it from its fork), sticks to + it -- so the same instance cannot be reused across assertions that + expect different commitment schemes. + """ + alloc = Alloc( + { + Address(0x1000): Account(balance=1000, nonce=1), + Address(0x2000): Account(balance=2, code=b"\x00"), + } + ) + if commitment is not None: + alloc.migrate_state_commitment(commitment) + return alloc + + +@pytest.fixture +def pbt_session() -> Iterator[None]: + """ + Run the test under the PBT session override, as `--state-trie + binary` would, restoring the fork-defined default afterwards. + """ + BaseFork.set_state_commitment_override(StateCommitment.PBT) + yield + BaseFork.set_state_commitment_override(None) + + +@pytest.mark.usefixtures("pbt_session") +def test_calculate_genesis_uses_the_session_state_commitment() -> None: + """ + A builder's genesis `state_root` follows the session state + commitment (`--state-trie`), not always the plain MPT root of the + same allocation. + + Regression test: `calculate_genesis` used to always compute the + MPT root regardless of the active commitment; the builder now + seeds the pre-alloc's scheme from `fork.state_commitment()` -- + which honors the session override -- on construction. + """ + builder = PreAllocGroupBuilder( + environment=Environment().set_fork_requirements(Amsterdam), + fork=Amsterdam, + pre=_commitment_pre(), + ) + + genesis = builder.calculate_genesis() + + assert ( + genesis.state_root == _commitment_pre(StateCommitment.PBT).state_root() + ) + + +@pytest.mark.usefixtures("pbt_session") +def test_group_pre_alloc_state_root_is_commitment_correct_after_reload( + tmp_path: Path, +) -> None: + """ + `GroupPreAlloc.state_root()` returns the commitment-correct root + even after a disk round trip, where private attrs -- including the + migrated state commitment -- do not survive + `model_dump`/`model_validate`. + + `PreAllocGroup.model_post_init` re-seeds the commitment from the + group's fork -- honoring the session override -- and caches + `_cached_state_root` from `genesis.state_root` on every + construction (including the one `from_file` performs after its + dump/validate round trip), so this pins that a reloaded group's + pre-allocation stays commitment-correct. + """ + builder = PreAllocGroupBuilder( + test_ids=["tests/a.py::test_a"], + environment=Environment().set_fork_requirements(Amsterdam), + fork=Amsterdam, + pre=_commitment_pre(), + ) + group_file = tmp_path / "0x01.json" + group_file.write_text( + builder.model_dump_json(by_alias=True, exclude_none=True, indent=2) + ) + + group = PreAllocGroup.from_file(group_file) + + assert ( + group.pre.state_root() + == _commitment_pre(StateCommitment.PBT).state_root() + ) diff --git a/packages/testing/src/execution_testing/forks/base_fork.py b/packages/testing/src/execution_testing/forks/base_fork.py index b1a5026acfb..537097b819f 100644 --- a/packages/testing/src/execution_testing/forks/base_fork.py +++ b/packages/testing/src/execution_testing/forks/base_fork.py @@ -345,6 +345,7 @@ class BaseFork(ForkOpcodeInterface, metaclass=BaseForkMeta): _deployed: ClassVar[bool] = True _enabled_eips: ClassVar[Set[int]] = set() _enabling_forks: ClassVar[Set[Type["BaseFork"]]] = set() + _state_commitment_override: ClassVar[Optional[StateCommitment]] = None # Method version bumps _engine_new_payload_version_bump: ClassVar[bool] = False @@ -458,9 +459,27 @@ def __init_subclass__( @classmethod def state_commitment(cls) -> StateCommitment: - """Return the state-commitment scheme for the state root.""" + """ + Return the state-commitment scheme for the state root. + + The `--state-trie` command-line option overrides the scheme for + every fork. + """ + if BaseFork._state_commitment_override is not None: + return BaseFork._state_commitment_override return StateCommitment.MPT + @classmethod + def set_state_commitment_override( + cls, commitment: Optional[StateCommitment] + ) -> None: + """ + Force every fork to report `commitment` from `state_commitment`. + + `None` restores the fork-defined scheme. + """ + BaseFork._state_commitment_override = commitment + # Header information abstract methods @classmethod @abstractmethod diff --git a/packages/testing/src/execution_testing/forks/tests/test_forks.py b/packages/testing/src/execution_testing/forks/tests/test_forks.py index c9c06f0a7b4..2494d755413 100644 --- a/packages/testing/src/execution_testing/forks/tests/test_forks.py +++ b/packages/testing/src/execution_testing/forks/tests/test_forks.py @@ -5,9 +5,10 @@ import pytest from pydantic import BaseModel -from execution_testing.base_types import BlobSchedule +from execution_testing.base_types import BlobSchedule, StateCommitment from execution_testing.vm import Opcodes +from ..base_fork import BaseFork from ..forks.eips.paris.eip_3675 import EIP3675 from ..forks.forks import ( BPO1, @@ -826,3 +827,36 @@ def test_oog_budget_lift() -> None: ) == 3 * sstore + 2 * create + code_64 ) + + +def test_state_commitment_defaults() -> None: + """ + Every fork defaults to the MPT commitment: the scheme is a session + property (`--state-trie`), not a fork property. The alloc seeding + sites rely on this default to pick the right state module. + """ + assert all( + fork.state_commitment() is StateCommitment.MPT for fork in get_forks() + ) + + +def test_state_commitment_override() -> None: + """ + `--state-trie` forces the scheme on every fork -- transition forks + included -- and `None` restores the fork-defined default. + """ + BaseFork.set_state_commitment_override(StateCommitment.PBT) + try: + assert all( + fork.state_commitment() is StateCommitment.PBT + for fork in get_forks() + ) + assert ( + BerlinToLondonAt5.transitions_from().state_commitment() + is StateCommitment.PBT + ) + finally: + BaseFork.set_state_commitment_override(None) + assert all( + fork.state_commitment() is StateCommitment.MPT for fork in get_forks() + ) From 924ae43cd5bc10a05b02706496777a8d101ee851 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Fri, 7 Aug 2026 17:55:28 +0200 Subject: [PATCH 3/4] feat(ci): run binary trie unit tests and a PBT-mode fill --- .github/workflows/test.yaml | 15 +++++++++++++++ Justfile | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e26aeddaf57..f4054500243 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -216,3 +216,18 @@ jobs: - uses: ./.github/actions/setup-uv - name: Run test-ci-scripts run: just test-ci-scripts + + binary-trie: + runs-on: ubuntu-latest + needs: static + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/setup-uv + - name: Run binary trie unit tests + run: just binary-trie-unit-test + env: + PYTEST_XDIST_AUTO_NUM_WORKERS: auto + - name: Fill in PBT mode + run: just fill-state-trie-pbt + env: + PYTEST_XDIST_AUTO_NUM_WORKERS: auto diff --git a/Justfile b/Justfile index 01467ef46fe..4d6d79bfd3b 100644 --- a/Justfile +++ b/Justfile @@ -155,6 +155,23 @@ fill-release *args: --log-level=DEBUG \ "$@" +# Fill the consensus tests using EELS in PBT mode (--state-trie pbt) +[group('consensus tests')] +fill-state-trie-pbt *args: (_tmp-logs "fill-state-trie-pbt") + uv run fill \ + -m "not slow" \ + -n {{ xdist_workers }} --dist=loadgroup \ + --skip-index \ + --state-trie pbt \ + --output="{{ output_dir }}/fill-state-trie-pbt/fixtures" \ + --basetemp="{{ output_dir }}/fill-state-trie-pbt/tmp" \ + --log-to "{{ output_dir }}/fill-state-trie-pbt/logs" \ + --clean \ + --until "{{ latest_fork }}" \ + --durations=50 \ + "$@" \ + tests + # --- Integration Tests --- # Fill the base coverage consensus tests using EELS with PyPy @@ -206,6 +223,7 @@ json-loader *args: (_tmp "json-loader") --cov-report "xml:{{ output_dir }}/json-loader/coverage.xml" \ --durations=50 \ --basetemp="{{ output_dir }}/json-loader/tmp" \ + --ignore=tests/binary_trie \ "$@" \ tests/json_loader @@ -245,6 +263,21 @@ test-tests-pypy *args: (_tmp "test-tests-pypy") test-ci-scripts *args: uv run pytest "$@" .github/scripts/tests/ +# Run the binary trie state-provider unit tests +[group('unit tests')] +binary-trie-unit-test *args: (_tmp "binary-trie-unit-test") + uv run pytest \ + -n {{ xdist_workers }} \ + --cov=ethereum.partitioned_binary_tree \ + --cov=ethereum.state_pbt \ + --cov-branch \ + --cov-report=term \ + --cov-report "xml:{{ output_dir }}/binary-trie-unit-test/coverage.xml" \ + --no-cov-on-fail \ + --basetemp="{{ output_dir }}/binary-trie-unit-test/tmp" \ + "$@" \ + tests/binary_trie + # --- Benchmarks --- # test_return_revert is excluded: its max-size INVALID-padded callees make From d46566726b12986424d3e51084cabc8b1d7fb171 Mon Sep 17 00:00:00 2001 From: spencer-tb Date: Fri, 7 Aug 2026 18:06:59 +0200 Subject: [PATCH 4/4] feat(ci): attach variant fixture tarballs to mainnet releases --- .github/actions/build-fixtures/action.yaml | 5 ++- .github/configs/feature.yaml | 11 ++++++ .github/scripts/generate_build_matrix.py | 29 ++++++++++++++ .github/scripts/tests/test_release_scripts.py | 39 ++++++++++++++++++- .github/workflows/release_fixtures.yaml | 25 +++++++++++- 5 files changed, 106 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 61d255c22ab..f84ab69847d 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -13,6 +13,9 @@ inputs: split_label: description: "Label for this fork-range split. Empty for unsplit builds." default: "" + extra_params: + description: "Extra fill params appended after the feature's own (last flag wins). Empty for none." + default: "" split_retention_days: description: "retention-days for the split fixture artifact. Empty for the repo default." default: "" @@ -72,7 +75,7 @@ runs: # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests. EXIT_CODE=0 - just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$? + just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS ${{ inputs.extra_params }} $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$? if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then exit "$EXIT_CODE" fi diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index 735fffa0cd9..158175e6fd8 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -26,3 +26,14 @@ benchmark_fast: devnet: evm-type: eels fill-params: --until=Amsterdam --generate-all-formats + +# Variant tarballs attached beside the main fixtures asset of `tests` +# and `-devnet` releases. Each variant fills its own fork range +# with extra fill params on its own runner and is packaged by the +# combine job as `_.tar.gz`. A failed variant fill +# never blocks the release; its tarball is simply absent. +variants: + binary: + from: Amsterdam + until: Amsterdam + fill-params: --state-trie pbt diff --git a/.github/scripts/generate_build_matrix.py b/.github/scripts/generate_build_matrix.py index 200bdb788ad..7e84b16d51b 100644 --- a/.github/scripts/generate_build_matrix.py +++ b/.github/scripts/generate_build_matrix.py @@ -107,6 +107,11 @@ def validate_inputs(feature: str, version: str, branch: str, evm: str) -> None: if feature in ("devnet", "-devnet"): fail("devnet releases require a - prefix, e.g. bal-devnet") + # `variants` configures the extra tarballs of mainnet-type + # releases; it is not a releasable feature itself. + if feature == "variants": + fail("'variants' is a reserved config key, not a feature") + # `-devnet-`: the devnet index belongs in the version (X of # vX.Y.Z), not in the feature name. if "-devnet-" in feature: @@ -244,9 +249,33 @@ def main() -> None: build, labels = build_matrix(config[lookup], name, fork_ranges) + # Mainnet-type releases (`tests` and `-devnet`) also fill + # each configured variant on its own runner. Variant entries ride + # the same matrix; the combine job packages each one into its own + # `_.tar.gz` beside the main fixtures asset. + for entry in build: + entry.setdefault("variant", "") + entry.setdefault("extra_params", "") + variants = config.get("variants") or {} + variant_labels = "" + if lookup in ("tests", "devnet") and variants: + for variant_name, variant in variants.items(): + build.append( + { + "feature": name, + "label": variant_name, + "from_fork": variant["from"], + "until_fork": variant.get("until", variant["from"]), + "variant": variant_name, + "extra_params": variant["fill-params"], + } + ) + variant_labels = " ".join(variants) + print(f"build_matrix={json.dumps(build)}") print(f"feature_name={name}") print(f"combine_labels={labels}") + print(f"variant_labels={variant_labels}") if __name__ == "__main__": diff --git a/.github/scripts/tests/test_release_scripts.py b/.github/scripts/tests/test_release_scripts.py index 44e6dbcab77..3e7d1a4ea6c 100644 --- a/.github/scripts/tests/test_release_scripts.py +++ b/.github/scripts/tests/test_release_scripts.py @@ -84,6 +84,42 @@ def test_devnet_name_resolves_to_shared_feature(self): # Entries keep the friendly name, not the shared "devnet" key. assert all(e["feature"] == "bal-devnet" for e in matrix) + def test_mainnet_features_gain_variant_entries(self): + """Verify tests/devnet matrices append the configured variants.""" + for args in ( + ("tests", "v24.0.0"), + ("bal-devnet", "v7.0.0", "devnets/bal/7"), + ): + result = run_script(BUILD_MATRIX_SCRIPT, *args) + assert result.returncode == 0 + out = parse_matrix_output(result.stdout) + matrix = json.loads(out["build_matrix"]) + assert out["variant_labels"] == "binary" + (binary,) = [e for e in matrix if e["variant"] == "binary"] + assert binary["label"] == "binary" + assert binary["from_fork"] == "Amsterdam" + assert binary["until_fork"] == "Amsterdam" + assert "--state-trie pbt" in binary["extra_params"] + # Non-variant entries carry the uniform empty fields. + assert all( + e["extra_params"] == "" for e in matrix if e["variant"] == "" + ) + + def test_feature_only_features_gain_no_variants(self): + """Verify non-mainnet features are variant-free.""" + result = run_script(BUILD_MATRIX_SCRIPT, "benchmark", "v24.0.0") + assert result.returncode == 0 + out = parse_matrix_output(result.stdout) + matrix = json.loads(out["build_matrix"]) + assert out["variant_labels"] == "" + assert all(e["variant"] == "" for e in matrix) + + def test_variants_is_not_a_releasable_feature(self): + """Verify the reserved `variants` key is rejected.""" + result = run_script(BUILD_MATRIX_SCRIPT, "variants", "v1.0.0") + assert result.returncode == 1 + assert "reserved" in result.stderr + def test_unknown_feature_fails(self): """Verify error exit for unknown feature name.""" result = run_script(BUILD_MATRIX_SCRIPT, "nonexistent", "v1.0.0") @@ -101,10 +137,11 @@ def test_output_is_valid_github_actions_format(self): result = run_script(BUILD_MATRIX_SCRIPT, "tests", "v24.0.0") assert result.returncode == 0 lines = result.stdout.strip().splitlines() - assert len(lines) == 3 + assert len(lines) == 4 assert lines[0].startswith("build_matrix=") assert lines[1].startswith("feature_name=") assert lines[2].startswith("combine_labels=") + assert lines[3].startswith("variant_labels=") class TestValidateInputs: diff --git a/.github/workflows/release_fixtures.yaml b/.github/workflows/release_fixtures.yaml index d56a9dae918..35ed645b51c 100644 --- a/.github/workflows/release_fixtures.yaml +++ b/.github/workflows/release_fixtures.yaml @@ -73,6 +73,7 @@ jobs: build_matrix: ${{ steps.matrix.outputs.build_matrix }} feature_name: ${{ steps.matrix.outputs.feature_name }} combine_labels: ${{ steps.matrix.outputs.combine_labels }} + variant_labels: ${{ steps.matrix.outputs.variant_labels }} target_sha: ${{ steps.cached.outputs.target_sha || steps.target_sha.outputs.sha }} short_sha: ${{ steps.target_sha.outputs.short_sha }} artifact_run_id: ${{ steps.cached.outputs.run_id }} @@ -143,6 +144,9 @@ jobs: if: needs.setup.outputs.run == 'true' runs-on: [self-hosted-ghr, size-gigachungus-x64] timeout-minutes: 1440 + # A variant tarball is a bonus asset: its fill failing must never + # block the main release, whose tarball it merely rides beside. + continue-on-error: ${{ matrix.variant != '' }} strategy: # A release must be complete, so abort on the first failed range; a # nightly wants every range's result for debugging. @@ -161,6 +165,7 @@ jobs: from_fork: ${{ matrix.from_fork }} until_fork: ${{ matrix.until_fork }} split_label: ${{ matrix.label }} + extra_params: ${{ matrix.extra_params }} # Nightly splits are intermediates consumed by `combine` right # away; don't retain them for the repo-default period. split_retention_days: ${{ github.event_name == 'schedule' && '1' || '' }} @@ -225,6 +230,23 @@ jobs: fi uv run -q .github/scripts/create_release_tarball.py combined "$TARBALL" echo "path=$TARBALL" >> "$GITHUB_OUTPUT" + - name: Create variant tarballs + if: needs.setup.outputs.variant_labels != '' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TARBALL: ${{ steps.tarball.outputs.path }} + run: | + for variant in ${{ needs.setup.outputs.variant_labels }}; do + echo "Downloading: fixtures__${variant}" + if gh run download ${{ github.run_id }} -n "fixtures__${variant}" --dir "variant_artifacts/${variant}"; then + uv run -q .github/scripts/create_release_tarball.py \ + "variant_artifacts/${variant}" \ + "${TARBALL%.tar.gz}_${variant}.tar.gz" + else + echo "No artifact for variant ${variant} (fill failed or empty), skipping" + fi + done - name: Upload combined fixture tarball uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: @@ -233,7 +255,8 @@ jobs: # cached-release resolver derives this exact name from each # run's head SHA. The tarball inside carries the feature name. name: fixtures_${{ needs.setup.outputs.short_sha }} - path: ${{ steps.tarball.outputs.path }} + # The main tarball plus any variant tarballs beside it. + path: fixtures*.tar.gz # Keep nightly tarballs for five days; a quiet nightly re-runs # after four (see check_new_commits.py), so a live artifact # always exists. Release tarballs keep the repo default since