diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py index 37184d04e91..1644a5a0439 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py @@ -2147,8 +2147,6 @@ def _log_timing(msg: str) -> None: # Log immediately when hook is entered (before any early returns) _log_timing(f"pytest_sessionfinish ENTERED (worker={is_worker})") - del exitstatus - # Save pre-allocation groups after phase 1 fixture_output: FixtureOutput = session.config.fixture_output # type: ignore[attr-defined] session_instance: FillingSession = session.config.filling_session # type: ignore[attr-defined] @@ -2241,36 +2239,42 @@ def _log_timing(msg: str) -> None: _log_timing(f"Lock files removed in {time.time() - t0:.1f}s") # Loudly fail the fill if pre-alloc group packing changed any Engine X - # test's execution (raises on drift, like a pre-alloc collision). - _log_timing("verify_engine_x_execution: starting...") - t0 = time.time() - engine_x_check = verify_engine_x_execution(fixture_output.directory) - engine_x_warning: str | None = None - if engine_x_check is not None: - if engine_x_check.compared > 0: - logger.info(engine_x_check.summary) - elif engine_x_check.skipped > 0: + # test's execution (raises on drift, like a pre-alloc collision). Only + # checked on an otherwise clean session: raising from this hook aborts + # the terminal FAILURES/short-summary sections, so it would hide any + # test failures (which already fail the fill and must surface first). + if exitstatus == pytest.ExitCode.OK: + _log_timing("verify_engine_x_execution: starting...") + t0 = time.time() + engine_x_check = verify_engine_x_execution(fixture_output.directory) + engine_x_warning: str | None = None + if engine_x_check is not None: + if engine_x_check.compared > 0: + logger.info(engine_x_check.summary) + elif engine_x_check.skipped > 0: + engine_x_warning = ( + "Engine X execution consistency check skipped: none of " + f"the {engine_x_check.skipped} Engine X fixtures have a " + "blockchain_tests_engine sibling fixture to compare " + "against. Leaks from pre-alloc group packing are not " + "verified for this output." + ) + elif (fixture_output.directory / ENGINE_X_FIXTURES_DIR).is_dir(): engine_x_warning = ( - "Engine X execution consistency check skipped: none of " - f"the {engine_x_check.skipped} Engine X fixtures have a " - "blockchain_tests_engine sibling fixture to compare " - "against. Leaks from pre-alloc group packing are not " - "verified for this output." + "Engine X execution consistency check skipped: this fill " + "generated no blockchain_tests_engine fixtures to compare " + "against (e.g. filling with `-m blockchain_test_engine_x`). " + "Leaks from pre-alloc group packing are not verified for " + "this output." ) - elif (fixture_output.directory / ENGINE_X_FIXTURES_DIR).is_dir(): - engine_x_warning = ( - "Engine X execution consistency check skipped: this fill " - "generated no blockchain_tests_engine fixtures to compare " - "against (e.g. filling with `-m blockchain_test_engine_x`). " - "Leaks from pre-alloc group packing are not verified for this " - "output." + if engine_x_warning is not None: + logger.warning(engine_x_warning) + # Repeated in the terminal summary; a log line alone is easy + # to miss. + session.config.engine_x_check_warning = engine_x_warning # type: ignore[attr-defined] # noqa: E501 + _log_timing( + f"verify_engine_x_execution: done in {time.time() - t0:.1f}s" ) - if engine_x_warning is not None: - logger.warning(engine_x_warning) - # Repeated in the terminal summary; a log line alone is easy to - # miss. - session.config.engine_x_check_warning = engine_x_warning # type: ignore[attr-defined] # noqa: E501 - _log_timing(f"verify_engine_x_execution: done in {time.time() - t0:.1f}s") # Verify fixtures after merge if verification is enabled if session.config.getoption("verify_fixtures"): diff --git a/packages/testing/src/execution_testing/fixtures/engine_x_checks.py b/packages/testing/src/execution_testing/fixtures/engine_x_checks.py index b81eaf38363..b884bac6522 100644 --- a/packages/testing/src/execution_testing/fixtures/engine_x_checks.py +++ b/packages/testing/src/execution_testing/fixtures/engine_x_checks.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from ethereum_rlp import rlp + ENGINE_X_FIXTURES_DIR = "blockchain_tests_engine_x" SIBLING_FIXTURES_DIR = "blockchain_tests_engine" @@ -12,6 +14,40 @@ # payload is a pure function of the test's execution. _STATE_ROOT_DERIVED_FIELDS = ("stateRoot", "blockHash", "parentHash") +# Placeholder for the parent-hash value embedded in a block access list. +_PARENT_HASH_PLACEHOLDER = "" + + +def _masked(node: Any, parent_hash: bytes) -> Any: + """Mask every ``parent_hash`` leaf in a decoded BAL, hex the rest.""" + if isinstance(node, bytes): + if parent_hash and node == parent_hash: + return _PARENT_HASH_PLACEHOLDER + return node.hex() + return [_masked(child, parent_hash) for child in node] + + +def _scrubbed_bal(bal_hex: str, parent_hash_hex: str) -> Any: + """ + Return a comparable form of a BAL with its parent hash masked out. + + The EIP-2935 system call writes the parent hash into the history + contract on every block, so each payload's BAL embeds one + state-root-derived value (at payload 0, the genesis hash itself). The + BAL is decoded and that value masked rather than the whole field + dropped, so the rest of the BAL still participates in the comparison: + a leaked account shows up in the BAL before anywhere else. Storage + values are RLP-encoded with leading zeros trimmed, so the trimmed + parent hash is masked. An undecodable BAL (an intentionally malformed + one from a negative test) is compared verbatim. + """ + parent_hash = bytes.fromhex(parent_hash_hex.removeprefix("0x")) + try: + decoded = rlp.decode(bytes.fromhex(bal_hex.removeprefix("0x"))) + except Exception: + return bal_hex + return _masked(decoded, parent_hash.lstrip(b"\x00")) + class EngineXExecutionDriftError(Exception): """ @@ -74,8 +110,13 @@ def _scrubbed_payloads(fixture: Dict[str, Any]) -> List[Any]: entry = json.loads(json.dumps(entry)) params = entry.get("params") if params and isinstance(params[0], dict): + payload = params[0] + bal = payload.get("blockAccessList") + parent_hash = payload.get("parentHash") + if isinstance(bal, str) and isinstance(parent_hash, str): + payload["blockAccessList"] = _scrubbed_bal(bal, parent_hash) for field in _STATE_ROOT_DERIVED_FIELDS: - params[0].pop(field, None) + payload.pop(field, None) payloads.append(entry) return payloads @@ -114,7 +155,10 @@ def verify_engine_x_execution( `blockchain_test_engine` sibling fixture (filled against the test's own pre-allocation in the same session, with an independent `t8n` execution: Engine X fixtures never share the transition tool output cache). All - payload fields except the state-root-derived ones must match exactly. + payload fields except the state-root-derived ones must match exactly; + each payload's block access list is compared with its own parent-hash + bytes normalized out, since the EIP-2935 system write embeds that + state-root-derived value in every BAL. Return the comparison counts, or ``None`` when one of the two fixture format trees was not generated at all (e.g. when filling with diff --git a/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py b/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py index 7e44afee03c..cbc7f2c282f 100644 --- a/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py +++ b/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List import pytest +from ethereum_rlp import rlp from execution_testing.fixtures.engine_x_checks import ( ENGINE_X_FIXTURES_DIR, @@ -22,22 +23,30 @@ def _payload( - *, gas_used: str, state_root: str, block_hash: str + *, + gas_used: str, + state_root: str, + block_hash: str, + parent_hash: str = f"0x{'00' * 31}aa", + block_access_list: str | None = None, ) -> Dict[str, Any]: """Build a single newPayload entry.""" + payload = { + "parentHash": parent_hash, + "stateRoot": state_root, + "blockHash": block_hash, + "gasUsed": gas_used, + "receiptsRoot": f"0x{'11' * 32}", + "logsBloom": f"0x{'00' * 256}", + "transactions": ["0xf86b..."], + } + if block_access_list is not None: + payload["blockAccessList"] = block_access_list return { "newPayloadVersion": "4", "forkchoiceUpdatedVersion": "3", "params": [ - { - "parentHash": f"0x{'00' * 31}aa", - "stateRoot": state_root, - "blockHash": block_hash, - "gasUsed": gas_used, - "receiptsRoot": f"0x{'11' * 32}", - "logsBloom": f"0x{'00' * 256}", - "transactions": ["0xf86b..."], - }, + payload, [], f"0x{'00' * 32}", ], @@ -78,6 +87,134 @@ def test_identical_execution_passes(tmp_path: Path) -> None: assert "1 Engine X fixtures execute identically" in result.summary +def _bal(parent_hash: bytes, *extra: bytes) -> str: + """ + Encode a minimal BAL-shaped RLP structure embedding a parent hash. + + Storage values are RLP-encoded with leading zeros trimmed, matching + the EIP-2935 history write of the parent hash in a real BAL. + """ + values = [parent_hash.lstrip(b"\x00"), *extra] + return "0x" + rlp.encode([values, []]).hex() + + +def test_bal_embedded_parent_hash_passes(tmp_path: Path) -> None: + """ + The EIP-2935 write embeds each side's own parent hash in its BAL; a BAL + differing only by that embedded value does not trip the check, even + when one side's hash is stored with its leading zero byte trimmed. + """ + sibling_parent = bytes.fromhex("aa" * 32) + engine_x_parent = bytes.fromhex("00" + "bb" * 31) + _write_fixture( + tmp_path, + SIBLING_FIXTURES_DIR, + SIBLING_ID, + [ + _payload( + gas_used="0x5208", + state_root="0x01", + block_hash="0x02", + parent_hash="0x" + sibling_parent.hex(), + block_access_list=_bal(sibling_parent), + ) + ], + ) + _write_fixture( + tmp_path, + ENGINE_X_FIXTURES_DIR, + ENGINE_X_ID, + [ + _payload( + gas_used="0x5208", + state_root="0xaa", + block_hash="0xbb", + parent_hash="0x" + engine_x_parent.hex(), + block_access_list=_bal(engine_x_parent), + ) + ], + ) + + result = verify_engine_x_execution(tmp_path) + + assert result is not None + assert result.compared == 1 + + +def test_bal_drift_raises(tmp_path: Path) -> None: + """A BAL difference beyond the embedded parent hash fails loudly.""" + sibling_parent = bytes.fromhex("aa" * 32) + engine_x_parent = bytes.fromhex("bb" * 32) + _write_fixture( + tmp_path, + SIBLING_FIXTURES_DIR, + SIBLING_ID, + [ + _payload( + gas_used="0x5208", + state_root="0x01", + block_hash="0x02", + parent_hash="0x" + sibling_parent.hex(), + block_access_list=_bal(sibling_parent), + ) + ], + ) + _write_fixture( + tmp_path, + ENGINE_X_FIXTURES_DIR, + ENGINE_X_ID, + [ + _payload( + gas_used="0x5208", + state_root="0xaa", + block_hash="0xbb", + parent_hash="0x" + engine_x_parent.hex(), + block_access_list=_bal(engine_x_parent, b"\x01"), + ) + ], + ) + + with pytest.raises(EngineXExecutionDriftError) as exc_info: + verify_engine_x_execution(tmp_path) + + assert "blockAccessList" in str(exc_info.value) + + +def test_malformed_bal_compared_verbatim(tmp_path: Path) -> None: + """An undecodable BAL (negative test) is compared verbatim.""" + _write_fixture( + tmp_path, + SIBLING_FIXTURES_DIR, + SIBLING_ID, + [ + _payload( + gas_used="0x5208", + state_root="0x01", + block_hash="0x02", + block_access_list="0xdeadbeef", + ) + ], + ) + _write_fixture( + tmp_path, + ENGINE_X_FIXTURES_DIR, + ENGINE_X_ID, + [ + _payload( + gas_used="0x5208", + state_root="0xaa", + block_hash="0xbb", + block_access_list="0xdeadbeef", + ) + ], + ) + + result = verify_engine_x_execution(tmp_path) + + assert result is not None + assert result.compared == 1 + + def test_execution_drift_raises(tmp_path: Path) -> None: """A gas difference (a leaked account changed execution) fails loudly.""" _write_fixture( diff --git a/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py b/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py index 64c7d7225ee..0ed2c038f80 100644 --- a/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py +++ b/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py @@ -136,7 +136,17 @@ def test_bal_2935_empty_block( @pytest.mark.parametrize( "query_block_number,is_valid", [ - pytest.param(0, True, id="valid_block_number"), + pytest.param( + 0, + True, + id="valid_block_number", + marks=pytest.mark.pre_alloc_group( + "separate", + reason="Queries the genesis hash from the history " + "contract and stores it, so the BAL contains the genesis " + "hash itself, which changes under any shared genesis.", + ), + ), pytest.param(1042, False, id="block_number_out_of_range"), ], ) diff --git a/tests/ported_static/conftest.py b/tests/ported_static/conftest.py index b26f10ade4a..efb35a532af 100644 --- a/tests/ported_static/conftest.py +++ b/tests/ported_static/conftest.py @@ -9,6 +9,7 @@ remove this skip list. """ +import re from pathlib import Path import pytest @@ -20,23 +21,17 @@ if line.strip() and not line.lstrip().startswith("#") ) -# Fixture format suffixes pytest appends inside the parametrize id. These -# must be stripped from the nodeid before substring-matching against the -# skip list, because the skip list predates these suffixes. -_FIXTURE_FORMAT_TOKENS: tuple[str, ...] = ( - "-blockchain_test_engine_from_state_test", - "-blockchain_test_from_state_test", - "-blockchain_test_engine", - "-blockchain_test", - "-state_test", -) +# Fixture format tokens pytest embeds in the parametrize id (e.g. +# `-blockchain_test_engine_x_from_state_test`). These must be stripped from +# the nodeid before substring-matching against the skip list, because the +# skip list predates these tokens. Matched by prefix so every format and +# label variant is covered. +_FIXTURE_FORMAT_TOKEN_RE = re.compile(r"-(?:blockchain|state)_test\w*") def _normalize_nodeid(nodeid: str) -> str: - """Strip pytest fixture-format suffixes to match the skip list format.""" - for token in _FIXTURE_FORMAT_TOKENS: - nodeid = nodeid.replace(token, "") - return nodeid + """Strip pytest fixture-format id tokens to match the skip list format.""" + return _FIXTURE_FORMAT_TOKEN_RE.sub("", nodeid) def pytest_collection_modifyitems( diff --git a/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py b/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py index cebe282fc29..9b8e0db1733 100644 --- a/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py +++ b/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py @@ -26,6 +26,12 @@ ) @pytest.mark.valid_from("Cancun") @pytest.mark.pre_alloc_mutable +@pytest.mark.pre_alloc_group( + "separate", + reason="Calls hardcoded addresses 0x1000 and 0x2000 without declaring " + "them, so gas usage depends on them staying empty; sharing a genesis " + "with a test that allocates either address changes the execution.", +) def test_gas_cost_return( state_test: StateTestFiller, pre: Alloc,