diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py index 37184d04e9..d961ca8062 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py @@ -2147,8 +2147,6 @@ def _log_timing(msg: str) -> None: # Log immediately when hook is entered (before any early returns) _log_timing(f"pytest_sessionfinish ENTERED (worker={is_worker})") - del exitstatus - # Save pre-allocation groups after phase 1 fixture_output: FixtureOutput = session.config.fixture_output # type: ignore[attr-defined] session_instance: FillingSession = session.config.filling_session # type: ignore[attr-defined] @@ -2241,36 +2239,29 @@ def _log_timing(msg: str) -> None: _log_timing(f"Lock files removed in {time.time() - t0:.1f}s") # Loudly fail the fill if pre-alloc group packing changed any Engine X - # test's execution (raises on drift, like a pre-alloc collision). - _log_timing("verify_engine_x_execution: starting...") - t0 = time.time() - engine_x_check = verify_engine_x_execution(fixture_output.directory) - engine_x_warning: str | None = None - if engine_x_check is not None: + # test's execution (raises on drift, like a pre-alloc collision). Only + # checked on an otherwise clean session: raising from this hook aborts + # the terminal FAILURES/short-summary sections, so it would hide any + # test failures (which already fail the fill and must surface first). + if exitstatus == pytest.ExitCode.OK: + _log_timing("verify_engine_x_execution: starting...") + t0 = time.time() + engine_x_check = verify_engine_x_execution(fixture_output.directory) if engine_x_check.compared > 0: logger.info(engine_x_check.summary) - elif engine_x_check.skipped > 0: - engine_x_warning = ( - "Engine X execution consistency check skipped: none of " - f"the {engine_x_check.skipped} Engine X fixtures have a " - "blockchain_tests_engine sibling fixture to compare " - "against. Leaks from pre-alloc group packing are not " - "verified for this output." - ) + if engine_x_check.skip_reason is not None: + logger.warning(engine_x_check.skip_reason) + # Repeated in the terminal summary; a log line alone is easy + # to miss. + session.config.engine_x_check_warning = engine_x_check.skip_reason # type: ignore[attr-defined] # noqa: E501 + _log_timing( + f"verify_engine_x_execution: done in {time.time() - t0:.1f}s" + ) elif (fixture_output.directory / ENGINE_X_FIXTURES_DIR).is_dir(): - engine_x_warning = ( - "Engine X execution consistency check skipped: this fill " - "generated no blockchain_tests_engine fixtures to compare " - "against (e.g. filling with `-m blockchain_test_engine_x`). " - "Leaks from pre-alloc group packing are not verified for this " - "output." + logger.info( + "Engine X execution consistency check skipped: the session " + "did not exit cleanly." ) - if engine_x_warning is not None: - logger.warning(engine_x_warning) - # Repeated in the terminal summary; a log line alone is easy to - # miss. - session.config.engine_x_check_warning = engine_x_warning # type: ignore[attr-defined] # noqa: E501 - _log_timing(f"verify_engine_x_execution: done in {time.time() - t0:.1f}s") # Verify fixtures after merge if verification is enabled if session.config.getoption("verify_fixtures"): diff --git a/packages/testing/src/execution_testing/fixtures/engine_x_checks.py b/packages/testing/src/execution_testing/fixtures/engine_x_checks.py index b81eaf3836..42cbf36ea7 100644 --- a/packages/testing/src/execution_testing/fixtures/engine_x_checks.py +++ b/packages/testing/src/execution_testing/fixtures/engine_x_checks.py @@ -1,59 +1,138 @@ -"""Fill-time execution-consistency check for Engine X fixtures.""" +""" +Fill-time execution-consistency check for Engine X fixtures. + +An Engine X fixture is filled against its packed pre-allocation group's +merged genesis, while the test's `blockchain_test_engine` sibling is +filled against the test's own pre-allocation. Their per-payload +execution outputs must be identical; `verify_engine_x_execution` +compares the two fixture trees after a fill and raises a classified, +per-cause report when they are not. + +The check runs post-fill on the fixture files because the two formats of +a test are separate pytest items that may fill on different xdist +workers; the output directory is the only place both reliably exist. It +only needs that directory, so it can also be re-run standalone against a +failed fill's artifacts without re-filling. +""" import json +from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from typing import Any, Dict, List, Tuple, Type, TypeVar + +from pydantic import ValidationError + +from execution_testing.base_types import Address, Bytes, Hash +from execution_testing.forks.forks.eips.prague.eip_2935 import ( + HISTORY_STORAGE_ADDRESS, +) +from execution_testing.test_types.block_access_list import BlockAccessList + +from .blockchain import ( + BlockchainEngineFixture, + BlockchainEngineXFixture, + FixtureEngineNewPayload, + FixtureExecutionPayload, +) +from .pre_alloc_groups import PreAllocGroup, PreAllocGroups + +ENGINE_X_FIXTURES_DIR = BlockchainEngineXFixture.output_base_dir_name() +SIBLING_FIXTURES_DIR = BlockchainEngineFixture.output_base_dir_name() + +# Path parts under a fixture format tree that do not contain fixture +# files (mirrors INDEX_EXCLUDED_PATH_PARTS in `cli/gen_index.py`). +_NON_FIXTURE_PATH_PARTS = frozenset({".meta", "pre_alloc"}) + +# Execution payload fields whose value is a function of the genesis +# state root and so legitimately differs between a test's own genesis +# and its packed group's genesis. Python field names of +# `FixtureExecutionPayload`; a unit test asserts they stay valid when +# the model changes. +STATE_ROOT_DERIVED_FIELDS = frozenset( + {"state_root", "block_hash", "parent_hash"} +) + +# Placeholder for the masked EIP-2935 parent-hash write in a BAL. +_PARENT_HASH_PLACEHOLDER = "" + +_VALUE_DISPLAY_LIMIT = 96 + +_FixtureT = TypeVar( + "_FixtureT", BlockchainEngineFixture, BlockchainEngineXFixture +) -ENGINE_X_FIXTURES_DIR = "blockchain_tests_engine_x" -SIBLING_FIXTURES_DIR = "blockchain_tests_engine" -# Every state-root-derived field of an execution payload. These are the only -# fields a packed (merged) genesis is allowed to change; everything else in a -# payload is a pure function of the test's execution. -_STATE_ROOT_DERIVED_FIELDS = ("stateRoot", "blockHash", "parentHash") +class EngineXCheckError(Exception): + """The Engine X execution check failed to run on the fixture output.""" -class EngineXExecutionDriftError(Exception): +@dataclass(frozen=True) +class ExecutionDrift: + """A fixture whose packed execution differs from its sibling's.""" + + test_id: str + signature: str + """One-line cause classification; identical causes share it.""" + detail: str + """Multi-line explanation rendered for the first example only.""" + + +class EngineXExecutionDriftError(EngineXCheckError): """ A packed pre-allocation group changed a test's execution. - An Engine X fixture is filled against its group's merged genesis, while - the test's `blockchain_test_engine` sibling is filled against the test's - own pre-allocation. Their per-payload execution outputs (gas used, - receipts root, logs bloom, ...) must be identical; a difference means - either an account introduced by pre-alloc group packing leaked into the - test's execution (see `pack_pre_alloc_groups`), or the test observes the - genesis hash itself (e.g. via `BLOCKHASH(0)`), which depends on every - account in the genesis and so cannot survive any grouping. + Either an account introduced by pre-alloc group packing leaked into + the test's execution (see `pack_pre_alloc_groups`), or the test + observes a block hash (e.g. via `BLOCKHASH`), which depends on every + account in the genesis and so cannot survive any grouping. Drifts + are grouped by cause signature so a single systemic cause reads as + one diagnosis, not one failure per fixture. """ - def __init__(self, mismatches: List[Tuple[str, str]], compared: int): - """Initialize with the mismatched test ids and the compared count.""" - self.mismatches = mismatches + MAX_SIGNATURES = 10 + MAX_EXAMPLE_IDS = 3 + + def __init__(self, drifts: List[ExecutionDrift], compared: int): + """Initialize with per-fixture drifts, grouped by signature.""" + self.drifts = drifts self.compared = compared - details = "\n".join( - f" {test_id}: {what}" for test_id, what in mismatches[:10] - ) - if len(mismatches) > 10: - details += f"\n ... and {len(mismatches) - 10} more" + grouped: Dict[str, List[ExecutionDrift]] = {} + for drift in drifts: + grouped.setdefault(drift.signature, []).append(drift) + sections = [] + for signature, group in list(grouped.items())[: self.MAX_SIGNATURES]: + ids = ", ".join( + drift.test_id for drift in group[: self.MAX_EXAMPLE_IDS] + ) + if len(group) > self.MAX_EXAMPLE_IDS: + ids += f", ... and {len(group) - self.MAX_EXAMPLE_IDS} more" + section = f"[{len(group)}x] {signature}\n tests: {ids}" + if group[0].detail: + section += "\n" + "\n".join( + f" {line}" for line in group[0].detail.splitlines() + ) + sections.append(section) + if len(grouped) > self.MAX_SIGNATURES: + sections.append( + f"... and {len(grouped) - self.MAX_SIGNATURES} more " + "distinct causes" + ) super().__init__( - f"{len(mismatches)} of {compared} Engine X fixtures execute " - "differently against their packed pre-allocation group's genesis " - "than against their own pre-allocation:\n" - f"{details}\n" - "Sharing a genesis changed these tests' execution: either an " - "account introduced by pre-alloc group packing leaked into " - "their execution, or they observe the genesis hash itself " - "(e.g. via BLOCKHASH(0)). Isolate the affected tests with " - '@pytest.mark.pre_alloc_group("separate") and re-fill.' + f"{len(drifts)} of {compared} Engine X fixtures execute " + "differently against their packed pre-allocation group's " + "genesis than against their own pre-allocation " + f"({len(grouped)} distinct cause(s)):\n\n" + "\n\n".join(sections) ) -class EngineXCheckResult(NamedTuple): - """Comparison counts from a completed Engine X execution check.""" +@dataclass(frozen=True) +class EngineXCheckResult: + """Outcome of a completed Engine X execution check.""" - compared: int - skipped: int + compared: int = 0 + skipped: int = 0 + skip_reason: str | None = None + """Set when Engine X fixtures exist but nothing could be compared.""" @property def summary(self) -> str: @@ -63,104 +142,454 @@ def summary(self) -> str: "against their packed group's genesis" ) if self.skipped: - summary += f" ({self.skipped} skipped: no sibling engine fixture)" + summary += f" ({self.skipped} skipped: no sibling fixture)" return summary -def _scrubbed_payloads(fixture: Dict[str, Any]) -> List[Any]: - """Return the fixture's payload entries minus state-root-derived fields.""" - payloads = [] - for entry in fixture.get("engineNewPayloads", []): - entry = json.loads(json.dumps(entry)) - params = entry.get("params") - if params and isinstance(params[0], dict): - for field in _STATE_ROOT_DERIVED_FIELDS: - params[0].pop(field, None) - payloads.append(entry) - return payloads - - -def _describe_mismatch(base: List[Any], packed: List[Any]) -> str: - """Return a short description of the first difference between payloads.""" - if len(base) != len(packed): - return f"payload count: {len(base)} != {len(packed)}" - for i, (base_entry, packed_entry) in enumerate( - zip(base, packed, strict=False) +def _sibling_test_id(engine_x_test_id: str) -> str: + """Return the sibling-format id of an Engine X test id.""" + return engine_x_test_id.replace( + BlockchainEngineXFixture.format_name, + BlockchainEngineFixture.format_name, + ) + + +def _sibling_file( + engine_x_file: Path, engine_x_dir: Path, sibling_dir: Path +) -> Path: + """ + Return the sibling fixture file for an Engine X fixture file. + + A `--single-fixture-per-file` fill embeds the fixture format name in + every file name, so the sibling's basename can differ. + """ + sibling = sibling_dir / engine_x_file.relative_to(engine_x_dir) + if not sibling.exists(): + sibling = sibling.with_name( + sibling.name.replace( + BlockchainEngineXFixture.format_name, + BlockchainEngineFixture.format_name, + ) + ) + return sibling + + +def _load_fixture_file( + file: Path, fixture_cls: Type[_FixtureT] +) -> Dict[str, _FixtureT]: + """Parse every fixture in a fixture file with its typed model.""" + try: + raw = json.loads(file.read_text()) + except json.JSONDecodeError as e: + raise EngineXCheckError(f"unreadable fixture file {file}: {e}") from e + fixtures: Dict[str, _FixtureT] = {} + for test_id, data in raw.items(): + try: + fixtures[test_id] = fixture_cls.model_validate(data) + except ValidationError as e: + raise EngineXCheckError( + f"cannot parse {fixture_cls.format_name!r} fixture " + f"{test_id!r} in {file}: {e}" + ) from e + return fixtures + + +def _comparable_payload(payload: FixtureEngineNewPayload) -> Dict[str, Any]: + """ + Return the payload as a dict without state-root-derived values. + + The block access list is replaced by its decoded, parent-hash-masked + form, see `_comparable_bal`. + """ + entry = payload.model_dump( + mode="json", + by_alias=True, + exclude={ + "params": { + 0: set(STATE_ROOT_DERIVED_FIELDS) | {"block_access_list"} + } + }, + ) + execution_payload = payload.params[0] + if execution_payload.block_access_list is not None: + entry["params"][0]["blockAccessList"] = _comparable_bal( + execution_payload.block_access_list, + execution_payload.parent_hash, + ) + return entry + + +def _comparable_bal(bal: Bytes, parent_hash: Hash) -> Any: + """ + Return the decoded BAL with the EIP-2935 history write masked. + + The EIP-2935 system call writes the block's parent hash into the + history contract on every block; at payload 0 that value is the + genesis hash itself, so it is the one BAL entry that legitimately + differs between a test's own genesis and its packed group's genesis. + Only that write is masked: a storage change of the history contract + whose written value equals the payload's own parent hash. A + parent-hash-valued write to any other account is genuine drift (the + test observes block hashes and cannot survive grouping). An + undecodable BAL (an intentionally malformed one from a negative + test) is compared verbatim. + """ + try: + accounts = BlockAccessList.from_rlp(bal) + except Exception: + return str(bal) + parent_hash_value = int.from_bytes(parent_hash, "big") + dumped = accounts.model_dump(mode="json") + for account in dumped: + if int(account["address"], 16) != HISTORY_STORAGE_ADDRESS: + continue + for slot in account["storage_changes"]: + for change in slot["slot_changes"]: + if int(change["post_value"], 16) == parent_hash_value: + change["post_value"] = _PARENT_HASH_PLACEHOLDER + return dumped + + +class _GroupLookup: + """Lazily load packed pre-allocation groups for drift attribution.""" + + def __init__(self, engine_x_dir: Path): + """Initialize with the Engine X fixture tree to look under.""" + self._folder = engine_x_dir / "pre_alloc" + self._groups: PreAllocGroups | None = None + + def get(self, pre_hash: str) -> PreAllocGroup | None: + """Return the group for a hash, or None if unavailable.""" + if self._groups is None: + if not self._folder.is_dir(): + return None + try: + self._groups = PreAllocGroups.from_folder( + self._folder, lazy_load=True + ) + except Exception: + return None + try: + return self._groups[pre_hash] + except (KeyError, ValidationError): + return None + + +def _short(value: Any) -> str: + """Render a value for an error message, truncated if long.""" + text = value if isinstance(value, str) else json.dumps(value) + if len(text) > _VALUE_DISPLAY_LIMIT: + text = f"{text[:_VALUE_DISPLAY_LIMIT]}... ({len(text)} chars)" + return text + + +def _hex_int(value: Any) -> int | None: + """Parse a hex string to an int, or return None.""" + try: + return int(value, 16) + except (TypeError, ValueError): + return None + + +def _diff_fields( + base_entry: Dict[str, Any], packed_entry: Dict[str, Any] +) -> List[Tuple[str, Any, Any]]: + """Return the (field, own, packed) diffs of two comparable payloads.""" + diffs: List[Tuple[str, Any, Any]] = [] + for key in sorted(set(base_entry) | set(packed_entry)): + base_value = base_entry.get(key) + packed_value = packed_entry.get(key) + if base_value == packed_value: + continue + if ( + key != "params" + or not isinstance(base_value, list) + or not isinstance(packed_value, list) + or len(base_value) != len(packed_value) + ): + diffs.append((key, base_value, packed_value)) + continue + for i, (base_param, packed_param) in enumerate( + zip(base_value, packed_value, strict=True) + ): + if base_param == packed_param: + continue + if ( + i == 0 + and isinstance(base_param, dict) + and isinstance(packed_param, dict) + ): + for field in sorted(set(base_param) | set(packed_param)): + if base_param.get(field) != packed_param.get(field): + diffs.append( + ( + field, + base_param.get(field), + packed_param.get(field), + ) + ) + else: + diffs.append((f"params[{i}]", base_param, packed_param)) + return diffs + + +def _parent_hash_write_slot( + base_account: Dict[str, Any], + packed_account: Dict[str, Any], + base_parent_hash: int, + packed_parent_hash: int, +) -> str | None: + """Return the slot where each side stored its own parent hash.""" + packed_slots = { + slot["slot"]: slot for slot in packed_account["storage_changes"] + } + for base_slot in base_account["storage_changes"]: + packed_slot = packed_slots.get(base_slot["slot"]) + if packed_slot is None: + continue + packed_changes = { + change["block_access_index"]: change + for change in packed_slot["slot_changes"] + } + for base_change in base_slot["slot_changes"]: + packed_change = packed_changes.get( + base_change["block_access_index"] + ) + if packed_change is None or base_change == packed_change: + continue + if ( + _hex_int(base_change["post_value"]) == base_parent_hash + and _hex_int(packed_change["post_value"]) == packed_parent_hash + ): + return str(base_slot["slot"]) + return None + + +def _diff_bal( + payload_index: int, + base_bal: Any, + packed_bal: Any, + base_payload: FixtureExecutionPayload, + packed_payload: FixtureExecutionPayload, + sibling: BlockchainEngineFixture, + engine_x: BlockchainEngineXFixture, + groups: _GroupLookup, +) -> Tuple[str, str]: + """Return (signature, detail) for a block-access-list difference.""" + prefix = f"payload {payload_index}" + if isinstance(base_bal, str) or isinstance(packed_bal, str): + return ( + f"{prefix}: blockAccessList differs (undecodable BAL " + "compared verbatim)", + f"own: {_short(base_bal)}\npacked: {_short(packed_bal)}", + ) + base_accounts = {account["address"]: account for account in base_bal} + packed_accounts = {account["address"]: account for account in packed_bal} + extra = sorted(set(packed_accounts) - set(base_accounts)) + missing = sorted(set(base_accounts) - set(packed_accounts)) + if extra: + address = extra[0] + signature = ( + f"{prefix}: account {address} appears in the packed " + "fixture's BAL only" + ) + if Address(address) in sibling.pre.root: + return signature, ( + "the account is declared in the test's own " + "pre-allocation but only touched under the packed " + "genesis: packing changed the execution path" + ) + group = groups.get(engine_x.pre_hash) + if group is not None and Address(address) in group.pre: + others = ", ".join(group.test_ids[:3]) + return signature, ( + "the account is absent from the test's own " + "pre-allocation but present in pre-alloc group " + f"{engine_x.pre_hash} ({group.test_count} tests, e.g. " + f"{others}): an account introduced by packing leaked " + "into this test's execution. Isolate the test with " + '@pytest.mark.pre_alloc_group("separate") or declare ' + "the account in its pre-allocation" + ) + return signature, ( + "the account is absent from the test's own pre-allocation " + "and from its packed pre-alloc group: packing changed the " + "execution path" + ) + if missing: + return ( + f"{prefix}: account {missing[0]} is missing from the " + "packed fixture's BAL", + "the test touches the account only under its own genesis: " + "packing changed the execution path", + ) + base_parent_hash = int.from_bytes(base_payload.parent_hash, "big") + packed_parent_hash = int.from_bytes(packed_payload.parent_hash, "big") + for address, base_account in base_accounts.items(): + packed_account = packed_accounts[address] + if base_account == packed_account: + continue + slot = _parent_hash_write_slot( + base_account, + packed_account, + base_parent_hash, + packed_parent_hash, + ) + if slot is not None: + return ( + f"{prefix}: account {address} writes its block's " + f"parent hash to storage (slot {slot})", + "each fixture's BAL stores its own parent hash: the " + "test observes block hashes (e.g. via BLOCKHASH), " + "which cannot survive pre-alloc grouping. Isolate the " + 'test with @pytest.mark.pre_alloc_group("separate")', + ) + return ( + f"{prefix}: BAL differs for account {address}", + f"own: {_short(base_account)}\n" + f"packed: {_short(packed_account)}", + ) + return f"{prefix}: blockAccessList differs", "" + + +def _diagnose( + test_id: str, + engine_x: BlockchainEngineXFixture, + sibling: BlockchainEngineFixture, + base_payloads: List[Dict[str, Any]], + packed_payloads: List[Dict[str, Any]], + groups: _GroupLookup, +) -> ExecutionDrift: + """Classify the first difference between two payload sequences.""" + if len(base_payloads) != len(packed_payloads): + return ExecutionDrift( + test_id, + signature="payload count differs", + detail=( + f"sibling has {len(base_payloads)} payloads, packed " + f"fixture has {len(packed_payloads)}: packing changed " + "block-level outcomes" + ), + ) + for index, (base_entry, packed_entry) in enumerate( + zip(base_payloads, packed_payloads, strict=True) ): if base_entry == packed_entry: continue - base_payload = base_entry.get("params", [{}])[0] - packed_payload = packed_entry.get("params", [{}])[0] - if isinstance(base_payload, dict) and isinstance(packed_payload, dict): - fields = sorted( - field - for field in set(base_payload) | set(packed_payload) - if base_payload.get(field) != packed_payload.get(field) + diffs = _diff_fields(base_entry, packed_entry) + fields = sorted({field for field, _, _ in diffs}) + if "blockAccessList" in fields: + base_bal, packed_bal = next( + (base_value, packed_value) + for field, base_value, packed_value in diffs + if field == "blockAccessList" + ) + signature, detail = _diff_bal( + index, + base_bal, + packed_bal, + sibling.payloads[index].params[0], + engine_x.payloads[index].params[0], + sibling, + engine_x, + groups, ) - if fields: - return f"payload {i} differs in: {', '.join(fields)}" - return f"payload {i} differs" - return "payloads differ" + other_fields = [f for f in fields if f != "blockAccessList"] + if other_fields: + detail += ( + f"\nthe payload also differs in: {', '.join(other_fields)}" + ) + return ExecutionDrift(test_id, signature, detail) + detail_lines = [] + for field, base_value, packed_value in diffs[:5]: + detail_lines.append(f"{field}:") + detail_lines.append(f" own: {_short(base_value)}") + detail_lines.append(f" packed: {_short(packed_value)}") + return ExecutionDrift( + test_id, + signature=f"payload {index} differs in: {', '.join(fields)}", + detail="\n".join(detail_lines), + ) + return ExecutionDrift(test_id, "payloads differ", "") -def verify_engine_x_execution( - output_dir: Path, -) -> Optional[EngineXCheckResult]: +def verify_engine_x_execution(output_dir: Path) -> EngineXCheckResult: """ - Verify that pre-alloc group packing did not change any test's execution. + Verify that pre-alloc group packing did not change any test's + execution. For every Engine X fixture (filled against its packed group's merged - genesis), compare its `engineNewPayloads` against the test's - `blockchain_test_engine` sibling fixture (filled against the test's own - pre-allocation in the same session, with an independent `t8n` execution: - Engine X fixtures never share the transition tool output cache). All - payload fields except the state-root-derived ones must match exactly. + genesis), compare its payloads against the test's + `blockchain_test_engine` sibling fixture (filled against the test's + own pre-allocation in the same session, with an independent `t8n` + execution: Engine X fixtures never share the transition tool output + cache). All payload fields except the state-root-derived ones must + match exactly; each payload's block access list is compared with the + EIP-2935 history write of its own parent hash masked out, see + `_comparable_bal`. - Return the comparison counts, or ``None`` when one of the two fixture - format trees was not generated at all (e.g. when filling with - ``-m blockchain_test_engine_x``, which produces no siblings). + Return the comparison counts; `skip_reason` is set when Engine X + fixtures exist but nothing could be compared (e.g. when filling with + `-m blockchain_test_engine_x`, which produces no siblings). - Raise `EngineXExecutionDriftError` if any test executed differently. + Raise `EngineXExecutionDriftError` if any test executed differently, + or `EngineXCheckError` if a fixture file cannot be parsed. """ engine_x_dir = output_dir / ENGINE_X_FIXTURES_DIR sibling_dir = output_dir / SIBLING_FIXTURES_DIR - if not engine_x_dir.is_dir() or not sibling_dir.is_dir(): - return None + if not engine_x_dir.is_dir(): + return EngineXCheckResult() + if not sibling_dir.is_dir(): + return EngineXCheckResult( + skip_reason=( + "Engine X execution consistency check skipped: this " + f"fill generated no {SIBLING_FIXTURES_DIR} fixtures to " + "compare against (e.g. filling with `-m " + f"{BlockchainEngineXFixture.format_name}`). Leaks from " + "pre-alloc group packing are not verified for this " + "output." + ) + ) + groups = _GroupLookup(engine_x_dir) compared = 0 skipped = 0 - mismatches: List[Tuple[str, str]] = [] - for engine_x_file in engine_x_dir.rglob("*.json"): - if "pre_alloc" in engine_x_file.parts: + drifts: List[ExecutionDrift] = [] + for engine_x_file in sorted(engine_x_dir.rglob("*.json")): + relative_parts = engine_x_file.relative_to(engine_x_dir).parts + if _NON_FIXTURE_PATH_PARTS.intersection(relative_parts): continue - sibling_file = sibling_dir / engine_x_file.relative_to(engine_x_dir) - if not sibling_file.exists(): - # A --single-fixture-per-file fill embeds the fixture format - # name in every file name, so the sibling's basename differs. - sibling_file = sibling_file.with_name( - sibling_file.name.replace( - "blockchain_test_engine_x", "blockchain_test_engine" - ) - ) - sibling_fixtures = ( - json.loads(sibling_file.read_text()) + sibling_file = _sibling_file(engine_x_file, engine_x_dir, sibling_dir) + siblings = ( + _load_fixture_file(sibling_file, BlockchainEngineFixture) if sibling_file.exists() else {} ) - for test_id, fixture in json.loads(engine_x_file.read_text()).items(): - sibling_id = test_id.replace( - "blockchain_test_engine_x", "blockchain_test_engine" - ) - sibling = sibling_fixtures.get(sibling_id) + for test_id, fixture in _load_fixture_file( + engine_x_file, BlockchainEngineXFixture + ).items(): + sibling = siblings.get(_sibling_test_id(test_id)) if sibling is None: skipped += 1 continue compared += 1 - base = _scrubbed_payloads(sibling) - packed = _scrubbed_payloads(fixture) + base = [_comparable_payload(p) for p in sibling.payloads] + packed = [_comparable_payload(p) for p in fixture.payloads] if base != packed: - mismatches.append((test_id, _describe_mismatch(base, packed))) + drifts.append( + _diagnose(test_id, fixture, sibling, base, packed, groups) + ) - if mismatches: - raise EngineXExecutionDriftError(mismatches, compared) - return EngineXCheckResult(compared=compared, skipped=skipped) + if drifts: + raise EngineXExecutionDriftError(drifts, compared) + skip_reason = None + if compared == 0 and skipped > 0: + skip_reason = ( + "Engine X execution consistency check skipped: none of the " + f"{skipped} Engine X fixtures have a {SIBLING_FIXTURES_DIR} " + "sibling fixture to compare against. Leaks from pre-alloc " + "group packing are not verified for this output." + ) + return EngineXCheckResult( + compared=compared, skipped=skipped, skip_reason=skip_reason + ) diff --git a/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py b/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py index 7e44afee03..3cfa54057a 100644 --- a/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py +++ b/packages/testing/src/execution_testing/fixtures/tests/test_engine_x_checks.py @@ -6,12 +6,36 @@ import pytest +from execution_testing.base_types import Account, Address, Bytes, Hash +from execution_testing.fixtures.blockchain import ( + BlockchainEngineFixture, + BlockchainEngineXFixture, + FixtureConfig, + FixtureEngineNewPayload, + FixtureExecutionPayload, + FixtureHeader, +) from execution_testing.fixtures.engine_x_checks import ( ENGINE_X_FIXTURES_DIR, SIBLING_FIXTURES_DIR, + STATE_ROOT_DERIVED_FIELDS, + EngineXCheckError, EngineXExecutionDriftError, verify_engine_x_execution, ) +from execution_testing.fixtures.pre_alloc_groups import PreAllocGroupBuilder +from execution_testing.forks import Prague +from execution_testing.forks.forks.eips.prague.eip_2935 import ( + HISTORY_STORAGE_ADDRESS, +) +from execution_testing.test_types import Alloc, Environment +from execution_testing.test_types.block_access_list import ( + BalAccountChange, + BalNonceChange, + BalStorageChange, + BalStorageSlot, + BlockAccessList, +) ENGINE_X_ID = ( "tests/a.py::test_a[fork_Prague-blockchain_test_engine_x_from_state_test]" @@ -19,124 +43,488 @@ SIBLING_ID = ( "tests/a.py::test_a[fork_Prague-blockchain_test_engine_from_state_test]" ) +ENGINE_X_ID_B = ENGINE_X_ID.replace("test_a[", "test_b[") +SIBLING_ID_B = SIBLING_ID.replace("test_a[", "test_b[") + +PRE_HASH = "0xf00" + +# One parent hash with no leading zero bytes and one with 31 of them: +# masking must be insensitive to the RLP canonical trimming of either. +SIBLING_PARENT_HASH = Hash(bytes.fromhex("aa" * 32)) +ENGINE_X_PARENT_HASH = Hash(0xBB) + +SENDER = Address(0xA) +UNDECLARED_ACCOUNT = Address(0x1000) +TEST_CONTRACT = Address(0xC0DE) + + +def _genesis_header() -> FixtureHeader: + """Build a minimal valid Prague genesis header.""" + return FixtureHeader( + fork=Prague, + fee_recipient=Address(0), + state_root=Hash(0), + number=0, + gas_limit=30_000_000, + gas_used=0, + timestamp=0, + extra_data=b"\x00", + base_fee_per_gas=7, + withdrawals_root=Hash(0), + blob_gas_used=0, + excess_blob_gas=0, + parent_beacon_block_root=Hash(0), + requests_hash=Hash(0), + ) def _payload( - *, gas_used: str, state_root: str, block_hash: str -) -> Dict[str, Any]: - """Build a single newPayload entry.""" - return { - "newPayloadVersion": "4", - "forkchoiceUpdatedVersion": "3", - "params": [ - { - "parentHash": f"0x{'00' * 31}aa", - "stateRoot": state_root, - "blockHash": block_hash, - "gasUsed": gas_used, - "receiptsRoot": f"0x{'11' * 32}", - "logsBloom": f"0x{'00' * 256}", - "transactions": ["0xf86b..."], - }, - [], - f"0x{'00' * 32}", - ], - } + *, + parent_hash: Hash, + state_root: Hash, + block_hash: Hash, + gas_used: int, + block_access_list: Bytes | None, +) -> FixtureEngineNewPayload: + """Build a payload whose execution outputs are deterministic.""" + execution_payload = FixtureExecutionPayload( + parent_hash=parent_hash, + fee_recipient=Address(0), + state_root=state_root, + receipts_root=Hash(0x11), + logs_bloom=b"\x00" * 256, + number=1, + gas_limit=30_000_000, + gas_used=gas_used, + timestamp=12, + extra_data=b"", + prev_randao=Hash(0), + base_fee_per_gas=7, + block_hash=block_hash, + transactions=[Bytes(b"\x01")], + block_access_list=block_access_list, + ) + return FixtureEngineNewPayload( + params=(execution_payload,), + new_payload_version=1, + forkchoice_updated_version=1, + ) + +def _sibling_payload( + *, + gas_used: int = 21_000, + block_access_list: Bytes | None = None, +) -> FixtureEngineNewPayload: + """Build a payload as filled against the test's own genesis.""" + return _payload( + parent_hash=SIBLING_PARENT_HASH, + state_root=Hash(1), + block_hash=Hash(2), + gas_used=gas_used, + block_access_list=block_access_list, + ) -def _write_fixture( + +def _engine_x_payload( + *, + gas_used: int = 21_000, + block_access_list: Bytes | None = None, +) -> FixtureEngineNewPayload: + """Build the same payload as filled against the packed genesis.""" + return _payload( + parent_hash=ENGINE_X_PARENT_HASH, + state_root=Hash(3), + block_hash=Hash(4), + gas_used=gas_used, + block_access_list=block_access_list, + ) + + +def _write_fixture_file(file: Path, fixtures: Dict[str, Any]) -> None: + """Write (or extend) a fixture file with serialized fixtures.""" + file.parent.mkdir(parents=True, exist_ok=True) + existing: Dict[str, Any] = {} + if file.exists(): + existing = json.loads(file.read_text()) + existing.update( + { + test_id: fixture.json_dict_with_info() + for test_id, fixture in fixtures.items() + } + ) + file.write_text(json.dumps(existing)) + + +def _write_sibling( folder: Path, - fixture_dir: str, - test_id: str, - payloads: List[Dict[str, Any]], + payloads: List[FixtureEngineNewPayload], + *, + test_id: str = SIBLING_ID, + file_name: str = "test_a.json", + pre: Alloc | None = None, ) -> None: - """Write a single-fixture file into a format tree.""" - file = folder / fixture_dir / "prague" / "module" / "test_a.json" - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(json.dumps({test_id: {"engineNewPayloads": payloads}})) + """Write a sibling engine fixture into its format tree.""" + fixture = BlockchainEngineFixture( + fork=Prague, + last_block_hash=Hash(0), + config=FixtureConfig(fork=Prague), + pre=pre if pre is not None else Alloc({SENDER: Account(balance=1)}), + post_state=Alloc({SENDER: Account(balance=1)}), + genesis=_genesis_header(), + payloads=payloads, + ) + _write_fixture_file( + folder / SIBLING_FIXTURES_DIR / "prague" / "module" / file_name, + {test_id: fixture}, + ) + + +def _write_engine_x( + folder: Path, + payloads: List[FixtureEngineNewPayload], + *, + test_id: str = ENGINE_X_ID, + file_name: str = "test_a.json", + pre_hash: str = PRE_HASH, +) -> None: + """Write an Engine X fixture into its format tree.""" + fixture = BlockchainEngineXFixture( + fork=Prague, + last_block_hash=Hash(0), + config=FixtureConfig(fork=Prague), + pre_hash=pre_hash, + post_state_diff=Alloc({}), + payloads=payloads, + ) + _write_fixture_file( + folder / ENGINE_X_FIXTURES_DIR / "prague" / "module" / file_name, + {test_id: fixture}, + ) + + +def _write_group( + folder: Path, + accounts: Dict[Address, Account | None], + test_ids: List[str], + *, + pre_hash: str = PRE_HASH, +) -> None: + """Write a packed pre-alloc group file, as phase 1 would.""" + group_folder = folder / ENGINE_X_FIXTURES_DIR / "pre_alloc" + group_folder.mkdir(parents=True, exist_ok=True) + builder = PreAllocGroupBuilder( + test_ids=test_ids, + environment=Environment( + base_fee_per_gas=7, + excess_blob_gas=0, + blob_gas_used=0, + withdrawals=[], + parent_beacon_block_root=Hash(0), + ), + fork=Prague, + pre=Alloc(accounts), + ) + (group_folder / f"{pre_hash}.json").write_text( + builder.model_dump_json(by_alias=True, exclude_none=True) + ) + + +def _bal(*accounts: BalAccountChange) -> Bytes: + """RLP-encode a BAL from account changes.""" + return BlockAccessList(list(accounts)).rlp + + +def _history_write(parent_hash: Hash) -> BalAccountChange: + """Build the EIP-2935 system write of the block's parent hash.""" + return _storage_write( + Address(HISTORY_STORAGE_ADDRESS), + slot=0, + value=int.from_bytes(parent_hash, "big"), + ) + + +def _storage_write( + address: Address, *, slot: int, value: int +) -> BalAccountChange: + """Build a single storage write of an account in a BAL.""" + return BalAccountChange( + address=address, + storage_changes=[ + BalStorageSlot( + slot=slot, + slot_changes=[ + BalStorageChange(block_access_index=0, post_value=value) + ], + ) + ], + ) + + +def _nonce_touch(address: Address) -> BalAccountChange: + """Build a minimal appearance of an account in a BAL.""" + return BalAccountChange( + address=address, + nonce_changes=[BalNonceChange(block_access_index=0, post_nonce=1)], + ) def test_identical_execution_passes(tmp_path: Path) -> None: """State-root-derived differences alone do not trip the check.""" - _write_fixture( + _write_sibling(tmp_path, [_sibling_payload()]) + _write_engine_x(tmp_path, [_engine_x_payload()]) + + result = verify_engine_x_execution(tmp_path) + + assert result.compared == 1 + assert result.skipped == 0 + assert result.skip_reason is None + assert "1 Engine X fixtures execute identically" in result.summary + + +def test_bal_embedded_parent_hash_passes(tmp_path: Path) -> None: + """ + The EIP-2935 write embeds each side's own parent hash in its BAL; a + BAL differing only by that history-contract write does not trip the + check, whatever the leading-zero shape of either hash. + """ + _write_sibling( tmp_path, - SIBLING_FIXTURES_DIR, - SIBLING_ID, - [_payload(gas_used="0x5208", state_root="0x01", block_hash="0x02")], + [ + _sibling_payload( + block_access_list=_bal(_history_write(SIBLING_PARENT_HASH)), + ) + ], ) - _write_fixture( + _write_engine_x( tmp_path, - ENGINE_X_FIXTURES_DIR, - ENGINE_X_ID, - [_payload(gas_used="0x5208", state_root="0xaa", block_hash="0xbb")], + [ + _engine_x_payload( + block_access_list=_bal(_history_write(ENGINE_X_PARENT_HASH)), + ) + ], ) result = verify_engine_x_execution(tmp_path) - assert result is not None assert result.compared == 1 - assert "1 Engine X fixtures execute identically" in result.summary -def test_execution_drift_raises(tmp_path: Path) -> None: - """A gas difference (a leaked account changed execution) fails loudly.""" - _write_fixture( +def test_bal_leaked_account_raises(tmp_path: Path) -> None: + """An account appearing only in the packed BAL fails loudly.""" + _write_sibling( + tmp_path, + [ + _sibling_payload( + block_access_list=_bal(_history_write(SIBLING_PARENT_HASH)), + ) + ], + ) + _write_engine_x( + tmp_path, + [ + _engine_x_payload( + block_access_list=_bal( + _nonce_touch(UNDECLARED_ACCOUNT), + _history_write(ENGINE_X_PARENT_HASH), + ), + ) + ], + ) + + with pytest.raises(EngineXExecutionDriftError) as exc_info: + verify_engine_x_execution(tmp_path) + + message = str(exc_info.value).lower() + assert str(UNDECLARED_ACCOUNT).lower() in message + assert "packed fixture's bal only" in message + + +def test_bal_leak_is_attributed_to_the_group(tmp_path: Path) -> None: + """A leaked account is traced to its packed pre-alloc group.""" + _write_sibling( + tmp_path, + [ + _sibling_payload( + block_access_list=_bal(_history_write(SIBLING_PARENT_HASH)), + ) + ], + ) + _write_engine_x( + tmp_path, + [ + _engine_x_payload( + block_access_list=_bal( + _nonce_touch(UNDECLARED_ACCOUNT), + _history_write(ENGINE_X_PARENT_HASH), + ), + ) + ], + ) + _write_group( + tmp_path, + accounts={ + SENDER: Account(balance=1), + UNDECLARED_ACCOUNT: Account(balance=1), + }, + test_ids=["tests/b.py::test_leaker[fork_Prague-foo]"], + ) + + with pytest.raises(EngineXExecutionDriftError) as exc_info: + verify_engine_x_execution(tmp_path) + + message = str(exc_info.value) + assert PRE_HASH in message + assert "test_leaker" in message + assert 'pre_alloc_group("separate")' in message + + +def test_parent_hash_write_outside_history_contract_raises( + tmp_path: Path, +) -> None: + """ + A test storing its block's parent hash in its own contract is drift: + only the EIP-2935 history-contract write is masked. + """ + _write_sibling( tmp_path, - SIBLING_FIXTURES_DIR, - SIBLING_ID, - [_payload(gas_used="0x5208", state_root="0x01", block_hash="0x02")], + [ + _sibling_payload( + block_access_list=_bal( + _storage_write( + TEST_CONTRACT, + slot=1, + value=int.from_bytes(SIBLING_PARENT_HASH, "big"), + ), + ), + ) + ], ) - _write_fixture( + _write_engine_x( tmp_path, - ENGINE_X_FIXTURES_DIR, - ENGINE_X_ID, - [_payload(gas_used="0xbeef", state_root="0xaa", block_hash="0xbb")], + [ + _engine_x_payload( + block_access_list=_bal( + _storage_write( + TEST_CONTRACT, + slot=1, + value=int.from_bytes(ENGINE_X_PARENT_HASH, "big"), + ), + ), + ) + ], ) + with pytest.raises(EngineXExecutionDriftError) as exc_info: + verify_engine_x_execution(tmp_path) + + message = str(exc_info.value) + assert "writes its block's parent hash" in message + assert "BLOCKHASH" in message + + +def test_malformed_bal_compared_verbatim(tmp_path: Path) -> None: + """An undecodable BAL (negative test) is compared verbatim.""" + garbage = Bytes(b"\xde\xad\xbe\xef") + _write_sibling(tmp_path, [_sibling_payload(block_access_list=garbage)]) + _write_engine_x(tmp_path, [_engine_x_payload(block_access_list=garbage)]) + + result = verify_engine_x_execution(tmp_path) + + assert result.compared == 1 + + +def test_malformed_bal_drift_raises(tmp_path: Path) -> None: + """Differing undecodable BALs fail loudly.""" + _write_sibling( + tmp_path, + [_sibling_payload(block_access_list=Bytes(b"\xde\xad\xbe\xef"))], + ) + _write_engine_x( + tmp_path, + [_engine_x_payload(block_access_list=Bytes(b"\xde\xad\xbe\xee"))], + ) + + with pytest.raises(EngineXExecutionDriftError) as exc_info: + verify_engine_x_execution(tmp_path) + + assert "undecodable" in str(exc_info.value) + + +def test_execution_drift_raises(tmp_path: Path) -> None: + """A gas difference fails loudly and shows both values.""" + _write_sibling(tmp_path, [_sibling_payload()]) + _write_engine_x(tmp_path, [_engine_x_payload(gas_used=0xBEEF)]) + with pytest.raises(EngineXExecutionDriftError) as exc_info: verify_engine_x_execution(tmp_path) message = str(exc_info.value) assert ENGINE_X_ID in message assert "gasUsed" in message + assert "0x5208" in message + assert "0xbeef" in message def test_payload_count_drift_raises(tmp_path: Path) -> None: """A different number of payloads fails loudly.""" - payload = _payload(gas_used="0x5208", state_root="0x01", block_hash="0x02") - _write_fixture(tmp_path, SIBLING_FIXTURES_DIR, SIBLING_ID, [payload]) - _write_fixture( - tmp_path, ENGINE_X_FIXTURES_DIR, ENGINE_X_ID, [payload, payload] - ) + _write_sibling(tmp_path, [_sibling_payload()]) + _write_engine_x(tmp_path, [_engine_x_payload(), _engine_x_payload()]) with pytest.raises(EngineXExecutionDriftError) as exc_info: verify_engine_x_execution(tmp_path) - assert "payload count" in str(exc_info.value) + assert "payload count differs" in str(exc_info.value) -def test_no_sibling_fixtures_skips_check(tmp_path: Path) -> None: - """An Engine X only fill (no sibling format tree) skips the check.""" - _write_fixture( +def test_same_cause_drifts_aggregate(tmp_path: Path) -> None: + """Drifts with the same cause collapse into one diagnosis.""" + _write_sibling(tmp_path, [_sibling_payload()]) + _write_sibling( + tmp_path, + [_sibling_payload()], + test_id=SIBLING_ID_B, + file_name="test_b.json", + ) + _write_engine_x(tmp_path, [_engine_x_payload(gas_used=0xBEEF)]) + _write_engine_x( tmp_path, - ENGINE_X_FIXTURES_DIR, - ENGINE_X_ID, - [_payload(gas_used="0x5208", state_root="0x01", block_hash="0x02")], + [_engine_x_payload(gas_used=0xBEEF)], + test_id=ENGINE_X_ID_B, + file_name="test_b.json", ) - assert verify_engine_x_execution(tmp_path) is None + with pytest.raises(EngineXExecutionDriftError) as exc_info: + verify_engine_x_execution(tmp_path) + message = str(exc_info.value) + assert "2 of 2" in message + assert "1 distinct cause" in message + assert "[2x]" in message + assert ENGINE_X_ID in message + assert ENGINE_X_ID_B in message -def test_no_engine_x_fixtures_skips_check(tmp_path: Path) -> None: - """A fill without Engine X fixtures skips the check.""" - _write_fixture( - tmp_path, - SIBLING_FIXTURES_DIR, - SIBLING_ID, - [_payload(gas_used="0x5208", state_root="0x01", block_hash="0x02")], - ) - assert verify_engine_x_execution(tmp_path) is None +def test_no_sibling_tree_sets_skip_reason(tmp_path: Path) -> None: + """An Engine X only fill (no sibling format tree) skips the check.""" + _write_engine_x(tmp_path, [_engine_x_payload()]) + + result = verify_engine_x_execution(tmp_path) + + assert result.compared == 0 + assert result.skip_reason is not None + assert f"generated no {SIBLING_FIXTURES_DIR}" in result.skip_reason + + +def test_no_engine_x_fixtures_is_silent(tmp_path: Path) -> None: + """A fill without Engine X fixtures has nothing to check or warn.""" + _write_sibling(tmp_path, [_sibling_payload()]) + + result = verify_engine_x_execution(tmp_path) + + assert result.compared == 0 + assert result.skipped == 0 + assert result.skip_reason is None def test_single_fixture_per_file_sibling_lookup(tmp_path: Path) -> None: @@ -144,70 +532,109 @@ def test_single_fixture_per_file_sibling_lookup(tmp_path: Path) -> None: A `--single-fixture-per-file` fill embeds the fixture format name in every file name; the sibling is still found under its own basename. """ - payload = _payload(gas_used="0x5208", state_root="0x01", block_hash="0x02") - sibling_file = ( - tmp_path - / SIBLING_FIXTURES_DIR - / "prague" - / "module" - / "a__fork_Prague_blockchain_test_engine_from_state_test.json" - ) - sibling_file.parent.mkdir(parents=True, exist_ok=True) - sibling_file.write_text( - json.dumps({SIBLING_ID: {"engineNewPayloads": [payload]}}) - ) - engine_x_file = ( - tmp_path - / ENGINE_X_FIXTURES_DIR - / "prague" - / "module" - / "a__fork_Prague_blockchain_test_engine_x_from_state_test.json" - ) - engine_x_file.parent.mkdir(parents=True, exist_ok=True) - engine_x_file.write_text( - json.dumps({ENGINE_X_ID: {"engineNewPayloads": [payload]}}) + _write_sibling( + tmp_path, + [_sibling_payload()], + file_name=( + "a__fork_Prague_blockchain_test_engine_from_state_test.json" + ), + ) + _write_engine_x( + tmp_path, + [_engine_x_payload()], + file_name=( + "a__fork_Prague_blockchain_test_engine_x_from_state_test.json" + ), ) result = verify_engine_x_execution(tmp_path) - assert result is not None assert result.compared == 1 assert result.skipped == 0 def test_missing_sibling_fixture_is_skipped(tmp_path: Path) -> None: """A test filtered from the sibling format is skipped, not failed.""" - payload = _payload(gas_used="0x5208", state_root="0x01", block_hash="0x02") - _write_fixture(tmp_path, SIBLING_FIXTURES_DIR, SIBLING_ID, [payload]) - other_engine_x_id = ENGINE_X_ID.replace("test_a[", "test_b[") - _write_fixture(tmp_path, ENGINE_X_FIXTURES_DIR, ENGINE_X_ID, [payload]) - file = ( - tmp_path / ENGINE_X_FIXTURES_DIR / "prague" / "module" / "test_b.json" - ) - file.write_text( - json.dumps({other_engine_x_id: {"engineNewPayloads": [payload]}}) + _write_sibling(tmp_path, [_sibling_payload()]) + _write_engine_x(tmp_path, [_engine_x_payload()]) + _write_engine_x( + tmp_path, + [_engine_x_payload()], + test_id=ENGINE_X_ID_B, + file_name="test_b.json", ) result = verify_engine_x_execution(tmp_path) - assert result is not None assert result.compared == 1 assert result.skipped == 1 assert "1 skipped" in result.summary -def test_no_matching_siblings_reports_skip_count(tmp_path: Path) -> None: +def test_no_matching_siblings_sets_skip_reason(tmp_path: Path) -> None: """ - Sibling fixtures exist but none match: The check reports the skip + Sibling fixtures exist but none match: the check reports the skip count instead of pretending no siblings were generated. """ - payload = _payload(gas_used="0x5208", state_root="0x01", block_hash="0x02") - other_sibling_id = SIBLING_ID.replace("test_a[", "test_b[") - _write_fixture(tmp_path, SIBLING_FIXTURES_DIR, other_sibling_id, [payload]) - _write_fixture(tmp_path, ENGINE_X_FIXTURES_DIR, ENGINE_X_ID, [payload]) + _write_sibling( + tmp_path, + [_sibling_payload()], + test_id=SIBLING_ID_B, + file_name="test_b.json", + ) + _write_engine_x(tmp_path, [_engine_x_payload()]) result = verify_engine_x_execution(tmp_path) - assert result is not None assert result.compared == 0 assert result.skipped == 1 + assert result.skip_reason is not None + assert "none of the 1" in result.skip_reason + + +def test_unparseable_fixture_raises(tmp_path: Path) -> None: + """A fixture that fails typed validation is a loud error.""" + _write_sibling(tmp_path, [_sibling_payload()]) + file = ( + tmp_path / ENGINE_X_FIXTURES_DIR / "prague" / "module" / "test_a.json" + ) + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(json.dumps({ENGINE_X_ID: {"engineNewPayloads": []}})) + + with pytest.raises(EngineXCheckError, match="cannot parse"): + verify_engine_x_execution(tmp_path) + + +def test_non_fixture_files_are_ignored(tmp_path: Path) -> None: + """`pre_alloc` group files and `.meta` files are not fixtures.""" + _write_sibling(tmp_path, [_sibling_payload()]) + _write_engine_x(tmp_path, [_engine_x_payload()]) + _write_group( + tmp_path, + accounts={SENDER: Account(balance=1)}, + test_ids=[ENGINE_X_ID], + ) + meta = tmp_path / ENGINE_X_FIXTURES_DIR / ".meta" + meta.mkdir(parents=True) + (meta / "index.json").write_text('{"not": "a fixture"}') + + result = verify_engine_x_execution(tmp_path) + + assert result.compared == 1 + assert result.skipped == 0 + + +def test_state_root_derived_fields_exist_on_the_payload_model() -> None: + """The exclusion set must track `FixtureExecutionPayload` renames.""" + model_fields = set(FixtureExecutionPayload.model_fields) + assert STATE_ROOT_DERIVED_FIELDS <= model_fields + assert "block_access_list" in model_fields + + +def test_bal_dump_keys_match_the_masking_walk() -> None: + """The BAL mask walks these keys; they must track the BAL models.""" + account = _history_write(SIBLING_PARENT_HASH).model_dump(mode="json") + assert {"address", "storage_changes"} <= set(account) + slot = account["storage_changes"][0] + assert {"slot", "slot_changes"} <= set(slot) + assert {"block_access_index", "post_value"} <= set(slot["slot_changes"][0]) diff --git a/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py b/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py index 64c7d7225e..0ed2c038f8 100644 --- a/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py +++ b/tests/amsterdam/eip7928_block_level_access_lists/test_block_access_lists_eip2935.py @@ -136,7 +136,17 @@ def test_bal_2935_empty_block( @pytest.mark.parametrize( "query_block_number,is_valid", [ - pytest.param(0, True, id="valid_block_number"), + pytest.param( + 0, + True, + id="valid_block_number", + marks=pytest.mark.pre_alloc_group( + "separate", + reason="Queries the genesis hash from the history " + "contract and stores it, so the BAL contains the genesis " + "hash itself, which changes under any shared genesis.", + ), + ), pytest.param(1042, False, id="block_number_out_of_range"), ], ) diff --git a/tests/ported_static/conftest.py b/tests/ported_static/conftest.py index c5d9aff2a9..b08ef30d39 100644 --- a/tests/ported_static/conftest.py +++ b/tests/ported_static/conftest.py @@ -10,6 +10,7 @@ remove this skip list. """ +import re from pathlib import Path import pytest @@ -22,23 +23,17 @@ if line.strip() and not line.lstrip().startswith("#") ) -# Fixture format suffixes pytest appends inside the parametrize id. These -# must be stripped from the nodeid before substring-matching against the -# skip list, because the skip list predates these suffixes. -_FIXTURE_FORMAT_TOKENS: tuple[str, ...] = ( - "-blockchain_test_engine_from_state_test", - "-blockchain_test_from_state_test", - "-blockchain_test_engine", - "-blockchain_test", - "-state_test", -) +# Fixture format tokens pytest embeds in the parametrize id (e.g. +# `-blockchain_test_engine_x_from_state_test`). These must be stripped from +# the nodeid before substring-matching against the skip list, because the +# skip list predates these tokens. Matched by prefix so every format and +# label variant is covered. +_FIXTURE_FORMAT_TOKEN_RE = re.compile(r"-(?:blockchain|state)_test\w*") def _normalize_nodeid(nodeid: str) -> str: - """Strip pytest fixture-format suffixes to match the skip list format.""" - for token in _FIXTURE_FORMAT_TOKENS: - nodeid = nodeid.replace(token, "") - return nodeid + """Strip pytest fixture-format id tokens to match the skip list format.""" + return _FIXTURE_FORMAT_TOKEN_RE.sub("", nodeid) def pytest_collection_modifyitems( diff --git a/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py b/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py index cebe282fc2..9b8e0db173 100644 --- a/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py +++ b/tests/ported_static/stEIP150singleCodeGasPrices/test_gas_cost_return.py @@ -26,6 +26,12 @@ ) @pytest.mark.valid_from("Cancun") @pytest.mark.pre_alloc_mutable +@pytest.mark.pre_alloc_group( + "separate", + reason="Calls hardcoded addresses 0x1000 and 0x2000 without declaring " + "them, so gas usage depends on them staying empty; sharing a genesis " + "with a test that allocates either address changes the execution.", +) def test_gas_cost_return( state_test: StateTestFiller, pre: Alloc,