Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,17 @@ past roughly six lines it belongs in the PR the entry links.

### Fixed

- **(common)** Order a pre-release version when importing a report, instead of treating it as no
version at all. `SmdaFunction.fromDict` gated its escape-version comparisons on
`v?\d+(\.\d+)*`, which no PEP 440 pre-release matches, so a report written by `4.8.0rc1` fell to
the "no valid version information" branch and had every function's `pic_hash` and nesting depth
recomputed on every import -- correct values at a cost proportional to the report, and the wrong
reading of a report from the newest build in existence. A pre-release now sorts directly below its
own release, so a candidate for the release that changed an escaper is still recomputed. *Measured
on `669153c` on the bundled `cutwail_xored` report:* 33 of 33 functions recomputed before, 0 after,
with hashes identical either way; a version this cannot order, such as `4.4.4-dev`, still
recomputes. (#345)

### Security

### Compatibility
Expand Down
38 changes: 29 additions & 9 deletions src/smda/common/SmdaFunction.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import re
import struct
from operator import itemgetter
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, NoReturn, Optional
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, NoReturn, Optional, Tuple

from smda.aarch64.AArch64InstructionEscaper import AArch64InstructionEscaper
from smda.aarch64.definitions import CALL_INS as AARCH64_CALL_INS
Expand Down Expand Up @@ -59,6 +59,23 @@
# position-dependent Dalvik pic_hash values and must recalculate on import.
DALVIK_PIC_HASH_ESCAPE_VERSION = [4, 4, 2]


# A report version as something the escape-version gates can be compared against: the
# release numbers, then 1 for a release and 0 for a PEP 440 pre-release, so 4.2.0rc1 sorts
# below 4.2.0 and a report written by a candidate for the release that changed an escaper
# is recomputed rather than trusted. None means the string states no version this can order,
# which is what sends the report down the recalculate-everything path.
def _reportVersionKey(version: str) -> Optional[Tuple[List[int], int]]:
match = re.fullmatch(r"v?(?P<release>\d+(?:\.\d+)*)(?P<pre>(?:a|b|rc)\d+)?", version)
if match is None:
return None
return ([int(part) for part in match.group("release").split(".")], 0 if match.group("pre") else 1)


def _isBeforeEscapeVersion(version_key: Tuple[List[int], int], escape_version: List[int]) -> bool:
return version_key < (escape_version, 1)


MAX_ADDRESS_VALUE = 1 << 64
REQUIRED_FUNCTION_FIELDS = frozenset({"offset", "blocks", "apirefs", "blockrefs", "inrefs", "outrefs", "metadata"})
REQUIRED_FUNCTION_METADATA = frozenset(
Expand Down Expand Up @@ -804,28 +821,31 @@ def fromDict(cls, function_dict, binary_info=None, version=None, smda_report=Non
if version and version.startswith("MCRIT4IDA"):
version = version.rsplit(" ", 1)[-1]
# modernize older reports on import
if version and re.fullmatch(r"v?\d+(\.\d+)*", version):
version = version.replace("v", "")
version = [int(v) for v in version.split(".")]
recalculate_pic_hash = version < [1, 3, 0]
version_key = _reportVersionKey(version) if version else None
if version_key is not None:
recalculate_pic_hash = _isBeforeEscapeVersion(version_key, [1, 3, 0])
if (
not recalculate_pic_hash
and function_architecture == "aarch64"
and version < AARCH64_PIC_HASH_ESCAPE_VERSION
and _isBeforeEscapeVersion(version_key, AARCH64_PIC_HASH_ESCAPE_VERSION)
):
recalculate_pic_hash = True
if (
not recalculate_pic_hash
and function_architecture == "intel"
and version < INTEL_PIC_HASH_ESCAPE_VERSION
and _isBeforeEscapeVersion(version_key, INTEL_PIC_HASH_ESCAPE_VERSION)
):
recalculate_pic_hash = True
if not recalculate_pic_hash and function_architecture == "cil" and version < CIL_PIC_HASH_ESCAPE_VERSION:
if (
not recalculate_pic_hash
and function_architecture == "cil"
and _isBeforeEscapeVersion(version_key, CIL_PIC_HASH_ESCAPE_VERSION)
):
recalculate_pic_hash = True
if (
not recalculate_pic_hash
and function_architecture == "dalvik"
and version < DALVIK_PIC_HASH_ESCAPE_VERSION
and _isBeforeEscapeVersion(version_key, DALVIK_PIC_HASH_ESCAPE_VERSION)
):
recalculate_pic_hash = True
if recalculate_pic_hash:
Expand Down
63 changes: 62 additions & 1 deletion tests/testCommonModels.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,13 @@

from smda.common.BinaryInfo import BinaryInfo
from smda.common.SmdaBasicBlock import SmdaBasicBlock
from smda.common.SmdaFunction import INTEL_PIC_HASH_ESCAPE_VERSION, LazyIntKeyDict, SmdaFunction
from smda.common.SmdaFunction import (
INTEL_PIC_HASH_ESCAPE_VERSION,
LazyIntKeyDict,
SmdaFunction,
_isBeforeEscapeVersion,
_reportVersionKey,
)
from smda.common.SmdaInstruction import SmdaInstruction
from smda.common.SmdaReport import REQUIRED_REPORT_FIELDS, SmdaReport
from smda.Disassembler import Disassembler
Expand Down Expand Up @@ -384,6 +390,61 @@ def test_imported_report_with_a_suffixed_version_recalculates_instead_of_raising

self.assertEqual(imported.pic_hash, expected_pic_hash)

def test_imported_report_from_a_prerelease_above_the_escape_version_keeps_pic_hash(self):
# a pre-release states a version, so it is ordered rather than treated as unversioned:
# without that, every report a release candidate wrote is rehashed on every import
report = _disassemble_intel_bytes(b"\xc3") # ret
function_dict = report.getFunction(INTEL_BASE).toDict()
stored_pic_hash = 0x0123456789ABCDEF
function_dict["metadata"]["pic_hash"] = stored_pic_hash
above_gate = [*INTEL_PIC_HASH_ESCAPE_VERSION[:-1], INTEL_PIC_HASH_ESCAPE_VERSION[-1] + 1]

for suffix in ("a1", "b2", "rc1"):
with self.subTest(suffix=suffix):
version = ".".join(str(v) for v in above_gate) + suffix

imported = SmdaFunction.fromDict(function_dict, version=version, smda_report=report)

self.assertEqual(imported.pic_hash, stored_pic_hash)

def test_imported_report_from_a_prerelease_of_the_escape_version_recalculates(self):
# 4.3.5rc1 precedes 4.3.5, so a candidate for the release that changed the escaper
# cannot be assumed to carry the change
report = _disassemble_intel_bytes(b"\xc3") # ret
function = report.getFunction(INTEL_BASE)
expected_pic_hash = function.pic_hash
function_dict = function.toDict()
function_dict["metadata"]["pic_hash"] = 0x0123456789ABCDEF
gate_version = ".".join(str(v) for v in INTEL_PIC_HASH_ESCAPE_VERSION)

for suffix in ("a1", "b2", "rc1"):
with self.subTest(suffix=suffix):
imported = SmdaFunction.fromDict(function_dict, version=gate_version + suffix, smda_report=report)

self.assertEqual(imported.pic_hash, expected_pic_hash)

def test_report_version_ordering_is_unchanged_for_release_and_unorderable_strings(self):
# the shapes that were already ordered have to keep ordering the same way, a version
# shorter than the gate included: [4, 3] precedes [4, 3, 5] on the prefix rule
ordered = {
"4.3.5": False,
"v4.3.5": False,
"4.3.4": True,
"4.3": True,
"4.3.6": False,
"5": False,
}
for version, expected in ordered.items():
with self.subTest(version=version):
key = _reportVersionKey(version)

self.assertIsNotNone(key)
self.assertEqual(_isBeforeEscapeVersion(key, INTEL_PIC_HASH_ESCAPE_VERSION), expected)

for version in ("4.4.4-dev", "MCRIT4IDA", "", "4..5", "4.3.5rc"):
with self.subTest(version=version):
self.assertIsNone(_reportVersionKey(version))

def test_imported_report_skips_hashing_against_an_unlocated_binary_info(self):
report = _disassemble_intel_bytes(b"\xc3") # ret
function_dict = report.getFunction(INTEL_BASE).toDict()
Expand Down