diff --git a/CHANGELOG.md b/CHANGELOG.md index 1837c41f..873a524b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,17 @@ past roughly six lines it belongs in the PR the entry links. ### Fixed +- **(common)** Order a pre-release version when importing a report, instead of treating it as no + version at all. `SmdaFunction.fromDict` gated its escape-version comparisons on + `v?\d+(\.\d+)*`, which no PEP 440 pre-release matches, so a report written by `4.8.0rc1` fell to + the "no valid version information" branch and had every function's `pic_hash` and nesting depth + recomputed on every import -- correct values at a cost proportional to the report, and the wrong + reading of a report from the newest build in existence. A pre-release now sorts directly below its + own release, so a candidate for the release that changed an escaper is still recomputed. *Measured + on `669153c` on the bundled `cutwail_xored` report:* 33 of 33 functions recomputed before, 0 after, + with hashes identical either way; a version this cannot order, such as `4.4.4-dev`, still + recomputes. (#345) + ### Security ### Compatibility diff --git a/src/smda/common/SmdaFunction.py b/src/smda/common/SmdaFunction.py index ce569129..22bf4c3d 100644 --- a/src/smda/common/SmdaFunction.py +++ b/src/smda/common/SmdaFunction.py @@ -5,7 +5,7 @@ import re import struct from operator import itemgetter -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, NoReturn, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, NoReturn, Optional, Tuple from smda.aarch64.AArch64InstructionEscaper import AArch64InstructionEscaper from smda.aarch64.definitions import CALL_INS as AARCH64_CALL_INS @@ -59,6 +59,23 @@ # position-dependent Dalvik pic_hash values and must recalculate on import. DALVIK_PIC_HASH_ESCAPE_VERSION = [4, 4, 2] + +# A report version as something the escape-version gates can be compared against: the +# release numbers, then 1 for a release and 0 for a PEP 440 pre-release, so 4.2.0rc1 sorts +# below 4.2.0 and a report written by a candidate for the release that changed an escaper +# is recomputed rather than trusted. None means the string states no version this can order, +# which is what sends the report down the recalculate-everything path. +def _reportVersionKey(version: str) -> Optional[Tuple[List[int], int]]: + match = re.fullmatch(r"v?(?P\d+(?:\.\d+)*)(?P
(?:a|b|rc)\d+)?", version)
+    if match is None:
+        return None
+    return ([int(part) for part in match.group("release").split(".")], 0 if match.group("pre") else 1)
+
+
+def _isBeforeEscapeVersion(version_key: Tuple[List[int], int], escape_version: List[int]) -> bool:
+    return version_key < (escape_version, 1)
+
+
 MAX_ADDRESS_VALUE = 1 << 64
 REQUIRED_FUNCTION_FIELDS = frozenset({"offset", "blocks", "apirefs", "blockrefs", "inrefs", "outrefs", "metadata"})
 REQUIRED_FUNCTION_METADATA = frozenset(
@@ -804,28 +821,31 @@ def fromDict(cls, function_dict, binary_info=None, version=None, smda_report=Non
         if version and version.startswith("MCRIT4IDA"):
             version = version.rsplit(" ", 1)[-1]
         # modernize older reports on import
-        if version and re.fullmatch(r"v?\d+(\.\d+)*", version):
-            version = version.replace("v", "")
-            version = [int(v) for v in version.split(".")]
-            recalculate_pic_hash = version < [1, 3, 0]
+        version_key = _reportVersionKey(version) if version else None
+        if version_key is not None:
+            recalculate_pic_hash = _isBeforeEscapeVersion(version_key, [1, 3, 0])
             if (
                 not recalculate_pic_hash
                 and function_architecture == "aarch64"
-                and version < AARCH64_PIC_HASH_ESCAPE_VERSION
+                and _isBeforeEscapeVersion(version_key, AARCH64_PIC_HASH_ESCAPE_VERSION)
             ):
                 recalculate_pic_hash = True
             if (
                 not recalculate_pic_hash
                 and function_architecture == "intel"
-                and version < INTEL_PIC_HASH_ESCAPE_VERSION
+                and _isBeforeEscapeVersion(version_key, INTEL_PIC_HASH_ESCAPE_VERSION)
             ):
                 recalculate_pic_hash = True
-            if not recalculate_pic_hash and function_architecture == "cil" and version < CIL_PIC_HASH_ESCAPE_VERSION:
+            if (
+                not recalculate_pic_hash
+                and function_architecture == "cil"
+                and _isBeforeEscapeVersion(version_key, CIL_PIC_HASH_ESCAPE_VERSION)
+            ):
                 recalculate_pic_hash = True
             if (
                 not recalculate_pic_hash
                 and function_architecture == "dalvik"
-                and version < DALVIK_PIC_HASH_ESCAPE_VERSION
+                and _isBeforeEscapeVersion(version_key, DALVIK_PIC_HASH_ESCAPE_VERSION)
             ):
                 recalculate_pic_hash = True
             if recalculate_pic_hash:
diff --git a/tests/testCommonModels.py b/tests/testCommonModels.py
index 1c1652ab..35013317 100644
--- a/tests/testCommonModels.py
+++ b/tests/testCommonModels.py
@@ -9,7 +9,13 @@
 
 from smda.common.BinaryInfo import BinaryInfo
 from smda.common.SmdaBasicBlock import SmdaBasicBlock
-from smda.common.SmdaFunction import INTEL_PIC_HASH_ESCAPE_VERSION, LazyIntKeyDict, SmdaFunction
+from smda.common.SmdaFunction import (
+    INTEL_PIC_HASH_ESCAPE_VERSION,
+    LazyIntKeyDict,
+    SmdaFunction,
+    _isBeforeEscapeVersion,
+    _reportVersionKey,
+)
 from smda.common.SmdaInstruction import SmdaInstruction
 from smda.common.SmdaReport import REQUIRED_REPORT_FIELDS, SmdaReport
 from smda.Disassembler import Disassembler
@@ -384,6 +390,61 @@ def test_imported_report_with_a_suffixed_version_recalculates_instead_of_raising
 
         self.assertEqual(imported.pic_hash, expected_pic_hash)
 
+    def test_imported_report_from_a_prerelease_above_the_escape_version_keeps_pic_hash(self):
+        # a pre-release states a version, so it is ordered rather than treated as unversioned:
+        # without that, every report a release candidate wrote is rehashed on every import
+        report = _disassemble_intel_bytes(b"\xc3")  # ret
+        function_dict = report.getFunction(INTEL_BASE).toDict()
+        stored_pic_hash = 0x0123456789ABCDEF
+        function_dict["metadata"]["pic_hash"] = stored_pic_hash
+        above_gate = [*INTEL_PIC_HASH_ESCAPE_VERSION[:-1], INTEL_PIC_HASH_ESCAPE_VERSION[-1] + 1]
+
+        for suffix in ("a1", "b2", "rc1"):
+            with self.subTest(suffix=suffix):
+                version = ".".join(str(v) for v in above_gate) + suffix
+
+                imported = SmdaFunction.fromDict(function_dict, version=version, smda_report=report)
+
+                self.assertEqual(imported.pic_hash, stored_pic_hash)
+
+    def test_imported_report_from_a_prerelease_of_the_escape_version_recalculates(self):
+        # 4.3.5rc1 precedes 4.3.5, so a candidate for the release that changed the escaper
+        # cannot be assumed to carry the change
+        report = _disassemble_intel_bytes(b"\xc3")  # ret
+        function = report.getFunction(INTEL_BASE)
+        expected_pic_hash = function.pic_hash
+        function_dict = function.toDict()
+        function_dict["metadata"]["pic_hash"] = 0x0123456789ABCDEF
+        gate_version = ".".join(str(v) for v in INTEL_PIC_HASH_ESCAPE_VERSION)
+
+        for suffix in ("a1", "b2", "rc1"):
+            with self.subTest(suffix=suffix):
+                imported = SmdaFunction.fromDict(function_dict, version=gate_version + suffix, smda_report=report)
+
+                self.assertEqual(imported.pic_hash, expected_pic_hash)
+
+    def test_report_version_ordering_is_unchanged_for_release_and_unorderable_strings(self):
+        # the shapes that were already ordered have to keep ordering the same way, a version
+        # shorter than the gate included: [4, 3] precedes [4, 3, 5] on the prefix rule
+        ordered = {
+            "4.3.5": False,
+            "v4.3.5": False,
+            "4.3.4": True,
+            "4.3": True,
+            "4.3.6": False,
+            "5": False,
+        }
+        for version, expected in ordered.items():
+            with self.subTest(version=version):
+                key = _reportVersionKey(version)
+
+                self.assertIsNotNone(key)
+                self.assertEqual(_isBeforeEscapeVersion(key, INTEL_PIC_HASH_ESCAPE_VERSION), expected)
+
+        for version in ("4.4.4-dev", "MCRIT4IDA", "", "4..5", "4.3.5rc"):
+            with self.subTest(version=version):
+                self.assertIsNone(_reportVersionKey(version))
+
     def test_imported_report_skips_hashing_against_an_unlocated_binary_info(self):
         report = _disassemble_intel_bytes(b"\xc3")  # ret
         function_dict = report.getFunction(INTEL_BASE).toDict()