diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bcbefc4..9c4d219c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,18 @@ past roughly six lines it belongs in the PR the entry links. ### Changed +- **(common)** Let a PE exception directory refuse an interior candidate too, not only an ELF's + `.eh_frame`. A `RUNTIME_FUNCTION` extent names the addresses inside a routine the way an FDE range + does, and it now answers where analysis would begin on a candidate from any source, under the same + guards: the record's own function has to be recovered, and its recovered extent has to surround the + address. A fragment record is not the shortcut it is in the gap scan -- its own start is not the + function covering the address, so it declines rather than refusing. *Measured on `fb510c4` against + compiler symbol tables:* 120 MinGW PE x64 cells 93.191 -> 93.235 PPV (-47 false positives, 109/120 + bit-identical) and 2 Rust windows-gnu-x64 cells (-2), both at identical true positives and false + negatives; the 140 C/C++ ELF cells and all 57 malpedia dumps are bit-identical. Three of the dumps + do declare an exception directory, and the rule is consulted on all three and refuses nothing, which + is what makes that a control rather than an absence. (#329) + - **(tests)** `make test` now runs the fast tier and `make test-all` runs the whole suite. The `slow` marker already existed, was already applied to the eleven fixture-corpus files, and was documented in `pyproject.toml` with its own deselect recipe — but no entry point used it, so every local run paid for @@ -119,10 +131,21 @@ past roughly six lines it belongs in the PR the entry links. before the memo was written, so a section naming one dead pointer from every record read it once per record; remembered by address, that is now one read. *All 447 cells across the six corpora are bit-identical.* (#335) +- **(common)** Let each backend decide whether its own exception-directory interior rule is enabled. Both native + backends fill the same `_pdata_ranges` list, but the shared rule named `USE_PE_X64_PDATA_INTERIOR_GAPS` alone, + so on an ARM64 PE the refusal answered to a switch for a format it never sees: the ARM64 flag no longer turned + it off, and the x64 flag would have. Default output does not move -- with both flags on the gap scan reaches + those addresses first -- so this is flag semantics rather than a recovery change, and the bundled ARM64 PE + fixture is what makes it visible. (#329) + ### Security ### Compatibility +- Recovery output moves on PE images that declare an exception directory: an address a + `RUNTIME_FUNCTION` extent covers is no longer reported as a function start unless it is that + record's own start. No true positive was lost on the corpora this was measured over. (#329) + - Recovery output moves on Intel PE and ELF images whose gap scan meets a failed candidate: the bytes after it are now scanned instead of skipped. No true positive was lost on the corpora this was measured over. (#338) diff --git a/src/smda/SmdaConfig.py b/src/smda/SmdaConfig.py index 545d9f29..c18979f2 100644 --- a/src/smda/SmdaConfig.py +++ b/src/smda/SmdaConfig.py @@ -137,6 +137,21 @@ class SmdaConfig: # an analysis that ran past the end of the routine the address sat inside and absorbed # the small aligned functions after it. On the cell examined, all eight functions # recovered sit 11 to 79 bytes past a declared extent's end. + # The same extents also answer for a candidate from any other source, at the point analysis + # would begin on it, under the guards the .eh_frame arm carries: the record's own function + # has to be recovered and its recovered extent has to surround the address. A fragment is + # not the shortcut it is in the gap scan - its start is not the function covering the + # address, so it declines here rather than refusing. + # Worth much less than the .eh_frame arm, and the numbers are why it is a change of its own: + # 120 MinGW PE x64 cells -47 FP at identical TP and FN, 109 of 120 bit-identical + # 2 Rust windows-gnu-x64 -2 FP at identical TP and FN + # Nothing else moves at all. The 140 x86-64 ELF cells, 72 AArch64 ELF cells, 23 Go cells, + # 11 ARM64 Mach-O cells, the 32-bit and ELF Rust cells and all 57 malpedia dumps are + # bit-identical. The dumps are the reading worth keeping, and not because the evidence is + # absent: three of the 57 do declare an exception directory, 282 to 430 records each, and on + # all three the rule is consulted and refuses nothing - every address its guards see is + # declined. Reachable and declining is the stronger control; an earlier note here claimed + # none of the dumps carried a directory, which is not so. USE_PE_X64_PDATA_INTERIOR_GAPS = True # Refuse a gap candidate that an ARM64 PE image's own exception directory places inside a # routine. The same evidence and the same rule as the x64 flag above, reached differently: diff --git a/src/smda/aarch64/FunctionCandidateManager.py b/src/smda/aarch64/FunctionCandidateManager.py index 44a84916..1335025d 100644 --- a/src/smda/aarch64/FunctionCandidateManager.py +++ b/src/smda/aarch64/FunctionCandidateManager.py @@ -1032,6 +1032,10 @@ def _isLikelyInteriorBtiCandidate(self, addr, word): or auth_or_exception_return ) + def _pdataInteriorRefusalEnabled(self): + """The flag this backend's gap scan already answers to, for the analysis-time rule.""" + return self.config.USE_PE_ARM64_PDATA_INTERIOR_GAPS + def nextGapCandidate(self, start_gap_pointer=None): # AArch64 gap scan: a fixed-stride linear sweep of unanalyzed executable bytes # for functions that no prologue, call reference or stored pointer reached diff --git a/src/smda/common/FunctionCandidateManager.py b/src/smda/common/FunctionCandidateManager.py index c8b7920a..629ad571 100644 --- a/src/smda/common/FunctionCandidateManager.py +++ b/src/smda/common/FunctionCandidateManager.py @@ -733,8 +733,23 @@ def declaredFdeRangeContaining(self, addr): index -= 1 return None + def _pdataInteriorRefusalEnabled(self): + """Whether this backend's exception-directory interior rule is enabled. + + `_pdata_ranges` is filled by whichever backend read the directory, and the two that do + answer to separate flags: the x64 and ARM64 PE rules are separate features with separate + defaults. Naming either flag here would leave one architecture's rule answering to the + other architecture's switch, so each backend says instead. Those two are also the only + managers this engine builds today, so what is inherited here is the answer for one that + carves no directory at all: off, there being no extents for the rule to read. + """ + return False + def declaredInteriorOwner(self, addr): - """The recovered function whose declared `.eh_frame` range contains `addr`, or None. + """The recovered function whose declared range contains `addr`, or None. + + Either structure that declares one answers: an ELF's `.eh_frame` FDE ranges, or a PE + exception directory's `RUNTIME_FUNCTION` extents. The same evidence the gap scan already refuses on, asked at the point a candidate from any source is about to be analysed rather than only where the gap pointer reaches. Both @@ -744,6 +759,17 @@ def declaredInteriorOwner(self, addr): begin in the alignment padding ahead of its function, which leaves the real entry a few bytes in interior to nothing. + Both structures answer here, as they do in the gap scan, and they are format-disjoint: + `_pdata_ranges` is only ever filled from a PE exception directory and the `.eh_frame` + ranges decode nothing unless lief reports an ELF, so no address is arbitrated between + them. Disjointness is load-bearing rather than merely tidy, unlike in the gap scan: a + declared FDE range whose start was never recovered sets `owner` and then fails the + recovered-owner test, so on an image carrying both structures the exception directory + would never be asked about that address. A fragment record is not the shortcut it is in + the gap scan: its own start is not the function that covers the address, so it fails the + recovered-owner test below and declines, which is the conservative reading at a point + where a better one is available. + A third guard the gap scan does not need: the owner's own recovered extent has to surround the address. An FDE can reach past everything its function's control flow arrives at, and refusing an address out there discards bytes nothing else claims -- @@ -758,17 +784,21 @@ def declaredInteriorOwner(self, addr): nothing in the format forbids that; what bounds it is measurement rather than structure, no true positive lost across the ELF corpora this was measured over. """ - if not self.config.USE_ELF_FDE_INTERIOR_GAPS: - return None - if self.isInDeclaredPltSection(addr): - return None - containing = self.declaredFdeRangeContaining(addr) - if containing is None or containing[0] not in self.disassembly.functions: + owner = None + if self.config.USE_ELF_FDE_INTERIOR_GAPS and not self.isInDeclaredPltSection(addr): + declared = self.declaredFdeRangeContaining(addr) + if declared is not None: + owner = declared[0] + if owner is None and self._pdata_ranges and self._pdataInteriorRefusalEnabled(): + declared = self.declaredExceptionRangeContaining(addr) + if declared is not None: + owner = declared[0] + if owner is None or owner not in self.disassembly.functions: return None - borders = self.disassembly.function_borders.get(containing[0]) + borders = self.disassembly.function_borders.get(owner) if borders is None or not borders[0] <= addr < borders[1]: return None - return containing[0] + return owner def opensInsideDeclaredFdeRange(self, addr): """Whether `addr` falls inside a declared FDE range without being that range's start.""" diff --git a/src/smda/intel/FunctionCandidateManager.py b/src/smda/intel/FunctionCandidateManager.py index 89d3bc42..ff36a262 100644 --- a/src/smda/intel/FunctionCandidateManager.py +++ b/src/smda/intel/FunctionCandidateManager.py @@ -239,6 +239,10 @@ def _failedGapResumeTarget(self): return None return target + def _pdataInteriorRefusalEnabled(self): + """The flag this backend's gap scan already answers to, for the analysis-time rule.""" + return self.config.USE_PE_X64_PDATA_INTERIOR_GAPS + def nextGapCandidate(self, start_gap_pointer=None): if self.language_candidates_only: return None diff --git a/tests/testFdeInteriorGaps.py b/tests/testFdeInteriorGaps.py index ad660a94..e5e7a7a8 100644 --- a/tests/testFdeInteriorGaps.py +++ b/tests/testFdeInteriorGaps.py @@ -14,9 +14,11 @@ import lief +from smda.aarch64.FunctionCandidateManager import FunctionCandidateManager as Aarch64FunctionCandidateManager from smda.common.EhFrameDecoder import decodeEhFrameFdeRanges from smda.common.FunctionCandidateManager import FunctionCandidateManager from smda.Disassembler import Disassembler +from smda.intel.FunctionCandidateManager import FunctionCandidateManager as IntelFunctionCandidateManager from smda.SmdaConfig import SmdaConfig logging.disable(logging.CRITICAL) @@ -196,13 +198,31 @@ def __init__(self, functions, borders): self.function_borders = borders -def declaredOwnerOf(address, ranges, functions, borders, plt=(), enabled=True): - manager = FunctionCandidateManager(SmdaConfig()) +def declaredOwnerOf( + address, + ranges, + functions, + borders, + plt=(), + enabled=True, + pdata=(), + pe_enabled=True, + arm64_pe_enabled=True, + manager_class=FunctionCandidateManager, +): + # the exception-directory arm asks the backend whether its own rule is on, and the common + # class answers no, so a PE case has to be posed to the backend that fills those ranges -- + # otherwise it passes by declining for the wrong reason + manager = manager_class(SmdaConfig()) manager.config.USE_ELF_FDE_INTERIOR_GAPS = enabled + manager.config.USE_PE_X64_PDATA_INTERIOR_GAPS = pe_enabled + manager.config.USE_PE_ARM64_PDATA_INTERIOR_GAPS = arm64_pe_enabled manager.disassembly = _RecoveredDisassembly(dict.fromkeys(functions), dict(borders)) manager._eh_frame_fde_ranges = list(ranges) manager._eh_frame_fde_starts = [start for start, _ in ranges] manager._plt_ranges = list(plt) + manager._pdata_ranges = list(pdata) + manager._pdata_range_starts = None return manager.declaredInteriorOwner(address) @@ -249,5 +269,94 @@ def testTheFlagTurnsTheRefusalOff(self): self.assertIsNone(declaredOwnerOf(0x1500, self.RANGES, self.FUNCTIONS, self.BORDERS, enabled=False)) +class DeclaredInteriorOwnerPeTest(unittest.TestCase): + """The same question asked of a PE exception directory rather than an `.eh_frame`. + + The two structures never describe the same image, so the arms are exercised apart: an + ELF names no `RUNTIME_FUNCTION` extents and a PE decodes no FDE ranges. + """ + + PDATA = [(0x1000, 0x2000, False)] + FUNCTIONS = [0x1000] + BORDERS = {0x1000: (0x1000, 0x1F00)} + + MANAGER_CLASS = IntelFunctionCandidateManager + + def ownerOf(self, address, **kwargs): + options = { + "ranges": [], + "functions": self.FUNCTIONS, + "borders": self.BORDERS, + "pdata": self.PDATA, + "manager_class": self.MANAGER_CLASS, + **kwargs, + } + return declaredOwnerOf(address, **options) + + def testAnAddressInsideADeclaredExtentIsRefused(self): + self.assertEqual(self.ownerOf(0x1500), 0x1000) + + def testAnExtentStartIsNotInteriorToItself(self): + self.assertIsNone(self.ownerOf(0x1000)) + + def testAnExtentWhoseOwnerWasNotRecoveredRefusesNothing(self): + self.assertIsNone(self.ownerOf(0x1500, functions=[])) + + def testAnAddressPastTheOwnersRecoveredExtentIsKept(self): + self.assertIsNone(self.ownerOf(0x1500, borders={0x1000: (0x1000, 0x1100)})) + + def testAFragmentRecordDeclinesRatherThanRefusing(self): + # the gap scan takes a fragment as evidence on its own; here the record's own start is + # not the function covering the address, so it fails the recovered-owner test instead + self.assertIsNone(self.ownerOf(0x1500, pdata=[(0x1400, 0x1600, True)])) + + def testTheFlagTurnsTheRefusalOff(self): + self.assertIsNone(self.ownerOf(0x1500, pe_enabled=False)) + + def testTheElfFlagDoesNotGateTheExceptionDirectory(self): + # the two arms carry their own switches; turning the ELF one off leaves this one alone + self.assertEqual(self.ownerOf(0x1500, enabled=False), 0x1000) + + def testTheOtherArchitecturesFlagDoesNotGateThisOne(self): + self.assertEqual(self.ownerOf(0x1500, arm64_pe_enabled=False), 0x1000) + + +class DeclaredInteriorOwnerArm64PeTest(DeclaredInteriorOwnerPeTest): + """The same extents read by the backend that carves an ARM64 PE's `.pdata`. + + Both backends fill `_pdata_ranges` and each gap scan gates on its own architecture's flag, + so the analysis-time rule has to as well: naming one flag in the shared helper leaves the + other architecture's rule answering to a switch for an image format it never sees. + """ + + MANAGER_CLASS = Aarch64FunctionCandidateManager + + def testTheFlagTurnsTheRefusalOff(self): + self.assertIsNone(self.ownerOf(0x1500, arm64_pe_enabled=False)) + + def testTheOtherArchitecturesFlagDoesNotGateThisOne(self): + self.assertEqual(self.ownerOf(0x1500, pe_enabled=False), 0x1000) + + +class DeclaredInteriorOwnerUnbackedPeTest(unittest.TestCase): + """A manager that carves no exception directory refuses nothing on one. + + The two native backends are the only ones that reach the rule today, so this pins what the + inherited answer is: a manager added without an override cannot start refusing on extents + it never filled. + """ + + def testTheCommonManagerDeclinesAnExtentItCouldNotHaveFilled(self): + self.assertIsNone( + declaredOwnerOf( + 0x1500, + ranges=[], + functions=[0x1000], + borders={0x1000: (0x1000, 0x1F00)}, + pdata=[(0x1000, 0x2000, False)], + ) + ) + + if __name__ == "__main__": unittest.main()