diff --git a/CHANGELOG.md b/CHANGELOG.md index dd5d2950..5bcbefc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,16 @@ past roughly six lines it belongs in the PR the entry links. travels, not the seconds. CI is unchanged and still runs the whole suite on every leg, so the gate does not move — `make test-all` before pushing is what keeps a slow-tier failure from reaching the PR. (#340) +- **(common)** Refuse a candidate from any source that the image's `.eh_frame` declares interior to a function, + not only a gap-scan candidate -- the gap pointer reaches only what the gap scan walks to, so the prologue, + reference and symbol scans seeded the rest unchecked. A procedure linkage table is exempt, the declared + range's own start has to be a recovered function, and that owner's recovered extent has to surround the + address. *Measured on `0a0a4c6` against compiler symbol tables:* 72 AArch64 ELF cells 95.994 -> 97.063 PPV + (-683 false positives) and 140 built C/C++ ELF cells 98.903 -> 98.969 (-77), both at identical true positives + and false negatives; the PE, Go, ARM64 Mach-O and 57 malpedia cells are bit-identical, which is the control + that it reaches only images carrying an `.eh_frame`. *Not reproducible from the bundled fixtures, which + produce 0 refusals.* (#327) + ### Deprecated ### Removed @@ -116,6 +126,9 @@ past roughly six lines it belongs in the PR the entry links. - Recovery output moves on Intel PE and ELF images whose gap scan meets a failed candidate: the bytes after it are now scanned instead of skipped. No true positive was lost on the corpora this was measured over. (#338) +- Recovery output moves on ELF images carrying an `.eh_frame`: an address a declared FDE range covers is no + longer reported as a function start unless it is that range's own start. No true positive was lost on the + corpora this was measured over. (#327) ## Older releases diff --git a/src/smda/SmdaConfig.py b/src/smda/SmdaConfig.py index 59501096..545d9f29 100644 --- a/src/smda/SmdaConfig.py +++ b/src/smda/SmdaConfig.py @@ -211,6 +211,21 @@ class SmdaConfig: # table names `dispatch`, and aarch64_static drops 0x400350 and 0x40DF30, both mid-function # instructions inside a declared FDE. None of the six carries a symbol or is a declared # start, so both baselines moved toward the truth. + # The same evidence also refuses a candidate from any other source, at the point analysis + # would begin on it, since the gap pointer reaches only what the gap scan walks to and the + # seeding scans reach the rest. That arm carries a third condition the gap scan does not + # need: the owner's own recovered extent has to surround the address. A declared range can + # reach past everything its function's control flow arrives at, and refusing an address out + # there discards bytes nothing else claims along with any reference only those bytes carry + # - without it the AArch64 corpus loses three functions reached through exactly that shape. + # Measured against compiler symbol tables, no corpus losing a true positive: + # 72 AArch64 ELF cells PPV 95.994 -> 97.063 -683 FP at identical TP and FN + # 140 built C/C++ ELF PPV 98.903 -> 98.969 -77 FP at identical TP and FN + # The 120 MinGW PE cells, 23 Go cells, 11 ARM64 Mach-O cells and all 57 malpedia dumps are + # bit-identical, the control that it reaches only images carrying an .eh_frame. Rust is too, + # and not because the rule is inert there: its images decode their ranges and 25 of 26 false + # positives on the first cell are interior to one, but the two conditions above decline all + # of them - 78 of 94 on the owner's extent, the rest on the owner not being recovered. USE_ELF_FDE_INTERIOR_GAPS = True RESOLVE_REGISTER_CALLS = True # resolve "call/jmp dword ptr [ + ]" against a runtime-built import table and diff --git a/src/smda/common/FunctionCandidateManager.py b/src/smda/common/FunctionCandidateManager.py index 6094dbd1..c8b7920a 100644 --- a/src/smda/common/FunctionCandidateManager.py +++ b/src/smda/common/FunctionCandidateManager.py @@ -733,6 +733,43 @@ def declaredFdeRangeContaining(self, addr): index -= 1 return None + def declaredInteriorOwner(self, addr): + """The recovered function whose declared `.eh_frame` range contains `addr`, or None. + + The same evidence the gap scan already refuses on, asked at the point a candidate from + any source is about to be analysed rather than only where the gap pointer reaches. Both + of that rule's guards apply unchanged. A PLT is exempt because the whole table sits + under one FDE, so the range test reads every stub after the first as interior to the + first. And the range's own start has to be a recovered function, because an FDE can + begin in the alignment padding ahead of its function, which leaves the real entry a few + bytes in interior to nothing. + + A third guard the gap scan does not need: the owner's own recovered extent has to + surround the address. An FDE can reach past everything its function's control flow + arrives at, and refusing an address out there discards bytes nothing else claims -- + along with any reference only those bytes carry, which costs real functions elsewhere. + Inside the extent, the address is never one the owner decoded: a candidate landing on + recovered code is refused a step earlier against a byte-level code map, and + `function_borders` records only the extremes of that code, not its coverage. What + reaches here sits in the holes -- alignment between blocks, an unreached tail, a data + island -- so the exposure is a real entry with no FDE of its own in one of them, which + loses whatever reference only its bytes carry. An entry that carries its own FDE is + never interior to it, so the shape needs a routine covered by a neighbour's range, and + nothing in the format forbids that; what bounds it is measurement rather than + structure, no true positive lost across the ELF corpora this was measured over. + """ + if not self.config.USE_ELF_FDE_INTERIOR_GAPS: + return None + if self.isInDeclaredPltSection(addr): + return None + containing = self.declaredFdeRangeContaining(addr) + if containing is None or containing[0] not in self.disassembly.functions: + return None + borders = self.disassembly.function_borders.get(containing[0]) + if borders is None or not borders[0] <= addr < borders[1]: + return None + return containing[0] + def opensInsideDeclaredFdeRange(self, addr): """Whether `addr` falls inside a declared FDE range without being that range's start.""" return self.declaredFdeRangeContaining(addr) is not None diff --git a/src/smda/common/RecursiveDisassembler.py b/src/smda/common/RecursiveDisassembler.py index 6e2557b9..f515693a 100644 --- a/src/smda/common/RecursiveDisassembler.py +++ b/src/smda/common/RecursiveDisassembler.py @@ -261,6 +261,13 @@ def analyzeFunction(self, start_addr, as_gap=False): # state.getBlocks(); this collision path is unreachable from the gap pass today, so # the change is behavior-neutral (output stays byte-for-byte identical). return state + declared_owner = self.fc_manager.declaredInteriorOwner(start_addr) + if declared_owner is not None: + self.fc_manager.updateAnalysisAborted( + start_addr, + f"inside the declared range of function 0x{declared_owner:08x}", + ) + return state blocks_processed = 0 while state.hasUnprocessedBlocks(): blocks_processed += 1 diff --git a/tests/testFdeInteriorGaps.py b/tests/testFdeInteriorGaps.py index f0023a0d..ad660a94 100644 --- a/tests/testFdeInteriorGaps.py +++ b/tests/testFdeInteriorGaps.py @@ -15,6 +15,7 @@ import lief from smda.common.EhFrameDecoder import decodeEhFrameFdeRanges +from smda.common.FunctionCandidateManager import FunctionCandidateManager from smda.Disassembler import Disassembler from smda.SmdaConfig import SmdaConfig @@ -86,10 +87,6 @@ def testNothingTheImageNamesIsRefused(self): self.assertEqual((off & named) - on, set()) -if __name__ == "__main__": - unittest.main() - - class FdeInteriorGapRuleAArch64Test(unittest.TestCase): """The rule lives in the shared candidate manager and both gap scans consult it. @@ -138,3 +135,119 @@ def testEveryRefusedAddressSatisfiesTheRulesOwnContract(self): self.assertIsNotNone(owner, f"0x{address:x} was refused but is inside no declared range") # the condition that keeps an FDE opening in padding from refusing its own function self.assertIn(owner[0], on, f"0x{address:x} was refused by a range whose start is not a function") + + def analysedFixture(self, claim=None): + """Run the fixture, optionally having the rule claim one address, and keep the manager.""" + config = SmdaConfig() + config.CALCULATE_SCC = False + config.CALCULATE_NESTING = False + config.CALCULATE_HASHING = False + original = FunctionCandidateManager.declaredInteriorOwner + if claim is not None: + address, owner = claim + + def claiming(manager, candidate_address): + if candidate_address == address: + return owner + return original(manager, candidate_address) + + FunctionCandidateManager.declaredInteriorOwner = claiming + temp_path = None + try: + with tempfile.NamedTemporaryFile(suffix=".elf", delete=False) as handle: + handle.write(self.data) + temp_path = handle.name + disassembler = Disassembler(config) + report = disassembler.disassembleFile(temp_path) + finally: + if temp_path is not None: + os.unlink(temp_path) + # restore inside the finally, and with everything after the patch inside the try: + # a failure while writing the sample or building the disassembler would otherwise + # leave the claim installed on the class for every later test in the process + FunctionCandidateManager.declaredInteriorOwner = original + return disassembler.disassembler, {function.offset for function in report.getFunctions()} + + def testAnalysisItselfDeclinesWhatTheRuleClaims(self): + """The refusal is wired into analysis, not only into the gap scan. + + On this fixture every address the rule would claim is one the gap scan already + refused or the collision check already caught, so asserting over what it happens to + drop here would assert nothing. Claiming one address the fixture does recover is what + shows analysis consults the rule at all, and that it declines rather than reports. + """ + _, baseline = self.analysedFixture() + self.assertGreater(len(baseline), 200) + owner, target = sorted(baseline)[0], sorted(baseline)[1] + + backend, recovered = self.analysedFixture(claim=(target, owner)) + self.assertNotIn(target, recovered, "analysis reported an address the rule claimed") + self.assertEqual(baseline - recovered, {target}, "the claim moved more than the address it named") + # the reason is not asserted: a refused address is offered again as a gap candidate, + # and that pass records its own outcome over this one + self.assertTrue(backend.fc_manager.candidates[target].analysis_aborted) + + +class _RecoveredDisassembly: + """The two things `declaredInteriorOwner` reads about what analysis has recovered.""" + + def __init__(self, functions, borders): + self.functions = functions + self.function_borders = borders + + +def declaredOwnerOf(address, ranges, functions, borders, plt=(), enabled=True): + manager = FunctionCandidateManager(SmdaConfig()) + manager.config.USE_ELF_FDE_INTERIOR_GAPS = enabled + manager.disassembly = _RecoveredDisassembly(dict.fromkeys(functions), dict(borders)) + manager._eh_frame_fde_ranges = list(ranges) + manager._eh_frame_fde_starts = [start for start, _ in ranges] + manager._plt_ranges = list(plt) + return manager.declaredInteriorOwner(address) + + +class DeclaredInteriorOwnerTest(unittest.TestCase): + """The conditions the analysis-time refusal requires, one at a time. + + The gap scan reaches a candidate only where its pointer walks; this answers the same + question for a candidate from any source, so each condition needs a case of its own + rather than whatever a corpus happens to exercise. + """ + + RANGES = [(0x1000, 0x2000)] + FUNCTIONS = [0x1000] + BORDERS = {0x1000: (0x1000, 0x1F00)} + + def testAnAddressItsOwnerSurroundsIsRefused(self): + self.assertEqual(declaredOwnerOf(0x1500, self.RANGES, self.FUNCTIONS, self.BORDERS), 0x1000) + + def testARangeStartIsNotInteriorToItself(self): + self.assertIsNone(declaredOwnerOf(0x1000, self.RANGES, self.FUNCTIONS, self.BORDERS)) + + def testAnAddressOutsideEveryRangeIsKept(self): + self.assertIsNone(declaredOwnerOf(0x2500, self.RANGES, self.FUNCTIONS, self.BORDERS)) + + def testARangeWhoseStartWasNotRecoveredRefusesNothing(self): + # an FDE can begin in the alignment padding ahead of its function, and then the real + # entry a few bytes in is interior to nothing + self.assertIsNone(declaredOwnerOf(0x1500, self.RANGES, [], self.BORDERS)) + + def testAnAddressPastTheOwnersRecoveredExtentIsKept(self): + # the declared range reaches further than the owner's control flow arrived; refusing + # out there discards bytes nothing else claims, and any reference only they carry + short = {0x1000: (0x1000, 0x1100)} + self.assertIsNone(declaredOwnerOf(0x1500, self.RANGES, self.FUNCTIONS, short)) + + def testAnOwnerWithNoRecordedExtentRefusesNothing(self): + self.assertIsNone(declaredOwnerOf(0x1500, self.RANGES, self.FUNCTIONS, {})) + + def testAStubInADeclaredPltIsExempt(self): + # the whole table sits under one FDE, so every stub after the first reads as interior + self.assertIsNone(declaredOwnerOf(0x1500, self.RANGES, self.FUNCTIONS, self.BORDERS, plt=[(0x1400, 0x1600)])) + + def testTheFlagTurnsTheRefusalOff(self): + self.assertIsNone(declaredOwnerOf(0x1500, self.RANGES, self.FUNCTIONS, self.BORDERS, enabled=False)) + + +if __name__ == "__main__": + unittest.main()