From 36721f8ba75fcb46599ed3e1151ba2f06685f5f7 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 02:15:59 -0500 Subject: [PATCH 01/20] tools: make TU stock controls follow production policies --- tools/test_tubuild.py | 35 ++++++++++++++++++++++++++++ tools/tubuild.py | 53 +++++++++++++++++++++++++++++++------------ 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index c634de0a0f..3f213c9e17 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -737,6 +737,41 @@ def test_unknown_id_fails_closed_with_a_clear_reason(): import tubuild +def test_linkcheck_compile_passes_production_compiler_only_policies(): + original_policies = tubuild.RB.compiler_only_policies + original_compile = tubuild.RB.compile_one + policy = {"src/actors/Promoted.cpp": {"deadstrip": ["helper"]}} + seen = [] + + try: + tubuild.RB.compiler_only_policies = lambda enrolled: ( + policy if list(enrolled) == ["src/actors/Promoted.cpp"] else None) + + def fake_compile(rel, vers, cache, init_srcs, syms, build_root=None, + compiler_only=None): + seen.append((rel, build_root, compiler_only)) + return rel, None, "hit" + + tubuild.RB.compile_one = fake_compile + failures, outcomes = tubuild.compile_linkcheck_sources( + ["src/actors/Promoted.cpp"], {}, None, set(), {}, pathlib.Path("scratch"), 1) + finally: + tubuild.RB.compiler_only_policies = original_policies + tubuild.RB.compile_one = original_compile + + assert failures == [] + assert outcomes["hit"] == 1 + assert seen == [("src/actors/Promoted.cpp", pathlib.Path("scratch"), policy)] + + +def test_linkcheck_symbol_verdict_uses_the_stock_failure_inventory(): + assert tubuild.linkcheck_symbol_verdict(True, False, None) + assert tubuild.linkcheck_symbol_verdict(False, False, []) + assert not tubuild.linkcheck_symbol_verdict(False, False, ["new error"]) + assert tubuild.linkcheck_symbol_verdict(False, True, None) + assert not tubuild.linkcheck_symbol_verdict(False, False, None) + + def test_vtable_storage_address_requires_an_explicit_consistent_bias(): if not _toolchain(): return diff --git a/tools/tubuild.py b/tools/tubuild.py index fda62a5735..d6891fb5ba 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -3588,6 +3588,42 @@ def shared_build_bin_snapshot(): for path in sorted((REPO / "build").glob("*.bin")) if path.is_file()} +def compile_linkcheck_sources(srcs, vers, cache, init_srcs, syms, build_root, jobs): + """Compile a scratch linkcheck with the normal production object policies. + + A baseline substitutes no candidate TU, but it still compiles production's + already-promoted multi-symbol objects. Those objects rely on manifest-backed + compiler-only policies for exact duplicate functions and data. Omitting the + policies makes the control fail before it reaches the candidate, even though the + normal ROM build accepts and verifies the same objects. + """ + compiler_only = RB.compiler_only_policies(srcs) + failures, outcomes = [], collections.Counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as ex: + for rel, err, outcome in ex.map( + lambda s: RB.compile_one( + s, vers, cache, init_srcs, syms, build_root=build_root, + compiler_only=compiler_only), srcs): + outcomes[outcome] += 1 + if err: + failures.append((rel, err)) + return failures, outcomes + + +def linkcheck_symbol_verdict(baseline, command_ok, new_errors): + """Whether the symbol phase is attributable-clean for this linkcheck. + + The stock control's job is to inventory the tree's existing dsd errors. A + candidate is clean only when it adds none to that inventory; without a control it + must make the command itself pass. + """ + if baseline: + return True + if new_errors is not None: + return not new_errors + return command_ok + + def cmd_linkcheck(args): data = load_manifest() entry = manifest_entry(data, args.id) if args.id else None @@ -3804,14 +3840,8 @@ def cmd_linkcheck(args): init_srcs = RB.init_section_sources() syms = RB.enrolled_symbols() t0 = time.time() - failures, outcomes = [], collections.Counter() - with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as ex: - for rel, err, outcome in ex.map( - lambda s: RB.compile_one(s, vers, cache, init_srcs, syms, build_root=scratch), - srcs): - outcomes[outcome] += 1 - if err: - failures.append((rel, err)) + failures, outcomes = compile_linkcheck_sources( + srcs, vers, cache, init_srcs, syms, scratch, args.jobs) dt = time.time() - t0 report["phases"]["compile"] = {"ok": not failures, "seconds": round(dt, 1), "outcomes": dict(outcomes)} @@ -4320,12 +4350,7 @@ def cmd_linkcheck(args): report["strayOutputs"] = changed_shared # ------------------------------------------------------------------------ verdict - if baseline: - symbols_verdict = symbols_ok - elif symbols_new is not None: - symbols_verdict = not symbols_new - else: - symbols_verdict = symbols_ok + symbols_verdict = linkcheck_symbol_verdict(baseline, symbols_ok, symbols_new) equivalent = all(v["identical"] for _o, _s, v in partial_rows) if partial_rows else False verified = bool(module_ok and symbols_verdict and (rom_ok is not False)) if partitioned: From 3c1173ac95d780a820c76d74855551baeb2aecd0 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 02:34:20 -0500 Subject: [PATCH 02/20] tools: preserve live references across vtable rebias --- tools/objisolate.py | 109 +++++++++++++++++++++++++++++++-------- tools/test_objisolate.py | 54 ++++++++++++++++++- tools/tubuild.py | 43 ++++++++++++++- 3 files changed, 182 insertions(+), 24 deletions(-) diff --git a/tools/objisolate.py b/tools/objisolate.py index 53e19df3b4..e0fe4169b7 100644 --- a/tools/objisolate.py +++ b/tools/objisolate.py @@ -901,11 +901,13 @@ def rebias_object_symbols(raw, symbol_policies): isolation already rewrites imports to that public convention; a separately linked data partition must therefore expose its strong definition at the same point. - Content and relocations are deliberately untouched. A policy may additionally - split the preamble into an exact storage-alias symbol by reusing one explicitly - deadstripped compiler-only symbol-table slot. Reusing a slot keeps every ELF - offset stable; it is allowed only when the old name is an unreferenced GLOBAL/FUNC - import with enough exclusive string-table storage for the alias. + Content bytes are deliberately untouched. Surviving RELA/ABS32 references to a + rebased definition have their addends reduced by the same bias, preserving the + resolved address exactly while changing the public symbol convention. A policy + may additionally split the preamble into an exact storage-alias symbol by reusing + one explicitly deadstripped compiler-only symbol-table slot. Reusing a slot keeps + every ELF offset stable; it is allowed only when the old name is an unreferenced + GLOBAL/FUNC import with enough exclusive string-table storage for the alias. """ requested = {} for name, policy in dict(symbol_policies).items(): @@ -942,11 +944,6 @@ def rebias_object_symbols(raw, symbol_policies): if symtab is None: return None, {"rebased": [], "aliases": [], "error": "no .symtab"} syms = list(symtab.iter_symbols()) - protected_sections = { - i: sec.data() for i, sec in enumerate(elf.iter_sections()) - if ((sec.header["sh_type"] in CONTENT and sec.header["sh_size"]) - or (isinstance(sec, RelocationSection) and sec.header["sh_size"])) - } by_name = {name: [(i, sym) for i, sym in enumerate(syms) if sym.name == name and sym["st_shndx"] not in ("SHN_UNDEF", "SHN_ABS")] @@ -981,22 +978,76 @@ def rebias_object_symbols(raw, symbol_policies): f"0x{sec.header['sh_size']:x}, manifest=" f"0x{requested[name]['size']:x})"} - # Rebiasing changes what a relocation to this definition means. No current owned - # data needs such a self-reference, so refuse it rather than guessing whether its - # addend used storage-start or public-address-point convention. - for relsec in elf.iter_sections(): + import struct + endian = "<" if elf.little_endian else ">" + + # Moving the definition from storage start to public address point must not move + # any resolved reference. mwcc's live references use RELA/ABS32 with the ABI + # preamble bias in r_addend, so subtract exactly that bias and leave the content, + # relocation offset, and target symbol index untouched. + relocation_rewrites = [] + mutable_relocation_sections = set() + for relsec_index, relsec in enumerate(elf.iter_sections()): if not isinstance(relsec, RelocationSection): continue source = elf.get_section(relsec.header["sh_info"]) if source.header["sh_type"] not in CONTENT or not source.header["sh_size"]: continue - for reloc in relsec.iter_relocations(): + for reloc_index, reloc in enumerate(relsec.iter_relocations()): target = symtab.get_symbol(reloc["r_info_sym"]) - if target.name in requested: + if target.name not in requested: + continue + if relsec.header["sh_type"] != "SHT_RELA" or not reloc.is_RELA(): + return None, {"rebased": [], "aliases": [], + "relocations": [], + "error": f"surviving {source.name} relocation at " + f"0x{reloc['r_offset']:x} targets rebased symbol " + f"{target.name} but is not SHT_RELA"} + if reloc["r_info_type"] != R_ARM_ABS32: + return None, {"rebased": [], "aliases": [], + "relocations": [], + "error": f"surviving {source.name} relocation at " + f"0x{reloc['r_offset']:x} targets rebased symbol " + f"{target.name} with unsupported type " + f"{reloc['r_info_type']}"} + bias = requested[target.name]["bias"] + old_addend = int(reloc["r_addend"]) + if old_addend < bias: + return None, {"rebased": [], "aliases": [], + "relocations": [], + "error": f"surviving {source.name} relocation at " + f"0x{reloc['r_offset']:x} targets rebased symbol " + f"{target.name} with addend {old_addend}, smaller than " + f"bias {bias}"} + entry_size = int(relsec.header["sh_entsize"]) + if entry_size < 12: return None, {"rebased": [], "aliases": [], - "error": f"surviving {source.name} " - f"relocation at 0x{reloc['r_offset']:x} targets rebased " - f"symbol {target.name}"} + "relocations": [], + "error": f"{relsec.name} has invalid RELA entry size " + f"{entry_size}"} + entry_offset = reloc_index * entry_size + relocation_rewrites.append({ + "sectionIndex": relsec_index, "section": source.name, + "relocationSection": relsec.name, "offset": reloc["r_offset"], + "symbol": target.name, "type": "R_ARM_ABS32", + "oldAddend": old_addend, "newAddend": old_addend - bias, + "entryOffset": entry_offset, + "fileOffset": relsec.header["sh_offset"] + entry_offset + 8, + }) + mutable_relocation_sections.add(relsec_index) + + protected_sections = { + i: sec.data() for i, sec in enumerate(elf.iter_sections()) + if i not in mutable_relocation_sections + and ((sec.header["sh_type"] in CONTENT and sec.header["sh_size"]) + or (isinstance(sec, RelocationSection) and sec.header["sh_size"])) + } + expected_relocation_sections = { + i: bytearray(elf.get_section(i).data()) for i in mutable_relocation_sections + } + for row in relocation_rewrites: + struct.pack_into(endian + "i", expected_relocation_sections[row["sectionIndex"]], + row["entryOffset"] + 8, row["newAddend"]) aliases = {} used_donors = set() @@ -1087,8 +1138,6 @@ def rebias_object_symbols(raw, symbol_policies): for i, sym in enumerate(syms) if i not in mutable_symbol_indices } - import struct - endian = "<" if elf.little_endian else ">" base = symtab.header["sh_offset"] rows, alias_rows = [], [] for name in sorted(requested): @@ -1117,6 +1166,8 @@ def rebias_object_symbols(raw, symbol_policies): rows.append({"symbol": name, "bias": bias, "oldValue": sym["st_value"], "newValue": new_value, "oldSize": sym["st_size"], "newSize": new_size}) + for row in relocation_rewrites: + struct.pack_into(endian + "i", raw_out, row["fileOffset"], row["newAddend"]) out = bytes(raw_out) checked = ELFFile(io.BytesIO(out)) checked_symtab = checked.get_section_by_name(".symtab") @@ -1137,6 +1188,16 @@ def rebias_object_symbols(raw, symbol_policies): if checked_sections != protected_sections: return None, {"rebased": rows, "aliases": alias_rows, "error": "storage rewrite changed content or relocation bytes"} + checked_relocation_sections = { + i: checked.get_section(i).data() for i in mutable_relocation_sections + } + expected_relocation_sections = { + i: bytes(data) for i, data in expected_relocation_sections.items() + } + if checked_relocation_sections != expected_relocation_sections: + return None, {"rebased": rows, "aliases": alias_rows, + "relocations": relocation_rewrites, + "error": "storage rewrite changed more than licensed RELA addends"} for name, alias in aliases.items(): matches = [sym for sym in checked_symbols if sym.name == alias["symbol"]] @@ -1154,7 +1215,11 @@ def rebias_object_symbols(raw, symbol_policies): return None, {"rebased": rows, "aliases": alias_rows, "error": f"post-rewrite storage split is not exact for " f"{alias['symbol']}/{name}"} - return out, {"rebased": rows, "aliases": alias_rows, "error": None} + public_relocations = [{k: v for k, v in row.items() + if k not in ("sectionIndex", "entryOffset", "fileOffset")} + for row in relocation_rewrites] + return out, {"rebased": rows, "aliases": alias_rows, + "relocations": public_relocations, "error": None} def isolate(obj, keep_symbol): diff --git a/tools/test_objisolate.py b/tools/test_objisolate.py index 43e233f583..0162b3e3d1 100644 --- a/tools/test_objisolate.py +++ b/tools/test_objisolate.py @@ -741,7 +741,7 @@ def test_rebias_vtable_requires_one_exact_dedicated_global_object(self): self.assertTrue(patched) refused, why = OI.rebias_object_symbols(bytes(bad), policy) self.assertIsNone(refused) - self.assertIn("targets rebased symbol", why["error"]) + self.assertIn("smaller than bias", why["error"]) bad = bytearray(raw) patched = False @@ -761,6 +761,58 @@ def test_rebias_vtable_requires_one_exact_dedicated_global_object(self): self.assertIsNone(refused) self.assertIn("still referenced", why["error"]) + def test_rebias_vtable_preserves_live_reference_targets(self): + """Whole-object vptr stores keep their target while _ZTV moves by eight.""" + import io + from elftools.elf.elffile import ELFFile + from elftools.elf.relocation import RelocationSection + + raw = self.build("struct P { virtual ~P(); virtual int f(); }; " + "P::~P(){} int P::f(){ return 1; }\n").read_bytes() + + def inspect(blob): + parsed = ELFFile(io.BytesIO(blob)) + table = parsed.get_section_by_name(".symtab") + symbols = list(table.iter_symbols()) + vtable = next(s for s in symbols if s.name == "_ZTV1P" + and s["st_shndx"] != "SHN_UNDEF") + content = {i: sec.data() for i, sec in enumerate(parsed.iter_sections()) + if sec.header["sh_type"] in OI.CONTENT and sec.header["sh_size"]} + references = [] + for sec in parsed.iter_sections(): + if not isinstance(sec, RelocationSection): + continue + source = parsed.get_section(sec.header["sh_info"]) + for reloc in sec.iter_relocations(): + if table.get_symbol(reloc["r_info_sym"]).name != "_ZTV1P": + continue + references.append({ + "section": source.name, "offset": reloc["r_offset"], + "type": reloc["r_info_type"], "addend": reloc["r_addend"], + "resolved": vtable["st_value"] + reloc["r_addend"], + }) + return vtable, content, references + + before_vtable, before_content, before_refs = inspect(raw) + self.assertTrue(before_refs) + self.assertTrue(all(row["type"] == OI.R_ARM_ABS32 and row["addend"] >= 8 + for row in before_refs)) + policy = {"_ZTV1P": {"bias": 8, "size": before_vtable["st_size"], + "section": ".data"}} + out, report = OI.rebias_object_symbols(raw, policy) + self.assertIsNone(report["error"]) + after_vtable, after_content, after_refs = inspect(out) + self.assertEqual(after_vtable["st_value"], before_vtable["st_value"] + 8) + self.assertEqual(after_vtable["st_size"], before_vtable["st_size"] - 8) + self.assertEqual(after_content, before_content) + self.assertEqual([(r["section"], r["offset"], r["type"], r["resolved"]) + for r in after_refs], + [(r["section"], r["offset"], r["type"], r["resolved"]) + for r in before_refs]) + self.assertEqual([r["addend"] for r in after_refs], + [r["addend"] - 8 for r in before_refs]) + self.assertEqual(len(report["relocations"]), len(before_refs)) + def test_vtable_addend_is_corrected_to_zero(self): """8 -> 0, because the ROM symbol is already past the preamble.""" from elftools.elf.elffile import ELFFile diff --git a/tools/tubuild.py b/tools/tubuild.py index d6891fb5ba..ba6656f5a0 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -4107,7 +4107,48 @@ def cmd_linkcheck(args): print(f" externalized {externalized['externalized']} to their exact " "configured canonical homes in the SCRATCH object only") - owned = verify_owned_sections(linked_tu, entry, claims) + owned_before = verify_owned_sections(linked_tu, entry, claims) + report["ownedSectionsBeforeRebias"] = owned_before + if not owned_before["ok"]: + print(" REFUSED -- licensed non-text contribution is not exact:") + for reason in owned_before.get("errors", []): + print(f" {reason}") + report["result"] = "data-refused" + _write_link_report(scratch, report) + _record_linkcheck(data, entry, report, baseline) + return 1 + + biases, bias_reasons = partition_vtable_rebiases(entry, claims) + vtable_policies = biases + if bias_reasons: + print(" REFUSED -- retained vtable address point is not explicit:") + for reason in bias_reasons: + print(f" {reason}") + report["result"] = "vtable-rebias-refused" + report["vtableRebias"] = {"requested": biases, + "errors": bias_reasons} + _write_link_report(scratch, report) + _record_linkcheck(data, entry, report, baseline) + return 1 + rebased_tu, bias_report = OI.rebias_object_symbols(linked_tu, biases) + report["vtableRebias"] = bias_report + if rebased_tu is None: + print(f" REFUSED -- vtable symbol rebias: {bias_report.get('error')}") + report["result"] = "vtable-rebias-refused" + _write_link_report(scratch, report) + _record_linkcheck(data, entry, report, baseline) + return 1 + if rebased_tu != linked_tu: + linked_tu = rebased_tu + scratch_rewrite = True + tu_obj.write_bytes(linked_tu) + print(f" rebased {len(bias_report.get('rebased', []))} retained vtable " + f"symbol(s) and compensated " + f"{len(bias_report.get('relocations', []))} live relocation addend(s) " + "in the SCRATCH object only") + + owned = verify_owned_sections(linked_tu, entry, claims, + public_address_points=True) report["ownedSections"] = owned for row in owned["rows"]: print(f" {row['section']:8} {row.get('start', '-')}..{row.get('end', '-')} " From 9302b80d05945661c5c50cfbc41e699985c38b75 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 02:39:10 -0500 Subject: [PATCH 03/20] tools: resolve raw RTTI vtable relocations --- tools/test_tubuild.py | 6 +++++- tools/tubuild.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index 3f213c9e17..ec92beb1b9 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -509,7 +509,11 @@ def _vague_externalization_fixture(): }) config["arm9"][address + reloc["r_offset"]] = ( "load", next_destination, "arm9") - name_index[target.name] = ("arm9", next_destination - reloc["r_addend"]) + raw_bias = (reloc["r_addend"] - tubuild.OI.VTABLE_PREAMBLE + if target.name.startswith("_ZTV") + and reloc["r_addend"] >= tubuild.OI.VTABLE_PREAMBLE + else reloc["r_addend"]) + name_index[target.name] = ("arm9", next_destination - raw_bias) next_destination += 4 policies.append({ "symbol": name, "disposition": "canonical-import", diff --git a/tools/tubuild.py b/tools/tubuild.py index ba6656f5a0..c1f3b1a322 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -2348,6 +2348,16 @@ def verify_externalized_output(obj_bytes, entry, policies=None, homes=None, candidate_module = (RL.normalize_module(resolved[0]) if resolved[0] is not None else None) candidate_address = resolved[1] + emitted["addend"] + # mwcc's raw `_ZTV` relocation is relative to the storage + # object, while symbols.txt names the public slot-array address + # after the two-word ABI preamble. This is the same raw-object + # convention used by reloc_audit.object_reloc_dests: addend 8 + # resolves to the configured address point, not eight bytes past + # it. Explicit addend-zero references already use the public + # convention and remain unchanged. + if emitted["symbol"].startswith("_ZTV") \ + and emitted["addend"] >= OI.VTABLE_PREAMBLE: + candidate_address -= OI.VTABLE_PREAMBLE if (candidate_module, candidate_address) != \ (expected["target_module"], expected["target_address"]): row_reasons.append(f"relocation +0x{offset:x} resolves to " From 326281dbc106ac7df3dde42b0e4df26e44decf7d Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 02:44:08 -0500 Subject: [PATCH 04/20] tools: normalize intact TU vtable imports --- tools/objisolate.py | 33 ++++++++++++++++++++++++--------- tools/test_objisolate.py | 40 ++++++++++++++++++++++++++++++++++++++++ tools/tubuild.py | 3 ++- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/tools/objisolate.py b/tools/objisolate.py index e0fe4169b7..ed4fbf4235 100644 --- a/tools/objisolate.py +++ b/tools/objisolate.py @@ -892,7 +892,7 @@ def derive_section_partition(raw, keep_section_names, licensed_symbols, return _apply(raw, p, None), p -def rebias_object_symbols(raw, symbol_policies): +def rebias_object_symbols(raw, symbol_policies, normalize_undefined=False): """Move exact retained object symbols from storage start to public address point. C++ vtable definitions are the motivating case. mwcc's symbol covers the full @@ -903,11 +903,15 @@ def rebias_object_symbols(raw, symbol_policies): Content bytes are deliberately untouched. Surviving RELA/ABS32 references to a rebased definition have their addends reduced by the same bias, preserving the - resolved address exactly while changing the public symbol convention. A policy - may additionally split the preamble into an exact storage-alias symbol by reusing - one explicitly deadstripped compiler-only symbol-table slot. Reusing a slot keeps - every ELF offset stable; it is allowed only when the old name is an unreferenced - GLOBAL/FUNC import with enough exclusive string-table storage for the alias. + resolved address exactly while changing the public symbol convention. With + ``normalize_undefined``, raw references to undefined ``_ZTV`` imports lose the + same ABI preamble from their addend; unlike a rebased local definition this fixes + the address the repository's already-public import would otherwise resolve to. + A policy may additionally split the preamble into an exact storage-alias symbol by + reusing one explicitly deadstripped compiler-only symbol-table slot. Reusing a + slot keeps every ELF offset stable; it is allowed only when the old name is an + unreferenced GLOBAL/FUNC import with enough exclusive string-table storage for the + alias. """ requested = {} for name, policy in dict(symbol_policies).items(): @@ -930,7 +934,7 @@ def rebias_object_symbols(raw, symbol_policies): return None, {"rebased": [], "aliases": [], "error": f"{name} needs bias/size/section and valid optional " "storageAlias fields"} - if not requested: + if not requested and not normalize_undefined: return bytes(raw), {"rebased": [], "aliases": [], "error": None} bad_bias = sorted(name for name, policy in requested.items() if policy["bias"] <= 0 or policy["bias"] >= policy["size"]) @@ -995,7 +999,11 @@ def rebias_object_symbols(raw, symbol_policies): continue for reloc_index, reloc in enumerate(relsec.iter_relocations()): target = symtab.get_symbol(reloc["r_info_sym"]) - if target.name not in requested: + target_is_rebased = target.name in requested + target_is_undefined = (normalize_undefined + and target.name.startswith("_ZTV") + and target["st_shndx"] in ("SHN_UNDEF", SHN_UNDEF)) + if not target_is_rebased and not target_is_undefined: continue if relsec.header["sh_type"] != "SHT_RELA" or not reloc.is_RELA(): return None, {"rebased": [], "aliases": [], @@ -1010,8 +1018,13 @@ def rebias_object_symbols(raw, symbol_policies): f"0x{reloc['r_offset']:x} targets rebased symbol " f"{target.name} with unsupported type " f"{reloc['r_info_type']}"} - bias = requested[target.name]["bias"] old_addend = int(reloc["r_addend"]) + # An addend-zero undefined import was written explicitly against the + # repository's public address-point convention and needs no correction. + if target_is_undefined and old_addend == 0: + continue + bias = (requested[target.name]["bias"] if target_is_rebased + else VTABLE_PREAMBLE) if old_addend < bias: return None, {"rebased": [], "aliases": [], "relocations": [], @@ -1031,6 +1044,8 @@ def rebias_object_symbols(raw, symbol_policies): "relocationSection": relsec.name, "offset": reloc["r_offset"], "symbol": target.name, "type": "R_ARM_ABS32", "oldAddend": old_addend, "newAddend": old_addend - bias, + "mode": "rebased-definition" if target_is_rebased + else "undefined-public-import", "entryOffset": entry_offset, "fileOffset": relsec.header["sh_offset"] + entry_offset + 8, }) diff --git a/tools/test_objisolate.py b/tools/test_objisolate.py index 0162b3e3d1..5e7382a68d 100644 --- a/tools/test_objisolate.py +++ b/tools/test_objisolate.py @@ -813,6 +813,46 @@ def inspect(blob): [r["addend"] - 8 for r in before_refs]) self.assertEqual(len(report["relocations"]), len(before_refs)) + def test_rebias_vtable_normalizes_undefined_base_import(self): + """An inlined base dtor's raw +8 import becomes the public +0 form.""" + import io + from elftools.elf.elffile import ELFFile + from elftools.elf.relocation import RelocationSection + + raw = self.build("struct B { virtual ~B(){} virtual int f(); }; " + "struct D : B { virtual ~D(); }; D::~D(){}\n").read_bytes() + + def addends(blob, name): + parsed = ELFFile(io.BytesIO(blob)) + table = parsed.get_section_by_name(".symtab") + return sorted(reloc["r_addend"] for sec in parsed.iter_sections() + if isinstance(sec, RelocationSection) + for reloc in sec.iter_relocations() + if table.get_symbol(reloc["r_info_sym"]).name == name) + + parsed = ELFFile(io.BytesIO(raw)) + table = parsed.get_section_by_name(".symtab") + symbols = list(table.iter_symbols()) + own = next(s for s in symbols if s.name == "_ZTV1D" + and s["st_shndx"] != "SHN_UNDEF") + base = next(s for s in symbols if s.name == "_ZTV1B") + self.assertEqual(base["st_shndx"], "SHN_UNDEF") + before = addends(raw, "_ZTV1B") + self.assertTrue(before) + self.assertTrue(all(value >= OI.VTABLE_PREAMBLE for value in before)) + out, report = OI.rebias_object_symbols( + raw, {"_ZTV1D": {"bias": 8, "size": own["st_size"], + "section": ".data"}}, + normalize_undefined=True) + self.assertIsNone(report["error"]) + self.assertEqual(addends(out, "_ZTV1B"), + [value - OI.VTABLE_PREAMBLE for value in before]) + imported = [row for row in report["relocations"] + if row["symbol"] == "_ZTV1B"] + self.assertEqual(len(imported), len(before)) + self.assertTrue(all(row["mode"] == "undefined-public-import" + for row in imported)) + def test_vtable_addend_is_corrected_to_zero(self): """8 -> 0, because the ROM symbol is already past the preamble.""" from elftools.elf.elffile import ELFFile diff --git a/tools/tubuild.py b/tools/tubuild.py index c1f3b1a322..83c33f4fb6 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -4140,7 +4140,8 @@ def cmd_linkcheck(args): _write_link_report(scratch, report) _record_linkcheck(data, entry, report, baseline) return 1 - rebased_tu, bias_report = OI.rebias_object_symbols(linked_tu, biases) + rebased_tu, bias_report = OI.rebias_object_symbols( + linked_tu, biases, normalize_undefined=True) report["vtableRebias"] = bias_report if rebased_tu is None: print(f" REFUSED -- vtable symbol rebias: {bias_report.get('error')}") From 85298b3b0e5bb58d81f9d4f429e68eecc6c39b8b Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 02:53:58 -0500 Subject: [PATCH 05/20] tools: enroll compiler-owned intact C++ TUs --- tools/rombuild.py | 84 ++++++++++++++++++++++++++++++++++--- tools/test_rombuild.py | 34 +++++++++++++++ tools/test_tu_production.py | 29 +++++++++++++ tools/test_tu_promote.py | 76 +++++++++++++++++++++++++++++++++ tools/tu_production.py | 53 +++++++++++++++++++++++ tools/tu_promote.py | 34 +++++++++++---- 6 files changed, 295 insertions(+), 15 deletions(-) create mode 100644 tools/test_tu_promote.py diff --git a/tools/rombuild.py b/tools/rombuild.py index b0107b0da8..e147a6ebda 100644 --- a/tools/rombuild.py +++ b/tools/rombuild.py @@ -291,7 +291,7 @@ def init_section_sources(): return {rel for (_d, _name, rel, _addr, _size, sec) in cands if sec == ".init"} -def _isolate(obj, rel, syms, data_sink=None, compiler_only=None): +def _isolate(obj, rel, syms, data_sink=None, compiler_only=None, intact_tus=None): """Reduce a compiled `src/` object to its declared function(s). A legacy source owns one function and takes the unchanged singular isolation path. @@ -319,6 +319,17 @@ def _isolate(obj, rel, syms, data_sink=None, compiler_only=None): # can fail the build. See tools/romdata_check.py for why it is not a gate. data_sink.extend(RDC.check_object(obj, rel)) selected = (syms or {}).get(rel, pathlib.Path(rel).stem) + intact = (intact_tus or {}).get(rel.replace("\\", "/")) + if intact is not None: + if not isinstance(selected, (list, tuple)) or not selected: + return "intact TU policy requires one or more enrolled function symbols" + try: + import tu_production as TP # noqa: PLC0415 - only intact TUs need it + prepared, _evidence = TP.prepare_intact_object(obj.read_bytes(), intact) + except Exception as exc: # noqa: BLE001 - one source gets one build verdict + return f"intact TU preparation refused: {exc}" + obj.write_bytes(prepared) + return None if isinstance(selected, (list, tuple)): if not selected: return "source owns no enrolled functions" @@ -713,6 +724,63 @@ def compiler_only_policies(enrolled=None, manifest=None, homes=None): return out +def intact_tu_policies(enrolled=None, manifest=None): + """Manifest entries admitted to the normal build as one intact compiler object. + + This is deliberately opt-in. A non-text manifest is not enough: the entry must + be promoted, request ``production_mode: intact-object``, and carry a successful + ordinary scratch link proving every claimed range, all modules, and the full ROM. + The compile path re-runs the exact object/data/relocation checks on every raw object; + this admission record prevents an unverified research manifest from selecting the + policy in the first place. + """ + data = TUM.load() if manifest is None else manifest + active = None if enrolled is None else { + str(rel).replace("\\", "/") for rel in enrolled + } + out, errors = {}, [] + for entry in data.get("entries", []): + if entry.get("production_mode") != "intact-object": + continue + source = str(entry.get("promoted_source") + or entry.get("source", "")).replace("\\", "/") + if active is not None and source not in active: + continue + label = entry.get("id", source or "") + if not source.startswith("src/"): + errors.append(f"{label}: intact-object policy has no production src/ path") + continue + if source in out: + errors.append(f"{source}: intact-object policy is declared by multiple entries") + continue + if entry.get("status") != "promoted": + errors.append(f"{label}: enrolled intact-object entry is not promoted") + section_names = {row.get("name") for row in entry.get("sections", []) + if isinstance(row, dict)} + if ".text" not in section_names or not (section_names - {".text"}): + errors.append(f"{label}: intact-object policy needs .text and non-text claims") + linkcheck = (entry.get("verification") or {}).get("linkcheck") or {} + phases = linkcheck.get("phases") or {} + if linkcheck.get("result") != "scratch-data-verified": + errors.append(f"{label}: intact-object policy needs scratch-data-verified " + "ordinary link evidence") + for phase in ("delink", "lcf", "compile", "link", "checkModules", "rom"): + if phases.get(phase) is not True: + errors.append(f"{label}: intact-object proof phase {phase} is not green") + if linkcheck.get("symbolCheckNewVsBaseline") != []: + errors.append(f"{label}: intact-object proof has new or unknown symbol errors") + ranges = linkcheck.get("tuRanges") or [] + if not ranges or any(row.get("differingBytes") != 0 for row in ranges): + errors.append(f"{label}: intact-object proof does not show every range exact") + sha = (linkcheck.get("rom") or {}).get("sha256") + if not isinstance(sha, str) or len(sha) != 64: + errors.append(f"{label}: intact-object proof has no full-ROM SHA-256") + out[source] = entry + if errors: + raise BuildError("intact TU policy", 1, "\n".join(errors)) + return out + + def _definition_symbols(rel, rows): """Collapse owned records only for one exact legacy outer-owner alias shape. @@ -784,7 +852,7 @@ def retarget_text_section(obj, section=".init"): def compile_one(rel, vers=None, cache=None, init_srcs=None, syms=None, build_root=None, - data_sink=None, prebuilt=None, compiler_only=None): + data_sink=None, prebuilt=None, compiler_only=None, intact_tus=None): """Compile one enrolled source file to the object path dsd's objects.txt names. Returns (rel, error-or-None, outcome), where outcome is how the object was @@ -824,7 +892,7 @@ def compile_one(rel, vers=None, cache=None, init_srcs=None, syms=None, build_roo err = _retarget(obj, rel, init_srcs) if err: return rel, f"retarget: {err}", "error" - err = _isolate(obj, rel, syms, data_sink, compiler_only) + err = _isolate(obj, rel, syms, data_sink, compiler_only, intact_tus) if err: return rel, f"isolate: {err}", "error" return rel, None, "hit" @@ -858,18 +926,18 @@ def compile_one(rel, vers=None, cache=None, init_srcs=None, syms=None, build_roo if err: return rel, f"retarget: {err}", "error" if key is None: - err = _isolate(obj, rel, syms, data_sink, compiler_only) + err = _isolate(obj, rel, syms, data_sink, compiler_only, intact_tus) return (rel, f"isolate: {err}", "error") if err else (rel, None, "miss") deps = cache.deps_from(scratch) if deps is None: - err = _isolate(obj, rel, syms, data_sink, compiler_only) + err = _isolate(obj, rel, syms, data_sink, compiler_only, intact_tus) return (rel, f"isolate: {err}", "error") if err else (rel, None, "uncacheable") # Cache the RAW object, then isolate the working copy. Storing the reduced # form instead would bake this transformation into every entry, so any later # fix to it would be masked by isolate()'s own idempotence -- which is exactly # how the STB_LOPROC bug survived a rebuild and forced SCHEMA 2. cache.put(key, deps, obj) - err = _isolate(obj, rel, syms, data_sink, compiler_only) + err = _isolate(obj, rel, syms, data_sink, compiler_only, intact_tus) return (rel, f"isolate: {err}", "error") if err else (rel, None, "miss") finally: if scratch: @@ -1015,6 +1083,9 @@ def save_report(): init_srcs = init_section_sources() syms = enrolled_symbols() compiler_only = compiler_only_policies(srcs) + intact_tus = intact_tu_policies(srcs) + report["intactTus"] = sorted(entry.get("id", source) + for source, entry in intact_tus.items()) # Collected during the compile because that is the only point at which the # objects still carry the data mwcc emitted -- see _isolate. None switches the # measurement off entirely; it never affects what gets linked either way. @@ -1033,6 +1104,7 @@ def save_report(): lambda s: compile_one(s, vers, cache, init_srcs, syms, data_sink=data_sink, compiler_only=compiler_only, + intact_tus=intact_tus, prebuilt=tu_overrides.get( s.replace("\\", "/"))), srcs): outcomes[outcome] = outcomes.get(outcome, 0) + 1 diff --git a/tools/test_rombuild.py b/tools/test_rombuild.py index 30605c6013..b7ef065dd9 100644 --- a/tools/test_rombuild.py +++ b/tools/test_rombuild.py @@ -291,6 +291,40 @@ def test_compiler_only_policy_ignores_unenrolled_shadow_manifests(self): enrolled=["src/Pair.cpp"], manifest=manifest, homes={"ConfiguredElsewhere": [("arm9", 0x02000008)]}), {}) + def test_intact_policy_requires_promoted_full_ordinary_link_proof(self): + entry = { + "id": "ov047/TU", "status": "promoted", + "production_mode": "intact-object", + "source": "src/actors/TU.cpp", "promoted_source": "src/actors/TU.cpp", + "sections": [{"name": ".text"}, {"name": ".data"}], + "verification": {"linkcheck": { + "result": "scratch-data-verified", + "phases": {name: True for name in + ("delink", "lcf", "compile", "link", "checkModules", "rom")}, + "symbolCheckNewVsBaseline": [], + "tuRanges": [{"section": ".text", "differingBytes": 0}, + {"section": ".data", "differingBytes": 0}], + "rom": {"sha256": "a" * 64}, + }}, + } + manifest = {"entries": [entry]} + self.assertEqual(RB.intact_tu_policies( + {"src/actors/TU.cpp"}, manifest=manifest), + {"src/actors/TU.cpp": entry}) + entry["verification"]["linkcheck"]["tuRanges"][1]["differingBytes"] = 1 + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) + self.assertIn("every range exact", raised.exception.output) + + def test_intact_policy_ignores_unenrolled_shadow(self): + manifest = {"entries": [{ + "id": "ov047/Shadow", "status": "text-verified", + "production_mode": "intact-object", + "promoted_source": "src/actors/Shadow.cpp", + }]} + self.assertEqual(RB.intact_tu_policies( + {"src/actors/Elsewhere.cpp"}, manifest=manifest), {}) + def _compiler(): exe = RB.MW / RB.VERSION / "mwccarm.exe" diff --git a/tools/test_tu_production.py b/tools/test_tu_production.py index 2decdd428a..31163de785 100644 --- a/tools/test_tu_production.py +++ b/tools/test_tu_production.py @@ -30,6 +30,35 @@ def test_missing_content_bound_baseline_refuses(self): class ProductionTuObjects(unittest.TestCase): + def test_intact_object_runs_all_fail_closed_policy_gates(self): + entry = {"id": "ov047/Thing"} + claims = [{"name": ".text", "start": 0x1000, "end": 0x1010}, + {"name": ".data", "start": 0x2000, "end": 0x2010}] + owned = {"ok": True, "rows": [], "errors": []} + with mock.patch.object(TP.TB, "manifest_section_claims", + return_value=(claims, [])), \ + mock.patch.object(TP.TB, "apply_compiler_only_policy", + return_value=(b"compiler", {"deadstripped": []}, [])), \ + mock.patch.object(TP.TB, "apply_externalized_output_policy", + return_value=(b"external", {"externalized": []}, [])), \ + mock.patch.object(TP.TB, "verify_owned_sections", + side_effect=[owned, owned]) as verify, \ + mock.patch.object(TP.TB, "partition_vtable_rebiases", + return_value=({"_ZTV1T": {"bias": 8}}, [])), \ + mock.patch.object(TP.TB.OI, "rebias_object_symbols", + return_value=(b"linked", {"error": None})) as rebias, \ + mock.patch.object(TP.TB, "complete_ranges", return_value={}), \ + mock.patch.object(TP.TB, "audit_tu_object", + return_value=([], [], [], True)), \ + mock.patch.object(TP.TB, "object_audit_refusals", return_value=[]): + output, evidence = TP.prepare_intact_object(b"raw", entry) + self.assertEqual(output, b"linked") + rebias.assert_called_once_with( + b"external", {"_ZTV1T": {"bias": 8}}, normalize_undefined=True) + self.assertEqual(verify.call_count, 2) + self.assertEqual(evidence["sha256"], + __import__("hashlib").sha256(b"linked").hexdigest()) + def test_compile_one_installs_prepared_object_without_compiler(self): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) diff --git a/tools/test_tu_promote.py b/tools/test_tu_promote.py new file mode 100644 index 0000000000..74ada31082 --- /dev/null +++ b/tools/test_tu_promote.py @@ -0,0 +1,76 @@ +import pathlib +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import tu_promote as TP # noqa: E402 + + +class IntactPromotion(unittest.TestCase): + def test_plan_and_rewrite_keep_nontext_claims_for_intact_object(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + config = root / "config" + delinks = config / "arm9/overlays/ov047/delinks.txt" + delinks.parent.mkdir(parents=True) + delinks.write_text( + " .text start:0x00001000 end:0x00001100 kind:code align:4\n" + " .data start:0x00002000 end:0x00002100 kind:data align:4\n\n" + "src/First.cpp:\n complete\n" + " .text start:0x00001000 end:0x00001010\n\n" + "src/Second.cpp:\n complete\n" + " .text start:0x00001010 end:0x00001020\n\n", + encoding="utf-8") + (root / "src_tu").mkdir() + (root / "src_tu/TU.cpp").write_text("//cpp\n", encoding="utf-8") + (root / "src").mkdir() + (root / "src/First.cpp").write_text("//cpp\n", encoding="utf-8") + (root / "src/Second.cpp").write_text("//cpp\n", encoding="utf-8") + entry = { + "id": "ov047/TU", "module": "ov047", + "status": "scratch-data-verified", + "production_mode": "intact-object", + "source": "src_tu/TU.cpp", "promoted_source": "src/TU.cpp", + "sections": [ + {"name": ".text", "start": "0x00001000", "end": "0x00001020"}, + {"name": ".data", "start": "0x00002040", "end": "0x00002050"}, + ], + "data": [{"symbol": "owned", "address": "0x2040", "size": "0x10"}], + "functions": [ + {"symbol": "First", "address": "0x1000", "size": "0x10", + "legacy_source": "src/First.cpp"}, + {"symbol": "Second", "address": "0x1010", "size": "0x10", + "legacy_source": "src/Second.cpp"}, + ], + } + with mock.patch.object(TP, "REPO", root), \ + mock.patch.object(TP, "CONFIG", config): + planned = TP.plan(entry) + TP.rewrite_delinks(planned) + written = delinks.read_text(encoding="utf-8") + self.assertNotIn("src/First.cpp:", written) + self.assertNotIn("src/Second.cpp:", written) + self.assertIn("src/TU.cpp:\n complete\n" + " .text start:0x00001000 end:0x00001020\n" + " .data start:0x00002040 end:0x00002050\n", written) + + def test_nontext_promotion_without_intact_mode_is_refused(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / "src_tu").mkdir() + (root / "src_tu/TU.cpp").write_text("//cpp\n", encoding="utf-8") + entry = {"id": "ov047/TU", "source": "src_tu/TU.cpp", + "promoted_source": "src/TU.cpp", "module": "ov047", + "sections": [ + {"name": ".text", "start": "0x1000", "end": "0x1010"}, + {"name": ".data", "start": "0x2000", "end": "0x2010"}, + ]} + with mock.patch.object(TP, "REPO", root): + with self.assertRaisesRegex(TP.PromoteError, "intact-object"): + TP.plan(entry) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tu_production.py b/tools/tu_production.py index c964ac8de4..794c86ab88 100644 --- a/tools/tu_production.py +++ b/tools/tu_production.py @@ -27,6 +27,59 @@ class ProductionTuError(RuntimeError): """A partitioned TU cannot safely enter the normal ROM build.""" +def prepare_intact_object(raw, entry): + """Apply and reverify the exact policies for one production intact TU object.""" + claims, reasons = TB.manifest_section_claims(entry) + if reasons: + _raise(f"{entry.get('id', '')} manifest section claims", reasons) + text = [claim for claim in claims if claim["name"] == ".text"] + nontext = [claim for claim in claims if claim["name"] != ".text"] + if len(text) != 1 or not nontext: + raise ProductionTuError( + f"{entry.get('id', '')}: intact production requires one .text " + "claim and at least one non-text claim") + + post_policy, compiler_only, reasons = TB.apply_compiler_only_policy(raw, entry) + if reasons: + _raise(f"{entry['id']} compiler-only output", reasons) + externalized_obj, externalized, reasons = \ + TB.apply_externalized_output_policy(post_policy, entry) + if reasons: + _raise(f"{entry['id']} exact RTTI externalization", reasons) + + owned_before = TB.verify_owned_sections(externalized_obj, entry, claims) + if not owned_before.get("ok"): + _raise(f"{entry['id']} licensed non-text contribution", + owned_before.get("errors", [])) + biases, reasons = TB.partition_vtable_rebiases(entry, claims) + if reasons: + _raise(f"{entry['id']} vtable address-point policy", reasons) + linked_obj, rebias = TB.OI.rebias_object_symbols( + externalized_obj, biases, normalize_undefined=True) + if linked_obj is None: + _raise(f"{entry['id']} vtable address-point rewrite", [rebias.get("error")]) + owned_after = TB.verify_owned_sections( + linked_obj, entry, claims, public_address_points=True) + if not owned_after.get("ok"): + _raise(f"{entry['id']} production non-text contribution", + owned_after.get("errors", [])) + + span_start, span_end = text[0]["start"], text[0]["end"] + rows, extra, emitted, order_ok = TB.audit_tu_object( + linked_obj, entry, span_start, span_end, TB.complete_ranges(TB.CFG_ARM9)) + audit_errors = TB.object_audit_refusals(rows, extra, order_ok) + if audit_errors: + _raise(f"{entry['id']} production object audit", audit_errors) + return linked_obj, { + "compilerOnly": compiler_only, "externalized": externalized, + "ownedBefore": owned_before, "ownedAfter": owned_after, + "vtableRebias": rebias, + "objectAudit": {"rows": rows, "extraSections": extra, + "emitted": emitted, "orderOk": order_ok}, + "sha256": hashlib.sha256(linked_obj).hexdigest(), + } + + def _raise(label, reasons): detail = "; ".join(str(reason) for reason in reasons if reason) raise ProductionTuError(f"{label}: {detail or 'refused without a reason'}") diff --git a/tools/tu_promote.py b/tools/tu_promote.py index dd28ac1054..4562fd8cfe 100644 --- a/tools/tu_promote.py +++ b/tools/tu_promote.py @@ -12,7 +12,7 @@ The mechanical steps, all of which this performs and none of which it guesses at: * replace the entry's per-function ``delinks.txt`` entries with one ``complete`` - entry spanning the manifest's sections; + entry spanning every manifest-owned section; * ``git mv`` the ``src_tu/`` source to its ``promoted_source`` path (R100, so the file's own credit follows) and ``git rm`` every ``legacy_source``; * rewrite the manifest entry to ``status: promoted`` with ``source`` at the @@ -87,19 +87,34 @@ def plan(entry): if (REPO / dest).exists(): raise PromoteError(f"{ident}: {dest} already exists") - sections = [s for s in entry.get("sections", []) if s.get("name") == ".text"] + raw_sections = [s for s in entry.get("sections", []) if isinstance(s, dict)] + sections = [s for s in raw_sections if s.get("name") == ".text"] if not sections: raise PromoteError(f"{ident}: no .text section in the manifest") - if entry.get("data") or entry.get("bss"): + owns_nontext = any(s.get("name") != ".text" for s in raw_sections) \ + or bool(entry.get("data") or entry.get("rodata") or entry.get("bss")) + if owns_nontext and entry.get("production_mode") != "intact-object": # Production isolation zeroes an object's data and rebinds the symbols to the - # cartridge's addresses; a TU that OWNS delinked data is a different problem - # and is not what this path models. - raise PromoteError(f"{ident}: entry owns data/bss; not a text-only promotion") + # cartridge's addresses unless the normal build has explicitly admitted this + # manifest-backed intact-object policy. + raise PromoteError(f"{ident}: entry owns non-text data without " + "production_mode: intact-object") + + claims = [] + for section in raw_sections: + name = section.get("name") + try: + lo, hi = int(section["start"], 16), int(section["end"], 16) + except (KeyError, TypeError, ValueError): + raise PromoteError(f"{ident}: invalid section claim {section!r}") from None + if not isinstance(name, str) or not name.startswith(".") or lo >= hi: + raise PromoteError(f"{ident}: invalid section claim {section!r}") + claims.append((name, lo, hi)) funcs = entry.get("functions", []) if not funcs: raise PromoteError(f"{ident}: entry licenses no functions") - spans = [(int(s["start"], 16), int(s["end"], 16)) for s in sections] + spans = [(lo, hi) for name, lo, hi in claims if name == ".text"] legacy = [] for f in funcs: addr, size = int(f["address"], 16), int(f["size"], 16) @@ -122,7 +137,7 @@ def plan(entry): raise PromoteError(f"{ident}: {source} has {n} entries in " f"{dl.relative_to(REPO)}, expected exactly 1") return {"id": ident, "source": src, "dest": dest, "delinks": dl, - "legacy": legacy, "spans": spans, "functions": funcs} + "legacy": legacy, "spans": spans, "claims": claims, "functions": funcs} def rewrite_delinks(p): @@ -130,7 +145,8 @@ def rewrite_delinks(p): text = p["delinks"].read_text(encoding="utf-8") for source in p["legacy"]: text = _entry_re(source).sub("", text, count=1) - body = "".join(f" .text start:0x{lo:08x} end:0x{hi:08x}\n" for lo, hi in p["spans"]) + body = "".join(f" {name} start:0x{lo:08x} end:0x{hi:08x}\n" + for name, lo, hi in p["claims"]) entry = f"{p['dest']}:\n complete\n{body}\n" # Address order is not cosmetic: it is how a reader of delinks.txt finds the entry # that owns an address, and dsd emits the file sorted. From 431a0ac8fa2beb9fcf08165f050e87744db433cd Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 02:57:00 -0500 Subject: [PATCH 06/20] tools: retain intact TU range evidence --- tools/test_tubuild.py | 27 +++++++++++++++++++++++++++ tools/tubuild.py | 1 + 2 files changed, 28 insertions(+) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index ec92beb1b9..76d0223096 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -1318,6 +1318,33 @@ def test_partitioned_cli_modes_are_mutually_exclusive_before_any_build(): assert code != 0 assert "not allowed with argument" in out or "mutually exclusive" in out + +def test_record_linkcheck_preserves_all_owned_ranges(): + entry = {"status": "text-verified"} + report = { + "result": "scratch-data-verified", + "scratch": "scratch/path", + "phases": {"link": {"ok": True}, "rom": {"ok": True}}, + "tuRange": {"section": ".text", "differingBytes": 0}, + "tuRanges": [ + {"section": ".text", "differingBytes": 0}, + {"section": ".data", "differingBytes": 0}, + ], + "objectAudit": {}, + "symbolsNew": [], + "rom": {"matchesStockRom": True, "sha256": "a" * 64}, + } + original = tubuild.save_manifest + try: + tubuild.save_manifest = lambda _data: None + tubuild._record_linkcheck({"entries": [entry]}, entry, report, False) + finally: + tubuild.save_manifest = original + + recorded = entry["verification"]["linkcheck"] + assert recorded["tuRange"] == report["tuRange"] + assert recorded["tuRanges"] == report["tuRanges"] + # ---------------------------------------------------------------- create repairs # The three assemble_shadow_source behaviors proven by six modules of # hand-assembly (222 byte-verified functions) before being folded into the diff --git a/tools/tubuild.py b/tools/tubuild.py index 83c33f4fb6..be7b1f3551 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -4672,6 +4672,7 @@ def _record_linkcheck(data, entry, report, baseline): "scratch": report["scratch"] + " (gitignored)", "phases": {k: v.get("ok") for k, v in report["phases"].items()}, "tuRange": report.get("tuRange"), + "tuRanges": report.get("tuRanges"), "objectAudit": { "counts": audit.get("counts"), "emittedTextOrderIsRomAscending": audit.get("orderOk"), From 9c740f0eade50448578bb8765d04e0f5d4d81034 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 03:57:09 -0500 Subject: [PATCH 07/20] tools: preserve converted TU member identities --- .github/workflows/converted-ratchet.yml | 8 ++- tools/test_tiers_ratchet.py | 47 ++++++++++++++++ tools/test_tu_promote.py | 30 ++++++++++ tools/tiers_ratchet.py | 75 ++++++++++++++++--------- tools/tu_promote.py | 60 +++++++++++++++++++- 5 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 tools/test_tiers_ratchet.py diff --git a/.github/workflows/converted-ratchet.yml b/.github/workflows/converted-ratchet.yml index bbca401591..80fe73297e 100644 --- a/.github/workflows/converted-ratchet.yml +++ b/.github/workflows/converted-ratchet.yml @@ -1,9 +1,11 @@ # Guards the CONVERTED tier against a silent backslide. # -# `tools/tiers.py` scores every file under src/ against five readability criteria (real +# `tools/tiers.py` scores every function under src/ against five readability criteria (real # function name, no raw offset arithmetic, no `unk_` fields, no codegen tricks, no -# calls through mangled names). `tools/tiers_ratchet.py` banks the SET of paths that pass -# all five in `config/converted-baseline.json` and fails when a banked path LEAVES it. +# calls through mangled names). `tools/tiers_ratchet.py` banks the SET of source/member +# identities that pass all five in `config/converted-baseline.json` and fails when one +# leaves it. One-function files use the source path itself; promoted TU members append +# `#symbol` to that path, matching attribution's ownership granularity. # # A SET, not a count, on purpose: a count ratchet is satisfied by converting one file # while wrecking another, which is the exact trade this gate exists to notice. diff --git a/tools/test_tiers_ratchet.py b/tools/test_tiers_ratchet.py new file mode 100644 index 0000000000..b0b50687cb --- /dev/null +++ b/tools/test_tiers_ratchet.py @@ -0,0 +1,47 @@ +import pathlib +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import tiers_ratchet as TR # noqa: E402 + + +class TranslationUnitIdentities(unittest.TestCase): + def test_multi_function_source_is_banked_per_member(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + source = root / "src/actors/TU.cpp" + source.parent.mkdir(parents=True) + source.write_text( + "//cpp\nint shared_helper;\nvoid First() {}\nvoid Second() {}\n", + encoding="utf-8") + ownership = {"src/actors/TU.cpp": ["First", "Second"]} + with mock.patch.object(TR, "REPO", root): + converted, scores = TR.scan(["src/actors/TU.cpp"], ownership) + + self.assertEqual(converted, { + "src/actors/TU.cpp#First", "src/actors/TU.cpp#Second"}) + self.assertEqual(set(scores), converted) + + def test_single_function_source_keeps_legacy_path_identity(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + source = root / "src/Only.cpp" + source.parent.mkdir(parents=True) + source.write_text("//cpp\nvoid Only() {}\n", encoding="utf-8") + ownership = {"src/Only.cpp": ["Only"]} + with mock.patch.object(TR, "REPO", root): + converted, scores = TR.scan(["src/Only.cpp"], ownership) + + self.assertEqual(converted, {"src/Only.cpp"}) + self.assertEqual(set(scores), converted) + + def test_missing_promoted_member_is_named_not_reported_as_unreadable(self): + why = TR.why("src/actors/TU.cpp#Missing", {}, {"src/actors/TU.cpp"}) + self.assertIn("no longer an enrolled member", why) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_tu_promote.py b/tools/test_tu_promote.py index 74ada31082..447417191a 100644 --- a/tools/test_tu_promote.py +++ b/tools/test_tu_promote.py @@ -1,3 +1,4 @@ +import json import pathlib import sys import tempfile @@ -9,6 +10,35 @@ class IntactPromotion(unittest.TestCase): + def test_converted_baseline_moves_only_banked_members_to_promoted_tu(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + config = root / "config" + config.mkdir() + baseline = config / "converted-baseline.json" + baseline.write_text( + '{"_note":"old","criteria":[],"count":2,"converted":[' + '"src/First.cpp","src/Unrelated.cpp"]}\n', encoding="utf-8") + original = baseline.read_text(encoding="utf-8") + plans = [{ + "dest": "src/actors/TU.cpp", + "functions": [ + {"symbol": "First", "legacy_source": "src/First.cpp"}, + {"symbol": "Second", "legacy_source": "src/Second.cpp"}, + ], + }] + with mock.patch.object(TP, "CONFIG", config): + prepared = TP.converted_baseline_update(plans) + self.assertEqual(baseline.read_text(encoding="utf-8"), original) + moved = TP.rewrite_converted_baseline(plans, prepared) + + data = json.loads(baseline.read_text(encoding="utf-8")) + self.assertEqual(moved, 1) + self.assertEqual(data["count"], 2) + self.assertEqual(data["converted"], [ + "src/Unrelated.cpp", "src/actors/TU.cpp#First"]) + self.assertEqual(data["_note"], TP.TR.NOTE) + def test_plan_and_rewrite_keep_nontext_claims_for_intact_object(self): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) diff --git a/tools/tiers_ratchet.py b/tools/tiers_ratchet.py index 6abb9903e3..101281ac01 100644 --- a/tools/tiers_ratchet.py +++ b/tools/tiers_ratchet.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 -"""Backslide gate for the CONVERTED tier: a readable file may not quietly stop being one. +"""Backslide gate for the CONVERTED tier: readable source ownership may not regress. WHAT IT GATES. `tools/tiers.py` scores every source file against the five CONVERTED criteria (real function name, no raw offset arithmetic, no `unk_` fields, no -codegen tricks, no calls through mangled names). This tool banks the SET of file paths -that pass all five and fails a PR when a path LEAVES that set. It reuses tiers.score_file -outright -- the classifier has exactly one implementation, and a second copy of those -regexes would be a second definition of a published percentage. +codegen tricks, no calls through mangled names). This tool banks the SET of source +identities that pass all five and fails a PR when an identity LEAVES that set. A +one-function source keeps its historical path identity. A promoted TU appends +``#symbol`` to that path for each enrolled member, matching attribution's ownership unit. +It reuses tiers.score_file outright -- the classifier has exactly one implementation, +and a second copy of those regexes would be a second definition of a published percentage. WHY BACKSLIDE-ONLY, AND NOT A COUNT. Two reasons, and the second is the important one. A count ratchet ("converted may not fall") is satisfied by converting one file while wrecking another, which is the trade this gate exists to notice. A set ratchet names - the file. + the source member. More importantly, this project's goal ordering is not negotiable: a historically accurate C++ source that reproduces the ROM's exact bytes comes FIRST, readability @@ -122,7 +124,7 @@ python tools/tiers_ratchet.py --check # exit 1 on any backslide python tools/tiers_ratchet.py --update # re-bank (additions only) python tools/tiers_ratchet.py --update --reason "..." # re-bank with removals - python tools/tiers_ratchet.py --list # the current CONVERTED paths + python tools/tiers_ratchet.py --list # current CONVERTED identities Exit codes: 0 ok, 1 backslide detected, 2 usage/configuration error (missing baseline, removal without a reason). It compiles nothing and reads no ROM: pure source text over @@ -142,10 +144,12 @@ BASELINE = REPO / "config" / "converted-baseline.json" EXCEPTIONS = REPO / "config" / "converted-backslide-exceptions.jsonl" -NOTE = ("The CONVERTED file set, banked. tools/tiers_ratchet.py --check fails when a " - "path here no longer passes all five criteria in tools/tiers.py. Removals need " - "--reason and land in config/converted-backslide-exceptions.jsonl. Regenerate " - "with `python tools/tiers_ratchet.py --update`.") +NOTE = ("The CONVERTED source/member identity set, banked. One-function sources use " + "their path; promoted TU members append #symbol to that path. " + "tools/tiers_ratchet.py --check fails when an identity no longer passes all " + "five criteria in tools/tiers.py. Removals need --reason and land in " + "config/converted-backslide-exceptions.jsonl. Regenerate with " + "`python tools/tiers_ratchet.py --update`.") def tracked_sources(): @@ -210,17 +214,31 @@ def score(rel): return tiers.score_file(rel, text) -def scan(paths=None): - """(converted_set, scores_by_path) for the whole tracked tree.""" +def scan(paths=None, ownership=None): + """(converted identities, scores by identity) for the tracked source tree. + + Physical paths remain the identity for ordinary one-function intake files so the + existing baseline stays valid. A production TU owns several enrolled functions; + those are independently banked as ``path#symbol`` so consolidating files cannot + masquerade as a readability backslide or let one readable member pay for another. + """ scores = {} converted = set() + if ownership is None: + ownership = tiers.srcpath.source_definition_index() for rel in (paths if paths is not None else tracked_sources()): - s = score(rel) - if s is None: + file_score = score(rel) + if file_score is None: continue - scores[rel] = s - if all(s[k] for k in tiers.CRITERIA): - converted.add(rel) + members = ownership.get(rel) or [pathlib.PurePosixPath(rel).stem] + multi = len(members) > 1 + for symbol in members: + identity = f"{rel}#{symbol}" if multi else rel + member_score = dict(file_score) + member_score["real_name"] = tiers._real_name_for_symbol(symbol) + scores[identity] = member_score + if all(member_score[k] for k in tiers.CRITERIA): + converted.add(identity) return converted, scores @@ -263,23 +281,24 @@ def append_exceptions(path, rows): f.write(json.dumps(r, sort_keys=True) + "\n") -def _failures(rel, scores): - """The criteria `rel` fails, as labels, or None when it passes all five.""" - s = scores.get(rel) +def _failures(identity, scores): + """The criteria `identity` fails, or None when it passes all five.""" + s = scores.get(identity) if s is None: return None failed = [k for k in tiers.CRITERIA if not s[k]] return failed or None -def why(rel, scores, tracked, moves=None): - """Why a banked path is no longer CONVERTED, in the words of the criteria. +def why(identity, scores, tracked, moves=None): + """Why a banked source/member identity is no longer CONVERTED. A path that is GONE gets one of two answers, and the difference is the whole point: someone deleted readable code, or a TU promotion absorbed it into the file it was always part of. The second names the absorbing file and says what that file does with the five criteria, because THAT is the thing a reviewer has to judge. """ + rel, marker, symbol = identity.partition("#") if rel not in tracked: moved = (moves or {}).get(rel) if not moved: @@ -295,9 +314,11 @@ def why(rel, scores, tracked, moves=None): "file passes all five, so nothing readable was lost") return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}), which " "fails: " + "; ".join(tiers.CRITERION_LABEL[k] for k in failed)) - failed = _failures(rel, scores) + failed = _failures(identity, scores) if failed is None: - if rel not in scores: + if marker: + return f"GONE -- {symbol} is no longer an enrolled member of {rel}" + if identity not in scores: return "UNREADABLE -- the file could not be read" # Cannot happen through --check, which derives both sides from one scan; it can # happen if a caller passes a hand-edited path list, so say so rather than lie. @@ -341,7 +362,7 @@ def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--check", action="store_true", - help="exit 1 if any banked path is no longer CONVERTED") + help="exit 1 if any banked identity is no longer CONVERTED") ap.add_argument("--update", action="store_true", help="rewrite the baseline from the current tree") ap.add_argument("--reason", metavar="TEXT", @@ -352,7 +373,7 @@ def main(): "on purpose: git already dates the commit that adds the row, " "and a live clock would make this tool untestable") ap.add_argument("--list", action="store_true", - help="print the current CONVERTED paths, one per line") + help="print the current CONVERTED source/member identities") ap.add_argument("--baseline", default=str(BASELINE), metavar="PATH") ap.add_argument("--exceptions", default=str(EXCEPTIONS), metavar="PATH") args = ap.parse_args() diff --git a/tools/tu_promote.py b/tools/tu_promote.py index 4562fd8cfe..bc7a60ec95 100644 --- a/tools/tu_promote.py +++ b/tools/tu_promote.py @@ -19,6 +19,8 @@ production path, matching the entries already enrolled; * add one ``attribution.json`` override per absorbed symbol, so a many-to-one consolidation reads as "consolidated with credit intact" instead of N lost. +* migrate banked CONVERTED legacy paths to ``promoted-path#symbol`` identities, + preserving the readability ratchet at function rather than physical-file granularity. It deliberately does NOT compile anything. The proof that a promotion is sound is ``rombuild.py`` reporting 106/106 with ``mismatching: 0`` afterwards, and running it @@ -41,6 +43,7 @@ sys.path.insert(0, str(REPO / "tools")) import tu_manifest as TUM # noqa: E402 +import tiers_ratchet as TR # noqa: E402 CONFIG = REPO / "config" @@ -198,6 +201,50 @@ def rewrite_attribution(plans, lineage): return added +def converted_baseline_update(plans): + """Prepare a CONVERTED identity rewrite without changing the worktree. + + The baseline historically keyed one-function intake by path. Consolidation deletes + those paths, while ``tiers.py`` continues counting the functions inside the promoted + TU. Only already-banked legacy identities move; an unbanked function does not become + readable merely because it now shares a file with one that was. + """ + path = CONFIG / "converted-baseline.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PromoteError(f"converted baseline is unreadable: {exc}") from exc + rows = data.get("converted") + if not isinstance(rows, list) or len(rows) != len(set(rows)): + raise PromoteError("converted baseline must contain a unique converted list") + + converted = set(rows) + moved = 0 + for p in plans: + for f in p["functions"]: + legacy = f["legacy_source"] + symbol = f["symbol"] + old_keys = (legacy, f"{legacy}#{symbol}") + if not any(key in converted for key in old_keys): + continue + converted.difference_update(old_keys) + converted.add(f"{p['dest']}#{symbol}") + moved += 1 + + data["_note"] = TR.NOTE + data["count"] = len(converted) + data["converted"] = sorted(converted) + return path, data, moved + + +def rewrite_converted_baseline(plans, prepared=None): + """Write a preflighted CONVERTED identity rewrite.""" + path, data, moved = prepared or converted_baseline_update(plans) + path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", newline="") + return moved + + def git(*args): subprocess.run(["git", *args], cwd=REPO, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) @@ -234,8 +281,15 @@ def main(): for p in plans: print(f" promote {p['id']:38s} {len(p['functions'])} function(s), " f"{len(p['legacy'])} legacy source(s) -> {p['dest']}") + try: + converted_update = converted_baseline_update(plans) + except PromoteError as exc: + print(f" refused {exc}") + return 1 if args.dry_run: - print(f"tu_promote: {len(plans)} entry(ies) would be promoted (dry run).") + print(f"tu_promote: {len(plans)} entry(ies) would be promoted and " + f"{converted_update[2]} CONVERTED member identity/identities retained " + "(dry run).") return 0 import prepush_attribution as PA @@ -249,10 +303,12 @@ def main(): for source in p["legacy"]: git("rm", "-q", source) rewrite_manifest(entry, p) + converted = rewrite_converted_baseline(plans, converted_update) added = rewrite_attribution(plans, lineage) print(f"tu_promote: {len(plans)} entry(ies) promoted, " f"{sum(len(p['functions']) for p in plans)} function(s) consolidated, " - f"{added} attribution override(s) added.") + f"{added} attribution override(s) added, " + f"{converted} CONVERTED member identity/identities retained.") print("tu_promote: now run `python tools/rombuild.py -j16 --no-rom` -- " "106/106 with mismatching 0 is the proof.") return 1 if refused else 0 From 1d9f73fbded5197bc28ecd3be304117a79e54d16 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 04:50:46 -0500 Subject: [PATCH 08/20] tools: fail closed on intact TU promotion --- tools/rombuild.py | 161 +++++++++++++++++++++++++-- tools/rombuild_check.py | 82 +++++++++++--- tools/test_rombuild.py | 78 ++++++++++++- tools/test_rombuild_check.py | 27 +++++ tools/test_tu_production.py | 47 +++++++- tools/test_tu_promote.py | 152 +++++++++++++++++++++++++- tools/test_tubuild.py | 14 ++- tools/tu_production.py | 50 ++++++++- tools/tu_promote.py | 206 ++++++++++++++++++++++++----------- tools/tubuild.py | 44 ++++++-- 10 files changed, 746 insertions(+), 115 deletions(-) diff --git a/tools/rombuild.py b/tools/rombuild.py index e147a6ebda..8344476c18 100644 --- a/tools/rombuild.py +++ b/tools/rombuild.py @@ -33,12 +33,14 @@ See notes/rom-build.md for the milestones and the enrollment rules. """ import argparse +import collections import concurrent.futures import hashlib import io import json import os import pathlib +import re import shutil import subprocess import sys @@ -253,9 +255,16 @@ def enrolled(config_root=CONFIG_ROOT, extra_roots=()): if path and saw_complete: files.append(path) path, saw_complete = None, False + normalized = [pathlib.PurePosixPath(rel.replace("\\", "/")).as_posix() + for rel in files] + duplicates = sorted(rel for rel, count in collections.Counter(normalized).items() + if count != 1) + if duplicates: + raise BuildError("profile", 1, "complete source path enrolled more than once: " + + ", ".join(duplicates)) checked = [] - for rel in sorted(set(files)): - pure = pathlib.PurePosixPath(rel.replace("\\", "/")) + for rel in sorted(normalized): + pure = pathlib.PurePosixPath(rel) if (pure.is_absolute() or ".." in pure.parts or len(pure.parts) < 2 or pure.parts[0] not in ("src", "mods", *extra_roots) or pure.suffix not in (".c", ".cpp")): @@ -735,18 +744,20 @@ def intact_tu_policies(enrolled=None, manifest=None): policy in the first place. """ data = TUM.load() if manifest is None else manifest - active = None if enrolled is None else { - str(rel).replace("\\", "/") for rel in enrolled - } + active_counts = None if enrolled is None else collections.Counter( + str(rel).replace("\\", "/") for rel in enrolled) out, errors = {}, [] for entry in data.get("entries", []): if entry.get("production_mode") != "intact-object": continue source = str(entry.get("promoted_source") or entry.get("source", "")).replace("\\", "/") - if active is not None and source not in active: - continue label = entry.get("id", source or "") + if active_counts is not None and active_counts[source] != 1: + if entry.get("status") == "promoted": + errors.append(f"{label}: promoted intact-object source {source!r} is " + f"enrolled {active_counts[source]} time(s), expected exactly 1") + continue if not source.startswith("src/"): errors.append(f"{label}: intact-object policy has no production src/ path") continue @@ -759,6 +770,20 @@ def intact_tu_policies(enrolled=None, manifest=None): if isinstance(row, dict)} if ".text" not in section_names or not (section_names - {".text"}): errors.append(f"{label}: intact-object policy needs .text and non-text claims") + mapped_sections = [ + f"{row.get('name')} -> {row.get('module_section')}" + for row in entry.get("sections", []) if isinstance(row, dict) + and row.get("module_section", row.get("name")) != row.get("name")] + if mapped_sections: + errors.append(f"{label}: intact-object input-section retargeting is not " + f"implemented: {', '.join(mapped_sections)}") + owned_fields = ("rodata", "init", "ctor", "data", "bss") + if any(isinstance(entry.get(field), list) + and any(isinstance(row, dict) and row.get("storage_alias") + for row in entry[field]) + for field in owned_fields): + errors.append(f"{label}: automatic intact-object storage aliases are not " + "supported until baseline bootstrapping is non-circular") linkcheck = (entry.get("verification") or {}).get("linkcheck") or {} phases = linkcheck.get("phases") or {} if linkcheck.get("result") != "scratch-data-verified": @@ -769,13 +794,77 @@ def intact_tu_policies(enrolled=None, manifest=None): errors.append(f"{label}: intact-object proof phase {phase} is not green") if linkcheck.get("symbolCheckNewVsBaseline") != []: errors.append(f"{label}: intact-object proof has new or unknown symbol errors") + recorded_errors = linkcheck.get("symbolCheckErrors") + baseline_errors = linkcheck.get("symbolCheckBaselineErrors") + if not isinstance(recorded_errors, list): + errors.append(f"{label}: intact-object proof has no symbol-error inventory") + if not isinstance(baseline_errors, list): + errors.append(f"{label}: intact-object proof has no baseline symbol-error " + "inventory") + if (isinstance(recorded_errors, list) and isinstance(baseline_errors, list) + and sorted(set(recorded_errors)) != sorted(set(baseline_errors))): + errors.append(f"{label}: intact-object proof's symbol errors differ from its " + "baseline inventory") ranges = linkcheck.get("tuRanges") or [] - if not ranges or any(row.get("differingBytes") != 0 for row in ranges): - errors.append(f"{label}: intact-object proof does not show every range exact") - sha = (linkcheck.get("rom") or {}).get("sha256") - if not isinstance(sha, str) or len(sha) != 64: + expected_ranges = [] + seen_sections = set() + for row in entry.get("sections", []): + try: + name = row["name"] + start, end = int(row["start"], 16), int(row["end"], 16) + except (KeyError, TypeError, ValueError): + errors.append(f"{label}: intact-object manifest has an invalid section claim") + continue + if name not in (".text", ".data", ".rodata", ".bss") or start >= end: + errors.append(f"{label}: intact-object manifest has unsupported/invalid " + f"section claim {name!r}") + if name in seen_sections: + errors.append(f"{label}: intact-object manifest repeats section {name}") + seen_sections.add(name) + expected_ranges.append((name, start, end)) + ordered_ranges = sorted(expected_ranges, key=lambda row: (row[1], row[2])) + if any(left[2] > right[1] + for left, right in zip(ordered_ranges, ordered_ranges[1:])): + errors.append(f"{label}: intact-object manifest section claims overlap") + proved_ranges = [] + ranges_exact = bool(ranges) + for row in ranges: + try: + key = (row["section"], int(row["start"], 16), int(row["end"], 16)) + except (KeyError, TypeError, ValueError): + ranges_exact = False + continue + proved_ranges.append(key) + if key[0] == ".bss": + ranges_exact = ranges_exact and row.get("comparison") == \ + "NOBITS -- symbol/module gates" + else: + ranges_exact = ranges_exact and row.get("differingBytes") == 0 + if (not ranges_exact or sorted(proved_ranges) != sorted(expected_ranges) + or len(proved_ranges) != len(expected_ranges)): + errors.append(f"{label}: intact-object proof does not show every current " + "manifest range exact") + rom = linkcheck.get("rom") or {} + sha = rom.get("sha256") + if not isinstance(sha, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", sha): errors.append(f"{label}: intact-object proof has no full-ROM SHA-256") + if rom.get("matchesStockRom") is not True: + errors.append(f"{label}: intact-object proof does not show the full ROM " + "identical to stock") out[source] = entry + proof_shas = { + (((entry.get("verification") or {}).get("linkcheck") or {}).get("rom") or {}) + .get("sha256") for entry in out.values() + } + if len(proof_shas) > 1: + errors.append("promoted intact-object entries disagree on the stock ROM SHA-256") + symbol_inventories = { + tuple(((entry.get("verification") or {}).get("linkcheck") or {}) + .get("symbolCheckErrors") or []) for entry in out.values() + } + if len(symbol_inventories) > 1: + errors.append("promoted intact-object entries disagree on the allowed symbol-error " + "inventory") if errors: raise BuildError("intact TU policy", 1, "\n".join(errors)) return out @@ -1086,6 +1175,24 @@ def save_report(): intact_tus = intact_tu_policies(srcs) report["intactTus"] = sorted(entry.get("id", source) for source, entry in intact_tus.items()) + intact_link_verification = None + if intact_tus and args.profile == "stock": + import tu_production as ITP + try: + intact_link_verification = ITP.prepare_intact_link_verification( + intact_tus) + except ITP.ProductionTuError as exc: + raise BuildError("intact TU link control", 1, str(exc)) from exc + elif intact_tus: + # `mods` deliberately permits module/ROM differences. The stock admission + # proof still governs how each intact source object is prepared, while its + # final fidelity is judged by rombuild_check's profile-aware allowances. + # Running retail-exact final gates here would make the documented mods + # profile impossible as soon as the first intact TU is promoted. + report["intactTuStockVerification"] = { + "status": "not-applicable", + "reason": f"{args.profile} profile permits intentional divergences", + } # Collected during the compile because that is the only point at which the # objects still carry the data mwcc emitted -- see _isolate. None switches the # measurement off entirely; it never affects what gets linked either way. @@ -1144,6 +1251,21 @@ def save_report(): "\n".join(detail) or "unknown verification failure") print(" partitioned TU gates: dsd modules PASS, zero new symbol " "errors, storage aliases exact") + if intact_link_verification: + verification = ITP.verify_link( + config_yaml, BUILD / "final_link.o", intact_link_verification) + report["intactTuLinkVerification"] = verification + if not verification["ok"]: + detail = [] + if not verification["modulesOk"]: + detail.append("dsd check modules --fail did not pass") + detail.extend(f"new symbol error: {line}" + for line in verification["newSymbolErrors"]) + detail.extend(verification["storageAliasErrors"]) + raise BuildError("intact TU link verification", 1, + "\n".join(detail) or "unknown verification failure") + print(" intact TU gates: dsd modules PASS, zero new symbol errors, " + "storage aliases exact") if args.no_rom: print("[5/6] skipped (--no-rom)") @@ -1168,6 +1290,23 @@ def save_report(): "bytes": rom_path.stat().st_size, "sha256": rom_sha256, } + if intact_link_verification: + expected = { + (((entry.get("verification") or {}).get("linkcheck") or {}) + .get("rom") or {}).get("sha256") + for entry in intact_tus.values() + } + expected.add(intact_link_verification["baseline"]["romSha256"]) + report["intactTuRom"] = { + "expectedSha256": next(iter(expected)) if len(expected) == 1 else None, + "actualSha256": rom_sha256, + "identical": len(expected) == 1 and rom_sha256 in expected, + } + if len(expected) != 1 or rom_sha256 not in expected: + raise BuildError( + "intact TU ROM comparison", 1, + f"built ROM sha256 {rom_sha256} differs from admitted " + f"stock proof(s) {sorted(expected)}") if tu_prepared: expected = tu_prepared["baseline"]["romSha256"] report["partitionedTuRom"] = { diff --git a/tools/rombuild_check.py b/tools/rombuild_check.py index 7039dd94ca..04d86477d7 100644 --- a/tools/rombuild_check.py +++ b/tools/rombuild_check.py @@ -77,32 +77,71 @@ def module_binaries(d, config_root=CONFIG, build_root=None): return None, None -def complete_entries_text(text): - """Return ``[(path, address, end)]`` for entries carrying ``complete``.""" - out, cur, done, sec = [], None, False, None +def complete_entry_sections_text(text): + """Return every ``(path, section, address, end)`` in complete entries.""" + out, cur, done, entry_sections = [], None, False, [] + + def flush(): + if cur and done: + out.extend((cur, name, start, end) + for name, start, end in entry_sections) + for line in text.splitlines(): if not line.strip(): continue if not line[0].isspace(): - if cur and done and sec: - out.append((cur, *sec)) - cur, done, sec = line.strip().rstrip(":"), False, None + flush() + cur, done, entry_sections = line.strip().rstrip(":"), False, [] elif cur is not None: if line.strip() == "complete": done = True else: m = ENTRY_SEC.match(line) if m: - sec = (int(m.group(2), 16), int(m.group(3), 16)) - if cur and done and sec: - out.append((cur, *sec)) + entry_sections.append((m.group(1), int(m.group(2), 16), + int(m.group(3), 16))) + flush() return out +def complete_entries_text(text): + """Return complete code contributions as ``[(path, address, end)]``. + + Callers of this historical API measure function/source enrollment. Intact C++ + entries can also own data; retaining the section name prevents a trailing .data + claim from replacing .text and being misreported as source code. + """ + return [(rel, start, end) + for rel, name, start, end in complete_entry_sections_text(text) + if name in (".text", ".init")] + + def complete_entries(path): return complete_entries_text(path.read_text(encoding="utf-8", errors="ignore")) +def complete_entry_sections(path): + return complete_entry_sections_text( + path.read_text(encoding="utf-8", errors="ignore")) + + +def _covered_bytes(ranges, base, size): + """Union length of address ranges clipped to one linked module image.""" + clipped = sorted((max(0, lo - base), min(size, hi - base)) + for lo, hi in ranges if hi > base and lo < base + size) + total = end = 0 + for lo, hi in clipped: + if hi <= lo: + continue + if lo > end: + total += hi - lo + end = hi + elif hi > end: + total += hi - end + end = hi + return total + + def _code_totals(config_root): funcs = size = 0 for sym in sorted(config_root.rglob("symbols.txt")): @@ -174,6 +213,7 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): missing_bins = [] per_module_bad = collections.Counter() source_functions = source_bytes = mod_functions = mod_bytes = 0 + source_data_bytes = 0 reproducing = reproducing_bytes = bad = bad_function_bytes = differing_source_bytes = 0 for sym in sorted(config_root.rglob("symbols.txt")): @@ -181,7 +221,9 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): dl = d / "delinks.txt" if not dl.is_file(): continue - entries = complete_entries(dl) + entry_sections = complete_entry_sections(dl) + entries = [(rel, start, end) for rel, name, start, end in entry_sections + if name in (".text", ".init")] built_p, retail_p = module_binaries(d, config_root, build_root) label = module_label(d, config_root) if not built_p or not built_p.is_file() or not retail_p.is_file(): @@ -193,8 +235,13 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): continue base = min(s[1] for s in secs) built, retail = built_p.read_bytes(), retail_p.read_bytes() - allowed_mod_ranges = [(addr - base, end - base) for rel, addr, end in entries + allowed_mod_ranges = [(addr - base, end - base) + for rel, _name, addr, end in entry_sections if rel.startswith("mods/")] + source_data_bytes += _covered_bytes( + [(addr, end) for rel, name, addr, end in entry_sections + if rel.startswith("src/") and name not in (".text", ".init")], + base, max(len(built), len(retail))) module_diff, unexpected_diff = _diff_counts(built, retail, allowed_mod_ranges) module_results.append({ "module": label, @@ -283,10 +330,16 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): "moduleBytes": compared_module_bytes, "codeBytes": module_code_bytes, "dataBytes": compared_module_bytes - module_code_bytes, + "sourceDataBytes": source_data_bytes, + "unownedDataBytes": max(0, compared_module_bytes - module_code_bytes + - source_data_bytes), "sourceBytes": source_bytes, "sourceBytesOfModulePercent": (100.0 * source_bytes / compared_module_bytes if compared_module_bytes else 0.0), "dataBytesVerified": 0, + "sourceDataBytesOfModulePercent": ( + 100.0 * source_data_bytes / compared_module_bytes + if compared_module_bytes else 0.0), "dataBytesOfModulePercent": (100.0 * (compared_module_bytes - module_code_bytes) / compared_module_bytes if compared_module_bytes else 0.0), @@ -337,9 +390,10 @@ def print_report(report, show=12): # construction for every byte dsd supplies from the ROM and says nothing at all # about them. This line is what it is a percentage OF. print(f" of {mc['moduleBytes']:,} module bytes: {mc['sourceBytes']:,} " - f"({mc['sourceBytesOfModulePercent']:.1f}%) built from source, " - f"{mc['dataBytes']:,} ({mc['dataBytesOfModulePercent']:.1f}%) are data " - f"no delink entry reaches ({mc['dataBytesVerified']:,} verified)") + f"({mc['sourceBytesOfModulePercent']:.1f}%) source-built code, " + f"{mc.get('sourceDataBytes', 0):,} source-owned data, " + f"{mc.get('unownedDataBytes', mc['dataBytes']):,} data bytes no complete " + f"source entry reaches ({mc['dataBytesVerified']:,} verified)") if report["missingModuleBinaries"]: print(f"missing module binaries: {report['missingModuleBinaries'][:8]}") for f in report["failures"][:show]: diff --git a/tools/test_rombuild.py b/tools/test_rombuild.py index b7ef065dd9..e168160a59 100644 --- a/tools/test_rombuild.py +++ b/tools/test_rombuild.py @@ -54,6 +54,17 @@ def test_enrolled_rejects_non_source_paths(self): RB.enrolled(self.config) self.assertIn("unsafe complete", raised.exception.output) + def test_enrolled_rejects_duplicate_complete_source_paths(self): + (self.config / "delinks.txt").write_text( + "src/Example.c:\n complete\n" + " .text start:0x02000000 end:0x02000004\n\n" + "src/Example.c:\n complete\n" + " .data start:0x02001000 end:0x02001004\n", + encoding="utf-8") + with self.assertRaises(RB.BuildError) as raised: + RB.enrolled(self.config) + self.assertIn("enrolled more than once", raised.exception.output) + def test_enrolled_symbols_group_one_source_in_rom_order(self): candidates = [ (self.config, "Second", "src/Pair.cpp", 0x02000004, 4, ".text"), @@ -296,25 +307,71 @@ def test_intact_policy_requires_promoted_full_ordinary_link_proof(self): "id": "ov047/TU", "status": "promoted", "production_mode": "intact-object", "source": "src/actors/TU.cpp", "promoted_source": "src/actors/TU.cpp", - "sections": [{"name": ".text"}, {"name": ".data"}], + "sections": [ + {"name": ".text", "start": "0x1000", "end": "0x1010"}, + {"name": ".data", "start": "0x2000", "end": "0x2010"}], "verification": {"linkcheck": { "result": "scratch-data-verified", "phases": {name: True for name in ("delink", "lcf", "compile", "link", "checkModules", "rom")}, "symbolCheckNewVsBaseline": [], - "tuRanges": [{"section": ".text", "differingBytes": 0}, - {"section": ".data", "differingBytes": 0}], - "rom": {"sha256": "a" * 64}, + "symbolCheckErrors": ["[ERROR] old"], + "symbolCheckBaselineErrors": ["[ERROR] old"], + "tuRanges": [ + {"section": ".text", "start": "0x1000", "end": "0x1010", + "differingBytes": 0}, + {"section": ".data", "start": "0x2000", "end": "0x2010", + "differingBytes": 0}], + "rom": {"sha256": "a" * 64, "matchesStockRom": True}, }}, } manifest = {"entries": [entry]} self.assertEqual(RB.intact_tu_policies( {"src/actors/TU.cpp"}, manifest=manifest), {"src/actors/TU.cpp": entry}) + duplicate = dict(entry) + duplicate["id"] = "ov047/Duplicate" + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies({"src/actors/TU.cpp"}, + manifest={"entries": [entry, duplicate]}) + self.assertIn("declared by multiple entries", raised.exception.output) + entry["verification"]["linkcheck"]["tuRanges"][1]["differingBytes"] = 1 with self.assertRaises(RB.BuildError) as raised: RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) - self.assertIn("every range exact", raised.exception.output) + self.assertIn("every current manifest range exact", raised.exception.output) + + entry["verification"]["linkcheck"]["tuRanges"][1]["differingBytes"] = 0 + entry["verification"]["linkcheck"]["tuRanges"].pop() + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) + self.assertIn("every current manifest range exact", raised.exception.output) + + entry["verification"]["linkcheck"]["tuRanges"].append( + {"section": ".data", "start": "0x2000", "end": "0x2010", + "differingBytes": 0}) + entry["verification"]["linkcheck"]["rom"]["matchesStockRom"] = False + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) + self.assertIn("identical to stock", raised.exception.output) + + entry["verification"]["linkcheck"]["rom"]["matchesStockRom"] = True + entry["sections"][1]["name"] = ".rodata" + entry["sections"][1]["module_section"] = ".data" + entry["verification"]["linkcheck"]["tuRanges"][1]["section"] = ".rodata" + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) + self.assertIn("input-section retargeting", raised.exception.output) + + entry["sections"][1] = {"name": ".data", "start": "0x2000", "end": "0x2010"} + entry["verification"]["linkcheck"]["tuRanges"][1]["section"] = ".data" + # Every licensed non-text field follows the same fail-closed rule, not just + # `.data`; vtables can be represented under `.rodata` in a manifest. + entry["rodata"] = [{"symbol": "_ZTV1T", "storage_alias": { + "symbol": "data_00002000", "address": "0x2000", "size": "0x8"}}] + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) + self.assertIn("baseline bootstrapping is non-circular", raised.exception.output) def test_intact_policy_ignores_unenrolled_shadow(self): manifest = {"entries": [{ @@ -325,6 +382,17 @@ def test_intact_policy_ignores_unenrolled_shadow(self): self.assertEqual(RB.intact_tu_policies( {"src/actors/Elsewhere.cpp"}, manifest=manifest), {}) + def test_intact_policy_refuses_promoted_source_missing_from_enrollment(self): + manifest = {"entries": [{ + "id": "ov047/Missing", "status": "promoted", + "production_mode": "intact-object", + "promoted_source": "src/actors/Missing.cpp", + }]} + with self.assertRaises(RB.BuildError) as raised: + RB.intact_tu_policies( + ["src/actors/Elsewhere.cpp"], manifest=manifest) + self.assertIn("enrolled 0 time(s), expected exactly 1", raised.exception.output) + def _compiler(): exe = RB.MW / RB.VERSION / "mwccarm.exe" diff --git a/tools/test_rombuild_check.py b/tools/test_rombuild_check.py index 0290286dc0..d98786c525 100644 --- a/tools/test_rombuild_check.py +++ b/tools/test_rombuild_check.py @@ -56,6 +56,33 @@ def test_shared_source_counts_every_owned_function(self): self.assertEqual(report["sourceBuild"]["reproducingFunctions"], 2) self.assertEqual(report["sourceBuild"]["sourceBytes"], 8) + def test_intact_entry_counts_code_members_and_nontext_separately(self): + (self.config / "delinks.txt").write_text( + " .text start:0x00001000 end:0x00001008 kind:code\n" + " .data start:0x00001008 end:0x0000100c kind:data\n\n" + "src/actors/Pair.cpp:\n" + " complete\n" + " .text start:0x00001000 end:0x00001008\n" + " .data start:0x00001008 end:0x0000100c\n", + encoding="utf-8") + (self.retail / "arm9" / "arm9.bin").write_bytes(b"ABCDEFGHIJKL") + (self.built / "arm9.bin").write_bytes(b"ABCDEFGHIJKL") + + text = (self.config / "delinks.txt").read_text(encoding="utf-8") + self.assertEqual(RBC.complete_entries_text(text), + [("src/actors/Pair.cpp", 0x1000, 0x1008)]) + self.assertEqual(RBC.complete_entry_sections_text(text), [ + ("src/actors/Pair.cpp", ".text", 0x1000, 0x1008), + ("src/actors/Pair.cpp", ".data", 0x1008, 0x100c), + ]) + + report = RBC.analyze(self.config, "stock") + self.assertTrue(report["passed"]) + self.assertEqual(report["sourceBuild"]["sourceFunctions"], 2) + self.assertEqual(report["sourceBuild"]["sourceBytes"], 8) + self.assertEqual(report["moduleComposition"]["sourceDataBytes"], 4) + self.assertEqual(report["moduleComposition"]["unownedDataBytes"], 0) + def test_module_paths_accept_config_or_arm9_as_the_root(self): self.assertEqual(RBC.module_label(self.config, self.config), "arm9") self.assertEqual(RBC.module_label(self.config, self.config.parent), "arm9") diff --git a/tools/test_tu_production.py b/tools/test_tu_production.py index 31163de785..af9907a13d 100644 --- a/tools/test_tu_production.py +++ b/tools/test_tu_production.py @@ -30,6 +30,42 @@ def test_missing_content_bound_baseline_refuses(self): class ProductionTuObjects(unittest.TestCase): + def test_automatic_intact_link_plan_uses_current_control_and_vtable_biases(self): + claims = [{"name": ".text", "start": 0x1000, "end": 0x1010}, + {"name": ".data", "start": 0x2000, "end": 0x2010}] + baseline = {"symbolErrors": ["[ERROR] old"], "romSha256": "ab" * 32} + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", + "verification": {"linkcheck": { + "symbolCheckErrors": ["[ERROR] old"], + "rom": {"sha256": "ab" * 32}}}, + }} + with mock.patch.object(TP, "_strict_baseline", return_value=baseline), \ + mock.patch.object(TP.TB, "manifest_section_claims", + return_value=(claims, [])), \ + mock.patch.object(TP.TB, "partition_vtable_rebiases", + return_value=({"_ZTV1T": {"bias": 8}}, [])): + prepared = TP.prepare_intact_link_verification(entries) + self.assertIs(prepared["baseline"], baseline) + self.assertEqual(prepared["entries"], [{ + "id": "ov047/Thing", "source": "src/actors/Thing.cpp", + "biases": {"_ZTV1T": {"bias": 8}}, + }]) + + def test_automatic_intact_link_plan_rejects_laundered_control_error(self): + baseline = {"symbolErrors": ["[ERROR] old", "[ERROR] new"], + "romSha256": "ab" * 32} + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", + "verification": {"linkcheck": { + "symbolCheckErrors": ["[ERROR] old"], + "rom": {"sha256": "ab" * 32}}}, + }} + with mock.patch.object(TP, "_strict_baseline", return_value=baseline): + with self.assertRaisesRegex(TP.ProductionTuError, + "pre-promotion inventory"): + TP.prepare_intact_link_verification(entries) + def test_intact_object_runs_all_fail_closed_policy_gates(self): entry = {"id": "ov047/Thing"} claims = [{"name": ".text", "start": 0x1000, "end": 0x1010}, @@ -78,7 +114,7 @@ def test_final_gate_allows_only_baseline_symbol_errors(self): } calls = [ (True, "modules ok", 0.1), - (False, "[ERROR] old\nError: Some symbol(s) did not match.", 0.1), + (True, "[ERROR] old\nError: Some symbol(s) did not match.", 0.1), ] with mock.patch.object(TP.TB, "_run_dsd", side_effect=calls), \ mock.patch.object(TP.TB, "verify_linked_storage_aliases", @@ -99,6 +135,15 @@ def test_final_gate_rejects_new_symbol_error(self): self.assertFalse(result["ok"]) self.assertEqual(result["newSymbolErrors"], ["[ERROR] new"]) + def test_final_gate_rejects_symbol_check_operational_failure(self): + prepared = {"baseline": {"symbolErrors": []}, "entries": []} + calls = [(True, "modules ok", 0.1), + (False, "tool crashed before producing an inventory", 0.1)] + with mock.patch.object(TP.TB, "_run_dsd", side_effect=calls): + result = TP.verify_link("config.yaml", "final_link.o", prepared) + self.assertFalse(result["ok"]) + self.assertFalse(result["symbolsCommandOk"]) + if __name__ == "__main__": unittest.main() diff --git a/tools/test_tu_promote.py b/tools/test_tu_promote.py index 447417191a..f77eeb5932 100644 --- a/tools/test_tu_promote.py +++ b/tools/test_tu_promote.py @@ -9,7 +9,69 @@ import tu_promote as TP # noqa: E402 +def intact_proof(sections): + return {"linkcheck": { + "result": "scratch-data-verified", + "phases": {name: True for name in + ("delink", "lcf", "compile", "link", "checkModules", "rom")}, + "symbolCheckNewVsBaseline": [], + "symbolCheckErrors": ["[ERROR] old"], + "symbolCheckBaselineErrors": ["[ERROR] old"], + "tuRanges": [ + {"section": row["name"], "start": row["start"], "end": row["end"], + "differingBytes": 0} + for row in sections], + "rom": {"sha256": "a" * 64, "matchesStockRom": True}, + }} + + class IntactPromotion(unittest.TestCase): + def test_batch_preflight_refuses_shared_legacy_before_mutation(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / "src_tu").mkdir() + (root / "src").mkdir() + (root / "src_tu/One.cpp").write_text("//cpp\n", encoding="utf-8") + (root / "src_tu/Two.cpp").write_text("//cpp\n", encoding="utf-8") + (root / "src/Shared.cpp").write_text("//cpp\n", encoding="utf-8") + plans = [ + {"id": "ov047/One", "source": "src_tu/One.cpp", + "dest": "src/One.cpp", "legacy": ["src/Shared.cpp"], + "delinks": root / "config/delinks.txt", "claims": []}, + {"id": "ov047/Two", "source": "src_tu/Two.cpp", + "dest": "src/Two.cpp", "legacy": ["src/Shared.cpp"], + "delinks": root / "config/delinks.txt", "claims": []}, + ] + with mock.patch.object(TP, "REPO", root): + with self.assertRaisesRegex(TP.PromoteError, + "consumed by both"): + TP.batch_preflight(plans) + + def test_batch_preflight_refuses_cross_plan_claim_overlap(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / "src_tu").mkdir() + (root / "src").mkdir() + for rel in ("src_tu/One.cpp", "src_tu/Two.cpp", + "src/First.cpp", "src/Second.cpp"): + (root / rel).write_text("//cpp\n", encoding="utf-8") + delinks = root / "config/delinks.txt" + plans = [ + {"id": "ov047/One", "source": "src_tu/One.cpp", + "dest": "src/One.cpp", "legacy": ["src/First.cpp"], + "delinks": delinks, + "claims": [(".text", 0x1000, 0x1010), + (".data", 0x2000, 0x2020)]}, + {"id": "ov047/Two", "source": "src_tu/Two.cpp", + "dest": "src/Two.cpp", "legacy": ["src/Second.cpp"], + "delinks": delinks, + "claims": [(".text", 0x1010, 0x1020), + (".data", 0x2010, 0x2030)]}, + ] + with mock.patch.object(TP, "REPO", root): + with self.assertRaisesRegex(TP.PromoteError, "overlaps"): + TP.batch_preflight(plans) + def test_converted_baseline_moves_only_banked_members_to_promoted_tu(self): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) @@ -39,6 +101,20 @@ def test_converted_baseline_moves_only_banked_members_to_promoted_tu(self): "src/Unrelated.cpp", "src/actors/TU.cpp#First"]) self.assertEqual(data["_note"], TP.TR.NOTE) + def test_single_member_converted_identity_stays_path_based(self): + with tempfile.TemporaryDirectory() as td: + config = pathlib.Path(td) / "config" + config.mkdir() + baseline = config / "converted-baseline.json" + baseline.write_text( + '{"converted":["src/Only.cpp"],"count":1}\n', encoding="utf-8") + plans = [{"dest": "src/actors/Only.cpp", "functions": [ + {"symbol": "Only", "legacy_source": "src/Only.cpp"}]}] + with mock.patch.object(TP, "CONFIG", config): + TP.rewrite_converted_baseline(plans) + data = json.loads(baseline.read_text(encoding="utf-8")) + self.assertEqual(data["converted"], ["src/actors/Only.cpp"]) + def test_plan_and_rewrite_keep_nontext_claims_for_intact_object(self): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) @@ -58,16 +134,18 @@ def test_plan_and_rewrite_keep_nontext_claims_for_intact_object(self): (root / "src").mkdir() (root / "src/First.cpp").write_text("//cpp\n", encoding="utf-8") (root / "src/Second.cpp").write_text("//cpp\n", encoding="utf-8") + sections = [ + {"name": ".text", "start": "0x00001000", "end": "0x00001020"}, + {"name": ".data", "start": "0x00002040", "end": "0x00002050"}, + ] entry = { "id": "ov047/TU", "module": "ov047", "status": "scratch-data-verified", "production_mode": "intact-object", "source": "src_tu/TU.cpp", "promoted_source": "src/TU.cpp", - "sections": [ - {"name": ".text", "start": "0x00001000", "end": "0x00001020"}, - {"name": ".data", "start": "0x00002040", "end": "0x00002050"}, - ], + "sections": sections, "data": [{"symbol": "owned", "address": "0x2040", "size": "0x10"}], + "verification": intact_proof(sections), "functions": [ {"symbol": "First", "address": "0x1000", "size": "0x10", "legacy_source": "src/First.cpp"}, @@ -86,6 +164,72 @@ def test_plan_and_rewrite_keep_nontext_claims_for_intact_object(self): " .text start:0x00001000 end:0x00001020\n" " .data start:0x00002040 end:0x00002050\n", written) + def test_intact_promotion_refuses_unimplemented_section_retargeting(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + config = root / "config" + delinks = config / "arm9/overlays/ov047/delinks.txt" + delinks.parent.mkdir(parents=True) + delinks.write_text( + " .text start:0x00001000 end:0x00001100 kind:code align:4\n" + " .data start:0x00002000 end:0x00002100 kind:data align:4\n\n" + "src/Only.cpp:\n complete\n" + " .text start:0x00001000 end:0x00001010\n\n", + encoding="utf-8") + (root / "src_tu").mkdir() + (root / "src_tu/TU.cpp").write_text("//cpp\n", encoding="utf-8") + (root / "src").mkdir() + (root / "src/Only.cpp").write_text("//cpp\n", encoding="utf-8") + entry = { + "id": "ov047/TU", "module": "ov047", "status": "text-verified", + "production_mode": "intact-object", "source": "src_tu/TU.cpp", + "promoted_source": "src/TU.cpp", + "sections": [ + {"name": ".text", "start": "0x1000", "end": "0x1010"}, + {"name": ".rodata", "module_section": ".data", + "start": "0x2000", "end": "0x2010"}], + "functions": [{"symbol": "Only", "address": "0x1000", "size": "0x10", + "legacy_source": "src/Only.cpp"}], + } + with mock.patch.object(TP, "REPO", root), \ + mock.patch.object(TP, "CONFIG", config): + with self.assertRaisesRegex(TP.PromoteError, + "input-section retargeting"): + TP.plan(entry) + + def test_intact_promotion_preflights_production_admission(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + config = root / "config" + delinks = config / "arm9/overlays/ov047/delinks.txt" + delinks.parent.mkdir(parents=True) + delinks.write_text( + " .text start:0x00001000 end:0x00001100 kind:code align:4\n" + " .data start:0x00002000 end:0x00002100 kind:data align:4\n\n" + "src/Only.cpp:\n complete\n" + " .text start:0x00001000 end:0x00001010\n\n", + encoding="utf-8") + (root / "src_tu").mkdir() + (root / "src_tu/TU.cpp").write_text("//cpp\n", encoding="utf-8") + (root / "src").mkdir() + (root / "src/Only.cpp").write_text("//cpp\n", encoding="utf-8") + entry = { + "id": "ov047/TU", "module": "ov047", "status": "text-verified", + "production_mode": "intact-object", "source": "src_tu/TU.cpp", + "promoted_source": "src/TU.cpp", + "sections": [ + {"name": ".text", "start": "0x00001000", "end": "0x00001010"}, + {"name": ".data", "start": "0x00002000", "end": "0x00002010"}], + "functions": [{"symbol": "Only", "address": "0x1000", "size": "0x10", + "legacy_source": "src/Only.cpp"}], + "verification": {"linkcheck": {}}, + } + with mock.patch.object(TP, "REPO", root), \ + mock.patch.object(TP, "CONFIG", config): + with self.assertRaisesRegex(TP.PromoteError, + "production admission preflight failed"): + TP.plan(entry) + def test_nontext_promotion_without_intact_mode_is_refused(self): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index 76d0223096..746de265aa 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -741,19 +741,23 @@ def test_unknown_id_fails_closed_with_a_clear_reason(): import tubuild -def test_linkcheck_compile_passes_production_compiler_only_policies(): +def test_linkcheck_compile_passes_all_production_object_policies(): original_policies = tubuild.RB.compiler_only_policies + original_intact = tubuild.RB.intact_tu_policies original_compile = tubuild.RB.compile_one policy = {"src/actors/Promoted.cpp": {"deadstrip": ["helper"]}} + intact = {"src/actors/Promoted.cpp": {"id": "ov047/Promoted"}} seen = [] try: tubuild.RB.compiler_only_policies = lambda enrolled: ( policy if list(enrolled) == ["src/actors/Promoted.cpp"] else None) + tubuild.RB.intact_tu_policies = lambda enrolled: ( + intact if list(enrolled) == ["src/actors/Promoted.cpp"] else None) def fake_compile(rel, vers, cache, init_srcs, syms, build_root=None, - compiler_only=None): - seen.append((rel, build_root, compiler_only)) + compiler_only=None, intact_tus=None): + seen.append((rel, build_root, compiler_only, intact_tus)) return rel, None, "hit" tubuild.RB.compile_one = fake_compile @@ -761,11 +765,13 @@ def fake_compile(rel, vers, cache, init_srcs, syms, build_root=None, ["src/actors/Promoted.cpp"], {}, None, set(), {}, pathlib.Path("scratch"), 1) finally: tubuild.RB.compiler_only_policies = original_policies + tubuild.RB.intact_tu_policies = original_intact tubuild.RB.compile_one = original_compile assert failures == [] assert outcomes["hit"] == 1 - assert seen == [("src/actors/Promoted.cpp", pathlib.Path("scratch"), policy)] + assert seen == [("src/actors/Promoted.cpp", pathlib.Path("scratch"), + policy, intact)] def test_linkcheck_symbol_verdict_uses_the_stock_failure_inventory(): diff --git a/tools/tu_production.py b/tools/tu_production.py index 794c86ab88..8539d739db 100644 --- a/tools/tu_production.py +++ b/tools/tu_production.py @@ -85,6 +85,52 @@ def _raise(label, reasons): raise ProductionTuError(f"{label}: {detail or 'refused without a reason'}") +def prepare_intact_link_verification(entries): + """Bind supported automatic intact TUs to the current strict stock control. + + Object preparation proves the compiler contribution before the link. This plan + carries the remaining facts needed after mwldarm: the content-bound current + baseline's symbol-error inventory and each vtable address-point mapping. Automatic + intact admission refuses storage aliases until their baseline bootstrap is + non-circular, so refreshing this control never depends on the control being kept. + """ + baseline = _strict_baseline() + admitted_errors = set() + admitted_roms = set() + for entry in entries.values(): + linkcheck = (entry.get("verification") or {}).get("linkcheck") or {} + errors = linkcheck.get("symbolCheckErrors") + rom_sha = (linkcheck.get("rom") or {}).get("sha256") + if not isinstance(errors, list) or not isinstance(rom_sha, str): + raise ProductionTuError( + f"{entry.get('id', '')}: admitted intact proof lacks its " + "symbol-error inventory or stock ROM SHA-256") + admitted_errors.add(tuple(sorted(set(errors)))) + admitted_roms.add(rom_sha) + current_errors = tuple(baseline["symbolErrors"]) + if admitted_errors != {current_errors}: + raise ProductionTuError( + "strict post-promotion control symbol errors do not equal the admitted " + f"pre-promotion inventory: current={list(current_errors)!r}, " + f"admitted={[list(rows) for rows in sorted(admitted_errors)]!r}") + if admitted_roms != {baseline["romSha256"]}: + raise ProductionTuError( + "strict post-promotion control ROM SHA-256 does not equal the admitted " + f"stock proof: current={baseline['romSha256']}, " + f"admitted={sorted(admitted_roms)!r}") + prepared = [] + for source, entry in sorted(entries.items()): + claims, reasons = TB.manifest_section_claims(entry) + if reasons: + _raise(f"{entry.get('id', source)} manifest section claims", reasons) + biases, reasons = TB.partition_vtable_rebiases(entry, claims) + if reasons: + _raise(f"{entry.get('id', source)} vtable address-point policy", reasons) + prepared.append({"id": entry.get("id", source), "source": source, + "biases": biases}) + return {"baseline": baseline, "entries": prepared} + + def _strict_baseline(): """Return the content-bound stock baseline or refuse stale/missing evidence.""" report_path = TB.BASELINE_LINK / "linkcheck.json" @@ -295,7 +341,7 @@ def verify_link(config_yaml, linked_elf, prepared): "dsd check modules") ok_symbols, symbols_out, _seconds = TB._run_dsd( [str(TB.RB.DSD), "check", "symbols", "-c", str(config_yaml), - "-e", str(linked_elf), "-f", "-m", "12"], + "-e", str(linked_elf), "-m", "12"], "dsd check symbols") symbol_errors = sorted({line.strip() for line in symbols_out.splitlines() if "[ERROR]" in line}) @@ -307,7 +353,7 @@ def verify_link(config_yaml, linked_elf, prepared): alias_rows.extend(result.get("rows", [])) alias_errors.extend(result.get("errors", [])) return { - "ok": bool(ok_modules and not new_errors and not alias_errors), + "ok": bool(ok_modules and ok_symbols and not new_errors and not alias_errors), "modulesOk": bool(ok_modules), "modulesOutput": modules_out[-4000:], "symbolsCommandOk": bool(ok_symbols), diff --git a/tools/tu_promote.py b/tools/tu_promote.py index bc7a60ec95..d9c2a68253 100644 --- a/tools/tu_promote.py +++ b/tools/tu_promote.py @@ -22,17 +22,16 @@ * migrate banked CONVERTED legacy paths to ``promoted-path#symbol`` identities, preserving the readability ratchet at function rather than physical-file granularity. -It deliberately does NOT compile anything. The proof that a promotion is sound is -``rombuild.py`` reporting 106/106 with ``mismatching: 0`` afterwards, and running it -once over a batch is far cheaper than once per entry. Refusals here are only about -facts that can be checked without a compiler: a missing file, a delinks entry that -does not look the way the manifest says it does, a section span that does not cover -the functions claimed inside it. +It deliberately does NOT compile anything. It preflights the tracked full-ROM proof; +after promotion, refresh the content-bound stock control, then require ``rombuild.py`` +to reproduce that stock ROM and report 106/106 with ``mismatching: 0`` plus the final +linked-symbol/address-point gates. Refusals here need no candidate compile. """ from __future__ import annotations import argparse +import copy import json import pathlib import re @@ -44,6 +43,8 @@ import tu_manifest as TUM # noqa: E402 import tiers_ratchet as TR # noqa: E402 +import rombuild as RB # noqa: E402 +import tubuild as TB # noqa: E402 CONFIG = REPO / "config" @@ -59,11 +60,6 @@ def delinks_path(module): return CONFIG / "arm9" / "overlays" / module / "delinks.txt" -def _entry_re(source): - """One whole delinks entry: the path line plus its indented body.""" - return re.compile(re.escape(source) + r":\n(?:[ \t]+[^\n]*\n)+\n?") - - def plan(entry): """Everything the promotion will touch, or a PromoteError explaining why not.""" ident = entry.get("id", "") @@ -90,11 +86,11 @@ def plan(entry): if (REPO / dest).exists(): raise PromoteError(f"{ident}: {dest} already exists") - raw_sections = [s for s in entry.get("sections", []) if isinstance(s, dict)] - sections = [s for s in raw_sections if s.get("name") == ".text"] - if not sections: - raise PromoteError(f"{ident}: no .text section in the manifest") - owns_nontext = any(s.get("name") != ".text" for s in raw_sections) \ + normalized_claims, claim_reasons = TB.manifest_section_claims(entry) + if claim_reasons: + raise PromoteError(f"{ident}: invalid manifest section claims: " + + "; ".join(claim_reasons)) + owns_nontext = any(s["name"] != ".text" for s in normalized_claims) \ or bool(entry.get("data") or entry.get("rodata") or entry.get("bss")) if owns_nontext and entry.get("production_mode") != "intact-object": # Production isolation zeroes an object's data and rebinds the symbols to the @@ -104,20 +100,23 @@ def plan(entry): "production_mode: intact-object") claims = [] - for section in raw_sections: - name = section.get("name") - try: - lo, hi = int(section["start"], 16), int(section["end"], 16) - except (KeyError, TypeError, ValueError): - raise PromoteError(f"{ident}: invalid section claim {section!r}") from None - if not isinstance(name, str) or not name.startswith(".") or lo >= hi: - raise PromoteError(f"{ident}: invalid section claim {section!r}") - claims.append((name, lo, hi)) + spans = [] + for section in normalized_claims: + name = section["name"] + module_section = section["module_section"] + lo, hi = section["start"], section["end"] + if (entry.get("production_mode") == "intact-object" + and module_section != name): + raise PromoteError(f"{ident}: intact-object claim {name} -> " + f"{module_section} needs input-section retargeting, " + "which is not implemented") + claims.append((module_section, lo, hi)) + if name == ".text": + spans.append((lo, hi)) funcs = entry.get("functions", []) if not funcs: raise PromoteError(f"{ident}: entry licenses no functions") - spans = [(lo, hi) for name, lo, hi in claims if name == ".text"] legacy = [] for f in funcs: addr, size = int(f["address"], 16), int(f["size"], 16) @@ -133,37 +132,42 @@ def plan(entry): dl = delinks_path(entry["module"]) if not dl.is_file(): raise PromoteError(f"{ident}: {dl} does not exist") - text = dl.read_text(encoding="utf-8") - for source in legacy: - n = len(_entry_re(source).findall(text)) - if n != 1: - raise PromoteError(f"{ident}: {source} has {n} entries in " - f"{dl.relative_to(REPO)}, expected exactly 1") - return {"id": ident, "source": src, "dest": dest, "delinks": dl, - "legacy": legacy, "spans": spans, "claims": claims, "functions": funcs} + if len(spans) != 1: + raise PromoteError(f"{ident}: production promotion needs exactly one .text " + f"span, got {len(spans)}") + _header, _entries, _inside, _validated_claims, splice_reasons = \ + TB.validate_tu_entry_splice(dl, spans[0][0], spans[0][1], dest, legacy, + normalized_claims) + if splice_reasons: + raise PromoteError(f"{ident}: current delinks ownership is not safe to splice: " + + "; ".join(splice_reasons)) + if entry.get("production_mode") == "intact-object": + prospective = copy.deepcopy(entry) + prospective["status"] = "promoted" + prospective["source"] = dest + try: + RB.intact_tu_policies([dest], manifest={"entries": [prospective]}) + except RB.BuildError as exc: + raise PromoteError(f"{ident}: production admission preflight failed: " + f"{exc.output}") from exc + + return {"id": ident, "module": entry["module"], "source": src, "dest": dest, + "delinks": dl, + "legacy": legacy, "spans": spans, "claims": claims, + "section_claims": normalized_claims, "functions": funcs} def rewrite_delinks(p): - """N per-function entries out, one spanning entry in, in address order.""" - text = p["delinks"].read_text(encoding="utf-8") - for source in p["legacy"]: - text = _entry_re(source).sub("", text, count=1) - body = "".join(f" {name} start:0x{lo:08x} end:0x{hi:08x}\n" - for name, lo, hi in p["claims"]) - entry = f"{p['dest']}:\n complete\n{body}\n" - # Address order is not cosmetic: it is how a reader of delinks.txt finds the entry - # that owns an address, and dsd emits the file sorted. - after = min(hi for _lo, hi in p["spans"]) - nxt = None - for m in re.finditer( - r"^[^\s:][^\n]*:\n[ \t]+complete\n[ \t]+\.text start:0x([0-9a-fA-F]{8})", - text, re.M): - if int(m.group(1), 16) >= after: - nxt = m - break - text = (text[:nxt.start()] + entry + text[nxt.start():]) if nxt \ - else text.rstrip("\n") + "\n\n" + entry - p["delinks"].write_text(text, encoding="utf-8", newline="") + """N per-function entries out, one spanning entry in, using the shared gate.""" + replaced, reasons = TB.splice_tu_entry( + p["delinks"], p["spans"][0][0], p["spans"][0][1], p["dest"], + p["legacy"], p["section_claims"]) + if reasons: + raise PromoteError(f"{p['id']}: delinks ownership changed after preflight: " + + "; ".join(reasons)) + if set(replaced) != set(p["legacy"]): + raise PromoteError(f"{p['id']}: shared splice replaced {replaced}, expected " + f"{p['legacy']}") def rewrite_manifest(entry, p): @@ -174,16 +178,21 @@ def rewrite_manifest(entry, p): encoding="utf-8", newline="") -def rewrite_attribution(plans, lineage): - """One override per absorbed symbol, so the consolidation keeps its authors. +def attribution_update(plans, lineage): + """Prepare attribution overrides without changing the worktree. Without these, prepush_attribution reports every legacy basename as CREDIT LOST and the merge gate needs a label to pass -- for a change that took nothing away from anyone. """ path = REPO / "attribution.json" - data = json.loads(path.read_text(encoding="utf-8")) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PromoteError(f"attribution data is unreadable: {exc}") from exc ov = data.setdefault("overrides", {}) + if not isinstance(ov, dict): + raise PromoteError("attribution overrides must be an object") added = 0 for p in plans: for f in p["functions"]: @@ -196,6 +205,12 @@ def rewrite_attribution(plans, lineage): ov[key] = who added += 1 data["overrides"] = dict(sorted(ov.items())) + return path, data, added + + +def rewrite_attribution(plans, lineage, prepared=None): + """Write a preflighted attribution update.""" + path, data, added = prepared or attribution_update(plans, lineage) path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="") return added @@ -228,7 +243,9 @@ def converted_baseline_update(plans): if not any(key in converted for key in old_keys): continue converted.difference_update(old_keys) - converted.add(f"{p['dest']}#{symbol}") + target = p["dest"] if len(p["functions"]) == 1 \ + else f"{p['dest']}#{symbol}" + converted.add(target) moved += 1 data["_note"] = TR.NOTE @@ -250,6 +267,56 @@ def git(*args): stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) +def batch_preflight(plans): + """Refuse a promotion batch before its first mutation if ownership is ambiguous.""" + errors = [] + destinations = {} + consumed = {} + claims_by_delinks = {} + required = set() + for p in plans: + dest = p["dest"] + if dest in destinations: + errors.append(f"{dest} is the destination of both {destinations[dest]} " + f"and {p['id']}") + destinations[dest] = p["id"] + inputs = [("shadow source", p["source"])] + inputs.extend(("legacy source", rel) for rel in p["legacy"]) + for role, rel in inputs: + required.add(rel) + owner = consumed.get(rel) + if owner and owner != p["id"]: + errors.append(f"{rel} is consumed by both {owner} and {p['id']}") + consumed[rel] = p["id"] + if not (REPO / rel).is_file(): + errors.append(f"{p['id']}: {role} {rel} is not on disk") + owned = claims_by_delinks.setdefault(str(p["delinks"]), []) + for section, start, end in p.get("claims", []): + for other_start, other_end, other_id, other_section in owned: + if max(start, other_start) < min(end, other_end): + errors.append( + f"{p['id']} {section} 0x{start:08x}..0x{end:08x} overlaps " + f"{other_id} {other_section} 0x{other_start:08x}.." + f"0x{other_end:08x} in the same delinks file") + owned.append((start, end, p["id"], section)) + for dest, owner in destinations.items(): + if dest in consumed: + errors.append(f"{owner}: destination {dest} is also a source consumed " + f"by {consumed[dest]}") + + if errors: + raise PromoteError("batch preflight failed: " + "; ".join(errors)) + + for rel in sorted(required): + tracked = subprocess.run( + ["git", "ls-files", "--error-unmatch", "--", rel], cwd=REPO, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 + if not tracked: + errors.append(f"{rel} is not tracked by git") + if errors: + raise PromoteError("batch preflight failed: " + "; ".join(errors)) + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -274,10 +341,17 @@ def main(): for why in refused: print(f" refused {why}") - if not plans: + if refused or not plans: print("tu_promote: nothing to promote.") return 1 if refused else 0 + try: + batch_preflight(plans) + except PromoteError as exc: + print(f" refused {exc}") + print("tu_promote: nothing to promote.") + return 1 + for p in plans: print(f" promote {p['id']:38s} {len(p['functions'])} function(s), " f"{len(p['legacy'])} legacy source(s) -> {p['dest']}") @@ -294,6 +368,11 @@ def main(): import prepush_attribution as PA lineage = PA.lineage("HEAD") + try: + attribution = attribution_update(plans, lineage) + except PromoteError as exc: + print(f" refused {exc}") + return 1 for p, entry in zip(plans, entries): rewrite_delinks(p) # `git mv` will not create the destination directory, and a promoted_source @@ -304,13 +383,16 @@ def main(): git("rm", "-q", source) rewrite_manifest(entry, p) converted = rewrite_converted_baseline(plans, converted_update) - added = rewrite_attribution(plans, lineage) + added = rewrite_attribution(plans, lineage, attribution) print(f"tu_promote: {len(plans)} entry(ies) promoted, " f"{sum(len(p['functions']) for p in plans)} function(s) consolidated, " f"{added} attribution override(s) added, " f"{converted} CONVERTED member identity/identities retained.") - print("tu_promote: now run `python tools/rombuild.py -j16 --no-rom` -- " - "106/106 with mismatching 0 is the proof.") + print("tu_promote: now refresh the content-bound control with " + f"`python tools/tubuild.py linkcheck --baseline --module " + f"{plans[0]['module']} -j16 --clean`, then run " + "`python tools/rombuild.py -j16` -- 106/106, mismatching 0, zero new " + "symbol errors, exact address points, and a stock-identical ROM are the proof.") return 1 if refused else 0 diff --git a/tools/tubuild.py b/tools/tubuild.py index be7b1f3551..f9ed6b7409 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -1573,17 +1573,9 @@ def span_entries(delinks_path, span_start, span_end, expected_legacy): return header, entries, inside, reasons -def splice_tu_entry(delinks_path, span_start, span_end, tu_rel, expected_legacy, - section_claims=None): - """Replace the per-function entries tiling [span_start, span_end) with ONE TU entry. - - Refuses -- returns (None, [reasons]) -- rather than producing a plausible-looking - scratch config, because every failure mode here is silent downstream: dsd fills any - range it has no object for with retail ROM bytes, so a mis-spliced delinks tree - links clean and compares green while contributing nothing (see - "unbuildable files are invisible to every gate", and layout_check's L1). `span_entries` holds - the checks; this adds the rewrite. - """ +def validate_tu_entry_splice(delinks_path, span_start, span_end, tu_rel, + expected_legacy, section_claims=None): + """Return the fully validated inputs for one destructive TU-entry splice.""" header, entries, inside, reasons = span_entries(delinks_path, span_start, span_end, expected_legacy) claims = list(section_claims or @@ -1598,6 +1590,9 @@ def splice_tu_entry(delinks_path, span_start, span_end, tu_rel, expected_legacy, # entry touches one, silently adding the TU would create two owners; deciding how # to retire a future data-source entry is promotion policy, not scratch plumbing. drop_indices = {i for i, _r, _s in inside} + for idx, (rel, _body) in enumerate(entries): + if rel == tu_rel and idx not in drop_indices: + reasons.append(f"TU destination {tu_rel} is already a delinks entry") for claim in (c for c in claims if c["name"] != ".text"): for idx, (rel, body) in enumerate(entries): for name, start, end in entry_sections(body): @@ -1606,6 +1601,23 @@ def splice_tu_entry(delinks_path, span_start, span_end, tu_rel, expected_legacy, f"0x{claim['end']:08x} overlaps existing entry {rel}'s " f"{name} 0x{start:08x}..0x{end:08x}; it is not pure " f"gap ownership") + return header, entries, inside, claims, reasons + + +def splice_tu_entry(delinks_path, span_start, span_end, tu_rel, expected_legacy, + section_claims=None): + """Replace the per-function entries tiling [span_start, span_end) with ONE TU entry. + + Refuses -- returns (None, [reasons]) -- rather than producing a plausible-looking + scratch config, because every failure mode here is silent downstream: dsd fills any + range it has no object for with retail ROM bytes, so a mis-spliced delinks tree + links clean and compares green while contributing nothing (see + "unbuildable files are invisible to every gate", and layout_check's L1). The + read-only validator above is also the production promotion preflight, so the two + paths cannot drift on current delinks ownership rules. + """ + header, entries, inside, claims, reasons = validate_tu_entry_splice( + delinks_path, span_start, span_end, tu_rel, expected_legacy, section_claims) if reasons: return None, reasons @@ -3608,12 +3620,13 @@ def compile_linkcheck_sources(srcs, vers, cache, init_srcs, syms, build_root, jo normal ROM build accepts and verifies the same objects. """ compiler_only = RB.compiler_only_policies(srcs) + intact_tus = RB.intact_tu_policies(srcs) failures, outcomes = [], collections.Counter() with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as ex: for rel, err, outcome in ex.map( lambda s: RB.compile_one( s, vers, cache, init_srcs, syms, build_root=build_root, - compiler_only=compiler_only), srcs): + compiler_only=compiler_only, intact_tus=intact_tus), srcs): outcomes[outcome] += 1 if err: failures.append((rel, err)) @@ -4349,6 +4362,7 @@ def cmd_linkcheck(args): for e in symbols_new[:15]: print(f" NEW | {e}") report["symbolsNew"] = symbols_new + report["symbolsBaseline"] = base_errors # ------------------------------------------------------------------- ROM build module_ok = bool(analysis["passed"]) and range_ok and not bad_modules @@ -4626,6 +4640,9 @@ def _partition_attempt_record(report): for key, value in report.get("phases", {}).items()}, "moduleFidelityPassed": bool((report.get("analysis") or {}).get("passed")), "symbolCheckNewVsBaseline": report.get("symbolsNew"), + "symbolCheckErrors": (((report.get("phases") or {}).get("checkSymbols") or {}) + .get("errors")), + "symbolCheckBaselineErrors": report.get("symbolsBaseline"), "rom": report.get("rom"), "strayOutputs": report.get("strayOutputs"), "scratch": report.get("scratch", "") + " (gitignored)", @@ -4683,6 +4700,9 @@ def _record_linkcheck(data, entry, report, baseline): "unlicensedSections": audit.get("unlicensedSections"), }, "symbolCheckNewVsBaseline": report.get("symbolsNew"), + "symbolCheckErrors": (((report.get("phases") or {}).get("checkSymbols") or {}) + .get("errors")), + "symbolCheckBaselineErrors": report.get("symbolsBaseline"), "rom": report.get("rom"), "linkerOutput": report["phases"].get("link", {}).get("output"), } From 2beb19f36f1984d2148c22fc792dd03323b3d49c Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 05:41:09 -0500 Subject: [PATCH 09/20] tools: preserve TU promotion registry order --- tools/test_tu_promote.py | 19 ++++++++++++++++++- tools/tu_promote.py | 14 ++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tools/test_tu_promote.py b/tools/test_tu_promote.py index f77eeb5932..0c4d6e7b3d 100644 --- a/tools/test_tu_promote.py +++ b/tools/test_tu_promote.py @@ -98,9 +98,26 @@ def test_converted_baseline_moves_only_banked_members_to_promoted_tu(self): self.assertEqual(moved, 1) self.assertEqual(data["count"], 2) self.assertEqual(data["converted"], [ - "src/Unrelated.cpp", "src/actors/TU.cpp#First"]) + "src/actors/TU.cpp#First", "src/Unrelated.cpp"]) self.assertEqual(data["_note"], TP.TR.NOTE) + def test_attribution_update_appends_without_reordering_existing_overrides(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + path = root / "attribution.json" + path.write_text( + '{"overrides":{"src/Z.cpp":"zed","src/A.cpp":"aye"}}\n', + encoding="utf-8") + plans = [{"dest": "src/actors/TU.cpp", "functions": [{ + "symbol": "First", "legacy_source": "src/First.cpp"}]}] + with mock.patch.object(TP, "REPO", root): + prepared = TP.attribution_update( + plans, {"src/First": "author"}) + TP.rewrite_attribution(plans, {}, prepared) + data = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(list(data["overrides"]), [ + "src/Z.cpp", "src/A.cpp", "src/actors/TU.cpp#First"]) + def test_single_member_converted_identity_stays_path_based(self): with tempfile.TemporaryDirectory() as td: config = pathlib.Path(td) / "config" diff --git a/tools/tu_promote.py b/tools/tu_promote.py index d9c2a68253..1de0986f1b 100644 --- a/tools/tu_promote.py +++ b/tools/tu_promote.py @@ -204,7 +204,6 @@ def attribution_update(plans, lineage): if key not in ov: ov[key] = who added += 1 - data["overrides"] = dict(sorted(ov.items())) return path, data, added @@ -233,24 +232,27 @@ def converted_baseline_update(plans): if not isinstance(rows, list) or len(rows) != len(set(rows)): raise PromoteError("converted baseline must contain a unique converted list") - converted = set(rows) + converted = list(rows) moved = 0 for p in plans: for f in p["functions"]: legacy = f["legacy_source"] symbol = f["symbol"] old_keys = (legacy, f"{legacy}#{symbol}") - if not any(key in converted for key in old_keys): + positions = [converted.index(key) for key in old_keys if key in converted] + if not positions: continue - converted.difference_update(old_keys) + insert_at = min(positions) + converted = [key for key in converted if key not in old_keys] target = p["dest"] if len(p["functions"]) == 1 \ else f"{p['dest']}#{symbol}" - converted.add(target) + if target not in converted: + converted.insert(min(insert_at, len(converted)), target) moved += 1 data["_note"] = TR.NOTE data["count"] = len(converted) - data["converted"] = sorted(converted) + data["converted"] = converted return path, data, moved From 73d9da498943b0c1fd1f768a5b708da4871ac9ff Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 05:57:18 -0500 Subject: [PATCH 10/20] tools: score promoted TU members independently --- tools/test_tiers_ratchet.py | 72 ++++++++++++++++++++ tools/tiers.py | 130 ++++++++++++++++++++++++++++++++++-- tools/tiers_ratchet.py | 22 +++--- 3 files changed, 208 insertions(+), 16 deletions(-) diff --git a/tools/test_tiers_ratchet.py b/tools/test_tiers_ratchet.py index b0b50687cb..b8abeb9121 100644 --- a/tools/test_tiers_ratchet.py +++ b/tools/test_tiers_ratchet.py @@ -25,6 +25,78 @@ def test_multi_function_source_is_banked_per_member(self): "src/actors/TU.cpp#First", "src/actors/TU.cpp#Second"}) self.assertEqual(set(scores), converted) + def test_multi_function_members_do_not_contaminate_each_other(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + source = root / "src/actors/TU.cpp" + source.parent.mkdir(parents=True) + source.write_text( + "//cpp\n" + "int unk_18;\n" + "extern int _ZN3Bad3UseEv();\n" + "// @symbol First\n" + "void First() {}\n" + "// @symbol Second\n" + "int Second() { return unk_18 + _ZN3Bad3UseEv(); }\n", + encoding="utf-8") + ownership = {"src/actors/TU.cpp": ["First", "Second"]} + with mock.patch.object(TR, "REPO", root): + converted, scores = TR.scan(["src/actors/TU.cpp"], ownership) + + self.assertEqual(converted, {"src/actors/TU.cpp#First"}) + self.assertTrue(scores["src/actors/TU.cpp#First"]["no_unk_field"]) + self.assertTrue(scores["src/actors/TU.cpp#First"]["no_mangled_refs"]) + self.assertFalse(scores["src/actors/TU.cpp#Second"]["no_unk_field"]) + self.assertFalse(scores["src/actors/TU.cpp#Second"]["no_mangled_refs"]) + + def test_inline_lifecycle_member_uses_its_header_definition(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + source = root / "src/actors/Thing.cpp" + header = root / "include/Thing.h" + source.parent.mkdir(parents=True) + header.parent.mkdir(parents=True) + header.write_text( + "struct Thing { virtual ~Thing() {} };\n", encoding="utf-8") + source.write_text( + "//cpp\n" + "#include \"Thing.h\"\n" + "int unk_18;\n" + "extern int _ZN3Bad3UseEv();\n" + "// @symbol Other\n" + "int Other() { return unk_18 + _ZN3Bad3UseEv(); }\n", + encoding="utf-8") + ownership = { + "src/actors/Thing.cpp": ["_ZN5ThingD1Ev", "Other"]} + with mock.patch.object(TR, "REPO", root): + converted, scores = TR.scan(["src/actors/Thing.cpp"], ownership) + + identity = "src/actors/Thing.cpp#_ZN5ThingD1Ev" + self.assertIn(identity, converted) + self.assertTrue(scores[identity]["no_unk_field"]) + self.assertTrue(scores[identity]["no_mangled_refs"]) + + def test_lifecycle_declaration_is_not_treated_as_its_definition(self): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + source = root / "src/actors/Thing.cpp" + header = root / "include/Thing.h" + source.parent.mkdir(parents=True) + header.parent.mkdir(parents=True) + header.write_text( + "struct Thing { virtual ~Thing(); };\n", encoding="utf-8") + source.write_text( + "//cpp\n#include \"Thing.h\"\nint unk_18;\n", + encoding="utf-8") + ownership = { + "src/actors/Thing.cpp": ["_ZN5ThingD1Ev", "Other"]} + with mock.patch.object(TR, "REPO", root): + converted, scores = TR.scan(["src/actors/Thing.cpp"], ownership) + + identity = "src/actors/Thing.cpp#_ZN5ThingD1Ev" + self.assertNotIn(identity, converted) + self.assertFalse(scores[identity]["no_unk_field"]) + def test_single_function_source_keeps_legacy_path_identity(self): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) diff --git a/tools/tiers.py b/tools/tiers.py index a44925c5dc..32755d421c 100644 --- a/tools/tiers.py +++ b/tools/tiers.py @@ -306,13 +306,129 @@ def score_file(path, text): } +def _marked_member_fragment(text, symbol): + """Return one explicitly marked member's source, or None if ambiguous. + + Production TUs put ``// @symbol `` immediately before each + hand-written member. The marker is stronger evidence than trying to parse + C++, and slicing at the next marker keeps an unrelated member's temporary + names or codegen constraints from changing this member's readability score. + """ + markers = list(SYMBOL.finditer(text)) + matches = [i for i, marker in enumerate(markers) + if marker.group(1) == symbol] + if len(matches) != 1: + return None + i = matches[0] + start = markers[i].end() + stop = markers[i + 1].start() if i + 1 < len(markers) else len(text) + return text[start:stop] + + +def _balanced_lifecycle_fragment(text, class_name, method_name): + """Find an inline ctor/dtor declaration or definition in one header. + + This is deliberately a small recognizer, not a C++ parser. It searches the + comment/string-masked text and returns only a balanced inline body. A mere + declaration is not ownership evidence. If the evidence is not this simple the + caller falls back to the whole + TU, which can under-credit a member but can never grant it speculatively. + """ + code = _code_only(text) + unqualified = class_name.rsplit("::", 1)[-1] + token = f"~{unqualified}" if method_name.startswith("~") else unqualified + pattern = re.compile(r"(?= 0 and (brace < 0 or semi < brace): + return None + if brace < 0: + return None + + depth = 0 + for i in range(brace, len(code)): + if code[i] == "{": + depth += 1 + elif code[i] == "}": + depth -= 1 + if depth == 0: + return text[start:i + 1] + return None + + +def _lifecycle_member_fragment(path, text, symbol, repo_root=None): + """Return an inline compiler-generated ctor/dtor's direct-header source.""" + parsed = demangle.demangle(symbol) + if not parsed or not (parsed.get("ctor") or parsed.get("dtor")): + return None + + root = pathlib.Path(repo_root or REPO) + source = pathlib.Path(path) + include_names = re.findall(r'^\s*#\s*include\s+"([^"]+)"', text, re.MULTILINE) + for name in include_names: + candidates = [] + if source.is_absolute(): + candidates.append(source.parent / name) + else: + candidates.append(root / source.parent / name) + candidates.append(root / "include" / name) + for header in candidates: + try: + header_text = header.read_text(errors="replace") + except OSError: + continue + fragment = _balanced_lifecycle_fragment( + header_text, parsed["class"], parsed["method"]) + if fragment is not None: + return fragment + return None + + +def score_member(path, text, symbol, repo_root=None): + """Score one member of a promoted multi-function translation unit. + + Hand-written members use exact ``@symbol`` boundaries. Compiler-generated + ctor/dtor variants use the inline lifecycle definition in a directly included + class header. Anything without either form of evidence is scored against the + entire file, preserving the old conservative behavior. + """ + fragment = _marked_member_fragment(text, symbol) + if fragment is None: + fragment = _lifecycle_member_fragment(path, text, symbol, repo_root) + score = score_file(path, fragment if fragment is not None else text) + score["real_name"] = _real_name_for_symbol(symbol) + # Header use describes the production source, not an individual body slice. + score["shared_header"] = bool(SHARED_HEADER.search(text)) + return score + + def converted(src_root=None): """Score every source-owned function. Returns CONVERTED plus its breakdown. Reads committed source only - no ROM, no build, no local state - so it - reproduces on a fresh checkout, which is what CI needs. File-wide criteria are - applied to every function the enrollment table assigns to that source; the name - criterion is evaluated per symbol. + reproduces on a fresh checkout, which is what CI needs. Ordinary intake files + retain file-wide scoring. Members of a promoted production TU are independently + scored from explicit source markers or inline lifecycle definitions, with a + conservative file-wide fallback when neither boundary is evidenced. """ root = pathlib.Path(src_root or SRC) files = sorted(str(p) for p in root.rglob("*") @@ -328,17 +444,19 @@ def converted(src_root=None): for p in files: with open(p, errors="replace") as f: text = f.read() - file_score = score_file(p, text) path = pathlib.Path(p) try: rel = path.absolute().relative_to(repo_root).as_posix() except ValueError: rel = None members = ownership.get(rel) or [path.stem] + multi = len(members) > 1 all_readable = True for sym in members: - s = dict(file_score) - s["real_name"] = _real_name_for_symbol(sym) + s = (score_member(p, text, sym, repo_root) + if multi else score_file(p, text)) + if not multi: + s["real_name"] = _real_name_for_symbol(sym) total += 1 for k in counts: counts[k] += s[k] diff --git a/tools/tiers_ratchet.py b/tools/tiers_ratchet.py index 101281ac01..afbf11c97f 100644 --- a/tools/tiers_ratchet.py +++ b/tools/tiers_ratchet.py @@ -7,8 +7,9 @@ identities that pass all five and fails a PR when an identity LEAVES that set. A one-function source keeps its historical path identity. A promoted TU appends ``#symbol`` to that path for each enrolled member, matching attribution's ownership unit. -It reuses tiers.score_file outright -- the classifier has exactly one implementation, -and a second copy of those regexes would be a second definition of a published percentage. +It reuses tiers.score_file/score_member outright -- the classifier has exactly one +implementation, and a second copy of those regexes would be a second definition of a +published percentage. WHY BACKSLIDE-ONLY, AND NOT A COUNT. Two reasons, and the second is the important one. @@ -205,13 +206,12 @@ def promoted_moves(root=None): return moves -def score(rel): - """tiers.score_file for one repo-relative path, or None if it is unreadable.""" +def source_text(rel): + """Text for one repo-relative path, or None if it is unreadable.""" try: - text = (REPO / rel).read_text(errors="replace") + return (REPO / rel).read_text(errors="replace") except OSError: return None - return tiers.score_file(rel, text) def scan(paths=None, ownership=None): @@ -227,15 +227,17 @@ def scan(paths=None, ownership=None): if ownership is None: ownership = tiers.srcpath.source_definition_index() for rel in (paths if paths is not None else tracked_sources()): - file_score = score(rel) - if file_score is None: + text = source_text(rel) + if text is None: continue members = ownership.get(rel) or [pathlib.PurePosixPath(rel).stem] multi = len(members) > 1 for symbol in members: identity = f"{rel}#{symbol}" if multi else rel - member_score = dict(file_score) - member_score["real_name"] = tiers._real_name_for_symbol(symbol) + member_score = (tiers.score_member(rel, text, symbol, REPO) + if multi else tiers.score_file(rel, text)) + if not multi: + member_score["real_name"] = tiers._real_name_for_symbol(symbol) scores[identity] = member_score if all(member_score[k] for k in tiers.CRITERIA): converted.add(identity) From b6c4b0210769cb5fa363220efcb8485ae1e90b1c Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 06:22:40 -0500 Subject: [PATCH 11/20] tools: bootstrap independent intact TU control --- tools/rombuild.py | 2 +- tools/test_tu_production.py | 65 ++++++++++++++++++++++++++++++++++++ tools/test_tubuild.py | 43 ++++++++++++++++++++++++ tools/tu_production.py | 66 +++++++++++++++++++++++++++++++++---- tools/tubuild.py | 54 ++++++++++++++++++++++++++++++ 5 files changed, 223 insertions(+), 7 deletions(-) diff --git a/tools/rombuild.py b/tools/rombuild.py index 8344476c18..fd1863dcd2 100644 --- a/tools/rombuild.py +++ b/tools/rombuild.py @@ -1180,7 +1180,7 @@ def save_report(): import tu_production as ITP try: intact_link_verification = ITP.prepare_intact_link_verification( - intact_tus) + intact_tus, jobs=args.jobs) except ITP.ProductionTuError as exc: raise BuildError("intact TU link control", 1, str(exc)) from exc elif intact_tus: diff --git a/tools/test_tu_production.py b/tools/test_tu_production.py index af9907a13d..bbb641a472 100644 --- a/tools/test_tu_production.py +++ b/tools/test_tu_production.py @@ -1,3 +1,4 @@ +import json import pathlib import sys import tempfile @@ -28,6 +29,70 @@ def test_missing_content_bound_baseline_refuses(self): "missing strict stock control"): TP._strict_baseline() + def test_clean_worker_bootstraps_rom_gap_control(self): + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", "module": "ov047"}} + baseline = {"symbolErrors": [], "romSha256": "ab" * 32} + with mock.patch.object( + TP, "_strict_baseline", + side_effect=[TP.ProductionTuError("missing"), baseline]) as strict, \ + mock.patch.object(TP.subprocess, "run", + return_value=mock.Mock(returncode=0)) as run: + self.assertIs( + TP._current_or_bootstrapped_intact_baseline(entries, 7), baseline) + + self.assertEqual(strict.call_count, 2) + self.assertEqual(strict.call_args_list, + [mock.call(entries), mock.call(entries)]) + command = run.call_args.args[0] + self.assertIn("--baseline", command) + self.assertIn("ov047", command) + self.assertIn("7", command) + self.assertIs(run.call_args.kwargs["check"], False) + + def test_clean_worker_refuses_failed_control_bootstrap(self): + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", "module": "ov047"}} + with mock.patch.object( + TP, "_strict_baseline", + side_effect=TP.ProductionTuError("missing")), \ + mock.patch.object(TP.subprocess, "run", + return_value=mock.Mock(returncode=3)): + with self.assertRaisesRegex( + TP.ProductionTuError, "baseline command exited 3"): + TP._current_or_bootstrapped_intact_baseline(entries, 7) + + def test_rom_gap_control_requires_exact_intact_inventory(self): + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", "module": "ov047"}} + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / "final_link.o").write_bytes(b"independent linked control") + report = { + "baseline": True, + "analysis": {"passed": True}, + "phases": { + "checkModules": {"ok": True}, + "checkSymbols": {"errors": ["[ERROR] old"]}, + }, + "intactTusDemoted": [{ + "id": "ov047/Thing", "source": "src/actors/Thing.cpp"}], + "rom": {"sha256": "ab" * 32}, + } + (root / "linkcheck.json").write_text( + json.dumps(report), encoding="utf-8") + with mock.patch.object(TP.TB, "BASELINE_LINK", root), \ + mock.patch.object( + TP.TB, "validate_partition_baseline_evidence", + return_value=("cd" * 32, None)): + baseline = TP._strict_baseline(entries) + self.assertEqual(baseline["romSha256"], "ab" * 32) + with self.assertRaisesRegex( + TP.ProductionTuError, "current intact TU inventory"): + TP._strict_baseline({ + "src/actors/Other.cpp": { + "id": "ov047/Other", "module": "ov047"}}) + class ProductionTuObjects(unittest.TestCase): def test_automatic_intact_link_plan_uses_current_control_and_vtable_biases(self): diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index 746de265aa..6ff1ca5972 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -774,6 +774,49 @@ def fake_compile(rel, vers, cache, init_srcs, syms, build_root=None, policy, intact)] +def test_strict_control_demotes_only_requested_complete_sources(): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + path = root / "overlays/ov047/delinks.txt" + path.parent.mkdir(parents=True) + path.write_text( + "src/actors/Promoted.cpp:\n" + " complete\n" + " .text start:0x1000 end:0x1010\n" + " .data start:0x2000 end:0x2010\n\n" + "src/Other.cpp:\n" + " complete\n" + " .text start:0x1010 end:0x1020\n", + encoding="utf-8") + + demoted, errors = tubuild.demote_complete_sources( + root, ["src/actors/Promoted.cpp"]) + + assert errors == [] + assert demoted == ["src/actors/Promoted.cpp"] + text = path.read_text(encoding="utf-8") + promoted = text.split("src/Other.cpp:", 1)[0] + assert "complete" not in promoted + assert ".text start:0x1000 end:0x1010" in promoted + assert ".data start:0x2000 end:0x2010" in promoted + assert "src/Other.cpp:\n complete" in text + + +def test_strict_control_refuses_a_source_that_was_not_complete(): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + path = root / "delinks.txt" + path.write_text( + "src/actors/Promoted.cpp:\n" + " .text start:0x1000 end:0x1010\n", + encoding="utf-8") + demoted, errors = tubuild.demote_complete_sources( + root, ["src/actors/Promoted.cpp"]) + assert demoted == [] + assert errors == [ + "src/actors/Promoted.cpp: delinks entry is not complete"] + + def test_linkcheck_symbol_verdict_uses_the_stock_failure_inventory(): assert tubuild.linkcheck_symbol_verdict(True, False, None) assert tubuild.linkcheck_symbol_verdict(False, False, []) diff --git a/tools/tu_production.py b/tools/tu_production.py index 8539d739db..59a7cb5571 100644 --- a/tools/tu_production.py +++ b/tools/tu_production.py @@ -19,6 +19,8 @@ import hashlib import json import pathlib +import subprocess +import sys import tubuild as TB @@ -85,7 +87,46 @@ def _raise(label, reasons): raise ProductionTuError(f"{label}: {detail or 'refused without a reason'}") -def prepare_intact_link_verification(entries): +def _demoted_inventory(entries): + return sorted(({"id": entry.get("id", source), "source": source} + for source, entry in entries.items()), + key=lambda row: row["source"]) + + +def _current_or_bootstrapped_intact_baseline(entries, jobs): + """Return a current source-independent control, building it if necessary. + + The baseline command removes every promoted intact source's ``complete`` marker + in its disposable config, so those ranges come directly from extracted retail + gap bytes. This makes the control independent of the C++ objects it will judge + and safe to create on a clean CI worker with no gitignored build artifacts. + """ + try: + return _strict_baseline(entries) + except ProductionTuError as first_error: + modules = sorted({entry.get("module") for entry in entries.values() + if entry.get("module")}) + module = modules[0] if modules else "ov002" + print("strict stock control is missing or stale; generating a fresh " + f"ROM-gap control for {module}") + command = [ + sys.executable, str(TB.REPO / "tools" / "tubuild.py"), + "linkcheck", "--baseline", "--module", module, + "-j", str(jobs), "--clean", + ] + result = subprocess.run(command, check=False) + if result.returncode: + raise ProductionTuError( + "could not bootstrap strict stock control: " + f"{first_error}; baseline command exited {result.returncode}") + try: + return _strict_baseline(entries) + except ProductionTuError as fresh_error: + raise ProductionTuError( + f"fresh strict stock control is invalid: {fresh_error}") from fresh_error + + +def prepare_intact_link_verification(entries, jobs=4): """Bind supported automatic intact TUs to the current strict stock control. Object preparation proves the compiler contribution before the link. This plan @@ -94,7 +135,6 @@ def prepare_intact_link_verification(entries): intact admission refuses storage aliases until their baseline bootstrap is non-circular, so refreshing this control never depends on the control being kept. """ - baseline = _strict_baseline() admitted_errors = set() admitted_roms = set() for entry in entries.values(): @@ -107,6 +147,7 @@ def prepare_intact_link_verification(entries): "symbol-error inventory or stock ROM SHA-256") admitted_errors.add(tuple(sorted(set(errors)))) admitted_roms.add(rom_sha) + baseline = _current_or_bootstrapped_intact_baseline(entries, jobs) current_errors = tuple(baseline["symbolErrors"]) if admitted_errors != {current_errors}: raise ProductionTuError( @@ -131,7 +172,7 @@ def prepare_intact_link_verification(entries): return {"baseline": baseline, "entries": prepared} -def _strict_baseline(): +def _strict_baseline(expected_intact=None): """Return the content-bound stock baseline or refuse stale/missing evidence.""" report_path = TB.BASELINE_LINK / "linkcheck.json" linked_elf = TB.BASELINE_LINK / "final_link.o" @@ -147,9 +188,21 @@ def _strict_baseline(): or (report.get("analysis") or {}).get("passed") is not True \ or ((report.get("phases") or {}).get("checkModules") or {}).get("ok") is not True: raise ProductionTuError("stock control did not pass module fidelity") + expected_demoted = _demoted_inventory(expected_intact or {}) + actual_demoted = report.get("intactTusDemoted") + if actual_demoted != expected_demoted: + raise ProductionTuError( + "stock control did not exclude the current intact TU inventory: " + f"expected={expected_demoted!r}, actual={actual_demoted!r}") rom = report.get("rom") or {} - if rom.get("matchesStockRom") is not True or not rom.get("sha256"): - raise ProductionTuError("stock control did not prove a ROM identical to build/sm64ds.nds") + if not rom.get("sha256"): + raise ProductionTuError("stock control did not produce a ROM SHA-256") + compared = rom.get("matchesStockRom") + if compared is False: + raise ProductionTuError("stock control ROM differs from build/sm64ds.nds") + if not expected_demoted and compared is not True: + raise ProductionTuError( + "stock control did not prove a ROM identical to build/sm64ds.nds") _sha, error = TB.validate_partition_baseline_evidence(report, linked_elf) if error: raise ProductionTuError(f"stock control is stale: {error}") @@ -297,7 +350,8 @@ def prepare(tu_ids, config_root, work_root, jobs=1): return {"entries": [], "overrides": {}, "baseline": None} if len(ids) != len(set(ids)): raise ProductionTuError("duplicate --partitioned-tu id") - baseline = _strict_baseline() + enrolled = TB.RB.enrolled(TB.CFG_ARM9) + baseline = _strict_baseline(TB.RB.intact_tu_policies(enrolled)) data = TB.load_manifest() config_root = pathlib.Path(config_root) work_root = pathlib.Path(work_root) diff --git a/tools/tubuild.py b/tools/tubuild.py index f9ed6b7409..1e8fd25d42 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -3633,6 +3633,42 @@ def compile_linkcheck_sources(srcs, vers, cache, init_srcs, syms, build_root, jo return failures, outcomes +def demote_complete_sources(config_root, sources): + """Make exact enrolled sources ROM-gap-owned in a disposable config tree. + + The section claims remain unchanged; only the indented ``complete`` marker is + removed. dsd then supplies those ranges from the extracted retail binaries. + Every requested path must name exactly one complete entry or the control build + is refused rather than silently compiling the source it was meant to exclude. + """ + wanted = {str(source).replace("\\", "/") for source in sources} + found, demoted = set(), set() + for path in sorted(pathlib.Path(config_root).rglob("delinks.txt")): + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + out, current, changed = [], None, False + for line in lines: + stripped = line.strip() + if line and not line[0].isspace() and stripped.endswith(":"): + current = stripped[:-1].replace("\\", "/") + if current in wanted: + found.add(current) + elif line and not line[0].isspace(): + current = None + if current in wanted and stripped == "complete": + if current in demoted: + return [], [f"{current}: duplicate complete marker"] + demoted.add(current) + changed = True + continue + out.append(line) + if changed: + path.write_text("\n".join(out) + "\n", encoding="utf-8", newline="\n") + errors = [f"{source}: no delinks entry" for source in sorted(wanted - found)] + errors.extend(f"{source}: delinks entry is not complete" + for source in sorted(found - demoted)) + return sorted(demoted), errors + + def linkcheck_symbol_verdict(baseline, command_ok, new_errors): """Whether the symbol phase is attributable-clean for this linkcheck. @@ -3703,6 +3739,24 @@ def cmd_linkcheck(args): f"{len(profile['modGapFallbacks'])} demoted to ROM gap bytes -- " f"rombuild_profile.prepare_profile, the same stock semantics as a real build)") + if baseline: + enrolled_before = RB.enrolled(cfg_root) + intact_before = RB.intact_tu_policies(enrolled_before) + demoted, reasons = demote_complete_sources(cfg_root, intact_before) + if reasons: + print("\nREFUSED -- strict control could not exclude every intact TU:") + for reason in reasons: + print(f" {reason}") + return 1 + report["intactTusDemoted"] = [ + {"id": intact_before[source].get("id", source), "source": source} + for source in demoted] + if demoted: + print(f" strict control demoted {len(demoted)} intact TU source(s) " + "to independently extracted ROM gap bytes:") + for source in demoted: + print(f" {source}") + span_start = span_end = None claims = [] replaced = [] From 25f26d97d5e270fb8086f5ffd3d76a106aaff163 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 06:26:08 -0500 Subject: [PATCH 12/20] tools: keep gap controls source independent --- tools/test_tubuild.py | 26 ++++++++++++++++++++++++++ tools/tubuild.py | 9 ++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index 6ff1ca5972..571dcb55a9 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -802,6 +802,32 @@ def test_strict_control_demotes_only_requested_complete_sources(): assert "src/Other.cpp:\n complete" in text +def test_strict_control_compile_does_not_re_admit_demoted_intact_tus(): + original_intact = tubuild.RB.intact_tu_policies + original_compile = tubuild.RB.compile_one + seen = [] + try: + tubuild.RB.intact_tu_policies = lambda _enrolled: (_ for _ in ()).throw( + AssertionError("demoted intact policy was recomputed")) + + def fake_compile(rel, vers, cache, init_srcs, syms, build_root=None, + compiler_only=None, intact_tus=None): + seen.append(intact_tus) + return rel, None, "hit" + + tubuild.RB.compile_one = fake_compile + failures, outcomes = tubuild.compile_linkcheck_sources( + ["src/Other.cpp"], {}, None, set(), {}, pathlib.Path("scratch"), 1, + intact_tus_override={}) + finally: + tubuild.RB.intact_tu_policies = original_intact + tubuild.RB.compile_one = original_compile + + assert failures == [] + assert outcomes["hit"] == 1 + assert seen == [{}] + + def test_strict_control_refuses_a_source_that_was_not_complete(): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) diff --git a/tools/tubuild.py b/tools/tubuild.py index 1e8fd25d42..3eca2b7f01 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -3610,7 +3610,8 @@ def shared_build_bin_snapshot(): for path in sorted((REPO / "build").glob("*.bin")) if path.is_file()} -def compile_linkcheck_sources(srcs, vers, cache, init_srcs, syms, build_root, jobs): +def compile_linkcheck_sources(srcs, vers, cache, init_srcs, syms, build_root, jobs, + intact_tus_override=None): """Compile a scratch linkcheck with the normal production object policies. A baseline substitutes no candidate TU, but it still compiles production's @@ -3620,7 +3621,8 @@ def compile_linkcheck_sources(srcs, vers, cache, init_srcs, syms, build_root, jo normal ROM build accepts and verifies the same objects. """ compiler_only = RB.compiler_only_policies(srcs) - intact_tus = RB.intact_tu_policies(srcs) + intact_tus = (RB.intact_tu_policies(srcs) if intact_tus_override is None + else intact_tus_override) failures, outcomes = [], collections.Counter() with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as ex: for rel, err, outcome in ex.map( @@ -3918,7 +3920,8 @@ def cmd_linkcheck(args): syms = RB.enrolled_symbols() t0 = time.time() failures, outcomes = compile_linkcheck_sources( - srcs, vers, cache, init_srcs, syms, scratch, args.jobs) + srcs, vers, cache, init_srcs, syms, scratch, args.jobs, + intact_tus_override={} if baseline else None) dt = time.time() - t0 report["phases"]["compile"] = {"ok": not failures, "seconds": round(dt, 1), "outcomes": dict(outcomes)} From 75b5230c87b6f0060ae0d05adbfe3488f5447f40 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 06:49:13 -0500 Subject: [PATCH 13/20] tools: bind TU controls to ROM inputs --- tools/test_tubuild.py | 22 ++++++++++++++++++---- tools/tu_production.py | 13 +++++++++++-- tools/tubuild.py | 18 +++++++++++++----- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index 571dcb55a9..29af386e6c 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -1285,25 +1285,31 @@ def test_partition_baseline_evidence_is_content_bound_not_mtime_bound(): with tempfile.TemporaryDirectory() as td: root = pathlib.Path(td) config = root / "config" + rom_inputs = root / "rom-inputs" config.mkdir() + rom_inputs.mkdir() cfg = config / "symbols.txt" + (rom_inputs / "header.yaml").write_bytes(b"ROM") linked, dsd, linker = root / "base.o", root / "dsd.exe", root / "mwld.exe" cfg.write_bytes(b"one") linked.write_bytes(b"ELF") dsd.write_bytes(b"DSD") linker.write_bytes(b"MWL") evidence = tubuild.partition_baseline_fingerprints( - linked, config, dsd_path=dsd, linker_path=linker) + linked, config, dsd_path=dsd, linker_path=linker, + rom_inputs=rom_inputs) report = {"baselineEvidence": evidence} digest, error = tubuild.validate_partition_baseline_evidence( - report, linked, config, dsd_path=dsd, linker_path=linker) + report, linked, config, dsd_path=dsd, linker_path=linker, + rom_inputs=rom_inputs) assert error is None and digest == evidence["linkedElfSha256"] stamp = cfg.stat().st_mtime_ns cfg.write_bytes(b"two") os.utime(cfg, ns=(stamp, stamp)) _digest, error = tubuild.validate_partition_baseline_evidence( - report, linked, config, dsd_path=dsd, linker_path=linker) + report, linked, config, dsd_path=dsd, linker_path=linker, + rom_inputs=rom_inputs) assert "configArm9Sha256" in error cfg.write_bytes(b"one") @@ -1311,9 +1317,17 @@ def test_partition_baseline_evidence_is_content_bound_not_mtime_bound(): linked.write_bytes(b"BAD") os.utime(linked, ns=(linked_stamp, linked_stamp)) _digest, error = tubuild.validate_partition_baseline_evidence( - report, linked, config, dsd_path=dsd, linker_path=linker) + report, linked, config, dsd_path=dsd, linker_path=linker, + rom_inputs=rom_inputs) assert "linkedElfSha256" in error + linked.write_bytes(b"ELF") + (rom_inputs / "header.yaml").write_bytes(b"CHANGED") + _digest, error = tubuild.validate_partition_baseline_evidence( + report, linked, config, dsd_path=dsd, linker_path=linker, + rom_inputs=rom_inputs) + assert "romInputsSha256" in error + def test_partitioned_result_gate_requires_every_full_rom_proof(): good = dict(equivalent=True, data_ok=True, storage_aliases_ok=True, diff --git a/tools/tu_production.py b/tools/tu_production.py index 59a7cb5571..528a24a7da 100644 --- a/tools/tu_production.py +++ b/tools/tu_production.py @@ -155,10 +155,18 @@ def prepare_intact_link_verification(entries, jobs=4): f"pre-promotion inventory: current={list(current_errors)!r}, " f"admitted={[list(rows) for rows in sorted(admitted_errors)]!r}") if admitted_roms != {baseline["romSha256"]}: + report = baseline["report"] + diagnostics = { + "rom": report.get("rom"), + "baselineEvidence": report.get("baselineEvidence"), + "intactTusDemoted": report.get("intactTusDemoted"), + "scratch": report.get("scratch"), + } raise ProductionTuError( "strict post-promotion control ROM SHA-256 does not equal the admitted " f"stock proof: current={baseline['romSha256']}, " - f"admitted={sorted(admitted_roms)!r}") + f"admitted={sorted(admitted_roms)!r}, " + f"control={json.dumps(diagnostics, sort_keys=True)}") prepared = [] for source, entry in sorted(entries.items()): claims, reasons = TB.manifest_section_claims(entry) @@ -203,7 +211,8 @@ def _strict_baseline(expected_intact=None): if not expected_demoted and compared is not True: raise ProductionTuError( "stock control did not prove a ROM identical to build/sm64ds.nds") - _sha, error = TB.validate_partition_baseline_evidence(report, linked_elf) + _sha, error = TB.validate_partition_baseline_evidence( + report, linked_elf, config_root=TB.BASELINE_LINK / "config" / "arm9") if error: raise ProductionTuError(f"stock control is stale: {error}") base_errors = (((report.get("phases") or {}).get("checkSymbols") or {}) diff --git a/tools/tubuild.py b/tools/tubuild.py index 3eca2b7f01..851c7c975e 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -2888,17 +2888,22 @@ def content_tree_sha256(root): def partition_baseline_fingerprints(linked_elf, config_root=CFG_ARM9, - dsd_path=None, linker_path=None): + dsd_path=None, linker_path=None, + rom_inputs=None): """Content identities that bind a baseline report to its actual inputs/output.""" linked_elf = pathlib.Path(linked_elf) dsd_path = pathlib.Path(dsd_path or RB.DSD) linker_path = pathlib.Path(linker_path or (RB.MW / RB.LD_VERSION / "mwldarm.exe")) + rom_inputs = pathlib.Path(rom_inputs or (REPO / "extracted" / "dsd")) required = [linked_elf, dsd_path, linker_path] missing = [str(path) for path in required if not path.is_file()] + if not rom_inputs.is_dir(): + missing.append(str(rom_inputs)) if missing: raise FileNotFoundError(f"baseline fingerprint input(s) missing: {missing}") return { "configArm9Sha256": content_tree_sha256(config_root), + "romInputsSha256": content_tree_sha256(rom_inputs), "linkedElfSha256": hashlib.sha256(linked_elf.read_bytes()).hexdigest(), "linkedElfBytes": linked_elf.stat().st_size, "dsdSha256": hashlib.sha256(dsd_path.read_bytes()).hexdigest(), @@ -2907,14 +2912,16 @@ def partition_baseline_fingerprints(linked_elf, config_root=CFG_ARM9, def validate_partition_baseline_evidence(report, linked_elf, config_root=CFG_ARM9, - dsd_path=None, linker_path=None): + dsd_path=None, linker_path=None, + rom_inputs=None): """Refuse a baseline whose report is detached from current bytes or tools.""" evidence = report.get("baselineEvidence") if not isinstance(evidence, dict): return None, "baseline report has no content-bound evidence" try: current = partition_baseline_fingerprints( - linked_elf, config_root, dsd_path=dsd_path, linker_path=linker_path) + linked_elf, config_root, dsd_path=dsd_path, linker_path=linker_path, + rom_inputs=rom_inputs) except (OSError, ValueError) as exc: return None, f"cannot fingerprint baseline: {exc}" mismatched = [key for key, value in current.items() if evidence.get(key) != value] @@ -2937,7 +2944,8 @@ def _baseline_partition_symbols(names): or (report.get("phases", {}).get("link") or {}).get("ok") is not True \ or (report.get("analysis") or {}).get("passed") is not True: return None, None, "baseline report does not prove a successful stock module link" - baseline_sha256, error = validate_partition_baseline_evidence(report, elf_path) + baseline_sha256, error = validate_partition_baseline_evidence( + report, elf_path, config_root=BASELINE_LINK / "config" / "arm9") if error: return None, None, error rows, error = linked_symbol_rows(elf_path, names) @@ -4303,7 +4311,7 @@ def cmd_linkcheck(args): if baseline: report["baselineEvidence"] = partition_baseline_fingerprints( - scratch / "final_link.o") + scratch / "final_link.o", config_root=cfg_root) if partitioned: linked_aliases = verify_linked_storage_aliases( From bccebfdb8c6a6af5061d310e9c75b0cdd1541644 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 07:12:43 -0500 Subject: [PATCH 14/20] tools: fingerprint exact executable module sets --- tools/rombuild_check.py | 9 +++++++++ tools/test_rombuild_check.py | 6 ++++++ tools/tu_production.py | 3 +++ 3 files changed, 18 insertions(+) diff --git a/tools/rombuild_check.py b/tools/rombuild_check.py index 04d86477d7..f487d4609e 100644 --- a/tools/rombuild_check.py +++ b/tools/rombuild_check.py @@ -14,6 +14,7 @@ """ import argparse import collections +import hashlib import json import pathlib import re @@ -215,6 +216,7 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): source_functions = source_bytes = mod_functions = mod_bytes = 0 source_data_bytes = 0 reproducing = reproducing_bytes = bad = bad_function_bytes = differing_source_bytes = 0 + module_set_digest = hashlib.sha256() for sym in sorted(config_root.rglob("symbols.txt")): d = sym.parent @@ -235,6 +237,11 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): continue base = min(s[1] for s in secs) built, retail = built_p.read_bytes(), retail_p.read_bytes() + label_bytes = label.encode("utf-8") + module_set_digest.update(len(label_bytes).to_bytes(4, "big")) + module_set_digest.update(label_bytes) + module_set_digest.update(len(built).to_bytes(8, "big")) + module_set_digest.update(built) allowed_mod_ranges = [(addr - base, end - base) for rel, _name, addr, end in entry_sections if rel.startswith("mods/")] @@ -319,6 +326,8 @@ def analyze(config_root=DEFAULT_CONFIG_ROOT, profile="stock", build_root=None): "unexpectedDifferingBytes": unexpected_module_bytes, "percent": (100.0 * (compared_module_bytes - differing_module_bytes) / compared_module_bytes) if compared_module_bytes else 0.0, + "moduleSetSha256": (module_set_digest.hexdigest() + if module_results else None), "results": module_results, }, # What the 106 module images are MADE OF, so the headline percentages cannot be diff --git a/tools/test_rombuild_check.py b/tools/test_rombuild_check.py index d98786c525..0f5437101a 100644 --- a/tools/test_rombuild_check.py +++ b/tools/test_rombuild_check.py @@ -44,14 +44,20 @@ def test_stock_exact_reports_fidelity_and_source_coverage(self): report = RBC.analyze(self.config, "stock") self.assertTrue(report["passed"]) self.assertEqual(report["moduleFidelity"]["percent"], 100.0) + digest = report["moduleFidelity"]["moduleSetSha256"] + self.assertEqual(len(digest), 64) self.assertEqual(report["sourceBuild"]["sourceFunctions"], 1) self.assertEqual(report["sourceBuild"]["sourceBytes"], 4) self.assertEqual(report["sourceBuild"]["sourceBytesPercent"], 50.0) + self.write_delinks("src/actors/Pair.cpp", end=0x00001008) + consolidated = RBC.analyze(self.config, "stock") + self.assertEqual(consolidated["moduleFidelity"]["moduleSetSha256"], digest) def test_shared_source_counts_every_owned_function(self): self.write_delinks("src/actors/Pair.cpp", end=0x00001008) report = RBC.analyze(self.config, "stock") self.assertTrue(report["passed"]) + self.assertEqual(len(report["moduleFidelity"]["moduleSetSha256"]), 64) self.assertEqual(report["sourceBuild"]["sourceFunctions"], 2) self.assertEqual(report["sourceBuild"]["reproducingFunctions"], 2) self.assertEqual(report["sourceBuild"]["sourceBytes"], 8) diff --git a/tools/tu_production.py b/tools/tu_production.py index 528a24a7da..cc72eb1427 100644 --- a/tools/tu_production.py +++ b/tools/tu_production.py @@ -159,6 +159,9 @@ def prepare_intact_link_verification(entries, jobs=4): diagnostics = { "rom": report.get("rom"), "baselineEvidence": report.get("baselineEvidence"), + "moduleSetSha256": (((report.get("analysis") or {}) + .get("moduleFidelity") or {}) + .get("moduleSetSha256")), "intactTusDemoted": report.get("intactTusDemoted"), "scratch": report.get("scratch"), } From 43f1af3589def0135ceb64a149c5275b3c368328 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 07:17:34 -0500 Subject: [PATCH 15/20] tools: invalidate stale TU control analysis --- tools/test_tubuild.py | 19 ++++++++++++++----- tools/tubuild.py | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index 29af386e6c..db4e2a7dac 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -1291,17 +1291,19 @@ def test_partition_baseline_evidence_is_content_bound_not_mtime_bound(): cfg = config / "symbols.txt" (rom_inputs / "header.yaml").write_bytes(b"ROM") linked, dsd, linker = root / "base.o", root / "dsd.exe", root / "mwld.exe" + control_tool = root / "analysis.py" cfg.write_bytes(b"one") linked.write_bytes(b"ELF") dsd.write_bytes(b"DSD") linker.write_bytes(b"MWL") + control_tool.write_bytes(b"ANALYZE") evidence = tubuild.partition_baseline_fingerprints( linked, config, dsd_path=dsd, linker_path=linker, - rom_inputs=rom_inputs) + rom_inputs=rom_inputs, control_tools=[control_tool]) report = {"baselineEvidence": evidence} digest, error = tubuild.validate_partition_baseline_evidence( report, linked, config, dsd_path=dsd, linker_path=linker, - rom_inputs=rom_inputs) + rom_inputs=rom_inputs, control_tools=[control_tool]) assert error is None and digest == evidence["linkedElfSha256"] stamp = cfg.stat().st_mtime_ns @@ -1309,7 +1311,7 @@ def test_partition_baseline_evidence_is_content_bound_not_mtime_bound(): os.utime(cfg, ns=(stamp, stamp)) _digest, error = tubuild.validate_partition_baseline_evidence( report, linked, config, dsd_path=dsd, linker_path=linker, - rom_inputs=rom_inputs) + rom_inputs=rom_inputs, control_tools=[control_tool]) assert "configArm9Sha256" in error cfg.write_bytes(b"one") @@ -1318,16 +1320,23 @@ def test_partition_baseline_evidence_is_content_bound_not_mtime_bound(): os.utime(linked, ns=(linked_stamp, linked_stamp)) _digest, error = tubuild.validate_partition_baseline_evidence( report, linked, config, dsd_path=dsd, linker_path=linker, - rom_inputs=rom_inputs) + rom_inputs=rom_inputs, control_tools=[control_tool]) assert "linkedElfSha256" in error linked.write_bytes(b"ELF") (rom_inputs / "header.yaml").write_bytes(b"CHANGED") _digest, error = tubuild.validate_partition_baseline_evidence( report, linked, config, dsd_path=dsd, linker_path=linker, - rom_inputs=rom_inputs) + rom_inputs=rom_inputs, control_tools=[control_tool]) assert "romInputsSha256" in error + (rom_inputs / "header.yaml").write_bytes(b"ROM") + control_tool.write_bytes(b"CHANGED") + _digest, error = tubuild.validate_partition_baseline_evidence( + report, linked, config, dsd_path=dsd, linker_path=linker, + rom_inputs=rom_inputs, control_tools=[control_tool]) + assert "controlToolsSha256" in error + def test_partitioned_result_gate_requires_every_full_rom_proof(): good = dict(equivalent=True, data_ok=True, storage_aliases_ok=True, diff --git a/tools/tubuild.py b/tools/tubuild.py index 851c7c975e..ec49cae8b0 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -2887,15 +2887,43 @@ def content_tree_sha256(root): return digest.hexdigest() +BASELINE_CONTROL_TOOLS = tuple( + REPO / "tools" / name for name in ( + "objisolate.py", "reloc_audit.py", "rombuild.py", "rombuild_cache.py", + "rombuild_check.py", "rombuild_profile.py", "tu_manifest.py", + "tu_production.py", "tubuild.py")) + + +def content_files_sha256(paths): + """Hash an ordered set of tool paths and bytes without timestamps.""" + rows = [] + for path in map(pathlib.Path, paths): + try: + label = path.resolve().relative_to(REPO.resolve()).as_posix() + except ValueError: + label = path.name + rows.append((label, path)) + digest = hashlib.sha256() + for label, path in sorted(rows): + raw_label = label.encode("utf-8") + raw = path.read_bytes() + digest.update(len(raw_label).to_bytes(4, "big")) + digest.update(raw_label) + digest.update(len(raw).to_bytes(8, "big")) + digest.update(raw) + return digest.hexdigest() + + def partition_baseline_fingerprints(linked_elf, config_root=CFG_ARM9, dsd_path=None, linker_path=None, - rom_inputs=None): + rom_inputs=None, control_tools=None): """Content identities that bind a baseline report to its actual inputs/output.""" linked_elf = pathlib.Path(linked_elf) dsd_path = pathlib.Path(dsd_path or RB.DSD) linker_path = pathlib.Path(linker_path or (RB.MW / RB.LD_VERSION / "mwldarm.exe")) rom_inputs = pathlib.Path(rom_inputs or (REPO / "extracted" / "dsd")) - required = [linked_elf, dsd_path, linker_path] + control_tools = tuple(control_tools or BASELINE_CONTROL_TOOLS) + required = [linked_elf, dsd_path, linker_path, *control_tools] missing = [str(path) for path in required if not path.is_file()] if not rom_inputs.is_dir(): missing.append(str(rom_inputs)) @@ -2904,6 +2932,7 @@ def partition_baseline_fingerprints(linked_elf, config_root=CFG_ARM9, return { "configArm9Sha256": content_tree_sha256(config_root), "romInputsSha256": content_tree_sha256(rom_inputs), + "controlToolsSha256": content_files_sha256(control_tools), "linkedElfSha256": hashlib.sha256(linked_elf.read_bytes()).hexdigest(), "linkedElfBytes": linked_elf.stat().st_size, "dsdSha256": hashlib.sha256(dsd_path.read_bytes()).hexdigest(), @@ -2913,7 +2942,7 @@ def partition_baseline_fingerprints(linked_elf, config_root=CFG_ARM9, def validate_partition_baseline_evidence(report, linked_elf, config_root=CFG_ARM9, dsd_path=None, linker_path=None, - rom_inputs=None): + rom_inputs=None, control_tools=None): """Refuse a baseline whose report is detached from current bytes or tools.""" evidence = report.get("baselineEvidence") if not isinstance(evidence, dict): @@ -2921,7 +2950,7 @@ def validate_partition_baseline_evidence(report, linked_elf, config_root=CFG_ARM try: current = partition_baseline_fingerprints( linked_elf, config_root, dsd_path=dsd_path, linker_path=linker_path, - rom_inputs=rom_inputs) + rom_inputs=rom_inputs, control_tools=control_tools) except (OSError, ValueError) as exc: return None, f"cannot fingerprint baseline: {exc}" mismatched = [key for key, value in current.items() if evidence.get(key) != value] From e078e41ebf54219a2305fda4fd80aa7eab2cdae1 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 10:14:58 -0500 Subject: [PATCH 16/20] tools: preserve legacy TU ratchet identities --- tools/test_tiers_ratchet.py | 12 ++++++++++++ tools/tiers_ratchet.py | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/test_tiers_ratchet.py b/tools/test_tiers_ratchet.py index b8abeb9121..0fd6af862f 100644 --- a/tools/test_tiers_ratchet.py +++ b/tools/test_tiers_ratchet.py @@ -114,6 +114,18 @@ def test_missing_promoted_member_is_named_not_reported_as_unreadable(self): why = TR.why("src/actors/TU.cpp#Missing", {}, {"src/actors/TU.cpp"}) self.assertIn("no longer an enrolled member", why) + def test_legacy_multi_function_path_upgrades_only_when_every_member_passes(self): + rel = "src/actors/TU.cpp" + ownership = {rel: ["First", "Second"]} + current = {f"{rel}#First", f"{rel}#Second"} + upgraded, backslid = TR.classify_missing( + [rel], current, {rel}, {}, ownership) + self.assertEqual((upgraded, backslid), ([rel], [])) + + upgraded, backslid = TR.classify_missing( + [rel], {f"{rel}#First"}, {rel}, {}, ownership) + self.assertEqual((upgraded, backslid), ([], [rel])) + if __name__ == "__main__": unittest.main() diff --git a/tools/tiers_ratchet.py b/tools/tiers_ratchet.py index afbf11c97f..5f499fc731 100644 --- a/tools/tiers_ratchet.py +++ b/tools/tiers_ratchet.py @@ -328,8 +328,8 @@ def why(identity, scores, tracked, moves=None): return "; ".join(tiers.CRITERION_LABEL[k] for k in failed) -def classify_missing(missing, current, tracked, moves): - """Split the banked-but-not-CONVERTED paths into (absorbed_clean, backslid). +def classify_missing(missing, current, tracked, moves, ownership=None): + """Split banked identities into clean ownership transitions and backslides. `absorbed_clean` is a banked path that stopped existing ONLY because a promoted TU absorbed it, and whose absorbing file is itself CONVERTED. Nothing left the @@ -348,8 +348,19 @@ def classify_missing(missing, current, tracked, moves): correct outcome and not a thing to "fix" by exempting mangled refs: byte-match outranks readability, and the exception log is where that trade gets recorded. """ + if ownership is None: + ownership = tiers.srcpath.source_definition_index() absorbed_clean, backslid = [], [] for rel in missing: + members = ownership.get(rel) or [] + member_ids = {f"{rel}#{symbol}" for symbol in members} + if len(members) > 1 and member_ids.issubset(current): + # The per-member scorer was introduced after some multi-function sources + # had already been banked by their physical path. Treat the first path -> + # member-identity rewrite as a lossless identity upgrade only when every + # enrolled member independently remains CONVERTED. + absorbed_clean.append(rel) + continue moved = moves.get(rel) if moved and rel not in tracked: _, dest = moved From 4dbbbc7fed012c2850cd63ae462a17dc970b0741 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 10:15:04 -0500 Subject: [PATCH 17/20] tools: admit same-worker stock TU controls --- tools/rombuild.py | 33 ++++++++------- tools/test_rombuild.py | 19 +++++++++ tools/test_tu_production.py | 84 +++++++++++++++++++++++++++++++++++-- tools/test_tubuild.py | 2 + tools/tu_production.py | 53 ++++++++++++++++++----- tools/tubuild.py | 3 ++ 6 files changed, 166 insertions(+), 28 deletions(-) diff --git a/tools/rombuild.py b/tools/rombuild.py index fd1863dcd2..790343fd80 100644 --- a/tools/rombuild.py +++ b/tools/rombuild.py @@ -214,6 +214,19 @@ def __init__(self, phase, returncode, output): self.output = output +def intact_rom_comparison(actual_sha256, verification): + """Describe the final same-worker production/control ROM comparison.""" + baseline = verification["baseline"] + expected = baseline["romSha256"] + return { + "expectedSha256": expected, + "admittedBootstrapSha256": verification["admittedRomSha256"], + "actualSha256": actual_sha256, + "moduleSetSha256": baseline["moduleSetSha256"], + "identical": actual_sha256 == expected, + } + + def run(cmd, what, quiet_patterns=()): r = subprocess.run(cmd, capture_output=True, text=True, env=dict(os.environ, LM_LICENSE_FILE=str(LICENSE)), cwd=REPO) @@ -1291,22 +1304,14 @@ def save_report(): "sha256": rom_sha256, } if intact_link_verification: - expected = { - (((entry.get("verification") or {}).get("linkcheck") or {}) - .get("rom") or {}).get("sha256") - for entry in intact_tus.values() - } - expected.add(intact_link_verification["baseline"]["romSha256"]) - report["intactTuRom"] = { - "expectedSha256": next(iter(expected)) if len(expected) == 1 else None, - "actualSha256": rom_sha256, - "identical": len(expected) == 1 and rom_sha256 in expected, - } - if len(expected) != 1 or rom_sha256 not in expected: + comparison = intact_rom_comparison( + rom_sha256, intact_link_verification) + report["intactTuRom"] = comparison + if not comparison["identical"]: raise BuildError( "intact TU ROM comparison", 1, - f"built ROM sha256 {rom_sha256} differs from admitted " - f"stock proof(s) {sorted(expected)}") + f"built ROM sha256 {rom_sha256} differs from same-worker " + f"independent control {comparison['expectedSha256']}") if tu_prepared: expected = tu_prepared["baseline"]["romSha256"] report["partitionedTuRom"] = { diff --git a/tools/test_rombuild.py b/tools/test_rombuild.py index e168160a59..0674e74fe9 100644 --- a/tools/test_rombuild.py +++ b/tools/test_rombuild.py @@ -373,6 +373,25 @@ def test_intact_policy_requires_promoted_full_ordinary_link_proof(self): RB.intact_tu_policies({"src/actors/TU.cpp"}, manifest=manifest) self.assertIn("baseline bootstrapping is non-circular", raised.exception.output) + def test_intact_rom_comparison_uses_current_same_worker_control(self): + verification = { + "baseline": { + "romSha256": "b" * 64, + "moduleSetSha256": "c" * 64, + }, + "admittedRomSha256": ["a" * 64], + } + result = RB.intact_rom_comparison("b" * 64, verification) + self.assertEqual(result, { + "expectedSha256": "b" * 64, + "admittedBootstrapSha256": ["a" * 64], + "actualSha256": "b" * 64, + "moduleSetSha256": "c" * 64, + "identical": True, + }) + self.assertFalse( + RB.intact_rom_comparison("d" * 64, verification)["identical"]) + def test_intact_policy_ignores_unenrolled_shadow(self): manifest = {"entries": [{ "id": "ov047/Shadow", "status": "text-verified", diff --git a/tools/test_tu_production.py b/tools/test_tu_production.py index bbb641a472..8004849782 100644 --- a/tools/test_tu_production.py +++ b/tools/test_tu_production.py @@ -70,7 +70,10 @@ def test_rom_gap_control_requires_exact_intact_inventory(self): (root / "final_link.o").write_bytes(b"independent linked control") report = { "baseline": True, - "analysis": {"passed": True}, + "analysis": { + "passed": True, + "moduleFidelity": {"moduleSetSha256": "ef" * 32}, + }, "phases": { "checkModules": {"ok": True}, "checkSymbols": {"errors": ["[ERROR] old"]}, @@ -87,6 +90,8 @@ def test_rom_gap_control_requires_exact_intact_inventory(self): return_value=("cd" * 32, None)): baseline = TP._strict_baseline(entries) self.assertEqual(baseline["romSha256"], "ab" * 32) + self.assertEqual(baseline["moduleSetSha256"], "ef" * 32) + self.assertIsNone(baseline["matchesStockRom"]) with self.assertRaisesRegex( TP.ProductionTuError, "current intact TU inventory"): TP._strict_baseline({ @@ -98,11 +103,17 @@ class ProductionTuObjects(unittest.TestCase): def test_automatic_intact_link_plan_uses_current_control_and_vtable_biases(self): claims = [{"name": ".text", "start": 0x1000, "end": 0x1010}, {"name": ".data", "start": 0x2000, "end": 0x2010}] - baseline = {"symbolErrors": ["[ERROR] old"], "romSha256": "ab" * 32} + baseline = { + "symbolErrors": ["[ERROR] old"], + "romSha256": "ab" * 32, + "matchesStockRom": True, + "moduleSetSha256": "ef" * 32, + } entries = {"src/actors/Thing.cpp": { "id": "ov047/Thing", "verification": {"linkcheck": { "symbolCheckErrors": ["[ERROR] old"], + "moduleSetSha256": "ef" * 32, "rom": {"sha256": "ab" * 32}}}, }} with mock.patch.object(TP, "_strict_baseline", return_value=baseline), \ @@ -112,6 +123,8 @@ def test_automatic_intact_link_plan_uses_current_control_and_vtable_biases(self) return_value=({"_ZTV1T": {"bias": 8}}, [])): prepared = TP.prepare_intact_link_verification(entries) self.assertIs(prepared["baseline"], baseline) + self.assertEqual(prepared["admittedRomSha256"], ["ab" * 32]) + self.assertEqual(prepared["admittedModuleSetSha256"], "ef" * 32) self.assertEqual(prepared["entries"], [{ "id": "ov047/Thing", "source": "src/actors/Thing.cpp", "biases": {"_ZTV1T": {"bias": 8}}, @@ -119,11 +132,13 @@ def test_automatic_intact_link_plan_uses_current_control_and_vtable_biases(self) def test_automatic_intact_link_plan_rejects_laundered_control_error(self): baseline = {"symbolErrors": ["[ERROR] old", "[ERROR] new"], - "romSha256": "ab" * 32} + "romSha256": "ab" * 32, "matchesStockRom": True, + "moduleSetSha256": "ef" * 32} entries = {"src/actors/Thing.cpp": { "id": "ov047/Thing", "verification": {"linkcheck": { "symbolCheckErrors": ["[ERROR] old"], + "moduleSetSha256": "ef" * 32, "rom": {"sha256": "ab" * 32}}}, }} with mock.patch.object(TP, "_strict_baseline", return_value=baseline): @@ -131,6 +146,69 @@ def test_automatic_intact_link_plan_rejects_laundered_control_error(self): "pre-promotion inventory"): TP.prepare_intact_link_verification(entries) + def test_same_worker_stock_control_allows_environment_specific_outer_rom(self): + baseline = { + "symbolErrors": [], "romSha256": "cd" * 32, + "matchesStockRom": True, "moduleSetSha256": "ef" * 32, + } + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", + "verification": {"linkcheck": { + "symbolCheckErrors": [], "moduleSetSha256": "ef" * 32, + "rom": {"sha256": "ab" * 32}}}, + }} + with mock.patch.object(TP, "_strict_baseline", return_value=baseline), \ + mock.patch.object(TP.TB, "manifest_section_claims", + return_value=([], [])), \ + mock.patch.object(TP.TB, "partition_vtable_rebiases", + return_value=({}, [])): + prepared = TP.prepare_intact_link_verification(entries) + self.assertEqual(prepared["baseline"]["romSha256"], "cd" * 32) + self.assertEqual(prepared["admittedRomSha256"], ["ab" * 32]) + + def test_environment_specific_control_without_stock_comparison_refuses(self): + baseline = { + "symbolErrors": [], "romSha256": "cd" * 32, + "matchesStockRom": None, "moduleSetSha256": "ef" * 32, + } + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", + "verification": {"linkcheck": { + "symbolCheckErrors": [], "moduleSetSha256": "ef" * 32, + "rom": {"sha256": "ab" * 32}}}, + }} + with mock.patch.object(TP, "_strict_baseline", return_value=baseline): + with self.assertRaisesRegex(TP.ProductionTuError, + "no same-worker stock-ROM comparison"): + TP.prepare_intact_link_verification(entries) + + def test_same_worker_stock_control_still_requires_admitted_module_set(self): + baseline = { + "symbolErrors": [], "romSha256": "cd" * 32, + "matchesStockRom": True, "moduleSetSha256": "12" * 32, + } + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", + "verification": {"linkcheck": { + "symbolCheckErrors": [], "moduleSetSha256": "ef" * 32, + "rom": {"sha256": "ab" * 32}}}, + }} + with mock.patch.object(TP, "_strict_baseline", return_value=baseline): + with self.assertRaisesRegex(TP.ProductionTuError, + "executable-module fingerprint"): + TP.prepare_intact_link_verification(entries) + + def test_admitted_intact_proof_requires_module_set_fingerprint(self): + entries = {"src/actors/Thing.cpp": { + "id": "ov047/Thing", + "verification": {"linkcheck": { + "symbolCheckErrors": [], + "rom": {"sha256": "ab" * 32}}}, + }} + with self.assertRaisesRegex(TP.ProductionTuError, + "executable-module fingerprint"): + TP.prepare_intact_link_verification(entries) + def test_intact_object_runs_all_fail_closed_policy_gates(self): entry = {"id": "ov047/Thing"} claims = [{"name": ".text", "start": 0x1000, "end": 0x1010}, diff --git a/tools/test_tubuild.py b/tools/test_tubuild.py index db4e2a7dac..07a6c8e065 100644 --- a/tools/test_tubuild.py +++ b/tools/test_tubuild.py @@ -1430,6 +1430,7 @@ def test_record_linkcheck_preserves_all_owned_ranges(): ], "objectAudit": {}, "symbolsNew": [], + "analysis": {"moduleFidelity": {"moduleSetSha256": "b" * 64}}, "rom": {"matchesStockRom": True, "sha256": "a" * 64}, } original = tubuild.save_manifest @@ -1442,6 +1443,7 @@ def test_record_linkcheck_preserves_all_owned_ranges(): recorded = entry["verification"]["linkcheck"] assert recorded["tuRange"] == report["tuRange"] assert recorded["tuRanges"] == report["tuRanges"] + assert recorded["moduleSetSha256"] == "b" * 64 # ---------------------------------------------------------------- create repairs # The three assemble_shadow_source behaviors proven by six modules of diff --git a/tools/tu_production.py b/tools/tu_production.py index cc72eb1427..b4fb8df2da 100644 --- a/tools/tu_production.py +++ b/tools/tu_production.py @@ -137,16 +137,21 @@ def prepare_intact_link_verification(entries, jobs=4): """ admitted_errors = set() admitted_roms = set() + admitted_module_sets = set() for entry in entries.values(): linkcheck = (entry.get("verification") or {}).get("linkcheck") or {} errors = linkcheck.get("symbolCheckErrors") rom_sha = (linkcheck.get("rom") or {}).get("sha256") - if not isinstance(errors, list) or not isinstance(rom_sha, str): + module_set_sha = linkcheck.get("moduleSetSha256") + if (not isinstance(errors, list) or not isinstance(rom_sha, str) + or not _is_sha256(module_set_sha)): raise ProductionTuError( f"{entry.get('id', '')}: admitted intact proof lacks its " - "symbol-error inventory or stock ROM SHA-256") + "symbol-error inventory, stock ROM SHA-256, or complete executable-" + "module fingerprint") admitted_errors.add(tuple(sorted(set(errors)))) admitted_roms.add(rom_sha) + admitted_module_sets.add(module_set_sha) baseline = _current_or_bootstrapped_intact_baseline(entries, jobs) current_errors = tuple(baseline["symbolErrors"]) if admitted_errors != {current_errors}: @@ -154,22 +159,26 @@ def prepare_intact_link_verification(entries, jobs=4): "strict post-promotion control symbol errors do not equal the admitted " f"pre-promotion inventory: current={list(current_errors)!r}, " f"admitted={[list(rows) for rows in sorted(admitted_errors)]!r}") - if admitted_roms != {baseline["romSha256"]}: - report = baseline["report"] + if admitted_module_sets != {baseline["moduleSetSha256"]}: + report = baseline.get("report") or {} diagnostics = { "rom": report.get("rom"), "baselineEvidence": report.get("baselineEvidence"), - "moduleSetSha256": (((report.get("analysis") or {}) - .get("moduleFidelity") or {}) - .get("moduleSetSha256")), + "moduleSetSha256": baseline["moduleSetSha256"], "intactTusDemoted": report.get("intactTusDemoted"), "scratch": report.get("scratch"), } raise ProductionTuError( - "strict post-promotion control ROM SHA-256 does not equal the admitted " - f"stock proof: current={baseline['romSha256']}, " - f"admitted={sorted(admitted_roms)!r}, " + "strict post-promotion control executable-module fingerprint does not " + f"equal the admitted proof: current={baseline['moduleSetSha256']}, " + f"admitted={sorted(admitted_module_sets)!r}, " f"control={json.dumps(diagnostics, sort_keys=True)}") + if baseline["matchesStockRom"] is not True \ + and admitted_roms != {baseline["romSha256"]}: + raise ProductionTuError( + "strict post-promotion control has no same-worker stock-ROM comparison " + "and its ROM SHA-256 does not equal the admitted bootstrap proof: " + f"current={baseline['romSha256']}, admitted={sorted(admitted_roms)!r}") prepared = [] for source, entry in sorted(entries.items()): claims, reasons = TB.manifest_section_claims(entry) @@ -180,7 +189,22 @@ def prepare_intact_link_verification(entries, jobs=4): _raise(f"{entry.get('id', source)} vtable address-point policy", reasons) prepared.append({"id": entry.get("id", source), "source": source, "biases": biases}) - return {"baseline": baseline, "entries": prepared} + return { + "baseline": baseline, + "entries": prepared, + "admittedRomSha256": sorted(admitted_roms), + "admittedModuleSetSha256": next(iter(admitted_module_sets)), + } + + +def _is_sha256(value): + if not isinstance(value, str) or len(value) != 64: + return False + try: + int(value, 16) + except ValueError: + return False + return True def _strict_baseline(expected_intact=None): @@ -222,6 +246,11 @@ def _strict_baseline(expected_intact=None): .get("errors")) if not isinstance(base_errors, list): raise ProductionTuError("stock control has no baseline symbol-error inventory") + module_set_sha = (((report.get("analysis") or {}).get("moduleFidelity") or {}) + .get("moduleSetSha256")) + if not _is_sha256(module_set_sha): + raise ProductionTuError( + "stock control has no complete executable-module fingerprint") return { "report": report, "reportPath": report_path, @@ -229,6 +258,8 @@ def _strict_baseline(expected_intact=None): "linkedElfSha256": hashlib.sha256(linked_elf.read_bytes()).hexdigest(), "symbolErrors": sorted(set(base_errors)), "romSha256": rom["sha256"], + "matchesStockRom": compared, + "moduleSetSha256": module_set_sha, } diff --git a/tools/tubuild.py b/tools/tubuild.py index ec49cae8b0..545a6f0a36 100644 --- a/tools/tubuild.py +++ b/tools/tubuild.py @@ -4797,6 +4797,9 @@ def _record_linkcheck(data, entry, report, baseline): "symbolCheckErrors": (((report.get("phases") or {}).get("checkSymbols") or {}) .get("errors")), "symbolCheckBaselineErrors": report.get("symbolsBaseline"), + "moduleSetSha256": (((report.get("analysis") or {}) + .get("moduleFidelity") or {}) + .get("moduleSetSha256")), "rom": report.get("rom"), "linkerOutput": report["phases"].get("link", {}).get("output"), } From af7997870a43ca52bb585601df1057b9d1a3122e Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 10:43:08 -0500 Subject: [PATCH 18/20] metadata: restore Kurumajiku contributor credits --- attribution.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/attribution.json b/attribution.json index cb7b554f81..dc57b67f5a 100644 --- a/attribution.json +++ b/attribution.json @@ -818,7 +818,10 @@ "src/_ZN19RotatingPlatformWdwD0Ev.cpp": "tangosdev", "src/_ZN20daObjFl_Fall_Block_cD0Ev.cpp": "tangosdev", "src/_ZN20daObjKm3_Kaitendai_cD0Ev.cpp": "tangosdev", + "src/RickshawBs_Spawn.c": "tangosdev", "src/_ZN21FloatingFloorLllSmallD0Ev.cpp": "tangosdev", + "src/_ZN21daObjKm3_Kurumajiku_c13InitResourcesEv.cpp": "lunavyqo", + "src/_ZN21daObjKm3_Kurumajiku_c16CleanupResourcesEv.cpp": "lunavyqo", "src/_ZN21daObjKm3_Kurumajiku_cD0Ev.cpp": "tangosdev", "src/_ZN23FloatOnWaterPlatformJrbD0Ev.cpp": "tangosdev", "src/_ZN29FloatOnWaterPlatformWdwSquareD0Ev.cpp": "tangosdev", From 4f4be74be22d566239aa0ec4fe9be1977e224685 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 11:02:05 -0500 Subject: [PATCH 19/20] tools: preserve promoted moves under TU member identities --- .github/workflows/converted-ratchet.yml | 2 +- tools/test_tiers_ratchet.py | 65 +++++++++++ tools/tiers_ratchet.py | 138 ++++++++++++++++-------- 3 files changed, 162 insertions(+), 43 deletions(-) diff --git a/.github/workflows/converted-ratchet.yml b/.github/workflows/converted-ratchet.yml index 80fe73297e..f1f42c77f5 100644 --- a/.github/workflows/converted-ratchet.yml +++ b/.github/workflows/converted-ratchet.yml @@ -1,6 +1,6 @@ # Guards the CONVERTED tier against a silent backslide. # -# `tools/tiers.py` scores every function under src/ against five readability criteria (real +# `tools/tiers.py` scores every enrolled member under src/ against five readability criteria (real # function name, no raw offset arithmetic, no `unk_` fields, no codegen tricks, no # calls through mangled names). `tools/tiers_ratchet.py` banks the SET of source/member # identities that pass all five in `config/converted-baseline.json` and fails when one diff --git a/tools/test_tiers_ratchet.py b/tools/test_tiers_ratchet.py index 0fd6af862f..ecbb313dc4 100644 --- a/tools/test_tiers_ratchet.py +++ b/tools/test_tiers_ratchet.py @@ -126,6 +126,71 @@ def test_legacy_multi_function_path_upgrades_only_when_every_member_passes(self) [rel], {f"{rel}#First"}, {rel}, {}, ownership) self.assertEqual((upgraded, backslid), ([], [rel])) + def test_legacy_identity_upgrade_names_every_passing_member(self): + rel = "src/actors/TU.cpp" + ownership = {rel: ["First", "Second"]} + clean = dict.fromkeys(TR.tiers.CRITERIA, True) + scores = {f"{rel}#First": clean, f"{rel}#Second": clean} + + reason = TR.why(rel, scores, {rel}, ownership=ownership) + + self.assertIn("IDENTITY UPGRADE", reason) + self.assertIn("2 independently passing member identities", reason) + self.assertIn(f"{rel}#First", reason) + self.assertIn(f"{rel}#Second", reason) + + def test_legacy_identity_upgrade_names_the_regressed_member(self): + rel = "src/actors/TU.cpp" + ownership = {rel: ["First", "Second"]} + clean = dict.fromkeys(TR.tiers.CRITERIA, True) + dirty = dict(clean, no_mangled_refs=False) + scores = {f"{rel}#First": clean, f"{rel}#Second": dirty} + + reason = TR.why(rel, scores, {rel}, ownership=ownership) + + self.assertIn("IDENTITY UPGRADE INCOMPLETE", reason) + self.assertIn(f"{rel}#Second", reason) + self.assertIn(TR.tiers.CRITERION_LABEL["no_mangled_refs"], reason) + self.assertNotIn("UNREADABLE", reason) + + def test_promoted_move_accepts_all_destination_member_identities(self): + legacy = "src/Legacy.cpp" + dest = "src/actors/TU.cpp" + moves = {legacy: ("ov001/TU", dest)} + ownership = {dest: ["First", "Second"]} + current = {f"{dest}#First", f"{dest}#Second"} + + moved, backslid = TR.classify_missing( + [legacy], current, {dest}, moves, ownership) + + self.assertEqual((moved, backslid), ([legacy], [])) + + def test_promoted_move_rejects_one_regressed_destination_member(self): + legacy = "src/Legacy.cpp" + dest = "src/actors/TU.cpp" + moves = {legacy: ("ov001/TU", dest)} + ownership = {dest: ["First", "Second"]} + + moved, backslid = TR.classify_missing( + [legacy], {f"{dest}#First"}, {dest}, moves, ownership) + + self.assertEqual((moved, backslid), ([], [legacy])) + + def test_promoted_move_names_the_regressed_destination_member(self): + legacy = "src/Legacy.cpp" + dest = "src/actors/TU.cpp" + moves = {legacy: ("ov001/TU", dest)} + ownership = {dest: ["First", "Second"]} + clean = dict.fromkeys(TR.tiers.CRITERIA, True) + dirty = dict(clean, no_raw_offset=False) + scores = {f"{dest}#First": clean, f"{dest}#Second": dirty} + + reason = TR.why(legacy, scores, {dest}, moves, ownership) + + self.assertIn("MOVED", reason) + self.assertIn(f"{dest}#Second", reason) + self.assertIn(TR.tiers.CRITERION_LABEL["no_raw_offset"], reason) + if __name__ == "__main__": unittest.main() diff --git a/tools/tiers_ratchet.py b/tools/tiers_ratchet.py index 5f499fc731..debcb0dc2d 100644 --- a/tools/tiers_ratchet.py +++ b/tools/tiers_ratchet.py @@ -44,14 +44,11 @@ entry with `"status": "promoted"` lists it as a `legacy_source`, the path is reported as a MOVE naming the `promoted_source` that absorbed it, and: - * if the absorbing file passes all five, that is NOT a backslide -- the same readable - code is simply scored under a different path, and the absorbing file enters the - baseline as an ordinary addition on the next `--update`; - * if it does not, that IS a backslide and still fails. The criteria are file-wide, so - merging a clean function into a file with one bad line genuinely costs it its - status, and the message says which criterion, e.g. "absorbed into - src/actors/.cpp by TU promotion (ov100/daObjPathLift_c), which fails: Calls - things by real names, not mangled _Z". + * if the absorbing source's enrolled members each pass all five, that is NOT a + backslide -- the same readable code is simply scored under member identities; + * if any member does not, that IS a backslide and still fails. The message names the + exact member and criterion, e.g. "src/actors/.cpp# fails: Calls things + by real names, not mangled _Z". A promotion is therefore never silently free. In practice it lands in the second case by construction: a reconstructed TU MUST spell vague-linkage symbols directly @@ -292,7 +289,36 @@ def _failures(identity, scores): return failed or None -def why(identity, scores, tracked, moves=None): +def _member_result(rel, scores, ownership): + """Return (passes, detail) for a source scored through member identities. + + A legacy baseline can still contain the physical path of a source which is now + scored as ``path#symbol``. Keep that identity transition observable: all members + passing is a lossless rewrite, while a mixed result names the exact member and + criterion that regressed. + """ + members = (ownership or {}).get(rel) or [] + if len(members) <= 1: + return None + failures = [] + identities = [f"{rel}#{symbol}" for symbol in members] + for member in identities: + score = scores.get(member) + if score is None: + failures.append(f"{member} is not scored") + continue + failed = [k for k in tiers.CRITERIA if not score[k]] + if failed: + failures.append( + f"{member} fails: " + + "; ".join(tiers.CRITERION_LABEL[k] for k in failed)) + if failures: + return False, "; ".join(failures) + return True, (f"rewritten as {len(identities)} independently passing member " + f"identities: {', '.join(identities)}") + + +def why(identity, scores, tracked, moves=None, ownership=None): """Why a banked source/member identity is no longer CONVERTED. A path that is GONE gets one of two answers, and the difference is the whole @@ -311,15 +337,33 @@ def why(identity, scores, tracked, moves=None): return (f"MOVED -- TU {tu_id} names {dest} as the file that absorbed it, " "but that file is not tracked; treat as a deletion") failed = _failures(dest, scores) - if failed is None: + if failed: + return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}), which " + "fails: " + "; ".join( + tiers.CRITERION_LABEL[k] for k in failed)) + if dest in scores: return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}); that " "file passes all five, so nothing readable was lost") - return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}), which " - "fails: " + "; ".join(tiers.CRITERION_LABEL[k] for k in failed)) + member_result = _member_result(dest, scores, ownership) + if member_result: + passes, detail = member_result + if passes: + return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}); " + f"{detail}, so nothing readable was lost") + return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}); " + f"member regression: {detail}") + return (f"MOVED -- absorbed into {dest} by TU promotion ({tu_id}), but the " + "tracked destination has no source or member score") failed = _failures(identity, scores) if failed is None: if marker: return f"GONE -- {symbol} is no longer an enrolled member of {rel}" + member_result = _member_result(rel, scores, ownership) + if member_result: + passes, detail = member_result + if passes: + return f"IDENTITY UPGRADE -- {rel} was {detail}" + return f"IDENTITY UPGRADE INCOMPLETE -- member regression: {detail}" if identity not in scores: return "UNREADABLE -- the file could not be read" # Cannot happen through --check, which derives both sides from one scan; it can @@ -331,22 +375,17 @@ def why(identity, scores, tracked, moves=None): def classify_missing(missing, current, tracked, moves, ownership=None): """Split banked identities into clean ownership transitions and backslides. - `absorbed_clean` is a banked path that stopped existing ONLY because a promoted - TU absorbed it, and whose absorbing file is itself CONVERTED. Nothing left the - CONVERTED set: the same readable code is scored under a different path, and the - absorbing file enters the baseline as a plain addition on the next `--update`. So - it is not a backslide and does not need an exception row. - - Everything else is `backslid`, and that deliberately INCLUDES a path absorbed into - a file that fails a criterion. The five criteria are file-wide, so consolidating a - clean function into a file with one bad line really does cost that function its - status, and this project's whole reason for a set ratchet is to name that instead - of averaging it away. In practice a reconstructed TU fails `no_mangled_refs` by - construction -- it MUST spell `_ZN7fBase_cnwEj`, `_ZN8dActor_cC2Ev` and - `_ZN8dActor_cD2Ev` directly or the range will not link -- so a promotion normally - lands here and is banked with a `--reason` saying exactly that. That is the - correct outcome and not a thing to "fix" by exempting mangled refs: byte-match - outranks readability, and the exception log is where that trade gets recorded. + `absorbed_clean` is either a banked path rewritten as independently passing member + identities, or a banked path that stopped existing only because a promoted TU + absorbed it and every destination member is CONVERTED. Nothing readable left the + set, so this is not a backslide and does not need an exception row. + + Everything else is `backslid`, including a path absorbed into a TU with one member + that fails a criterion. Member scoring keeps that failure local and the diagnostic + names it instead of averaging it away. Reconstructed members may still need direct + vague-linkage spellings such as `_ZN7fBase_cnwEj`, `_ZN8dActor_cC2Ev` and + `_ZN8dActor_cD2Ev` to link their range. Such a byte-match-driven regression belongs + in the exception log; it must not be hidden by the ownership transition. """ if ownership is None: ownership = tiers.srcpath.source_definition_index() @@ -364,7 +403,12 @@ def classify_missing(missing, current, tracked, moves, ownership=None): moved = moves.get(rel) if moved and rel not in tracked: _, dest = moved - if dest in current: + dest_members = ownership.get(dest) or [] + dest_member_ids = { + f"{dest}#{symbol}" for symbol in dest_members + } if len(dest_members) > 1 else set() + if (dest in current + or (dest_member_ids and dest_member_ids.issubset(current))): absorbed_clean.append(rel) continue backslid.append(rel) @@ -397,7 +441,8 @@ def main(): pass tracked = tracked_sources() - current, scores = scan(tracked) + ownership = tiers.srcpath.source_definition_index() + current, scores = scan(tracked, ownership) tracked_set = set(tracked) moves = promoted_moves() @@ -414,15 +459,16 @@ def main(): if removed and not args.reason: print(f"REFUSING to bank {len(removed)} removal(s) without --reason:\n") for rel in removed: - print(f" {rel}\n {why(rel, scores, tracked_set, moves)}") + print(f" {rel}\n " + f"{why(rel, scores, tracked_set, moves, ownership)}") print("\nA path leaving the CONVERTED set is allowed -- byte-match outranks\n" "readability and sometimes requires it -- but it is not allowed to be\n" "silent. Re-run with --reason \"\"; the reason\n" f"is appended to {args.exceptions} for every path above.") if absorbed_clean: - print(f"\n({len(absorbed_clean)} further path(s) left the baseline by TU\n" - "promotion into a file that is itself CONVERTED. Those are moves,\n" - "not backslides, and need no reason.)") + print(f"\n({len(absorbed_clean)} further path(s) made a lossless " + "ownership transition.\nThose are not backslides and need no " + "reason.)") return 2 if removed: append_exceptions(args.exceptions, @@ -456,11 +502,12 @@ def main(): print(f"CONVERTED backslide: {len(missing)} banked file(s) no longer pass " f"all {len(tiers.CRITERIA)} criteria\n") for rel in missing: - print(f" {rel}\n {why(rel, scores, tracked_set, moves)}") + print(f" {rel}\n " + f"{why(rel, scores, tracked_set, moves, ownership)}") if absorbed_clean: - print(f"\n({len(absorbed_clean)} further banked path(s) were absorbed " - "into a promoted TU that is\nitself CONVERTED -- moves, not " - "backslides. They are not counted above.)") + print(f"\n({len(absorbed_clean)} further banked path(s) made a " + "lossless ownership transition.\nThey are not backslides and " + "are not counted above.)") print(f"\nbaseline {len(banked)} current {len(current)} " f"(+{gained} gained, -{len(missing)} lost)") print("\nIf a byte match REQUIRED this -- and it legitimately can; raw-cast\n" @@ -476,10 +523,13 @@ def main(): "outranks readability -- bank it with that as the reason.") return 1 tail = f" (+{gained} gained, not yet banked)" if gained else "" - moved = (f" ({len(absorbed_clean)} moved into a promoted TU)" + moved = (f" ({len(absorbed_clean)} clean ownership transition(s))" if absorbed_clean else "") print(f"CONVERTED ratchet PASS baseline {len(banked)} " f"current {len(current)}{tail}{moved}") + for rel in absorbed_clean: + print(f" {rel}\n " + f"{why(rel, scores, tracked_set, moves, ownership)}") return 0 # No mode flag: a plain report. Says the same things --check would, without an @@ -494,12 +544,16 @@ def main(): absorbed_clean, missing = classify_missing(left, current, tracked_set, moves) print(f"baseline {len(banked):6d} {args.baseline}") print(f"gained, not banked {len(current - banked):6d}") - print(f"moved into a TU {len(absorbed_clean):6d} " - "(absorbed by a promoted TU that is itself CONVERTED)") + print(f"ownership transitions {len(absorbed_clean):6d} " + "(lossless TU move or path-to-member identity upgrade)") + for rel in absorbed_clean: + print(f" {rel}\n " + f"{why(rel, scores, tracked_set, moves, ownership)}") print(f"BACKSLID {len(missing):6d}" f"{' <- --check would fail' if missing else ''}") for rel in missing[:20]: - print(f" {rel}\n {why(rel, scores, tracked_set, moves)}") + print(f" {rel}\n " + f"{why(rel, scores, tracked_set, moves, ownership)}") if len(missing) > 20: print(f" ... and {len(missing) - 20} more") return 0 From ef16a307356cc88462842335a7ad24a3f122c2bb Mon Sep 17 00:00:00 2001 From: = Date: Sun, 30 Aug 2026 11:02:05 -0500 Subject: [PATCH 20/20] metadata: bank initial TU member identity migration --- config/converted-baseline.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config/converted-baseline.json b/config/converted-baseline.json index 29bf4afa72..53bbbcab69 100644 --- a/config/converted-baseline.json +++ b/config/converted-baseline.json @@ -1,5 +1,5 @@ { - "_note": "The CONVERTED file set, banked. tools/tiers_ratchet.py --check fails when a path here no longer passes all five criteria in tools/tiers.py. Removals need --reason and land in config/converted-backslide-exceptions.jsonl. Regenerate with `python tools/tiers_ratchet.py --update`.", + "_note": "The CONVERTED source/member identity set, banked. One-function sources use their path; promoted TU members append #symbol to that path. tools/tiers_ratchet.py --check fails when an identity no longer passes all five criteria in tools/tiers.py. Removals need --reason and land in config/converted-backslide-exceptions.jsonl. Regenerate with `python tools/tiers_ratchet.py --update`.", "criteria": [ "real_name", "no_raw_offset", @@ -7,7 +7,7 @@ "no_codegen_trick", "no_mangled_refs" ], - "count": 2553, + "count": 2554, "converted": [ "src/ARMMathLoadState.c", "src/ARMMathSaveState.c", @@ -2516,7 +2516,8 @@ "src/_ll_sdiv.c", "src/_s32_div_f.c", "src/_u32_div_f.c", - "src/actors/ActorBase_SceneNode.cpp", + "src/actors/ActorBase_SceneNode.cpp#_ZN7fBase_c9SceneNode5ResetEv", + "src/actors/ActorBase_SceneNode.cpp#_ZN7fBase_c9SceneNodeC1Ev", "src/actors/BigBooIcon/_ZN11daTrsIcon_c13InitResourcesEv.cpp", "src/actors/BigBooIcon/_ZN11daTrsIcon_cD0Ev.cpp", "src/actors/BigBooIcon/_ZN11daTrsIcon_cD1Ev.cpp",