From a8f2ac620ab84c5d726789da0ed1803a5697fee9 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 12:46:20 +0530 Subject: [PATCH 01/59] fix(scripts): write through a symlinked output, roll back every staged swap Two P1 review findings on the merged #305 (4a2d5f1e), both in write_outputs's atomic-replace path for an existing destination. write_outputs() staged a symlinked destination by renaming a temp file onto the *link's own name*: `os.replace(temporary, path)` where `path` was the symlink. `os.replace` does not follow a symlink at the destination -- it replaces the link itself, the same as `unlink` would -- so writing through a symlinked --out or --manifest silently turned a live link into a plain file, leaving whatever else reads through that link looking at stale content forever. It now resolves through the link first: the sibling temp file is staged next to, and the final rename targets, `os.path.realpath(path)`, so the swap lands on the link's target and the link itself is untouched. Windows behaviour is unchanged -- an existing destination is still refused there rather than staged, symlink or not. Second half: the final swap-in loop (`for temporary, path in staged: os.replace(temporary, path)`) ran *after* the try/except that covers claiming and writing, so a second target's swap failing left an already- succeeded first target's swap in place with no rollback -- a mid-sequence failure undid only the destinations it had not yet reached. Each existing destination is now backed up to a sibling name (an atomic rename, so it is never briefly missing) immediately before its swap, and the swap itself moved inside the same try; on any failure every backup made so far is restored, not only the one in progress, and a reserved-but-unused backup name is discarded rather than left behind. New assertions that would fail if either fix were reverted: * `test_write_outputs_writes_through_a_symlinked_destination` creates a real symlink, writes through it, and asserts the link itself survives (`link.is_symlink()`) and its target received the payload. * `test_a_failed_swap_rolls_back_every_staged_replacement` stages two existing destinations, lets the first's swap land, fails the second's, and asserts the first destination is restored to its pre-run content with no stray `.part`/`.bak` file left behind. Ran `python3 scripts/bank_statement_import.test.py`: 49 tests passed on origin/master before this change; 51 after, the two new ones included. Co-Authored-By: Claude Opus 5 --- scripts/bank_statement_import.py | 64 ++++++++++++++++++++++---- scripts/bank_statement_import.test.py | 66 +++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 9 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 32a57727c..82b72f96c 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1515,8 +1515,26 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): does not exist is created exclusively under its own name, which is both the reservation against a concurrent run and, on Windows, the rule itself — there an existing destination is refused outright rather than staged. + + A destination that is a **symlink** is written through to its target: the + sibling temporary file is created next to (and the final rename targets) + `os.path.realpath(path)`, not `path` itself. `os.replace` does not follow a + symlink at the destination — it replaces the link, the same as `unlink` + would — so renaming onto the link's own name would silently turn it into a + plain file and leave whatever else reads through that link looking at + stale content. + + The final swap is inside the same try as the write, and every swap is + itself preceded by backing its destination up to a sibling name. A run + with two staged destinations that fails swapping the second must not leave + the first swapped in with no way back: **each swap is undone in reverse** + on any failure, from the backups, so a mid-sequence failure restores every + destination this call has touched, not only the ones the failure had not + yet reached. """ claimed, staged = [], [] + replaced = [] # (backup, path) already swapped in — undone on failure + pending_backup = None # reserved backup name not yet holding content try: for path, _ in targets: if os.path.exists(path): @@ -1524,11 +1542,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): refusal = windows_destination_refusal(path, accept_inherited) if refusal: raise refusal + real_path = os.path.realpath(path) handle, temporary = tempfile.mkstemp( - dir=os.path.dirname(os.path.abspath(path)), - prefix=os.path.basename(path) + ".", suffix=".part") + dir=os.path.dirname(real_path), + prefix=os.path.basename(real_path) + ".", suffix=".part") claimed.append((temporary, handle)) - staged.append((temporary, path)) + staged.append((temporary, real_path)) else: claimed.append((path, _open_private(path, accept_inherited))) if after_claim is not None: @@ -1538,9 +1557,35 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) os.chmod(where, 0o600) + # Every payload is on disk. Swapping now cannot lose an existing output + # to a failure that has already been ruled out — but the swap itself + # can still fail partway through a multi-target run, so each existing + # destination is backed up (an atomic rename to a sibling name, so it + # is never briefly missing) before its replacement lands, and that + # backup is what the `except` below restores from. + for temporary, real_path in staged: + backup_handle, backup = tempfile.mkstemp( + dir=os.path.dirname(real_path), + prefix=os.path.basename(real_path) + ".", suffix=".bak") + os.close(backup_handle) + pending_backup = backup + os.replace(real_path, backup) + pending_backup = None + replaced.append((backup, real_path)) + os.replace(temporary, real_path) except BaseException: - # Only paths this call created are removed. A created destination is - # still this run's; a staged file never was the destination at all. + # Undo everything this call has done, most recent first: a backup + # name reserved by mkstemp but never populated (the rename into it + # failed) is discarded, destinations already swapped in are restored + # from their backup, files this call created outright are removed, + # and a temp file staged but never swapped in is removed too (a + # staged file never was the destination). + if pending_backup is not None: + with contextlib.suppress(OSError): + os.unlink(pending_backup) + for backup, real_path in reversed(replaced): + with contextlib.suppress(OSError): + os.replace(backup, real_path) for where, handle in claimed: if handle is not None: with contextlib.suppress(OSError): @@ -1548,10 +1593,11 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): with contextlib.suppress(OSError): os.unlink(where) raise - # Every payload is on disk. Replacing now cannot lose an existing output to - # a failure that has already been ruled out. - for temporary, path in staged: - os.replace(temporary, path) + # Every swap committed. The backups exist only to undo a failure that, by + # this point, cannot happen any more. + for backup, _ in replaced: + with contextlib.suppress(OSError): + os.unlink(backup) def _check_paths(args): diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index ec52173e7..8fc8c92a1 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1362,6 +1362,72 @@ def test_a_failed_run_does_not_destroy_the_previous_output(m): ["new.csv", "previous.xml"] +def test_write_outputs_writes_through_a_symlinked_destination(m): + """`os.replace` does not follow a symlink at the destination -- it + replaces the link itself, the same as `unlink` would. Renaming the staged + payload onto the link's own name would silently turn a live symlink into + a plain file, leaving whatever else reads through that link looking at + stale content forever. The swap must resolve through the link and land on + its target instead.""" + with tempfile.TemporaryDirectory() as directory: + real_target = pathlib.Path(directory, "real-target.xml") + real_target.write_text("old") + link = pathlib.Path(directory, "out.xml") + link.symlink_to(real_target) + + m.write_outputs([(str(link), "")]) + + assert link.is_symlink(), "the link itself must survive the write" + assert pathlib.Path(os.readlink(link)) == real_target + assert real_target.read_text() == "", \ + "the payload must reach the link's target, not replace the link" + + +def test_a_failed_swap_rolls_back_every_staged_replacement(m): + """Two existing destinations are both staged; the first's swap succeeds + and the second's fails. The rollback used to run only the un-staged + cleanup, so the first destination was left holding the new run's content + with no way back -- a mid-sequence failure must undo every replacement + this call already committed, not only the one in progress when it failed. + """ + with tempfile.TemporaryDirectory() as directory: + first = pathlib.Path(directory, "first.xml") + second = pathlib.Path(directory, "second.csv") + first.write_text("first old") + second.write_text("second old") + + real_replace = m.os.replace + calls = {"n": 0} + + def flaky_replace(src, dst): + calls["n"] += 1 + # calls 1-2 are the first destination's own backup-then-swap; + # let those land, then fail the second destination's backup -- + # after the first has already fully committed. + if calls["n"] == 3: + raise OSError("simulated failure mid-sequence") + return real_replace(src, dst) + + m.os.replace = flaky_replace + try: + try: + m.write_outputs([(str(first), ""), + (str(second), "new-second\n")]) + raise AssertionError("the second target's swap must fail") + except OSError: + pass + finally: + m.os.replace = real_replace + + assert first.read_text() == "first old", \ + "the first destination's already-committed swap must be rolled back" + assert second.read_text() == "second old" + # no stray .part/.bak file is left behind by either destination + assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == \ + ["first.xml", "second.csv"], \ + sorted(p.name for p in pathlib.Path(directory).iterdir()) + + def test_a_case_insensitive_collision_is_refused_before_anything_is_written(m): """`--out Result.xml --manifest result.XML` is one file on a case-insensitive volume. The lexical preflight cannot see it and `samefile` needs both paths From 5199f7c30b8999f1bb3a344a9f26c101af56235f Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 12:46:43 +0530 Subject: [PATCH 02/59] fix(scripts): distinct wrong tails per header field, not one stand-in P1 review finding on the merged #312 (80360d31): test_account_binding was supposed to prove the account binding rejects any *other* header field's number, but only ever exercised one field (the customer id, "4230") -- so the test would still pass against a version that read the whole header block, as long as it happened to land on a field the test never tried. Verified against the real capture first, to be precise about what it can and cannot prove: in scripts/fixtures/hdfc-bbox-capture.xml, every header field except phone/MICR sanitises to an unbroken run of the same digit as the real account number, so its own tail is a *suffix* of the correct answer and cannot be used as a wrong value there without also matching it; phone and MICR themselves sanitise to the identical tail ("1112"). So the real capture can supply at most one genuinely distinct wrong header value, and test_parse_real_hdfc_capture's comment claiming five ("phone, customer id, IFSC digits, MICR, postcode") was wrong -- three of its five loop values (1113-1115) are real, but from the transaction table, not the header. Fixed the comment to say so; the assertions themselves were not weakened, all five values are still real and are still refused. The discriminating test the real capture cannot provide belongs on the constructed-page fixture instead (this file's own stated tier for exactly this case). HDFC_PAGE now carries four more header lines -- phone, IFSC, MICR, postcode -- each with its own number, and test_account_binding checks each one's tail is refused independently rather than trying the customer id five times. New assertion that would fail if this were reverted: mutating `HDFC.account_anchors` to match the phone line instead of "Account No" (simulating a version that reads the wrong header line) still passed the old test -- confirmed by running it against that mutation before writing this fix. The new phone/IFSC/MICR/postcode cases in test_account_binding catch it: `require_account_match` incorrectly succeeds against the mutated anchor and `refuses()` raises `AssertionError`. Ran `python3 scripts/bank_statement_import.test.py`: 51 tests passed, same count as after the previous commit -- this changes what two existing tests assert, not how many tests exist. Co-Authored-By: Claude Opus 5 --- scripts/bank_statement_import.test.py | 38 ++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 8fc8c92a1..e6e201e4a 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -98,7 +98,16 @@ def page(*lines): # them: this is where a customer id or a phone number sits (56, [(340, 380, "Cust"), (382, 396, "ID"), (397, 400, ":"), (403, 470, "00000000004230")]), - (60, [(70, 200, "Statement"), (205, 260, "of"), (265, 340, "account")]), + # four more header lines, each with its own distinct number, so the wrong + # tails below exercise phone, IFSC, MICR and postcode independently rather + # than a single stand-in field + (60, [(340, 380, "Phone"), (382, 396, "no."), (397, 400, ":"), + (403, 470, "00000000005551")]), + (64, [(340, 375, "RTGS/NEFT"), (378, 396, "IFSC"), (397, 400, ":"), + (403, 460, "ZZZZ0005552")]), + (68, [(340, 372, "MICR"), (397, 400, ":"), (403, 460, "000000005553")]), + (72, [(340, 372, "City"), (397, 400, ":"), (403, 460, "ZZZZZ 005554")]), + (76, [(70, 200, "Statement"), (205, 260, "of"), (265, 340, "account")]), (100, [(5, 30, "Date"), (72, 120, "Narration"), (282, 340, "Chq./Ref.No."), (360, 380, "Value"), (382, 396, "Dt"), (402, 452, "Withdrawal"), (454, 474, "Amt."), (482, 522, "Deposit"), (524, 544, "Amt."), @@ -287,9 +296,17 @@ def test_parse_real_hdfc_capture(m): # the account number is bound from the header block, not from the table m.require_account_match(pages, bank, "HDFC CA xx1111") - # every one of these is a real number printed in this capture's header — - # phone, customer id, IFSC digits, MICR, postcode — and every one passed - # before the binding was narrowed to the account-number line + # 1112 is real: this capture's sanitised phone number and its MICR code + # both end in it, and neither line is the account-number line. Every + # *other* header field here (customer id, IFSC, postcode) sanitises to an + # unbroken run of the same digit as the account number itself, so its own + # tail cannot serve as a wrong value on this fixture without also + # matching the real account — `test_account_binding` below carries phone, + # IFSC, MICR and postcode as genuinely distinct fields, which a real + # capture this heavily redacted cannot. 1113-1115 are real too, but from + # the transaction table rather than the header — a different negative + # case (a table reference must not stand in for the account), not a sixth + # header field. 9876 is printed nowhere in the document at all. for wrong in ("xx1112", "xx1113", "xx1114", "xx1115", "xx9876"): refuses(m, "account_not_in_statement", m.require_account_match, pages, bank, f"HDFC CA {wrong}") @@ -392,9 +409,16 @@ def test_account_binding(m): # 9012 ends the UPI reference on row 1 refuses(m, "account_not_in_statement", m.require_account_match, [HDFC_PAGE], hdfc, "HDFC CA xx9012") - # nor may any other number in the header: 4230 is the customer id - refuses(m, "account_not_in_statement", m.require_account_match, - [HDFC_PAGE], hdfc, "HDFC CA xx4230") + # nor may any other number in the header — and each of these is its own + # field with its own distinct value, not one stand-in tried five times, so + # a version that fell back to reading the whole header block would be + # caught by whichever field it happened to read + for label, wrong in (("customer id", "xx4230"), ("phone", "xx5551"), + ("IFSC", "xx5552"), ("MICR", "xx5553"), + ("postcode", "xx5554")): + refusal = refuses(m, "account_not_in_statement", m.require_account_match, + [HDFC_PAGE], hdfc, f"HDFC CA {wrong}") + assert wrong[2:] in str(refusal), (label, refusal) # and a document with no account-number line fails closed refuses(m, "no_account_number_line", m.require_account_match, [page((10, [(2, 60, "nothing")]))], hdfc, "HDFC CA xx1234") From e25d008cebd92a7e150ddecfb9340c4972767c6c Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 16:24:50 +0530 Subject: [PATCH 03/59] fix(scripts): preserve output replacement recovery --- scripts/bank_statement_import.py | 162 ++++++++++++++++------ scripts/bank_statement_import.test.py | 191 +++++++++++++++++++++----- 2 files changed, 276 insertions(+), 77 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 8ae04df6d..81d25f2be 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -126,6 +126,21 @@ def __init__(self, category, message): super().__init__(f"{category}: {message}") +class OutputCleanupFailure(OSError): + """Committed output is present, but an old sensitive copy remains. + + `retained_paths` gives the operator the exact private backup location to + protect or remove. A normal return would conceal that copy. + """ + + def __init__(self, retained_paths): + self.retained_paths = tuple(retained_paths) + super().__init__( + "output cleanup failed; prior output retained at " + + ", ".join(self.retained_paths) + ) + + # --------------------------------------------------------------------------- # # PDF -> rows # # --------------------------------------------------------------------------- # @@ -1488,6 +1503,58 @@ def _open_private(path, accept_inherited=False): raise _existing_target_on_windows(path) from None +def _file_identity(path): + stat_result = os.stat(path) + return stat_result.st_dev, stat_result.st_ino + + +def _unlink_for_cleanup(path, failures): + try: + os.unlink(path) + except FileNotFoundError: + pass + except OSError: + # A filesystem call can report an error after taking effect. Only retain + # the path when reconciliation shows bytes may still be present. + if os.path.lexists(path): + failures.append(path) + + +def _restore_backup(backup, destination, original_identity, failures): + """Restore a hard-link backup after a caught swap failure. + + `os.replace` can report an exception after the filesystem call took effect. + If the destination is already the original inode, reconciliation proves the + old bytes are back; otherwise leave the backup in place and name it in the + error rather than guessing which bytes survived. + """ + try: + os.replace(backup, destination) + # POSIX rename is a no-op when source and destination already name the + # same inode. That is the expected recovery path when the interrupted + # replacement never took effect, and it leaves the hard-link backup to + # be removed explicitly. + _unlink_for_cleanup(backup, failures) + return + except OSError: + try: + restored = _file_identity(destination) == original_identity + except OSError: + restored = False + if restored: + _unlink_for_cleanup(backup, failures) + else: + failures.append(backup) + + +def _note_cleanup_failures(error, failures): + if failures: + retained = ", ".join(sorted(set(failures))) + error.add_note( + "output cleanup or rollback failed; retained path(s): " + retained + ) + + def write_outputs(targets, accept_inherited=False, after_claim=None): """Claim **every** destination, then write them. All of them or none. @@ -1524,17 +1591,24 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): plain file and leave whatever else reads through that link looking at stale content. - The final swap is inside the same try as the write, and every swap is - itself preceded by backing its destination up to a sibling name. A run - with two staged destinations that fails swapping the second must not leave - the first swapped in with no way back: **each swap is undone in reverse** - on any failure, from the backups, so a mid-sequence failure restores every - destination this call has touched, not only the ones the failure had not - yet reached. + Before replacing an existing POSIX destination, its old inode is hard-linked + to a private sibling backup. The destination therefore remains present until + one `os.replace` atomically changes it from old bytes to new bytes. This is + rollback for exceptions caught in this process, not a multi-file crash + transaction: a process or host crash can retain private `.bak` files and + leave different destinations at different committed versions. + + The supplied destination's resolved path and inode are revalidated right + before that replacement. A symlink (or symlinked parent) retargeted after + claiming is refused instead of silently writing the stale target. This + detects changes observed at the commit boundary; a hostile filesystem that + changes the path again after that check remains outside this CLI's locking + authority. """ claimed, staged = [], [] - replaced = [] # (backup, path) already swapped in — undone on failure - pending_backup = None # reserved backup name not yet holding content + replaced = [] # (backup, path, original identity) already swapped in + pending_backup = None # reservation which may now hold a hard link + pending_swap = None # backup preserved before a replace which may have run try: for path, _ in targets: if os.path.exists(path): @@ -1547,7 +1621,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") claimed.append((temporary, handle)) - staged.append((temporary, real_path)) + staged.append((temporary, path, real_path, _file_identity(real_path))) else: claimed.append((path, _open_private(path, accept_inherited))) if after_claim is not None: @@ -1557,47 +1631,57 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) os.chmod(where, 0o600) - # Every payload is on disk. Swapping now cannot lose an existing output - # to a failure that has already been ruled out — but the swap itself - # can still fail partway through a multi-target run, so each existing - # destination is backed up (an atomic rename to a sibling name, so it - # is never briefly missing) before its replacement lands, and that - # backup is what the `except` below restores from. - for temporary, real_path in staged: + # Every payload is on disk. Each backup is a second link to the old + # inode, so creating it never removes the requested destination. + for temporary, supplied_path, real_path, original_identity in staged: + if (os.path.realpath(supplied_path) != real_path + or _file_identity(real_path) != original_identity): + raise Refusal( + "output_path_changed", + f"{supplied_path} changed after it was claimed; no output was replaced", + ) backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") os.close(backup_handle) pending_backup = backup - os.replace(real_path, backup) + os.unlink(backup) + os.link(real_path, backup) + pending_swap = (backup, real_path, original_identity) pending_backup = None - replaced.append((backup, real_path)) + # The destination stays present until this one atomic replacement. + # `pending_swap` is set first because an interrupt may arrive after + # the filesystem call has taken effect but before it returns. os.replace(temporary, real_path) - except BaseException: - # Undo everything this call has done, most recent first: a backup - # name reserved by mkstemp but never populated (the rename into it - # failed) is discarded, destinations already swapped in are restored - # from their backup, files this call created outright are removed, - # and a temp file staged but never swapped in is removed too (a - # staged file never was the destination). + replaced.append(pending_swap) + pending_swap = None + except BaseException as error: + # Reconcile a swap which may have completed before raising, then undo + # earlier committed swaps. Cleanup failures remain attached to the + # original exception with their recoverable locations. + cleanup_failures = [] + if pending_swap is not None: + _restore_backup(*pending_swap, cleanup_failures) if pending_backup is not None: - with contextlib.suppress(OSError): - os.unlink(pending_backup) - for backup, real_path in reversed(replaced): - with contextlib.suppress(OSError): - os.replace(backup, real_path) + _unlink_for_cleanup(pending_backup, cleanup_failures) + for backup, real_path, original_identity in reversed(replaced): + _restore_backup(backup, real_path, original_identity, cleanup_failures) for where, handle in claimed: if handle is not None: - with contextlib.suppress(OSError): + try: os.close(handle) - with contextlib.suppress(OSError): - os.unlink(where) + except OSError: + cleanup_failures.append(where) + _unlink_for_cleanup(where, cleanup_failures) + _note_cleanup_failures(error, cleanup_failures) raise - # Every swap committed. The backups exist only to undo a failure that, by - # this point, cannot happen any more. - for backup, _ in replaced: - with contextlib.suppress(OSError): - os.unlink(backup) + # A successful replacement is not a successful command if an old statement + # survives under an undisclosed random name. + cleanup_failures = [] + for backup, _, _ in replaced: + _unlink_for_cleanup(backup, cleanup_failures) + if cleanup_failures: + raise OutputCleanupFailure(cleanup_failures) def _check_paths(args): diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index dba6e4903..68c495904 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -98,16 +98,7 @@ def page(*lines): # them: this is where a customer id or a phone number sits (56, [(340, 380, "Cust"), (382, 396, "ID"), (397, 400, ":"), (403, 470, "00000000004230")]), - # four more header lines, each with its own distinct number, so the wrong - # tails below exercise phone, IFSC, MICR and postcode independently rather - # than a single stand-in field - (60, [(340, 380, "Phone"), (382, 396, "no."), (397, 400, ":"), - (403, 470, "00000000005551")]), - (64, [(340, 375, "RTGS/NEFT"), (378, 396, "IFSC"), (397, 400, ":"), - (403, 460, "ZZZZ0005552")]), - (68, [(340, 372, "MICR"), (397, 400, ":"), (403, 460, "000000005553")]), - (72, [(340, 372, "City"), (397, 400, ":"), (403, 460, "ZZZZZ 005554")]), - (76, [(70, 200, "Statement"), (205, 260, "of"), (265, 340, "account")]), + (60, [(70, 200, "Statement"), (205, 260, "of"), (265, 340, "account")]), (100, [(5, 30, "Date"), (72, 120, "Narration"), (282, 340, "Chq./Ref.No."), (360, 380, "Value"), (382, 396, "Dt"), (402, 452, "Withdrawal"), (454, 474, "Amt."), (482, 522, "Deposit"), (524, 544, "Amt."), @@ -296,17 +287,10 @@ def test_parse_real_hdfc_capture(m): # the account number is bound from the header block, not from the table m.require_account_match(pages, bank, "HDFC CA xx1111") - # 1112 is real: this capture's sanitised phone number and its MICR code - # both end in it, and neither line is the account-number line. Every - # *other* header field here (customer id, IFSC, postcode) sanitises to an - # unbroken run of the same digit as the account number itself, so its own - # tail cannot serve as a wrong value on this fixture without also - # matching the real account — `test_account_binding` below carries phone, - # IFSC, MICR and postcode as genuinely distinct fields, which a real - # capture this heavily redacted cannot. 1113-1115 are real too, but from - # the transaction table rather than the header — a different negative - # case (a table reference must not stand in for the account), not a sixth - # header field. 9876 is printed nowhere in the document at all. + # 1112 is the captured MICR tail, not an account-number value. 1113-1115 + # occur in captured transaction-table references, a separate negative + # case: a table reference must not stand in for the account. 9876 is not + # printed in the capture. for wrong in ("xx1112", "xx1113", "xx1114", "xx1115", "xx9876"): refuses(m, "account_not_in_statement", m.require_account_match, pages, bank, f"HDFC CA {wrong}") @@ -395,9 +379,9 @@ def test_account_binding(m): It reads the line the statement labels as its account number, and nothing else. Reading the whole document lets a transaction reference stand in for - the account; reading the whole header block is barely better, because a - header prints a phone number, a customer id, an IFSC, a MICR code and a - postcode — on the real HDFC capture, four different wrong tails passed. + the account; reading the whole header block is barely better because it + includes non-account identifiers. The captured HDFC contract below keeps + that evidence tied to the bank-produced geometry. """ hdfc = m.HDFC() m.require_account_match([HDFC_PAGE], hdfc, "HDFC CA xx1234") @@ -409,16 +393,10 @@ def test_account_binding(m): # 9012 ends the UPI reference on row 1 refuses(m, "account_not_in_statement", m.require_account_match, [HDFC_PAGE], hdfc, "HDFC CA xx9012") - # nor may any other number in the header — and each of these is its own - # field with its own distinct value, not one stand-in tried five times, so - # a version that fell back to reading the whole header block would be - # caught by whichever field it happened to read - for label, wrong in (("customer id", "xx4230"), ("phone", "xx5551"), - ("IFSC", "xx5552"), ("MICR", "xx5553"), - ("postcode", "xx5554")): - refusal = refuses(m, "account_not_in_statement", m.require_account_match, - [HDFC_PAGE], hdfc, f"HDFC CA {wrong}") - assert wrong[2:] in str(refusal), (label, refusal) + # nor may the other constructed header value; fixture-specific account + # binding against real header geometry remains in test_parse_real_hdfc_capture. + refuses(m, "account_not_in_statement", m.require_account_match, + [HDFC_PAGE], hdfc, "HDFC CA xx4230") # and a document with no account-number line fails closed refuses(m, "no_account_number_line", m.require_account_match, [page((10, [(2, 60, "nothing")]))], hdfc, "HDFC CA xx1234") @@ -1407,6 +1385,144 @@ def test_write_outputs_writes_through_a_symlinked_destination(m): "the payload must reach the link's target, not replace the link" +def test_write_outputs_refuses_a_retargeted_symlink_before_commit(m): + """The destination observed after claiming must still name the same target + at the commit boundary. Otherwise a successful command updates a stale path + while the operator's requested path continues to expose old bytes.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.xml" + link = root / "out.xml" + first.write_text("first old") + second.write_text("second old") + link.symlink_to(first) + + def retarget(): + link.unlink() + link.symlink_to(second) + + refuses(m, "output_path_changed", m.write_outputs, + [(str(link), "new bytes")], False, retarget) + assert first.read_text() == "first old" + assert second.read_text() == "second old" + assert link.read_text() == "second old" + assert sorted(p.name for p in root.iterdir()) == ["first.xml", "out.xml", "second.xml"] + + +def test_write_outputs_keeps_destination_during_backup_preparation(m): + """A backup is a second link to the old inode, so the requested output is + still readable until the one atomic replacement. This catches a regression + back to moving the destination aside before the replacement is ready.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory, "previous.xml") + destination.write_text("old bytes") + real_link = m.os.link + observed = [] + + def link_while_observing(src, dst): + result = real_link(src, dst) + observed.append((destination.exists(), destination.read_text())) + return result + + m.os.link = link_while_observing + try: + m.write_outputs([(str(destination), "new bytes")]) + finally: + m.os.link = real_link + + assert observed == [(True, "old bytes")] + assert destination.read_text() == "new bytes" + + +def test_an_interrupt_after_a_backup_link_preserves_the_previous_output(m): + """Unlike a rename-to-backup, an interrupt after hard-link creation leaves + the requested destination intact; the exception cleanup may remove only the + extra link.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory, "previous.xml") + destination.write_text("old bytes") + real_link = m.os.link + + def interrupt_after_backup_link(src, dst): + result = real_link(src, dst) + raise KeyboardInterrupt("controlled interrupt after backup link") + + m.os.link = interrupt_after_backup_link + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + m.os.link = real_link + + assert destination.read_text() == "old bytes" + assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == ["previous.xml"] + + +def test_an_interrupt_after_a_swap_restores_the_previous_output(m): + """A caught interrupt may arrive after rename(2) took effect. Pending swap + state must therefore be restored, not discarded as if the call had failed + before touching the filesystem.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory, "previous.xml") + destination.write_text("old bytes") + real_replace = m.os.replace + + def interrupt_after_swap(src, dst): + result = real_replace(src, dst) + if str(src).endswith(".part"): + raise KeyboardInterrupt("controlled interrupt after swap") + return result + + m.os.replace = interrupt_after_swap + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + m.os.replace = real_replace + + assert destination.read_text() == "old bytes" + assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == ["previous.xml"] + + +def test_write_outputs_reports_a_retained_backup_after_commit(m): + """Successful replacement is not a successful command when cleanup leaves + prior bank-statement bytes at an undisclosed random backup path.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_unlink = m.os.unlink + + def fail_committed_backup(path): + if str(path).endswith(".bak") and destination.read_text() == "new bytes": + raise OSError("controlled backup cleanup failure") + return real_unlink(path) + + m.os.unlink = fail_committed_backup + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("a retained backup must be reported") + except m.OutputCleanupFailure as failure: + assert len(failure.retained_paths) == 1 + backup = pathlib.Path(failure.retained_paths[0]) + assert backup.exists() + assert backup.read_text() == "old bytes" + finally: + m.os.unlink = real_unlink + for path in root.glob("*.bak"): + path.unlink() + + assert destination.read_text() == "new bytes" + + def test_a_failed_swap_rolls_back_every_staged_replacement(m): """Two existing destinations are both staged; the first's swap succeeds and the second's fails. The rollback used to run only the un-staged @@ -1425,10 +1541,9 @@ def test_a_failed_swap_rolls_back_every_staged_replacement(m): def flaky_replace(src, dst): calls["n"] += 1 - # calls 1-2 are the first destination's own backup-then-swap; - # let those land, then fail the second destination's backup -- - # after the first has already fully committed. - if calls["n"] == 3: + # The first destination's swap lands, then the second swap fails. + # Its backup is a hard link, so only the replacement calls count. + if calls["n"] == 2: raise OSError("simulated failure mid-sequence") return real_replace(src, dst) From 37005ad9041bda5f6f775840a21fd489f351c586 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 16:29:42 +0530 Subject: [PATCH 04/59] fix(scripts): revalidate output at swap boundary --- scripts/bank_statement_import.py | 27 ++++++---- scripts/bank_statement_import.test.py | 77 +++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 81d25f2be..292ae8a81 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1550,9 +1550,15 @@ def _restore_backup(backup, destination, original_identity, failures): def _note_cleanup_failures(error, failures): if failures: retained = ", ".join(sorted(set(failures))) - error.add_note( - "output cleanup or rollback failed; retained path(s): " + retained - ) + message = "output cleanup or rollback failed; retained path(s): " + retained + # Python prints `SystemExit.code`, not exception notes. A Refusal is a + # SystemExit so that command-line validation exits without a traceback; + # put the retained location in its visible code rather than hiding it in + # an unrendered note. + if isinstance(error, Refusal): + error.code = f"{error.code}\n{message}" + else: + error.add_note(message) def write_outputs(targets, accept_inherited=False, after_claim=None): @@ -1634,12 +1640,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # Every payload is on disk. Each backup is a second link to the old # inode, so creating it never removes the requested destination. for temporary, supplied_path, real_path, original_identity in staged: - if (os.path.realpath(supplied_path) != real_path - or _file_identity(real_path) != original_identity): - raise Refusal( - "output_path_changed", - f"{supplied_path} changed after it was claimed; no output was replaced", - ) backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") @@ -1652,6 +1652,15 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after # the filesystem call has taken effect but before it returns. + # Revalidate *after* the backup operation: it is a filesystem call + # an attacker can use to retarget the supplied symlink before this + # commit. The pending backup lets the refusal cleanly undo itself. + if (os.path.realpath(supplied_path) != real_path + or _file_identity(real_path) != original_identity): + raise Refusal( + "output_path_changed", + f"{supplied_path} changed after it was claimed; no output was replaced", + ) os.replace(temporary, real_path) replaced.append(pending_swap) pending_swap = None diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 68c495904..cd988b755 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -37,6 +37,7 @@ import os import pathlib import stat +import subprocess import sys import tempfile import types @@ -1410,6 +1411,37 @@ def retarget(): assert sorted(p.name for p in root.iterdir()) == ["first.xml", "out.xml", "second.xml"] +def test_write_outputs_revalidates_a_symlink_after_backup_preparation(m): + """The check belongs directly before the swap, not before a backup syscall + which can itself be used to retarget the operator's path.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.xml" + link = root / "out.xml" + first.write_text("first old") + second.write_text("second old") + link.symlink_to(first) + real_link = m.os.link + + def retarget_after_backup(src, dst): + result = real_link(src, dst) + link.unlink() + link.symlink_to(second) + return result + + m.os.link = retarget_after_backup + try: + refuses(m, "output_path_changed", m.write_outputs, [(str(link), "new bytes")]) + finally: + m.os.link = real_link + + assert first.read_text() == "first old" + assert second.read_text() == "second old" + assert link.read_text() == "second old" + assert sorted(p.name for p in root.iterdir()) == ["first.xml", "out.xml", "second.xml"] + + def test_write_outputs_keeps_destination_during_backup_preparation(m): """A backup is a second link to the old inode, so the requested output is still readable until the one atomic replacement. This catches a regression @@ -1523,6 +1555,51 @@ def fail_committed_backup(path): assert destination.read_text() == "new bytes" +def test_refusal_reports_a_retained_backup_on_stderr(m): + """Refusal inherits SystemExit, whose unhandled rendering ignores + `BaseException.add_note`. Assert the CLI-visible error rather than the + in-process exception object so a sensitive retained backup is not hidden.""" + program = f'''\ +import importlib.util +import pathlib +import tempfile + +script = {str(SCRIPT)!r} +spec = importlib.util.spec_from_file_location("bank_statement_import_subprocess", script) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.xml" + link = root / "out.xml" + first.write_text("first old") + second.write_text("second old") + link.symlink_to(first) + real_link = module.os.link + real_cleanup = module._unlink_for_cleanup + def retarget_after_backup(src, dst): + result = real_link(src, dst) + link.unlink() + link.symlink_to(second) + return result + def retain_backup(path, failures): + if str(path).endswith(".bak"): + failures.append(path) + else: + real_cleanup(path, failures) + module.os.link = retarget_after_backup + module._unlink_for_cleanup = retain_backup + module.write_outputs([(str(link), "new bytes")]) +''' + done = subprocess.run([sys.executable, "-c", program], text=True, + capture_output=True, check=False) + assert done.returncode != 0 + assert "output_path_changed" in done.stderr, done.stderr + assert "retained path(s):" in done.stderr, done.stderr + assert ".bak" in done.stderr, done.stderr + + def test_a_failed_swap_rolls_back_every_staged_replacement(m): """Two existing destinations are both staged; the first's swap succeeds and the second's fails. The rollback used to run only the un-staged From e0ca932e1a49ede0a3e4dd162bbe8aa4d705cec2 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 16:35:10 +0530 Subject: [PATCH 05/59] test(scripts): pin captured account header geometry --- scripts/bank_statement_import.test.py | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index cd988b755..090907c93 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -297,6 +297,51 @@ def test_parse_real_hdfc_capture(m): pages, bank, f"HDFC CA {wrong}") +def test_real_hdfc_capture_binds_the_account_no_geometry_only(m): + """The unchanged capture pins the header field, not a convenient tail. + + Customer values are sanitised, so several header numbers intentionally end + alike. Its captured labels and coordinates still prove which field the + production selector reads. A postcode label is not present in this + capture; this test makes no claim about an absent field. + """ + bank = m.HDFC() + pages = capture("hdfc-bbox-capture.xml") + + selected = [ + [(round(x0, 3), round(y0, 3), round(x1, 3), round(y1, 3), text) + for x0, y0, x1, y1, text in group + if text in {"Account", "No"}] + for _, group in m._lines(pages[0]) + if m._matches(group, bank.account_anchors) + ] + assert selected == [[ + (340.157, 149.001, 367.261, 156.201, "Account"), + (369.261, 149.001, 379.037, 156.201, "No"), + ]], selected + account = m.require_account_match(pages, bank, "xx1111111") + assert account == "11111111111111" + + # Mutation controls select existing captured header geometry. They prove + # that the production Account/No selector excludes phone, customer-id, + # IFSC and MICR rows even where their sanitised numeric tails overlap. + original = bank.account_anchors + try: + bank.account_anchors = (("Phone", "no."),) + assert m.require_account_match(pages, bank, "xx1112") == "11111112" + + bank.account_anchors = (("Cust", "ID"),) + assert m.require_account_match(pages, bank, "xx111111111") == "111111111" + + bank.account_anchors = (("RTGS/NEFT", "IFSC"),) + assert m.require_account_match(pages, bank, "xx1111111") == "1111111" + + bank.account_anchors = (("MICR",),) + assert m.require_account_match(pages, bank, "xx1112") == "111111112" + finally: + bank.account_anchors = original + + def test_parse_real_sbi_capture(m): """SBI stacks the date over the year, repeats a three-line column header on every page, and wraps the narration mid-token across five lines. All three From 6cee19746543254bcd073779685ea02d835d1d14 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 17:19:47 +0530 Subject: [PATCH 06/59] fix(import): retain owned backups on rollback conflicts --- scripts/bank_statement_import.py | 215 ++++++++++++++++++++------ scripts/bank_statement_import.test.py | 156 +++++++++++++++---- 2 files changed, 291 insertions(+), 80 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 292ae8a81..e9dc73451 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1508,8 +1508,27 @@ def _file_identity(path): return stat_result.st_dev, stat_result.st_ino -def _unlink_for_cleanup(path, failures): +def _fd_identity(handle): + stat_result = os.fstat(handle) + return stat_result.st_dev, stat_result.st_ino + + +def _entry_identity(path): + """The inode `unlink` would remove, without following a replacement link.""" + stat_result = os.lstat(path) + return stat_result.st_dev, stat_result.st_ino + + +def _unlink_for_cleanup(path, owned_identity, failures): + """Remove a path only while it still names the inode this run created.""" try: + if _entry_identity(path) != owned_identity: + # The pathname has been reclaimed. It is not ours to delete, and + # reporting it gives the operator a chance to find the private copy + # if it still exists without making a claim about foreign bytes. + if os.path.lexists(path): + failures.append(str(path)) + return os.unlink(path) except FileNotFoundError: pass @@ -1517,33 +1536,92 @@ def _unlink_for_cleanup(path, failures): # A filesystem call can report an error after taking effect. Only retain # the path when reconciliation shows bytes may still be present. if os.path.lexists(path): - failures.append(path) + failures.append(str(path)) + + +def _copy_private_backup(source_path, original_identity, backup_handle): + """Copy the original inode into an owner-only backup already opened O_EXCL. + + The source descriptor pins the inode whose bytes are copied. The source + path is checked both before and after the copy; that catches an atomic path + replacement during preparation. Without filesystem locking, an adversary + that modifies the same inode while it is being read remains outside this + command's authority, so this does not promise a crash transaction. + """ + source_handle = os.open(source_path, os.O_RDONLY) + try: + if _fd_identity(source_handle) != original_identity: + raise Refusal( + "output_path_changed", + f"{source_path} changed while its rollback copy was prepared", + ) + copied = hashlib.sha256() + while True: + chunk = os.read(source_handle, 1024 * 1024) + if not chunk: + break + copied.update(chunk) + view = memoryview(chunk) + while view: + written = os.write(backup_handle, view) + if written == 0: + raise OSError("private backup write made no progress") + view = view[written:] + os.fsync(backup_handle) + os.lseek(backup_handle, 0, os.SEEK_SET) + verified = hashlib.sha256() + while True: + chunk = os.read(backup_handle, 1024 * 1024) + if not chunk: + break + verified.update(chunk) + if copied.digest() != verified.digest(): + raise OSError("private backup did not retain the copied bytes") + if (_fd_identity(source_handle) != original_identity + or _file_identity(source_path) != original_identity): + raise Refusal( + "output_path_changed", + f"{source_path} changed while its rollback copy was prepared", + ) + finally: + os.close(source_handle) -def _restore_backup(backup, destination, original_identity, failures): - """Restore a hard-link backup after a caught swap failure. +def _restore_backup(backup, backup_identity, destination, original_identity, + staged_identity, failures): + """Restore an owned private backup after a caught swap failure. `os.replace` can report an exception after the filesystem call took effect. - If the destination is already the original inode, reconciliation proves the - old bytes are back; otherwise leave the backup in place and name it in the - error rather than guessing which bytes survived. + Roll back only when the destination still names this run's staged inode. + A different inode may be a foreign writer's success, so keep the private + backup and report the conflict rather than overwriting it. """ try: - os.replace(backup, destination) - # POSIX rename is a no-op when source and destination already name the - # same inode. That is the expected recovery path when the interrupted - # replacement never took effect, and it leaves the hard-link backup to - # be removed explicitly. - _unlink_for_cleanup(backup, failures) + current_identity = _file_identity(destination) + except OSError: + current_identity = None + if current_identity == original_identity: + # The replace did not take effect, or a previous reconciliation already + # restored it. The extra private copy is ours to remove. + _unlink_for_cleanup(backup, backup_identity, failures) + return + if current_identity != staged_identity: + failures.append(backup) return + try: + if _entry_identity(backup) != backup_identity: + failures.append(backup) + return + # This observes ownership immediately before the replace. POSIX has no + # compare-and-swap rename, so a hostile concurrent rename after this + # check is still outside the CLI's locking authority. + os.replace(backup, destination) except OSError: try: restored = _file_identity(destination) == original_identity except OSError: restored = False - if restored: - _unlink_for_cleanup(backup, failures) - else: + if not restored: failures.append(backup) @@ -1597,12 +1675,13 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): plain file and leave whatever else reads through that link looking at stale content. - Before replacing an existing POSIX destination, its old inode is hard-linked - to a private sibling backup. The destination therefore remains present until - one `os.replace` atomically changes it from old bytes to new bytes. This is - rollback for exceptions caught in this process, not a multi-file crash - transaction: a process or host crash can retain private `.bak` files and - leave different destinations at different committed versions. + Before replacing an existing POSIX destination, its old inode is copied from + an opened descriptor to a private sibling backup. The destination therefore + remains present until one `os.replace` atomically changes it from old bytes + to new bytes. This is rollback for exceptions caught in this process, not a + multi-file crash transaction: a process or host crash can retain private + `.bak` files and leave different destinations at different committed + versions. The supplied destination's resolved path and inode are revalidated right before that replacement. A symlink (or symlinked parent) retargeted after @@ -1611,10 +1690,11 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): changes the path again after that check remains outside this CLI's locking authority. """ - claimed, staged = [], [] - replaced = [] # (backup, path, original identity) already swapped in - pending_backup = None # reservation which may now hold a hard link - pending_swap = None # backup preserved before a replace which may have run + claimed, staged, replaced = [], [], [] + # A record is the one ownership authority for a pathname: cleanup may + # unlink it only while its identity still equals record["identity"]. + pending_backup = None + pending_swap = None try: for path, _ in targets: if os.path.exists(path): @@ -1626,28 +1706,41 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle, temporary = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") - claimed.append((temporary, handle)) - staged.append((temporary, path, real_path, _file_identity(real_path))) + record = {"path": temporary, "handle": handle, + "identity": _fd_identity(handle)} + claimed.append(record) + staged.append({"temporary": record, "supplied_path": path, + "real_path": real_path, + "original_identity": _file_identity(real_path)}) else: - claimed.append((path, _open_private(path, accept_inherited))) + handle = _open_private(path, accept_inherited) + claimed.append({"path": path, "handle": handle, + "identity": _fd_identity(handle)}) if after_claim is not None: after_claim() - for index, ((_, text), (where, handle)) in enumerate(zip(targets, claimed)): - claimed[index] = (where, None) # fdopen owns the handle from here + for (_, text), record in zip(targets, claimed): + where, handle = record["path"], record["handle"] + record["handle"] = None # fdopen owns the handle from here with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) - os.chmod(where, 0o600) - # Every payload is on disk. Each backup is a second link to the old - # inode, so creating it never removes the requested destination. - for temporary, supplied_path, real_path, original_identity in staged: + # Every payload is on disk. A private copy preserves the old bytes while + # the requested destination stays present until the atomic replacement. + for state in staged: + temporary = state["temporary"] + supplied_path, real_path = state["supplied_path"], state["real_path"] + original_identity = state["original_identity"] backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") + os.fchmod(backup_handle, 0o600) + pending_backup = {"path": backup, "handle": backup_handle, + "identity": _fd_identity(backup_handle)} + _copy_private_backup(real_path, original_identity, backup_handle) os.close(backup_handle) - pending_backup = backup - os.unlink(backup) - os.link(real_path, backup) - pending_swap = (backup, real_path, original_identity) + pending_backup["handle"] = None + pending_swap = {"backup": pending_backup, "destination": real_path, + "original_identity": original_identity, + "staged_identity": temporary["identity"]} pending_backup = None # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after @@ -1661,7 +1754,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} changed after it was claimed; no output was replaced", ) - os.replace(temporary, real_path) + if _entry_identity(temporary["path"]) != temporary["identity"]: + raise Refusal( + "output_path_changed", + f"{supplied_path} staged output changed before replacement", + ) + os.replace(temporary["path"], real_path) replaced.append(pending_swap) pending_swap = None except BaseException as error: @@ -1669,26 +1767,41 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # earlier committed swaps. Cleanup failures remain attached to the # original exception with their recoverable locations. cleanup_failures = [] + if pending_backup is not None and pending_backup["handle"] is not None: + try: + os.close(pending_backup["handle"]) + except OSError: + cleanup_failures.append(pending_backup["path"]) + pending_backup["handle"] = None if pending_swap is not None: - _restore_backup(*pending_swap, cleanup_failures) + backup = pending_swap["backup"] + _restore_backup(backup["path"], backup["identity"], + pending_swap["destination"], + pending_swap["original_identity"], + pending_swap["staged_identity"], cleanup_failures) if pending_backup is not None: - _unlink_for_cleanup(pending_backup, cleanup_failures) - for backup, real_path, original_identity in reversed(replaced): - _restore_backup(backup, real_path, original_identity, cleanup_failures) - for where, handle in claimed: - if handle is not None: + _unlink_for_cleanup(pending_backup["path"], pending_backup["identity"], + cleanup_failures) + for swap in reversed(replaced): + backup = swap["backup"] + _restore_backup(backup["path"], backup["identity"], swap["destination"], + swap["original_identity"], swap["staged_identity"], + cleanup_failures) + for record in claimed: + if record["handle"] is not None: try: - os.close(handle) + os.close(record["handle"]) except OSError: - cleanup_failures.append(where) - _unlink_for_cleanup(where, cleanup_failures) + cleanup_failures.append(record["path"]) + _unlink_for_cleanup(record["path"], record["identity"], cleanup_failures) _note_cleanup_failures(error, cleanup_failures) raise # A successful replacement is not a successful command if an old statement # survives under an undisclosed random name. cleanup_failures = [] - for backup, _, _ in replaced: - _unlink_for_cleanup(backup, cleanup_failures) + for swap in replaced: + backup = swap["backup"] + _unlink_for_cleanup(backup["path"], backup["identity"], cleanup_failures) if cleanup_failures: raise OutputCleanupFailure(cleanup_failures) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 090907c93..18b6b4e21 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1467,19 +1467,19 @@ def test_write_outputs_revalidates_a_symlink_after_backup_preparation(m): first.write_text("first old") second.write_text("second old") link.symlink_to(first) - real_link = m.os.link + real_copy = m._copy_private_backup - def retarget_after_backup(src, dst): - result = real_link(src, dst) + def retarget_after_backup(src, identity, backup_handle): + result = real_copy(src, identity, backup_handle) link.unlink() link.symlink_to(second) return result - m.os.link = retarget_after_backup + m._copy_private_backup = retarget_after_backup try: refuses(m, "output_path_changed", m.write_outputs, [(str(link), "new bytes")]) finally: - m.os.link = real_link + m._copy_private_backup = real_copy assert first.read_text() == "first old" assert second.read_text() == "second old" @@ -1487,45 +1487,54 @@ def retarget_after_backup(src, dst): assert sorted(p.name for p in root.iterdir()) == ["first.xml", "out.xml", "second.xml"] -def test_write_outputs_keeps_destination_during_backup_preparation(m): - """A backup is a second link to the old inode, so the requested output is - still readable until the one atomic replacement. This catches a regression - back to moving the destination aside before the replacement is ready.""" +def test_write_outputs_keeps_destination_during_private_backup_preparation(m): + """The requested output remains readable while its private backup is made. + + `os.link` deliberately fails here: writable filesystems without hard-link + support still need the same caught-exception rollback behavior. + """ with tempfile.TemporaryDirectory() as directory: destination = pathlib.Path(directory, "previous.xml") destination.write_text("old bytes") + real_copy = m._copy_private_backup real_link = m.os.link observed = [] - def link_while_observing(src, dst): - result = real_link(src, dst) + def copy_while_observing(src, identity, backup_handle): + result = real_copy(src, identity, backup_handle) observed.append((destination.exists(), destination.read_text())) return result - m.os.link = link_while_observing + m._copy_private_backup = copy_while_observing + m.os.link = lambda *_: (_ for _ in ()).throw(OSError("hard links unavailable")) try: m.write_outputs([(str(destination), "new bytes")]) finally: + m._copy_private_backup = real_copy m.os.link = real_link assert observed == [(True, "old bytes")] assert destination.read_text() == "new bytes" -def test_an_interrupt_after_a_backup_link_preserves_the_previous_output(m): - """Unlike a rename-to-backup, an interrupt after hard-link creation leaves - the requested destination intact; the exception cleanup may remove only the - extra link.""" +def test_an_interrupt_after_private_backup_preserves_the_previous_output(m): + """The exclusive backup is private while interruption can still occur, and + cleanup removes only that owned copy while leaving the destination intact.""" with tempfile.TemporaryDirectory() as directory: destination = pathlib.Path(directory, "previous.xml") destination.write_text("old bytes") - real_link = m.os.link + root = pathlib.Path(directory) + real_copy = m._copy_private_backup + modes = [] - def interrupt_after_backup_link(src, dst): - result = real_link(src, dst) - raise KeyboardInterrupt("controlled interrupt after backup link") + def interrupt_after_backup_copy(src, identity, backup_handle): + result = real_copy(src, identity, backup_handle) + backups = list(root.glob("*.bak")) + assert len(backups) == 1 + modes.append(stat.S_IMODE(backups[0].stat().st_mode)) + raise KeyboardInterrupt("controlled interrupt after private backup") - m.os.link = interrupt_after_backup_link + m._copy_private_backup = interrupt_after_backup_copy try: try: m.write_outputs([(str(destination), "new bytes")]) @@ -1533,8 +1542,9 @@ def interrupt_after_backup_link(src, dst): except KeyboardInterrupt: pass finally: - m.os.link = real_link + m._copy_private_backup = real_copy + assert modes == [0o600] assert destination.read_text() == "old bytes" assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == ["previous.xml"] @@ -1621,19 +1631,19 @@ def test_refusal_reports_a_retained_backup_on_stderr(m): first.write_text("first old") second.write_text("second old") link.symlink_to(first) - real_link = module.os.link + real_copy = module._copy_private_backup real_cleanup = module._unlink_for_cleanup - def retarget_after_backup(src, dst): - result = real_link(src, dst) + def retarget_after_backup(src, identity, backup_handle): + result = real_copy(src, identity, backup_handle) link.unlink() link.symlink_to(second) return result - def retain_backup(path, failures): + def retain_backup(path, identity, failures): if str(path).endswith(".bak"): failures.append(path) else: - real_cleanup(path, failures) - module.os.link = retarget_after_backup + real_cleanup(path, identity, failures) + module._copy_private_backup = retarget_after_backup module._unlink_for_cleanup = retain_backup module.write_outputs([(str(link), "new bytes")]) ''' @@ -1664,7 +1674,6 @@ def test_a_failed_swap_rolls_back_every_staged_replacement(m): def flaky_replace(src, dst): calls["n"] += 1 # The first destination's swap lands, then the second swap fails. - # Its backup is a hard link, so only the replacement calls count. if calls["n"] == 2: raise OSError("simulated failure mid-sequence") return real_replace(src, dst) @@ -1689,6 +1698,95 @@ def flaky_replace(src, dst): sorted(p.name for p in pathlib.Path(directory).iterdir()) +def test_backup_copy_refuses_a_replaced_original_before_commit(m): + """Rollback bytes need the original inode's provenance, not whatever + happened to occupy its name while the private copy was being prepared.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + replacement = root / "foreign.xml" + destination.write_text("old bytes") + replacement.write_text("foreign writer bytes") + real_copy = m._copy_private_backup + real_replace = m.os.replace + + def replace_before_copy(src, identity, backup_handle): + real_replace(replacement, destination) + return real_copy(src, identity, backup_handle) + + m._copy_private_backup = replace_before_copy + try: + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + assert destination.read_text() == "foreign writer bytes" + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + +def test_cleanup_keeps_a_reclaimed_owned_path(m): + """A cleanup record proves only the path our run made. If that pathname + changes identity, reporting it is safe; unlinking it is not.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + owned = root / "output.xml.pending" + foreign = root / "foreign.xml" + owned.write_text("our temporary bytes") + identity = m._entry_identity(owned) + foreign.write_text("foreign writer bytes") + os.replace(foreign, owned) + failures = [] + + m._unlink_for_cleanup(owned, identity, failures) + + assert owned.read_text() == "foreign writer bytes" + assert failures == [str(owned)] + + +def test_rollback_keeps_a_foreign_destination_and_private_backup(m): + """When an external writer replaces an already-swapped destination before + another target fails, rollback must retain the owned backup rather than + overwriting that writer's bytes.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.csv" + foreign = root / "foreign.xml" + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + + def replace_then_conflict(src, dst): + if str(src).endswith(".part") and os.path.basename(dst) == "first.xml": + result = real_replace(src, dst) + foreign.write_text("foreign writer bytes") + real_replace(foreign, first) + return result + if str(src).endswith(".part") and os.path.basename(dst) == "second.csv": + raise OSError("simulated failure after foreign writer") + return real_replace(src, dst) + + m.os.replace = replace_then_conflict + try: + try: + m.write_outputs([(str(first), "new first"), + (str(second), "new second")]) + raise AssertionError("the second target's swap must fail") + except OSError as error: + assert any(str(path).endswith(".bak") + for path in getattr(error, "__notes__", [])) + finally: + m.os.replace = real_replace + + backups = list(root.glob("first.xml.*.bak")) + assert len(backups) == 1 + assert stat.S_IMODE(backups[0].stat().st_mode) == 0o600 + assert backups[0].read_text() == "first old" + assert first.read_text() == "foreign writer bytes" + assert second.read_text() == "second old" + + def test_a_case_insensitive_collision_is_refused_before_anything_is_written(m): """`--out Result.xml --manifest result.XML` is one file on a case-insensitive volume. The lexical preflight cannot see it and `samefile` needs both paths From 1bec9e6a995d5f902ea4fa37070ad41d87fcaa14 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 17:23:43 +0530 Subject: [PATCH 07/59] fix(import): reconcile post-effect backup restores --- scripts/bank_statement_import.py | 15 ++++-- scripts/bank_statement_import.test.py | 67 +++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index e9dc73451..9b6d604a3 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1618,11 +1618,15 @@ def _restore_backup(backup, backup_identity, destination, original_identity, os.replace(backup, destination) except OSError: try: - restored = _file_identity(destination) == original_identity + restored = _file_identity(destination) == backup_identity except OSError: restored = False if not restored: - failures.append(backup) + try: + backup_retained = _entry_identity(backup) == backup_identity + except OSError: + backup_retained = False + failures.append(backup if backup_retained else destination) def _note_cleanup_failures(error, failures): @@ -1732,7 +1736,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") - os.fchmod(backup_handle, 0o600) pending_backup = {"path": backup, "handle": backup_handle, "identity": _fd_identity(backup_handle)} _copy_private_backup(real_path, original_identity, backup_handle) @@ -1759,6 +1762,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} staged output changed before replacement", ) + if _entry_identity(pending_swap["backup"]["path"]) != \ + pending_swap["backup"]["identity"]: + raise Refusal( + "output_path_changed", + f"{supplied_path} rollback copy changed before replacement", + ) os.replace(temporary["path"], real_path) replaced.append(pending_swap) pending_swap = None diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 18b6b4e21..2e306bcd8 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1578,6 +1578,42 @@ def interrupt_after_swap(src, dst): assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == ["previous.xml"] +def test_restore_reconciles_a_backup_replace_that_raised_after_effect(m): + """A restore rename can report an error after it has moved the private + backup. Its new identity then proves recovery completed and must not be + reported as a missing retained backup.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.csv" + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + + def fail_second_swap_and_restore(src, dst): + if str(src).endswith(".part") and os.path.basename(dst) == "second.csv": + raise OSError("simulated second swap failure") + result = real_replace(src, dst) + if str(src).endswith(".bak"): + raise OSError("simulated restore failure after effect") + return result + + m.os.replace = fail_second_swap_and_restore + try: + try: + m.write_outputs([(str(first), "new first"), + (str(second), "new second")]) + raise AssertionError("the second target's swap must fail") + except OSError as error: + assert not getattr(error, "__notes__", []) + finally: + m.os.replace = real_replace + + assert first.read_text() == "first old" + assert second.read_text() == "second old" + assert sorted(path.name for path in root.iterdir()) == ["first.xml", "second.csv"] + + def test_write_outputs_reports_a_retained_backup_after_commit(m): """Successful replacement is not a successful command when cleanup leaves prior bank-statement bytes at an undisclosed random backup path.""" @@ -1744,6 +1780,37 @@ def test_cleanup_keeps_a_reclaimed_owned_path(m): assert failures == [str(owned)] +def test_writer_refuses_when_the_private_backup_path_is_reclaimed(m): + """The commit boundary must still name this run's backup; if it does not, + do not overwrite the destination without a recoverable owned copy.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + foreign = root / "foreign.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def reclaim_backup_after_copy(src, identity, backup_handle): + result = real_copy(src, identity, backup_handle) + backup, = root.glob("previous.xml.*.bak") + foreign.write_text("foreign writer bytes") + os.replace(foreign, backup) + return result + + m._copy_private_backup = reclaim_backup_after_copy + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + backups = list(root.glob("previous.xml.*.bak")) + assert destination.read_text() == "old bytes" + assert len(backups) == 1 + assert backups[0].read_text() == "foreign writer bytes" + assert str(backups[0]) in str(refusal.code) + + def test_rollback_keeps_a_foreign_destination_and_private_backup(m): """When an external writer replaces an already-swapped destination before another target fails, rollback must retain the owned backup rather than From cdd90345f03d7e7c6ef12eba8186be68bf7d3eca Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 17:39:19 +0530 Subject: [PATCH 08/59] fix(import): preserve rollback file identity --- scripts/bank_statement_import.py | 110 ++++++++++++++++++++++++-- scripts/bank_statement_import.test.py | 77 +++++++++++++++++- 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 9b6d604a3..d6a0f3e1c 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -99,6 +99,7 @@ import pathlib import re import shutil +import stat import subprocess import sys import tempfile @@ -1539,6 +1540,82 @@ def _unlink_for_cleanup(path, owned_identity, failures): failures.append(str(path)) +def _open_regular_output(path, expected_identity): + """Open and pin one existing regular output without waiting on a FIFO.""" + handle = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) + try: + stat_result = os.fstat(handle) + if not stat.S_ISREG(stat_result.st_mode): + raise Refusal( + "output_not_regular", + f"{path}: an existing output must be a regular file", + ) + if (stat_result.st_dev, stat_result.st_ino) != expected_identity: + raise Refusal( + "output_path_changed", + f"{path} changed while its rollback copy was prepared", + ) + return handle + except BaseException: + os.close(handle) + raise + + +def _metadata_from_handle(handle): + """Capture regular-output metadata while its original inode is pinned. + + The private backup remains mode 0600. These values are applied only after + that backup has been atomically restored to its original pathname. + """ + stat_result = os.fstat(handle) + metadata = { + "mode": stat.S_IMODE(stat_result.st_mode), + "uid": stat_result.st_uid, + "gid": stat_result.st_gid, + "atime_ns": stat_result.st_atime_ns, + "mtime_ns": stat_result.st_mtime_ns, + "xattrs": None, + } + if hasattr(os, "listxattr"): + try: + metadata["xattrs"] = { + name: os.getxattr(handle, name) + for name in os.listxattr(handle) + } + except OSError: + # Metadata restoration below still preserves portable mode and + # times. Some filesystems do not expose extended attributes. + pass + return metadata + + +def _restore_metadata(handle, metadata): + """Restore captured metadata to an already-restored regular output.""" + current = os.fstat(handle) + if (current.st_uid, current.st_gid) != (metadata["uid"], metadata["gid"]): + os.fchown(handle, metadata["uid"], metadata["gid"]) + os.fchmod(handle, metadata["mode"]) + os.utime(handle, ns=(metadata["atime_ns"], metadata["mtime_ns"])) + original_xattrs = metadata["xattrs"] + if original_xattrs is not None: + for name in os.listxattr(handle): + if name not in original_xattrs: + os.removexattr(handle, name) + for name, value in original_xattrs.items(): + os.setxattr(handle, name, value) + + +def _close_original_handle(swap, failures): + handle = swap.get("original_handle") + if handle is None: + return + swap["original_handle"] = None + try: + os.close(handle) + except OSError: + failures.append(swap["destination"]) + + def _copy_private_backup(source_path, original_identity, backup_handle): """Copy the original inode into an owner-only backup already opened O_EXCL. @@ -1548,7 +1625,7 @@ def _copy_private_backup(source_path, original_identity, backup_handle): that modifies the same inode while it is being read remains outside this command's authority, so this does not promise a crash transaction. """ - source_handle = os.open(source_path, os.O_RDONLY) + source_handle = _open_regular_output(source_path, original_identity) try: if _fd_identity(source_handle) != original_identity: raise Refusal( @@ -1588,7 +1665,7 @@ def _copy_private_backup(source_path, original_identity, backup_handle): def _restore_backup(backup, backup_identity, destination, original_identity, - staged_identity, failures): + staged_identity, metadata, failures): """Restore an owned private backup after a caught swap failure. `os.replace` can report an exception after the filesystem call took effect. @@ -1608,6 +1685,7 @@ def _restore_backup(backup, backup_identity, destination, original_identity, if current_identity != staged_identity: failures.append(backup) return + restored = False try: if _entry_identity(backup) != backup_identity: failures.append(backup) @@ -1616,6 +1694,7 @@ def _restore_backup(backup, backup_identity, destination, original_identity, # compare-and-swap rename, so a hostile concurrent rename after this # check is still outside the CLI's locking authority. os.replace(backup, destination) + restored = True except OSError: try: restored = _file_identity(destination) == backup_identity @@ -1627,6 +1706,15 @@ def _restore_backup(backup, backup_identity, destination, original_identity, except OSError: backup_retained = False failures.append(backup if backup_retained else destination) + if restored: + try: + restore_handle = _open_regular_output(destination, backup_identity) + try: + _restore_metadata(restore_handle, metadata) + finally: + os.close(restore_handle) + except (OSError, Refusal): + failures.append(destination) def _note_cleanup_failures(error, failures): @@ -1743,7 +1831,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_backup["handle"] = None pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, - "staged_identity": temporary["identity"]} + "staged_identity": temporary["identity"], + "original_handle": None, "metadata": None} pending_backup = None # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after @@ -1768,6 +1857,13 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} rollback copy changed before replacement", ) + # Hold the original inode through the whole transaction. Once a + # replacement unlinks its pathname, this prevents inode reuse from + # making a later foreign writer look like the old destination. + pending_swap["original_handle"] = _open_regular_output( + real_path, original_identity) + pending_swap["metadata"] = _metadata_from_handle( + pending_swap["original_handle"]) os.replace(temporary["path"], real_path) replaced.append(pending_swap) pending_swap = None @@ -1787,7 +1883,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): _restore_backup(backup["path"], backup["identity"], pending_swap["destination"], pending_swap["original_identity"], - pending_swap["staged_identity"], cleanup_failures) + pending_swap["staged_identity"], + pending_swap["metadata"], cleanup_failures) + _close_original_handle(pending_swap, cleanup_failures) if pending_backup is not None: _unlink_for_cleanup(pending_backup["path"], pending_backup["identity"], cleanup_failures) @@ -1795,7 +1893,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): backup = swap["backup"] _restore_backup(backup["path"], backup["identity"], swap["destination"], swap["original_identity"], swap["staged_identity"], - cleanup_failures) + swap["metadata"], cleanup_failures) + _close_original_handle(swap, cleanup_failures) for record in claimed: if record["handle"] is not None: try: @@ -1811,6 +1910,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): for swap in replaced: backup = swap["backup"] _unlink_for_cleanup(backup["path"], backup["identity"], cleanup_failures) + _close_original_handle(swap, cleanup_failures) if cleanup_failures: raise OutputCleanupFailure(cleanup_failures) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 2e306bcd8..0b2417824 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1761,6 +1761,23 @@ def replace_before_copy(src, identity, backup_handle): assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] +def test_backup_copy_refuses_a_fifo_before_reading_it(m): + """An existing output is data only when it is a regular file; opening a + FIFO for its rollback copy would otherwise wait for an unrelated writer.""" + if not hasattr(os, "mkfifo"): + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + os.mkfifo(destination) + + refuses(m, "output_not_regular", m.write_outputs, + [(str(destination), "new bytes")]) + + assert destination.is_fifo() + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + def test_cleanup_keeps_a_reclaimed_owned_path(m): """A cleanup record proves only the path our run made. If that pathname changes identity, reporting it is safe; unlinking it is not.""" @@ -1823,37 +1840,93 @@ def test_rollback_keeps_a_foreign_destination_and_private_backup(m): first.write_text("first old") second.write_text("second old") real_replace = m.os.replace + swaps = [] def replace_then_conflict(src, dst): if str(src).endswith(".part") and os.path.basename(dst) == "first.xml": + swaps.append("first") result = real_replace(src, dst) foreign.write_text("foreign writer bytes") real_replace(foreign, first) return result if str(src).endswith(".part") and os.path.basename(dst) == "second.csv": + swaps.append("second") raise OSError("simulated failure after foreign writer") return real_replace(src, dst) m.os.replace = replace_then_conflict + diagnostic_notes = "" try: try: m.write_outputs([(str(first), "new first"), (str(second), "new second")]) raise AssertionError("the second target's swap must fail") except OSError as error: - assert any(str(path).endswith(".bak") - for path in getattr(error, "__notes__", [])) + diagnostic_notes = "\n".join( + str(note) for note in getattr(error, "__notes__", []) + ) finally: m.os.replace = real_replace backups = list(root.glob("first.xml.*.bak")) assert len(backups) == 1 + assert str(backups[0]) in diagnostic_notes + assert swaps == ["first", "second"] assert stat.S_IMODE(backups[0].stat().st_mode) == 0o600 assert backups[0].read_text() == "first old" assert first.read_text() == "foreign writer bytes" assert second.read_text() == "second old" +def test_rollback_restores_original_output_metadata(m): + """A caught later swap failure restores the former bytes and portable + metadata, while a retained pre-rollback backup stays private.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.csv" + first.write_text("first old") + second.write_text("second old") + first.chmod(0o640) + old_mtime_ns = 1_700_000_000_123_456_789 + os.utime(first, ns=(old_mtime_ns, old_mtime_ns)) + xattr_name = "user.bridge_rollback_test" + xattr_value = b"original metadata" + preserves_xattr = False + if hasattr(os, "setxattr"): + try: + os.setxattr(first, xattr_name, xattr_value) + preserves_xattr = True + except OSError: + pass + real_replace = m.os.replace + + def fail_second_swap(src, dst): + if str(src).endswith(".part") and os.path.basename(dst) == "second.csv": + raise OSError("controlled second swap failure") + return real_replace(src, dst) + + m.os.replace = fail_second_swap + try: + try: + m.write_outputs([(str(first), "new first"), + (str(second), "new second")]) + raise AssertionError("the controlled swap failure must escape") + except OSError as error: + assert "controlled second swap failure" in str(error) + finally: + m.os.replace = real_replace + + restored = first.stat() + assert first.read_text() == "first old" + assert stat.S_IMODE(restored.st_mode) == 0o640 + assert restored.st_mtime_ns == old_mtime_ns + if preserves_xattr: + assert os.getxattr(first, xattr_name) == xattr_value + assert second.read_text() == "second old" + assert sorted(path.name for path in root.iterdir()) == ["first.xml", "second.csv"] + + def test_a_case_insensitive_collision_is_refused_before_anything_is_written(m): """`--out Result.xml --manifest result.XML` is one file on a case-insensitive volume. The lexical preflight cannot see it and `samefile` needs both paths From b5dd06c8caf40de21a50fabff56ca693f6d5e275 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 18:43:10 +0530 Subject: [PATCH 09/59] fix(import): pin rollback ownership records --- scripts/bank_statement_import.py | 142 +++++++++++++++----------- scripts/bank_statement_import.test.py | 67 +++++++++++- 2 files changed, 151 insertions(+), 58 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index d6a0f3e1c..0fe9cbee2 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1561,7 +1561,36 @@ def _open_regular_output(path, expected_identity): raise -def _metadata_from_handle(handle): +def _owned_path(path, handle): + """Record a pathname and retain the descriptor that pins its inode.""" + return {"path": path, "identity": _fd_identity(handle), "pin": handle} + + +def _close_owned_path(record, failures): + """Release an ownership pin only after its cleanup decision is complete.""" + handle = record.get("pin") + if handle is None: + return + record["pin"] = None + try: + os.close(handle) + except OSError: + failures.append(record["path"]) + + +def _close_write_handle(record, failures): + """Close a descriptor used for writing while retaining its ownership pin.""" + handle = record.get("write_handle") + if handle is None: + return + record["write_handle"] = None + try: + os.close(handle) + except OSError: + failures.append(record["path"]) + + +def _metadata_from_handle(path, handle): """Capture regular-output metadata while its original inode is pinned. The private backup remains mode 0600. These values are applied only after @@ -1582,10 +1611,11 @@ def _metadata_from_handle(handle): name: os.getxattr(handle, name) for name in os.listxattr(handle) } - except OSError: - # Metadata restoration below still preserves portable mode and - # times. Some filesystems do not expose extended attributes. - pass + except OSError as error: + raise Refusal( + "output_metadata_unavailable", + f"{path}: could not record extended attributes for rollback: {error}", + ) from None return metadata @@ -1605,25 +1635,15 @@ def _restore_metadata(handle, metadata): os.setxattr(handle, name, value) -def _close_original_handle(swap, failures): - handle = swap.get("original_handle") - if handle is None: - return - swap["original_handle"] = None - try: - os.close(handle) - except OSError: - failures.append(swap["destination"]) - - def _copy_private_backup(source_path, original_identity, backup_handle): """Copy the original inode into an owner-only backup already opened O_EXCL. - The source descriptor pins the inode whose bytes are copied. The source - path is checked both before and after the copy; that catches an atomic path - replacement during preparation. Without filesystem locking, an adversary - that modifies the same inode while it is being read remains outside this - command's authority, so this does not promise a crash transaction. + The caller already pins the original inode through the transaction; this + read descriptor pins it while bytes are copied. The source path is checked + both before and after the copy; that catches an atomic path replacement + during preparation. Without filesystem locking, an adversary that modifies + the same inode while it is being read remains outside this command's + authority, so this does not promise a crash transaction. """ source_handle = _open_regular_output(source_path, original_identity) try: @@ -1665,7 +1685,7 @@ def _copy_private_backup(source_path, original_identity, backup_handle): def _restore_backup(backup, backup_identity, destination, original_identity, - staged_identity, metadata, failures): + staged_identity, metadata, swap_started, failures): """Restore an owned private backup after a caught swap failure. `os.replace` can report an exception after the filesystem call took effect. @@ -1673,6 +1693,9 @@ def _restore_backup(backup, backup_identity, destination, original_identity, A different inode may be a foreign writer's success, so keep the private backup and report the conflict rather than overwriting it. """ + if not swap_started: + _unlink_for_cleanup(backup, backup_identity, failures) + return try: current_identity = _file_identity(destination) except OSError: @@ -1798,21 +1821,22 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle, temporary = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") - record = {"path": temporary, "handle": handle, - "identity": _fd_identity(handle)} + record = _owned_path(temporary, os.dup(handle)) + record["write_handle"] = handle claimed.append(record) staged.append({"temporary": record, "supplied_path": path, "real_path": real_path, "original_identity": _file_identity(real_path)}) else: handle = _open_private(path, accept_inherited) - claimed.append({"path": path, "handle": handle, - "identity": _fd_identity(handle)}) + record = _owned_path(path, os.dup(handle)) + record["write_handle"] = handle + claimed.append(record) if after_claim is not None: after_claim() for (_, text), record in zip(targets, claimed): - where, handle = record["path"], record["handle"] - record["handle"] = None # fdopen owns the handle from here + where, handle = record["path"], record["write_handle"] + record["write_handle"] = None # fdopen owns the handle from here with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) # Every payload is on disk. A private copy preserves the old bytes while @@ -1824,16 +1848,23 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") - pending_backup = {"path": backup, "handle": backup_handle, - "identity": _fd_identity(backup_handle)} - _copy_private_backup(real_path, original_identity, backup_handle) - os.close(backup_handle) - pending_backup["handle"] = None + pending_backup = _owned_path(backup, os.dup(backup_handle)) + pending_backup["write_handle"] = backup_handle pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, "staged_identity": temporary["identity"], - "original_handle": None, "metadata": None} + "original": None, "metadata": None, + "swap_started": False} pending_backup = None + # This ownership pin both prevents original-inode ABA reuse and + # captures metadata before the backup read can update atime. + original_handle = _open_regular_output(real_path, original_identity) + pending_swap["original"] = _owned_path(real_path, original_handle) + pending_swap["metadata"] = _metadata_from_handle( + real_path, original_handle) + _copy_private_backup(real_path, original_identity, backup_handle) + os.close(backup_handle) + pending_swap["backup"]["write_handle"] = None # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after # the filesystem call has taken effect but before it returns. @@ -1857,13 +1888,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} rollback copy changed before replacement", ) - # Hold the original inode through the whole transaction. Once a - # replacement unlinks its pathname, this prevents inode reuse from - # making a later foreign writer look like the old destination. - pending_swap["original_handle"] = _open_regular_output( - real_path, original_identity) - pending_swap["metadata"] = _metadata_from_handle( - pending_swap["original_handle"]) + pending_swap["swap_started"] = True os.replace(temporary["path"], real_path) replaced.append(pending_swap) pending_swap = None @@ -1872,36 +1897,36 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # earlier committed swaps. Cleanup failures remain attached to the # original exception with their recoverable locations. cleanup_failures = [] - if pending_backup is not None and pending_backup["handle"] is not None: - try: - os.close(pending_backup["handle"]) - except OSError: - cleanup_failures.append(pending_backup["path"]) - pending_backup["handle"] = None + if pending_backup is not None: + _close_write_handle(pending_backup, cleanup_failures) if pending_swap is not None: backup = pending_swap["backup"] + _close_write_handle(backup, cleanup_failures) _restore_backup(backup["path"], backup["identity"], pending_swap["destination"], pending_swap["original_identity"], pending_swap["staged_identity"], - pending_swap["metadata"], cleanup_failures) - _close_original_handle(pending_swap, cleanup_failures) + pending_swap["metadata"], + pending_swap["swap_started"], cleanup_failures) + _close_owned_path(backup, cleanup_failures) + if pending_swap["original"] is not None: + _close_owned_path(pending_swap["original"], cleanup_failures) if pending_backup is not None: _unlink_for_cleanup(pending_backup["path"], pending_backup["identity"], cleanup_failures) + _close_owned_path(pending_backup, cleanup_failures) for swap in reversed(replaced): backup = swap["backup"] + _close_write_handle(backup, cleanup_failures) _restore_backup(backup["path"], backup["identity"], swap["destination"], swap["original_identity"], swap["staged_identity"], - swap["metadata"], cleanup_failures) - _close_original_handle(swap, cleanup_failures) + swap["metadata"], swap["swap_started"], cleanup_failures) + _close_owned_path(backup, cleanup_failures) + _close_owned_path(swap["original"], cleanup_failures) for record in claimed: - if record["handle"] is not None: - try: - os.close(record["handle"]) - except OSError: - cleanup_failures.append(record["path"]) + _close_write_handle(record, cleanup_failures) _unlink_for_cleanup(record["path"], record["identity"], cleanup_failures) + _close_owned_path(record, cleanup_failures) _note_cleanup_failures(error, cleanup_failures) raise # A successful replacement is not a successful command if an old statement @@ -1910,7 +1935,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): for swap in replaced: backup = swap["backup"] _unlink_for_cleanup(backup["path"], backup["identity"], cleanup_failures) - _close_original_handle(swap, cleanup_failures) + _close_owned_path(backup, cleanup_failures) + _close_owned_path(swap["original"], cleanup_failures) + for record in claimed: + _close_owned_path(record, cleanup_failures) if cleanup_failures: raise OutputCleanupFailure(cleanup_failures) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 0b2417824..dd08e8249 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1778,6 +1778,31 @@ def test_backup_copy_refuses_a_fifo_before_reading_it(m): assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] +def test_backup_refuses_when_original_metadata_cannot_be_recorded(m): + """A rollback cannot claim to restore metadata it was unable to capture.""" + if not hasattr(m.os, "listxattr"): + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_listxattr = m.os.listxattr + + def unavailable_xattrs(_): + raise OSError("controlled xattr metadata failure") + + m.os.listxattr = unavailable_xattrs + try: + refusal = refuses(m, "output_metadata_unavailable", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m.os.listxattr = real_listxattr + + assert "controlled xattr metadata failure" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + def test_cleanup_keeps_a_reclaimed_owned_path(m): """A cleanup record proves only the path our run made. If that pathname changes identity, reporting it is safe; unlinking it is not.""" @@ -1840,11 +1865,40 @@ def test_rollback_keeps_a_foreign_destination_and_private_backup(m): first.write_text("first old") second.write_text("second old") real_replace = m.os.replace + real_open_regular = m._open_regular_output + real_owned_path = m._owned_path + real_close = m.os.close swaps = [] + first_handles, closed_handles = [], [] + owned_handles = {} + + def observe_first_handles(path, identity): + handle = real_open_regular(path, identity) + if os.path.samefile(path, first): + first_handles.append(handle) + return handle + + def observe_closes(handle): + closed_handles.append(handle) + return real_close(handle) + + def observe_owned_path(path, handle): + record = real_owned_path(path, handle) + if os.path.basename(path).startswith("first.xml."): + owned_handles[pathlib.Path(path).suffix] = record["pin"] + return record def replace_then_conflict(src, dst): if str(src).endswith(".part") and os.path.basename(dst) == "first.xml": swaps.append("first") + assert len(first_handles) == 2 + # The first open is the ownership pin captured before the + # backup read. It must survive the first swap and the foreign + # replacement so that its inode cannot be recycled. + assert first_handles[0] not in closed_handles + assert first_handles[1] in closed_handles + assert owned_handles[".part"] not in closed_handles + assert owned_handles[".bak"] not in closed_handles result = real_replace(src, dst) foreign.write_text("foreign writer bytes") real_replace(foreign, first) @@ -1855,6 +1909,9 @@ def replace_then_conflict(src, dst): return real_replace(src, dst) m.os.replace = replace_then_conflict + m._open_regular_output = observe_first_handles + m._owned_path = observe_owned_path + m.os.close = observe_closes diagnostic_notes = "" try: try: @@ -1867,11 +1924,17 @@ def replace_then_conflict(src, dst): ) finally: m.os.replace = real_replace + m._open_regular_output = real_open_regular + m._owned_path = real_owned_path + m.os.close = real_close backups = list(root.glob("first.xml.*.bak")) assert len(backups) == 1 assert str(backups[0]) in diagnostic_notes assert swaps == ["first", "second"] + assert first_handles[0] in closed_handles + assert owned_handles[".part"] in closed_handles + assert owned_handles[".bak"] in closed_handles assert stat.S_IMODE(backups[0].stat().st_mode) == 0o600 assert backups[0].read_text() == "first old" assert first.read_text() == "foreign writer bytes" @@ -1888,8 +1951,9 @@ def test_rollback_restores_original_output_metadata(m): first.write_text("first old") second.write_text("second old") first.chmod(0o640) + old_atime_ns = 1_600_000_000_123_456_789 old_mtime_ns = 1_700_000_000_123_456_789 - os.utime(first, ns=(old_mtime_ns, old_mtime_ns)) + os.utime(first, ns=(old_atime_ns, old_mtime_ns)) xattr_name = "user.bridge_rollback_test" xattr_value = b"original metadata" preserves_xattr = False @@ -1920,6 +1984,7 @@ def fail_second_swap(src, dst): restored = first.stat() assert first.read_text() == "first old" assert stat.S_IMODE(restored.st_mode) == 0o640 + assert restored.st_atime_ns == old_atime_ns assert restored.st_mtime_ns == old_mtime_ns if preserves_xattr: assert os.getxattr(first, xattr_name) == xattr_value From f85bb02dd4ed85d0db72634e47f79b68c2065419 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 18:47:35 +0530 Subject: [PATCH 10/59] fix(import): retain claimed output ownership --- scripts/bank_statement_import.py | 54 +++++++++---------- scripts/bank_statement_import.test.py | 78 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 30 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 0fe9cbee2..f5c4d048e 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1578,16 +1578,21 @@ def _close_owned_path(record, failures): failures.append(record["path"]) -def _close_write_handle(record, failures): - """Close a descriptor used for writing while retaining its ownership pin.""" - handle = record.get("write_handle") - if handle is None: - return - record["write_handle"] = None - try: - os.close(handle) - except OSError: - failures.append(record["path"]) +def _cleanup_owned_path(record, failures): + """Remove one owned pathname, releasing its pin first on Windows. + + POSIX keeps the descriptor open through the identity decision so an inode + cannot be recycled before cleanup. Windows does not permit unlinking an + open file, so fresh exclusive outputs close first and retain the existing + Windows cleanup behavior; actual Windows filesystem evidence remains + required for that platform-specific branch. + """ + if os.name == "nt": + _close_owned_path(record, failures) + _unlink_for_cleanup(record["path"], record["identity"], failures) + else: + _unlink_for_cleanup(record["path"], record["identity"], failures) + _close_owned_path(record, failures) def _metadata_from_handle(path, handle): @@ -1821,22 +1826,22 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle, temporary = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") - record = _owned_path(temporary, os.dup(handle)) - record["write_handle"] = handle + record = _owned_path(temporary, handle) claimed.append(record) staged.append({"temporary": record, "supplied_path": path, "real_path": real_path, "original_identity": _file_identity(real_path)}) else: handle = _open_private(path, accept_inherited) - record = _owned_path(path, os.dup(handle)) - record["write_handle"] = handle + record = _owned_path(path, handle) claimed.append(record) if after_claim is not None: after_claim() for (_, text), record in zip(targets, claimed): - where, handle = record["path"], record["write_handle"] - record["write_handle"] = None # fdopen owns the handle from here + where = record["path"] + # The record already owns the opened descriptor, so a duplicate + # failure here can still close it and unlink the created path. + handle = os.dup(record["pin"]) with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) # Every payload is on disk. A private copy preserves the old bytes while @@ -1848,8 +1853,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") - pending_backup = _owned_path(backup, os.dup(backup_handle)) - pending_backup["write_handle"] = backup_handle + pending_backup = _owned_path(backup, backup_handle) pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, "staged_identity": temporary["identity"], @@ -1863,8 +1867,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_swap["metadata"] = _metadata_from_handle( real_path, original_handle) _copy_private_backup(real_path, original_identity, backup_handle) - os.close(backup_handle) - pending_swap["backup"]["write_handle"] = None # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after # the filesystem call has taken effect but before it returns. @@ -1897,11 +1899,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # earlier committed swaps. Cleanup failures remain attached to the # original exception with their recoverable locations. cleanup_failures = [] - if pending_backup is not None: - _close_write_handle(pending_backup, cleanup_failures) if pending_swap is not None: backup = pending_swap["backup"] - _close_write_handle(backup, cleanup_failures) _restore_backup(backup["path"], backup["identity"], pending_swap["destination"], pending_swap["original_identity"], @@ -1912,21 +1911,16 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if pending_swap["original"] is not None: _close_owned_path(pending_swap["original"], cleanup_failures) if pending_backup is not None: - _unlink_for_cleanup(pending_backup["path"], pending_backup["identity"], - cleanup_failures) - _close_owned_path(pending_backup, cleanup_failures) + _cleanup_owned_path(pending_backup, cleanup_failures) for swap in reversed(replaced): backup = swap["backup"] - _close_write_handle(backup, cleanup_failures) _restore_backup(backup["path"], backup["identity"], swap["destination"], swap["original_identity"], swap["staged_identity"], swap["metadata"], swap["swap_started"], cleanup_failures) _close_owned_path(backup, cleanup_failures) _close_owned_path(swap["original"], cleanup_failures) for record in claimed: - _close_write_handle(record, cleanup_failures) - _unlink_for_cleanup(record["path"], record["identity"], cleanup_failures) - _close_owned_path(record, cleanup_failures) + _cleanup_owned_path(record, cleanup_failures) _note_cleanup_failures(error, cleanup_failures) raise # A successful replacement is not a successful command if an old statement diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index dd08e8249..100421b78 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1270,6 +1270,43 @@ def test_windows_refuses_to_claim_a_privacy_it_cannot_deliver(m): assert pathlib.Path(target).read_text() == "", "refused, so not truncated" +def test_windows_cleanup_closes_a_new_output_before_unlinking(m): + """Windows cannot unlink an open exclusive output, so caught cleanup + releases its pin before removal. The branch is simulated; ACL/filesystem + behavior still needs an affected Windows host.""" + with tempfile.TemporaryDirectory() as directory, pretending_windows(m) as shim: + target = pathlib.Path(directory, "out.xml") + events = [] + real_close = shim.close + real_unlink = m._unlink_for_cleanup + + def observe_close(handle): + events.append("close") + return real_close(handle) + + def observe_unlink(path, identity, failures): + assert events == ["close"] + events.append("unlink") + return real_unlink(path, identity, failures) + + shim.close = observe_close + m._unlink_for_cleanup = observe_unlink + try: + try: + m.write_outputs([(str(target), "")], True, + after_claim=lambda: (_ for _ in ()).throw( + OSError("controlled failure"))) + raise AssertionError("the controlled failure must escape") + except OSError as error: + assert "controlled failure" in str(error) + finally: + shim.close = real_close + m._unlink_for_cleanup = real_unlink + + assert events == ["close", "unlink"] + assert not target.exists() + + def test_a_windows_target_appearing_after_the_check_is_not_truncated(m): """The check and the create must be one operation. @@ -1778,6 +1815,47 @@ def test_backup_copy_refuses_a_fifo_before_reading_it(m): assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] +def test_duplicate_failure_after_claim_closes_and_removes_new_output(m): + """The claimed private descriptor is already recorded before a duplicate + is needed for writing, so descriptor exhaustion cannot leak the path.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory, "new.xml") + real_open_private = m._open_private + real_dup = m.os.dup + real_close = m.os.close + opened, closed = [], [] + + def observe_open(path, accept_inherited=False): + handle = real_open_private(path, accept_inherited) + opened.append(handle) + return handle + + def exhausted_dup(_): + raise OSError("controlled descriptor exhaustion") + + def observe_close(handle): + closed.append(handle) + return real_close(handle) + + m._open_private = observe_open + m.os.dup = exhausted_dup + m.os.close = observe_close + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the duplicate failure must escape") + except OSError as error: + assert "controlled descriptor exhaustion" in str(error) + finally: + m._open_private = real_open_private + m.os.dup = real_dup + m.os.close = real_close + + assert len(opened) == 1 + assert opened[0] in closed + assert not destination.exists() + + def test_backup_refuses_when_original_metadata_cannot_be_recorded(m): """A rollback cannot claim to restore metadata it was unable to capture.""" if not hasattr(m.os, "listxattr"): From 0ebb77428605afe6f4353ae5e1443597779cfca9 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 18:53:27 +0530 Subject: [PATCH 11/59] fix(import): disclose rollback metadata scope --- scripts/bank_statement_import.py | 27 +++++++++-- scripts/bank_statement_import.test.py | 70 ++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index f5c4d048e..9dcb4e57d 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1690,7 +1690,8 @@ def _copy_private_backup(source_path, original_identity, backup_handle): def _restore_backup(backup, backup_identity, destination, original_identity, - staged_identity, metadata, swap_started, failures): + staged_identity, metadata, swap_started, failures, + metadata_scope_warnings): """Restore an owned private backup after a caught swap failure. `os.replace` can report an exception after the filesystem call took effect. @@ -1741,6 +1742,7 @@ def _restore_backup(backup, backup_identity, destination, original_identity, _restore_metadata(restore_handle, metadata) finally: os.close(restore_handle) + metadata_scope_warnings.append(destination) except (OSError, Refusal): failures.append(destination) @@ -1759,6 +1761,21 @@ def _note_cleanup_failures(error, failures): error.add_note(message) +def _note_rollback_metadata_scope(error, restored_paths): + """Disclose metadata classes not established by this caught rollback.""" + if not restored_paths: + return + restored = ", ".join(sorted(set(restored_paths))) + message = ( + "rollback restored bytes and captured portable metadata for: " + f"{restored}; extended ACLs and file flags were not verified" + ) + if isinstance(error, Refusal): + error.code = f"{error.code}\n{message}" + else: + error.add_note(message) + + def write_outputs(targets, accept_inherited=False, after_claim=None): """Claim **every** destination, then write them. All of them or none. @@ -1899,6 +1916,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # earlier committed swaps. Cleanup failures remain attached to the # original exception with their recoverable locations. cleanup_failures = [] + metadata_scope_warnings = [] if pending_swap is not None: backup = pending_swap["backup"] _restore_backup(backup["path"], backup["identity"], @@ -1906,7 +1924,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_swap["original_identity"], pending_swap["staged_identity"], pending_swap["metadata"], - pending_swap["swap_started"], cleanup_failures) + pending_swap["swap_started"], cleanup_failures, + metadata_scope_warnings) _close_owned_path(backup, cleanup_failures) if pending_swap["original"] is not None: _close_owned_path(pending_swap["original"], cleanup_failures) @@ -1916,12 +1935,14 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): backup = swap["backup"] _restore_backup(backup["path"], backup["identity"], swap["destination"], swap["original_identity"], swap["staged_identity"], - swap["metadata"], swap["swap_started"], cleanup_failures) + swap["metadata"], swap["swap_started"], cleanup_failures, + metadata_scope_warnings) _close_owned_path(backup, cleanup_failures) _close_owned_path(swap["original"], cleanup_failures) for record in claimed: _cleanup_owned_path(record, cleanup_failures) _note_cleanup_failures(error, cleanup_failures) + _note_rollback_metadata_scope(error, metadata_scope_warnings) raise # A successful replacement is not a successful command if an old statement # survives under an undisclosed random name. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 100421b78..80a8fb5a0 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1642,7 +1642,11 @@ def fail_second_swap_and_restore(src, dst): (str(second), "new second")]) raise AssertionError("the second target's swap must fail") except OSError as error: - assert not getattr(error, "__notes__", []) + notes = getattr(error, "__notes__", []) + assert len(notes) == 1 + assert str(first) in notes[0] + assert "extended ACLs and file flags were not verified" in notes[0] + assert ".bak" not in notes[0] finally: m.os.replace = real_replace @@ -1651,6 +1655,36 @@ def fail_second_swap_and_restore(src, dst): assert sorted(path.name for path in root.iterdir()) == ["first.xml", "second.csv"] +def test_refusal_reports_rollback_metadata_scope_in_its_visible_code(m): + """Refusal is SystemExit, so rollback scope must be added to `code`, not + only an exception note that an unhandled process would omit.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.csv" + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + + def replace_first_then_refuse_second(src, dst): + if str(src).endswith(".part") and os.path.basename(dst) == "second.csv": + raise m.Refusal("controlled_refusal", "controlled second swap refusal") + return real_replace(src, dst) + + m.os.replace = replace_first_then_refuse_second + try: + refusal = refuses(m, "controlled_refusal", m.write_outputs, + [(str(first), "new first"), + (str(second), "new second")]) + finally: + m.os.replace = real_replace + + assert str(first) in str(refusal.code) + assert "extended ACLs and file flags were not verified" in str(refusal.code) + assert first.read_text() == "first old" + assert second.read_text() == "second old" + + def test_write_outputs_reports_a_retained_backup_after_commit(m): """Successful replacement is not a successful command when cleanup leaves prior bank-statement bytes at an undisclosed random backup path.""" @@ -1728,6 +1762,40 @@ def retain_backup(path, identity, failures): assert ".bak" in done.stderr, done.stderr +def test_refusal_reports_rollback_metadata_scope_on_stderr(m): + """The ACL/file-flag limitation must survive unhandled SystemExit + rendering, where Python omits ordinary exception notes.""" + program = f'''\ +import importlib.util +import os +import pathlib +import tempfile + +script = {str(SCRIPT)!r} +spec = importlib.util.spec_from_file_location("bank_statement_import_subprocess", script) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.csv" + first.write_text("first old") + second.write_text("second old") + real_replace = module.os.replace + def replace_first_then_refuse_second(src, dst): + if str(src).endswith(".part") and os.path.basename(dst) == "second.csv": + raise module.Refusal("controlled_refusal", "controlled second swap refusal") + return real_replace(src, dst) + module.os.replace = replace_first_then_refuse_second + module.write_outputs([(str(first), "new first"), (str(second), "new second")]) +''' + done = subprocess.run([sys.executable, "-c", program], text=True, + capture_output=True, check=False) + assert done.returncode != 0 + assert "controlled_refusal" in done.stderr, done.stderr + assert "extended ACLs and file flags were not verified" in done.stderr, done.stderr + + def test_a_failed_swap_rolls_back_every_staged_replacement(m): """Two existing destinations are both staged; the first's swap succeeds and the second's fails. The rollback used to run only the un-staged From 5e054071ff2529937dbd046bccf120e726c53ec0 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 19:16:59 +0530 Subject: [PATCH 12/59] fix(import): reconcile interrupted output cleanup --- scripts/bank_statement_import.py | 105 ++++++++++++---- scripts/bank_statement_import.test.py | 173 ++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 26 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 9dcb4e57d..41a3db277 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1747,18 +1747,31 @@ def _restore_backup(backup, backup_identity, destination, original_identity, failures.append(destination) +def _append_cleanup_detail(error, message): + """Keep recovery details visible on Python versions without add_note.""" + # Python prints `SystemExit.code`, not exception notes. A Refusal is a + # SystemExit so that command-line validation exits without a traceback; + # put the retained location in its visible code rather than hiding it in + # an unrendered note. + if isinstance(error, Refusal): + error.code = f"{error.code}\n{message}" + return + add_note = getattr(error, "add_note", None) + if callable(add_note): + add_note(message) + return + # BaseException.add_note arrived in Python 3.11. OSError can render cached + # errno, strerror, and filename fields instead of its mutable `args`, so + # retain the original exception intact and write the recovery detail where + # an unhandled CLI failure will still show it on Python 3.10. + print(message, file=sys.stderr) + + def _note_cleanup_failures(error, failures): if failures: retained = ", ".join(sorted(set(failures))) message = "output cleanup or rollback failed; retained path(s): " + retained - # Python prints `SystemExit.code`, not exception notes. A Refusal is a - # SystemExit so that command-line validation exits without a traceback; - # put the retained location in its visible code rather than hiding it in - # an unrendered note. - if isinstance(error, Refusal): - error.code = f"{error.code}\n{message}" - else: - error.add_note(message) + _append_cleanup_detail(error, message) def _note_rollback_metadata_scope(error, restored_paths): @@ -1770,10 +1783,39 @@ def _note_rollback_metadata_scope(error, restored_paths): "rollback restored bytes and captured portable metadata for: " f"{restored}; extended ACLs and file flags were not verified" ) - if isinstance(error, Refusal): - error.code = f"{error.code}\n{message}" - else: - error.add_note(message) + _append_cleanup_detail(error, message) + + +def _cleanup_committed_outputs(replaced, claimed, failures): + """Remove old private copies after every replacement has committed.""" + for swap in replaced: + backup = swap["backup"] + _unlink_for_cleanup(backup["path"], backup["identity"], failures) + _close_owned_path(backup, failures) + _close_owned_path(swap["original"], failures) + for record in claimed: + _close_owned_path(record, failures) + + +def _reconcile_interrupted_committed_cleanup(replaced, claimed, failures): + """Close pins and disclose owned old copies without undoing a commit.""" + for swap in replaced: + backup = swap["backup"] + try: + if _entry_identity(backup["path"]) == backup["identity"]: + failures.append(str(backup["path"])) + elif os.path.lexists(backup["path"]): + failures.append(str(backup["path"])) + except FileNotFoundError: + pass + except OSError: + if os.path.lexists(backup["path"]): + failures.append(str(backup["path"])) + finally: + _close_owned_path(backup, failures) + _close_owned_path(swap["original"], failures) + for record in claimed: + _close_owned_path(record, failures) def write_outputs(targets, accept_inherited=False, after_claim=None): @@ -1832,6 +1874,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # unlink it only while its identity still equals record["identity"]. pending_backup = None pending_swap = None + cleanup_failures = [] + metadata_scope_warnings = [] + committed = False try: for path, _ in targets: if os.path.exists(path): @@ -1911,12 +1956,26 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): os.replace(temporary["path"], real_path) replaced.append(pending_swap) pending_swap = None + # The final replacement is the boundary between rollback and committed + # cleanup. Keep it in this same handler so an interrupt before cleanup + # starts cannot skip both recovery paths. + committed = True + _cleanup_committed_outputs(replaced, claimed, cleanup_failures) + if cleanup_failures: + raise OutputCleanupFailure(cleanup_failures) except BaseException as error: + if committed: + # This is after the transaction committed. Never call + # `_restore_backup` here: an interrupt during old-copy cleanup must + # preserve the new output, close every ownership pin, and identify + # any old private copy that still needs operator cleanup. + _reconcile_interrupted_committed_cleanup( + replaced, claimed, cleanup_failures) + _note_cleanup_failures(error, cleanup_failures) + raise # Reconcile a swap which may have completed before raising, then undo # earlier committed swaps. Cleanup failures remain attached to the # original exception with their recoverable locations. - cleanup_failures = [] - metadata_scope_warnings = [] if pending_swap is not None: backup = pending_swap["backup"] _restore_backup(backup["path"], backup["identity"], @@ -1932,6 +1991,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if pending_backup is not None: _cleanup_owned_path(pending_backup, cleanup_failures) for swap in reversed(replaced): + # An interrupt can arrive after `_record_replaced_swap` appends but + # before its caller clears `pending_swap`. That one backup has + # already been reconciled above; restoring it twice risks treating + # the now-restored destination as a second transaction outcome. + if swap is pending_swap: + continue backup = swap["backup"] _restore_backup(backup["path"], backup["identity"], swap["destination"], swap["original_identity"], swap["staged_identity"], @@ -1944,18 +2009,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): _note_cleanup_failures(error, cleanup_failures) _note_rollback_metadata_scope(error, metadata_scope_warnings) raise - # A successful replacement is not a successful command if an old statement - # survives under an undisclosed random name. - cleanup_failures = [] - for swap in replaced: - backup = swap["backup"] - _unlink_for_cleanup(backup["path"], backup["identity"], cleanup_failures) - _close_owned_path(backup, cleanup_failures) - _close_owned_path(swap["original"], cleanup_failures) - for record in claimed: - _close_owned_path(record, cleanup_failures) - if cleanup_failures: - raise OutputCleanupFailure(cleanup_failures) def _check_paths(args): diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 80a8fb5a0..640ab2cf1 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -34,6 +34,7 @@ import hashlib import io import importlib.util +import inspect import os import pathlib import stat @@ -1615,6 +1616,52 @@ def interrupt_after_swap(src, dst): assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == ["previous.xml"] +def test_line_interrupt_after_recording_a_swap_restores_it_once(m): + """Trace the real line between append and clearing pending ownership.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_restore = m._restore_backup + restores = [] + _, start = inspect.getsourcelines(m.write_outputs) + clear_line = start + [ + index for index, line in enumerate(inspect.getsource(m.write_outputs).splitlines()) + if line.strip() == "pending_swap = None" + ][-1] + old_trace = sys.gettrace() + fired = False + + def interrupt_after_append(frame, event, _arg): + nonlocal fired + if (not fired and event == "line" and frame.f_code is m.write_outputs.__code__ + and frame.f_lineno == clear_line): + fired = True + raise KeyboardInterrupt("controlled interrupt after swap append") + return interrupt_after_append + + def observe_restore(*args, **kwargs): + restores.append(args[0]) + return real_restore(*args, **kwargs) + + m._restore_backup = observe_restore + sys.settrace(interrupt_after_append) + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + sys.settrace(old_trace) + m._restore_backup = real_restore + + assert fired + assert len(restores) == 1, "one swap must have one rollback owner" + assert destination.read_text() == "old bytes" + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + def test_restore_reconciles_a_backup_replace_that_raised_after_effect(m): """A restore rename can report an error after it has moved the private backup. Its new identity then proves recovery completed and must not be @@ -1717,6 +1764,132 @@ def fail_committed_backup(path): assert destination.read_text() == "new bytes" +def test_interrupt_before_committed_cleanup_keeps_new_output_and_reports_backup(m): + """The committed flag covers the line before old-copy cleanup starts.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_close = m._close_owned_path + closed_paths = [] + _, start = inspect.getsourcelines(m._cleanup_committed_outputs) + cleanup_line = start + next( + index for index, line in enumerate( + inspect.getsource(m._cleanup_committed_outputs).splitlines()) + if line.strip() == "for swap in replaced:") + old_trace = sys.gettrace() + fired = False + + def interrupt_before_cleanup(frame, event, _arg): + nonlocal fired + if (not fired and event == "line" + and frame.f_code is m._cleanup_committed_outputs.__code__ + and frame.f_lineno == cleanup_line): + fired = True + raise KeyboardInterrupt("controlled interrupt before cleanup") + return interrupt_before_cleanup + + def observe_close(record, failures): + if record.get("pin") is not None: + closed_paths.append(str(record["path"])) + return real_close(record, failures) + + m._close_owned_path = observe_close + sys.settrace(interrupt_before_cleanup) + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt as error: + notes = "\n".join(getattr(error, "__notes__", [])) + backups = list(root.glob("*.bak")) + assert len(backups) == 1 + assert backups[0].read_text() == "old bytes" + assert "retained path(s):" in notes + assert os.path.realpath(backups[0]) in notes + finally: + sys.settrace(old_trace) + m._close_owned_path = real_close + for path in root.glob("*.bak"): + path.unlink() + + assert fired + assert destination.read_text() == "new bytes" + assert os.path.realpath(destination) in closed_paths, closed_paths + assert any(path.endswith(".bak") for path in closed_paths) + assert any(path.endswith(".part") for path in closed_paths) + + +def test_interrupt_after_backup_unlink_does_not_report_a_phantom_path(m): + """A cleanup syscall may interrupt after deletion; name no absent backup.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_unlink = m.os.unlink + real_close = m._close_owned_path + closed_paths = [] + interrupted_backup = None + + def interrupt_after_backup_unlink(path): + nonlocal interrupted_backup + result = real_unlink(path) + if str(path).endswith(".bak"): + interrupted_backup = str(path) + raise KeyboardInterrupt("controlled interrupt after backup unlink") + return result + + def observe_close(record, failures): + if record.get("pin") is not None: + closed_paths.append(str(record["path"])) + return real_close(record, failures) + + m.os.unlink = interrupt_after_backup_unlink + m._close_owned_path = observe_close + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt as error: + notes = "\n".join(getattr(error, "__notes__", [])) + assert interrupted_backup is not None + assert not pathlib.Path(interrupted_backup).exists() + assert os.path.realpath(interrupted_backup) not in notes + finally: + m.os.unlink = real_unlink + m._close_owned_path = real_close + + assert destination.read_text() == "new bytes" + assert not list(root.glob("*.bak")) + assert os.path.realpath(destination) in closed_paths, closed_paths + assert any(path.endswith(".bak") for path in closed_paths) + assert any(path.endswith(".part") for path in closed_paths) + + +def test_legacy_oserror_cleanup_diagnostic_reaches_stderr(m): + """Python 3.10's OSError rendering ignores args mutations and needs stderr.""" + program = f'''\ +import errno +import importlib.util + +script = {str(SCRIPT)!r} +spec = importlib.util.spec_from_file_location("bank_statement_import_subprocess", script) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +class LegacyOSError(OSError): + add_note = None +error = LegacyOSError(errno.EIO, "controlled original failure", "/tmp/legacy-output.xml") +module._note_cleanup_failures(error, ["/tmp/owned-backup.bak"]) +raise error +''' + done = subprocess.run([sys.executable, "-c", program], text=True, + capture_output=True, check=False) + assert done.returncode != 0 + assert "controlled original failure" in done.stderr + assert "/tmp/legacy-output.xml" in done.stderr + assert "retained path(s): /tmp/owned-backup.bak" in done.stderr + + def test_refusal_reports_a_retained_backup_on_stderr(m): """Refusal inherits SystemExit, whose unhandled rendering ignores `BaseException.add_note`. Assert the CLI-visible error rather than the From 150cac91e6ed272a97ca3f4acc9ed3699a452a8e Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 19:53:01 +0530 Subject: [PATCH 13/59] fix importer ownership registration cleanup --- scripts/bank_statement_import.py | 39 ++++++++- scripts/bank_statement_import.test.py | 114 ++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 2 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 41a3db277..39d12f4dc 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1563,7 +1563,39 @@ def _open_regular_output(path, expected_identity): def _owned_path(path, handle): """Record a pathname and retain the descriptor that pins its inode.""" - return {"path": path, "identity": _fd_identity(handle), "pin": handle} + try: + identity = _fd_identity(handle) + except BaseException as error: + # The creator has already made `path`, but until its descriptor and + # pathname agree on one identity it has not entered any ownership list. + # Retry once for a transient fstat failure; if that still cannot prove + # ownership, preserve a possibly reclaimed pathname and say so rather + # than deleting a foreign file during failure cleanup. + failures = [] + try: + try: + identity = _fd_identity(handle) + except BaseException: + identity = None + if identity is None: + if os.path.lexists(path): + failures.append(str(path)) + else: + _unlink_for_cleanup(path, identity, failures) + finally: + try: + os.close(handle) + except OSError: + if os.path.lexists(path): + failures.append(str(path)) + if failures: + _append_cleanup_detail( + error, + "output ownership registration failed; retained path(s): " + + ", ".join(sorted(set(failures))), + ) + raise + return {"path": path, "identity": identity, "pin": handle} def _close_owned_path(record, failures): @@ -1988,7 +2020,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): _close_owned_path(backup, cleanup_failures) if pending_swap["original"] is not None: _close_owned_path(pending_swap["original"], cleanup_failures) - if pending_backup is not None: + if pending_backup is not None and ( + pending_swap is None + or pending_backup["path"] != pending_swap["backup"]["path"] + or pending_backup["identity"] != pending_swap["backup"]["identity"]): _cleanup_owned_path(pending_backup, cleanup_failures) for swap in reversed(replaced): # An interrupt can arrive after `_record_replaced_swap` appends but diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 640ab2cf1..726128640 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1662,6 +1662,120 @@ def observe_restore(*args, **kwargs): assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] +def test_line_interrupt_before_clearing_pending_backup_has_one_cleanup_owner(m): + """An interrupt in the alias window must not clean one backup twice.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + _, start = inspect.getsourcelines(m.write_outputs) + clear_line = start + [ + index for index, line in enumerate(inspect.getsource(m.write_outputs).splitlines()) + if line.strip() == "pending_backup = None" + ][-1] + real_cleanup = m._cleanup_owned_path + cleanup_paths = [] + old_trace = sys.gettrace() + fired = False + + def interrupt_before_clear(frame, event, _arg): + nonlocal fired + if (not fired and event == "line" and frame.f_code is m.write_outputs.__code__ + and frame.f_lineno == clear_line): + fired = True + raise KeyboardInterrupt("controlled pending-backup interrupt") + return interrupt_before_clear + + def observe_cleanup(record, failures): + cleanup_paths.append(str(record["path"])) + return real_cleanup(record, failures) + + m._cleanup_owned_path = observe_cleanup + sys.settrace(interrupt_before_clear) + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + sys.settrace(old_trace) + m._cleanup_owned_path = real_cleanup + + assert fired + assert destination.read_text() == "old bytes" + assert not any(path.endswith(".bak") for path in cleanup_paths) + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + +def test_ownership_registration_failure_reconciles_a_created_path_and_closes_its_pin(m): + """The creator is not in an outer cleanup list until fstat succeeds.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + real_identity = m._fd_identity + real_close = m.os.close + closed = [] + for position, failure in enumerate((OSError("fstat I/O"), + KeyboardInterrupt("fstat interrupt"))): + path = root / f"fresh-{position}.xml" + handle = m._open_private(path) + calls = 0 + + def fail_once(candidate): + nonlocal calls + calls += 1 + if calls == 1: + raise failure + return real_identity(candidate) + + def observe_close(candidate): + closed.append(candidate) + return real_close(candidate) + + m._fd_identity = fail_once + m.os.close = observe_close + try: + try: + m._owned_path(path, handle) + raise AssertionError("the original identity failure must escape") + except BaseException as error: + assert error is failure + finally: + m._fd_identity = real_identity + m.os.close = real_close + + assert not path.exists(), "a reconciled fresh output must not strand a refusal" + assert handle in closed + + +def test_ownership_registration_preserves_an_unproven_reclaimed_path(m): + """When fstat cannot establish ownership, foreign bytes survive visibly.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path = root / "fresh.xml" + foreign = root / "foreign.xml" + handle = m._open_private(path) + foreign.write_text("foreign writer bytes") + os.replace(foreign, path) + real_identity = m._fd_identity + + def fail_identity(_handle): + raise OSError("persistent fstat I/O") + + m._fd_identity = fail_identity + try: + try: + m._owned_path(path, handle) + raise AssertionError("the identity failure must escape") + except OSError as error: + notes = "\n".join(getattr(error, "__notes__", [])) + finally: + m._fd_identity = real_identity + + assert path.read_text() == "foreign writer bytes" + assert str(path) in notes + + def test_restore_reconciles_a_backup_replace_that_raised_after_effect(m): """A restore rename can report an error after it has moved the private backup. Its new identity then proves recovery completed and must not be From 009bd291f9b7937823a124db7ba37b51c582e4ea Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 19:55:57 +0530 Subject: [PATCH 14/59] preserve existing output on pin failure --- scripts/bank_statement_import.py | 35 +++++++++++++---------- scripts/bank_statement_import.test.py | 40 ++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 39d12f4dc..f60b89c68 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1561,8 +1561,13 @@ def _open_regular_output(path, expected_identity): raise -def _owned_path(path, handle): - """Record a pathname and retain the descriptor that pins its inode.""" +def _owned_path(path, handle, *, created): + """Record a pathname and retain the descriptor that pins its inode. + + `created` is explicit because a registration failure has opposite cleanup + authority for a new private path and an existing output. Only the former + may be unlinked while recovery establishes descriptor ownership. + """ try: identity = _fd_identity(handle) except BaseException as error: @@ -1577,16 +1582,17 @@ def _owned_path(path, handle): identity = _fd_identity(handle) except BaseException: identity = None - if identity is None: - if os.path.lexists(path): - failures.append(str(path)) - else: - _unlink_for_cleanup(path, identity, failures) + if created: + if identity is None: + if os.path.lexists(path): + failures.append(str(path)) + else: + _unlink_for_cleanup(path, identity, failures) finally: try: os.close(handle) except OSError: - if os.path.lexists(path): + if created and os.path.lexists(path): failures.append(str(path)) if failures: _append_cleanup_detail( @@ -1920,14 +1926,14 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle, temporary = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") - record = _owned_path(temporary, handle) + record = _owned_path(temporary, handle, created=True) claimed.append(record) staged.append({"temporary": record, "supplied_path": path, "real_path": real_path, "original_identity": _file_identity(real_path)}) else: handle = _open_private(path, accept_inherited) - record = _owned_path(path, handle) + record = _owned_path(path, handle, created=True) claimed.append(record) if after_claim is not None: after_claim() @@ -1947,7 +1953,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): backup_handle, backup = tempfile.mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") - pending_backup = _owned_path(backup, backup_handle) + pending_backup = _owned_path(backup, backup_handle, created=True) pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, "staged_identity": temporary["identity"], @@ -1957,7 +1963,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # This ownership pin both prevents original-inode ABA reuse and # captures metadata before the backup read can update atime. original_handle = _open_regular_output(real_path, original_identity) - pending_swap["original"] = _owned_path(real_path, original_handle) + pending_swap["original"] = _owned_path( + real_path, original_handle, created=False) pending_swap["metadata"] = _metadata_from_handle( real_path, original_handle) _copy_private_backup(real_path, original_identity, backup_handle) @@ -2021,9 +2028,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if pending_swap["original"] is not None: _close_owned_path(pending_swap["original"], cleanup_failures) if pending_backup is not None and ( - pending_swap is None - or pending_backup["path"] != pending_swap["backup"]["path"] - or pending_backup["identity"] != pending_swap["backup"]["identity"]): + pending_swap is None or pending_backup is not pending_swap["backup"]): _cleanup_owned_path(pending_backup, cleanup_failures) for swap in reversed(replaced): # An interrupt can arrive after `_record_replaced_swap` appends but diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 726128640..ce200e5ef 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1736,7 +1736,7 @@ def observe_close(candidate): m.os.close = observe_close try: try: - m._owned_path(path, handle) + m._owned_path(path, handle, created=True) raise AssertionError("the original identity failure must escape") except BaseException as error: assert error is failure @@ -1765,7 +1765,7 @@ def fail_identity(_handle): m._fd_identity = fail_identity try: try: - m._owned_path(path, handle) + m._owned_path(path, handle, created=True) raise AssertionError("the identity failure must escape") except OSError as error: notes = "\n".join(getattr(error, "__notes__", [])) @@ -1776,6 +1776,38 @@ def fail_identity(_handle): assert str(path) in notes +def test_original_pin_registration_failure_preserves_existing_output(m): + """The original pin is not newly created cleanup authority.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_identity = m._fd_identity + calls = 0 + + def fail_original_pin(handle): + nonlocal calls + calls += 1 + # Staged output and private backup register first. The third pin + # is the old destination opened for backup and metadata capture. + if calls == 3: + raise OSError("controlled original pin fstat failure") + return real_identity(handle) + + m._fd_identity = fail_original_pin + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled original-pin failure must escape") + except OSError as error: + assert "controlled original pin fstat failure" in str(error) + finally: + m._fd_identity = real_identity + + assert destination.read_text() == "old bytes" + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + def test_restore_reconciles_a_backup_replace_that_raised_after_effect(m): """A restore rename can report an error after it has moved the private backup. Its new identity then proves recovery completed and must not be @@ -2315,8 +2347,8 @@ def observe_closes(handle): closed_handles.append(handle) return real_close(handle) - def observe_owned_path(path, handle): - record = real_owned_path(path, handle) + def observe_owned_path(path, handle, *, created): + record = real_owned_path(path, handle, created=created) if os.path.basename(path).startswith("first.xml."): owned_handles[pathlib.Path(path).suffix] = record["pin"] return record From 633f8fcdf68be43061f801ccf527e0a497efb865 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 12 Sep 2026 23:58:16 +0530 Subject: [PATCH 15/59] Reconcile cleanup paths and preserve pre-swap metadata --- scripts/bank_statement_import.py | 56 +++++++++++------- scripts/bank_statement_import.test.py | 81 +++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 21 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index f60b89c68..d222bd366 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1550,6 +1550,12 @@ def _open_regular_output(path, expected_identity): "output_not_regular", f"{path}: an existing output must be a regular file", ) + if stat_result.st_nlink != 1: + raise Refusal( + "output_has_multiple_links", + f"{path}: replacement requires a single-link output; rollback " + "cannot preserve hard-link topology", + ) if (stat_result.st_dev, stat_result.st_ino) != expected_identity: raise Refusal( "output_path_changed", @@ -1613,7 +1619,16 @@ def _close_owned_path(record, failures): try: os.close(handle) except OSError: - failures.append(record["path"]) + # close can fail after taking effect. Only an extant owned entry is a + # retained-path failure; an already-unlinked backup has no such path. + try: + if _entry_identity(record["path"]) == record["identity"]: + failures.append(record["path"]) + except FileNotFoundError: + pass + except OSError: + if os.path.lexists(record["path"]): + failures.append(record["path"]) def _cleanup_owned_path(record, failures): @@ -1727,9 +1742,7 @@ def _copy_private_backup(source_path, original_identity, backup_handle): os.close(source_handle) -def _restore_backup(backup, backup_identity, destination, original_identity, - staged_identity, metadata, swap_started, failures, - metadata_scope_warnings): +def _restore_backup(swap, failures, metadata_scope_warnings): """Restore an owned private backup after a caught swap failure. `os.replace` can report an exception after the filesystem call took effect. @@ -1737,16 +1750,26 @@ def _restore_backup(backup, backup_identity, destination, original_identity, A different inode may be a foreign writer's success, so keep the private backup and report the conflict rather than overwriting it. """ - if not swap_started: - _unlink_for_cleanup(backup, backup_identity, failures) - return + backup, backup_identity = swap["backup"]["path"], swap["backup"]["identity"] + destination, original_identity = swap["destination"], swap["original_identity"] + staged_identity, metadata = swap["staged_identity"], swap["metadata"] try: current_identity = _file_identity(destination) except OSError: current_identity = None - if current_identity == original_identity: - # The replace did not take effect, or a previous reconciliation already - # restored it. The extra private copy is ours to remove. + if not swap["swap_started"] or current_identity == original_identity: + # Backup reads can update atime before any swap. Restore that effect + # through the original inode pin, never through a possibly foreign path. + # Keep the current mtime and all other metadata: this read did not alter + # them, and restoring their old values could erase an external change. + original = swap["original"] + if original is not None and metadata is not None: + try: + handle = original["pin"] + current = os.fstat(handle) + os.utime(handle, ns=(metadata["atime_ns"], current.st_mtime_ns)) + except OSError: + failures.append(destination) _unlink_for_cleanup(backup, backup_identity, failures) return if current_identity != staged_identity: @@ -2017,13 +2040,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # original exception with their recoverable locations. if pending_swap is not None: backup = pending_swap["backup"] - _restore_backup(backup["path"], backup["identity"], - pending_swap["destination"], - pending_swap["original_identity"], - pending_swap["staged_identity"], - pending_swap["metadata"], - pending_swap["swap_started"], cleanup_failures, - metadata_scope_warnings) + _restore_backup(pending_swap, cleanup_failures, metadata_scope_warnings) _close_owned_path(backup, cleanup_failures) if pending_swap["original"] is not None: _close_owned_path(pending_swap["original"], cleanup_failures) @@ -2038,10 +2055,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if swap is pending_swap: continue backup = swap["backup"] - _restore_backup(backup["path"], backup["identity"], swap["destination"], - swap["original_identity"], swap["staged_identity"], - swap["metadata"], swap["swap_started"], cleanup_failures, - metadata_scope_warnings) + _restore_backup(swap, cleanup_failures, metadata_scope_warnings) _close_owned_path(backup, cleanup_failures) _close_owned_path(swap["original"], cleanup_failures) for record in claimed: diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index ce200e5ef..0a12da86f 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2406,6 +2406,87 @@ def replace_then_conflict(src, dst): assert second.read_text() == "second old" +def test_committed_close_after_effect_does_not_report_missing_backup(m): + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + destination.write_text("old bytes") + real_close = m.os.close + real_owned = m._owned_path + backup_handles = set() + fired = False + + def observe_owned(path, handle, *, created): + record = real_owned(path, handle, created=created) + if str(path).endswith(".bak"): + backup_handles.add(handle) + return record + + def close_then_error(handle): + nonlocal fired + real_close(handle) + if handle in backup_handles and not fired: + fired = True + raise OSError("controlled close after effect") + + m._owned_path, m.os.close = observe_owned, close_then_error + try: + m.write_outputs([(str(destination), "new bytes")]) + finally: + m._owned_path, m.os.close = real_owned, real_close + assert fired + assert destination.read_text() == "new bytes" + assert list(pathlib.Path(directory).iterdir()) == [destination] + + +def test_existing_hard_link_output_is_refused_with_topology_unchanged(m): + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + alias = pathlib.Path(directory) / "alias.xml" + destination.write_text("old bytes") + os.link(destination, alias) + refuses(m, "output_has_multiple_links", m.write_outputs, + [(str(destination), "new bytes")]) + assert os.path.samefile(destination, alias) + assert destination.read_text() == alias.read_text() == "old bytes" + assert sorted(p.name for p in pathlib.Path(directory).iterdir()) == ["alias.xml", "output.xml"] + + +def test_preswap_abort_restores_access_time_on_the_original_pin(m): + # Inject the access-time effect explicitly so this covers noatime hosts too. + for abort_before_replace in (True, False): + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + destination.write_text("old bytes") + old_atime = 1_600_000_000_000_000_000 + old_mtime = 1_700_000_000_000_000_000 + os.utime(destination, ns=(old_atime, old_mtime)) + real_copy, real_replace = m._copy_private_backup, m.os.replace + + def copy_with_access_time_effect(*args): + real_copy(*args) + os.utime(destination, ns=(old_mtime, old_mtime)) + if abort_before_replace: + raise OSError("controlled copy abort") + + def refuse_replace(*_args): + raise OSError("controlled replace before effect") + + m._copy_private_backup, m.os.replace = copy_with_access_time_effect, refuse_replace + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled abort must escape") + except OSError as error: + assert str(error).startswith("controlled") + finally: + m._copy_private_backup, m.os.replace = real_copy, real_replace + final_stat = destination.stat() + assert final_stat.st_atime_ns == old_atime + assert final_stat.st_mtime_ns == old_mtime + assert destination.read_text() == "old bytes" + assert list(pathlib.Path(directory).iterdir()) == [destination] + + def test_rollback_restores_original_output_metadata(m): """A caught later swap failure restores the former bytes and portable metadata, while a retained pre-rollback backup stays private.""" From 3d3bb9871385858e8df2a7c4aafe7cbb8cfc97aa Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 00:27:09 +0530 Subject: [PATCH 16/59] fix(import): preserve link and cleanup diagnostics --- scripts/bank_statement_import.py | 76 +++++++++++++++++++++------ scripts/bank_statement_import.test.py | 67 +++++++++++++++++++++++ 2 files changed, 128 insertions(+), 15 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index d222bd366..46bc34cf2 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -131,7 +131,7 @@ class OutputCleanupFailure(OSError): """Committed output is present, but an old sensitive copy remains. `retained_paths` gives the operator the exact private backup location to - protect or remove. A normal return would conceal that copy. + protect or remove. A normal return would conceal that copy. """ def __init__(self, retained_paths): @@ -142,6 +142,17 @@ def __init__(self, retained_paths): ) +class OutputDescriptorCloseFailure(OSError): + """New output committed, but an ownership descriptor close was uncertain.""" + + def __init__(self, output_paths): + self.output_paths = tuple(output_paths) + super().__init__( + "output committed; ownership descriptor close failed for " + + ", ".join(self.output_paths) + ) + + # --------------------------------------------------------------------------- # # PDF -> rows # # --------------------------------------------------------------------------- # @@ -1693,6 +1704,22 @@ def _restore_metadata(handle, metadata): os.setxattr(handle, name, value) +def _pinned_original_still_has_one_link(record): + """Refuse if the original gained a hard link after its first pin.""" + stat_result = os.fstat(record["pin"]) + if (stat_result.st_dev, stat_result.st_ino) != record["identity"]: + raise Refusal( + "output_path_changed", + f"{record['path']} changed while its rollback copy was prepared", + ) + if stat_result.st_nlink != 1: + raise Refusal( + "output_has_multiple_links", + f"{record['path']}: replacement requires a single-link output; rollback " + "cannot preserve hard-link topology", + ) + + def _copy_private_backup(source_path, original_identity, backup_handle): """Copy the original inode into an owner-only backup already opened O_EXCL. @@ -1847,36 +1874,40 @@ def _note_rollback_metadata_scope(error, restored_paths): _append_cleanup_detail(error, message) -def _cleanup_committed_outputs(replaced, claimed, failures): +def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_failures): """Remove old private copies after every replacement has committed.""" for swap in replaced: backup = swap["backup"] - _unlink_for_cleanup(backup["path"], backup["identity"], failures) - _close_owned_path(backup, failures) - _close_owned_path(swap["original"], failures) + _unlink_for_cleanup(backup["path"], backup["identity"], retained_failures) + _close_owned_path(backup, retained_failures) + _close_owned_path(swap["original"], retained_failures) for record in claimed: - _close_owned_path(record, failures) + # A claimed path did not exist before this run. Its close failure cannot + # retain a prior sensitive copy, so keep that diagnostic distinct from + # a backup that an operator must protect or remove. + _close_owned_path(record, descriptor_failures) -def _reconcile_interrupted_committed_cleanup(replaced, claimed, failures): +def _reconcile_interrupted_committed_cleanup( + replaced, claimed, retained_failures, descriptor_failures): """Close pins and disclose owned old copies without undoing a commit.""" for swap in replaced: backup = swap["backup"] try: if _entry_identity(backup["path"]) == backup["identity"]: - failures.append(str(backup["path"])) + retained_failures.append(str(backup["path"])) elif os.path.lexists(backup["path"]): - failures.append(str(backup["path"])) + retained_failures.append(str(backup["path"])) except FileNotFoundError: pass except OSError: if os.path.lexists(backup["path"]): - failures.append(str(backup["path"])) + retained_failures.append(str(backup["path"])) finally: - _close_owned_path(backup, failures) - _close_owned_path(swap["original"], failures) + _close_owned_path(backup, retained_failures) + _close_owned_path(swap["original"], retained_failures) for record in claimed: - _close_owned_path(record, failures) + _close_owned_path(record, descriptor_failures) def write_outputs(targets, accept_inherited=False, after_claim=None): @@ -1936,6 +1967,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_backup = None pending_swap = None cleanup_failures = [] + descriptor_close_failures = [] metadata_scope_warnings = [] committed = False try: @@ -2014,6 +2046,11 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} rollback copy changed before replacement", ) + # The first pin checked that the original was single-linked. A + # backup hook can still add an alias before the commit boundary; + # recheck this pinned inode so replacement never detaches a new + # hard link while reporting a successful overwrite. + _pinned_original_still_has_one_link(pending_swap["original"]) pending_swap["swap_started"] = True os.replace(temporary["path"], real_path) replaced.append(pending_swap) @@ -2022,9 +2059,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. committed = True - _cleanup_committed_outputs(replaced, claimed, cleanup_failures) + _cleanup_committed_outputs( + replaced, claimed, cleanup_failures, descriptor_close_failures) if cleanup_failures: raise OutputCleanupFailure(cleanup_failures) + if descriptor_close_failures: + raise OutputDescriptorCloseFailure(descriptor_close_failures) except BaseException as error: if committed: # This is after the transaction committed. Never call @@ -2032,8 +2072,14 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # preserve the new output, close every ownership pin, and identify # any old private copy that still needs operator cleanup. _reconcile_interrupted_committed_cleanup( - replaced, claimed, cleanup_failures) + replaced, claimed, cleanup_failures, descriptor_close_failures) _note_cleanup_failures(error, cleanup_failures) + if descriptor_close_failures and not isinstance(error, OutputDescriptorCloseFailure): + _append_cleanup_detail( + error, + "ownership descriptor close failed for committed output(s): " + + ", ".join(sorted(set(descriptor_close_failures))), + ) raise # Reconcile a swap which may have completed before raising, then undo # earlier committed swaps. Cleanup failures remain attached to the diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 0a12da86f..1cb456b89 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2644,6 +2644,73 @@ def test_ledger_key_folds_exactly_what_its_docstring_claims(m): assert not same("A\u2013B", "A B"), "en dash is not an ASCII hyphen" +def test_write_outputs_refuses_a_hard_link_added_after_backup_copy(m): + """The commit check must see a link created by a backup-time hook.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "output.xml" + alias = root / "alias.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def add_link_after_backup(*args): + result = real_copy(*args) + os.link(destination, alias) + return result + + m._copy_private_backup = add_link_after_backup + try: + refuses( + m, + "output_has_multiple_links", + m.write_outputs, + [(str(destination), "new bytes")], + ) + finally: + m._copy_private_backup = real_copy + + assert os.path.samefile(destination, alias) + assert destination.read_text() == alias.read_text() == "old bytes" + assert sorted(path.name for path in root.iterdir()) == ["alias.xml", "output.xml"] + + +def test_committed_new_output_close_failure_is_not_a_retained_backup(m): + """A failed ownership-pin close does not create a prior-output backup.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + real_close = m.os.close + real_owned = m._owned_path + output_handles = set() + fired = False + + def observe_owned(path, handle, *, created): + record = real_owned(path, handle, created=created) + if created and pathlib.Path(path) == destination: + output_handles.add(handle) + return record + + def close_then_error(handle): + nonlocal fired + real_close(handle) + if handle in output_handles and not fired: + fired = True + raise OSError("controlled close after effect") + + m._owned_path, m.os.close = observe_owned, close_then_error + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled close failure must escape") + except m.OutputDescriptorCloseFailure as failure: + assert failure.output_paths == (str(destination),) + assert "prior output retained" not in str(failure) + finally: + m._owned_path, m.os.close = real_owned, real_close + + assert fired + assert destination.read_text() == "new bytes" + assert list(pathlib.Path(directory).iterdir()) == [destination] + def main(): module = load() for name, test in sorted(globals().items()): From 77d4051aa332c334ad8534c5f5f24b6222e2d37a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 00:57:37 +0530 Subject: [PATCH 17/59] fix(import): revalidate new output path authority --- scripts/bank_statement_import.py | 45 +++++++++++++++++++++++---- scripts/bank_statement_import.test.py | 32 ++++++++++++++++++- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 46bc34cf2..6a3caebfa 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1633,12 +1633,13 @@ def _close_owned_path(record, failures): # close can fail after taking effect. Only an extant owned entry is a # retained-path failure; an already-unlinked backup has no such path. try: - if _entry_identity(record["path"]) == record["identity"]: + cleanup_path = record.get("cleanup_path", record["path"]) + if _entry_identity(cleanup_path) == record["identity"]: failures.append(record["path"]) except FileNotFoundError: pass except OSError: - if os.path.lexists(record["path"]): + if os.path.lexists(record.get("cleanup_path", record["path"])): failures.append(record["path"]) @@ -1653,9 +1654,9 @@ def _cleanup_owned_path(record, failures): """ if os.name == "nt": _close_owned_path(record, failures) - _unlink_for_cleanup(record["path"], record["identity"], failures) + _unlink_for_cleanup(record.get("cleanup_path", record["path"]), record["identity"], failures) else: - _unlink_for_cleanup(record["path"], record["identity"], failures) + _unlink_for_cleanup(record.get("cleanup_path", record["path"]), record["identity"], failures) _close_owned_path(record, failures) @@ -1961,7 +1962,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): changes the path again after that check remains outside this CLI's locking authority. """ - claimed, staged, replaced = [], [], [] + claimed, staged, replaced, new_outputs = [], [], [], [] # A record is the one ownership authority for a pathname: cleanup may # unlink it only while its identity still equals record["identity"]. pending_backup = None @@ -1987,9 +1988,27 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "real_path": real_path, "original_identity": _file_identity(real_path)}) else: + supplied_path = path + canonical_path = str(pathlib.Path(supplied_path).resolve()) handle = _open_private(path, accept_inherited) - record = _owned_path(path, handle, created=True) + # Keep cleanup on the canonical inode path captured before the + # open. The supplied spelling remains an authority that must + # still resolve to that same inode at commit time. + if (str(pathlib.Path(supplied_path).resolve()) != canonical_path + or _file_identity(canonical_path) != _fd_identity(handle)): + _unlink_for_cleanup(canonical_path, _fd_identity(handle), []) + os.close(handle) + raise Refusal( + "output_path_changed", + f"{supplied_path} changed while it was being claimed", + ) + record = _owned_path(canonical_path, handle, created=True) + record["supplied_path"] = supplied_path + record["canonical_path"] = canonical_path + record["cleanup_path"] = canonical_path + record["path"] = supplied_path claimed.append(record) + new_outputs.append(record) if after_claim is not None: after_claim() for (_, text), record in zip(targets, claimed): @@ -1999,6 +2018,20 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle = os.dup(record["pin"]) with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) + for record in new_outputs: + supplied_path = record["supplied_path"] + canonical_path = record["canonical_path"] + try: + changed = (str(pathlib.Path(supplied_path).resolve()) != canonical_path + or _entry_identity(supplied_path) != record["identity"] + or _entry_identity(canonical_path) != record["identity"]) + except (FileNotFoundError, OSError): + changed = True + if changed: + raise Refusal( + "output_path_changed", + f"{supplied_path} changed before commit; no output was committed", + ) # Every payload is on disk. A private copy preserves the old bytes while # the requested destination stays present until the atomic replacement. for state in staged: diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 1cb456b89..04860d78c 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2685,7 +2685,7 @@ def test_committed_new_output_close_failure_is_not_a_retained_backup(m): def observe_owned(path, handle, *, created): record = real_owned(path, handle, created=created) - if created and pathlib.Path(path) == destination: + if created and pathlib.Path(path).resolve() == destination.resolve(): output_handles.add(handle) return record @@ -2711,6 +2711,36 @@ def close_then_error(handle): assert destination.read_text() == "new bytes" assert list(pathlib.Path(directory).iterdir()) == [destination] + +def test_new_output_unlink_after_claim_refuses_and_cleans_owned_canonical_path(m): + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + def unlink_after_claim(): + destination.unlink() + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")], False, unlink_after_claim) + assert not destination.exists() + + +def test_new_output_parent_retarget_refuses_and_preserves_foreign_path(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, foreign = root / "original", root / "foreign" + original.mkdir() + foreign.mkdir() + destination = original / "output.xml" + foreign_destination = foreign / "output.xml" + foreign_destination.write_text("foreign bytes") + def retarget_parent(): + destination.unlink() + original.rmdir() + original.symlink_to(foreign, target_is_directory=True) + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")], False, retarget_parent) + assert foreign_destination.read_text() == "foreign bytes" + assert (root / "original").is_symlink(), "foreign retarget must not be deleted" + assert (root / "original").joinpath("output.xml").read_text() == "foreign bytes" + def main(): module = load() for name, test in sorted(globals().items()): From 932887341a870d492814b2deb1dd59a201801ee5 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 01:04:13 +0530 Subject: [PATCH 18/59] fix(import): close new output path race --- scripts/bank_statement_import.py | 49 +++++++++++++++------------ scripts/bank_statement_import.test.py | 41 +++++++++++++++++++--- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 6a3caebfa..ecb2ce38d 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1994,14 +1994,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # Keep cleanup on the canonical inode path captured before the # open. The supplied spelling remains an authority that must # still resolve to that same inode at commit time. - if (str(pathlib.Path(supplied_path).resolve()) != canonical_path - or _file_identity(canonical_path) != _fd_identity(handle)): - _unlink_for_cleanup(canonical_path, _fd_identity(handle), []) - os.close(handle) - raise Refusal( - "output_path_changed", - f"{supplied_path} changed while it was being claimed", - ) record = _owned_path(canonical_path, handle, created=True) record["supplied_path"] = supplied_path record["canonical_path"] = canonical_path @@ -2009,6 +2001,17 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): record["path"] = supplied_path claimed.append(record) new_outputs.append(record) + try: + claimed_path_changed = ( + str(pathlib.Path(supplied_path).resolve()) != canonical_path + or _file_identity(canonical_path) != record["identity"]) + except (FileNotFoundError, OSError): + claimed_path_changed = True + if claimed_path_changed: + raise Refusal( + "output_path_changed", + f"{supplied_path} changed while it was being claimed", + ) if after_claim is not None: after_claim() for (_, text), record in zip(targets, claimed): @@ -2018,20 +2021,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle = os.dup(record["pin"]) with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) - for record in new_outputs: - supplied_path = record["supplied_path"] - canonical_path = record["canonical_path"] - try: - changed = (str(pathlib.Path(supplied_path).resolve()) != canonical_path - or _entry_identity(supplied_path) != record["identity"] - or _entry_identity(canonical_path) != record["identity"]) - except (FileNotFoundError, OSError): - changed = True - if changed: - raise Refusal( - "output_path_changed", - f"{supplied_path} changed before commit; no output was committed", - ) # Every payload is on disk. A private copy preserves the old bytes while # the requested destination stays present until the atomic replacement. for state in staged: @@ -2088,6 +2077,22 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): os.replace(temporary["path"], real_path) replaced.append(pending_swap) pending_swap = None + # Keep this after every staged filesystem operation and before the + # committed boundary. Any changed new path still rolls back swaps. + for record in new_outputs: + supplied_path = record["supplied_path"] + canonical_path = record["canonical_path"] + try: + changed = (str(pathlib.Path(supplied_path).resolve()) != canonical_path + or _entry_identity(supplied_path) != record["identity"] + or _entry_identity(canonical_path) != record["identity"]) + except (FileNotFoundError, OSError): + changed = True + if changed: + raise Refusal( + "output_path_changed", + f"{supplied_path} changed before commit; no output was committed", + ) # The final replacement is the boundary between rollback and committed # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 04860d78c..6028aa9b6 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2728,18 +2728,49 @@ def test_new_output_parent_retarget_refuses_and_preserves_foreign_path(m): original, foreign = root / "original", root / "foreign" original.mkdir() foreign.mkdir() - destination = original / "output.xml" foreign_destination = foreign / "output.xml" foreign_destination.write_text("foreign bytes") + destination = root / "linked" / "output.xml" + (root / "linked").symlink_to(original, target_is_directory=True) def retarget_parent(): destination.unlink() - original.rmdir() - original.symlink_to(foreign, target_is_directory=True) + (root / "linked").unlink() + (root / "linked").symlink_to(foreign, target_is_directory=True) refuses(m, "output_path_changed", m.write_outputs, [(str(destination), "new bytes")], False, retarget_parent) assert foreign_destination.read_text() == "foreign bytes" - assert (root / "original").is_symlink(), "foreign retarget must not be deleted" - assert (root / "original").joinpath("output.xml").read_text() == "foreign bytes" + assert original.exists() and not (original / "output.xml").exists() + assert (root / "linked").is_symlink(), "foreign retarget must not be deleted" + assert (root / "linked").joinpath("output.xml").read_text() == "foreign bytes" + + +def test_new_output_replace_after_claim_preserves_foreign_bytes(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, foreign = root / "output.xml", root / "foreign.xml" + def replace_after_claim(): + foreign.write_text("foreign bytes") + os.replace(foreign, destination) + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")], False, replace_after_claim) + assert destination.read_text() == "foreign bytes" + + +def test_new_output_claim_inspection_failure_cleans_owned_path(m): + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + real_identity = m._file_identity + def fail_identity(path): + if pathlib.Path(path).resolve() == destination.resolve(): + raise OSError("controlled claim inspection failure") + return real_identity(path) + m._file_identity = fail_identity + try: + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._file_identity = real_identity + assert not destination.exists() def main(): module = load() From 19d179fc6bc3285bfed83dc4e40c482bf8f36f0f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 01:15:56 +0530 Subject: [PATCH 19/59] fix: claim fresh outputs through the retained canonical path --- scripts/bank_statement_import.py | 10 +++--- scripts/bank_statement_import.test.py | 51 +++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index ecb2ce38d..62136511b 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1635,12 +1635,12 @@ def _close_owned_path(record, failures): try: cleanup_path = record.get("cleanup_path", record["path"]) if _entry_identity(cleanup_path) == record["identity"]: - failures.append(record["path"]) + failures.append(cleanup_path) except FileNotFoundError: pass except OSError: - if os.path.lexists(record.get("cleanup_path", record["path"])): - failures.append(record["path"]) + if os.path.lexists(cleanup_path): + failures.append(cleanup_path) def _cleanup_owned_path(record, failures): @@ -1990,7 +1990,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): else: supplied_path = path canonical_path = str(pathlib.Path(supplied_path).resolve()) - handle = _open_private(path, accept_inherited) + handle = _open_private(canonical_path, accept_inherited) # Keep cleanup on the canonical inode path captured before the # open. The supplied spelling remains an authority that must # still resolve to that same inode at commit time. @@ -2093,7 +2093,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} changed before commit; no output was committed", ) - # The final replacement is the boundary between rollback and committed + # Final path validation is the boundary between rollback and committed # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. committed = True diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 6028aa9b6..7446403bc 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2702,7 +2702,7 @@ def close_then_error(handle): m.write_outputs([(str(destination), "new bytes")]) raise AssertionError("the controlled close failure must escape") except m.OutputDescriptorCloseFailure as failure: - assert failure.output_paths == (str(destination),) + assert failure.output_paths == (str(destination.resolve()),) assert "prior output retained" not in str(failure) finally: m._owned_path, m.os.close = real_owned, real_close @@ -2733,7 +2733,6 @@ def test_new_output_parent_retarget_refuses_and_preserves_foreign_path(m): destination = root / "linked" / "output.xml" (root / "linked").symlink_to(original, target_is_directory=True) def retarget_parent(): - destination.unlink() (root / "linked").unlink() (root / "linked").symlink_to(foreign, target_is_directory=True) refuses(m, "output_path_changed", m.write_outputs, @@ -2772,6 +2771,54 @@ def fail_identity(path): m._file_identity = real_identity assert not destination.exists() + +def test_new_output_parent_retarget_during_open_cleans_actual_created_path(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, foreign = root / "original", root / "foreign" + original.mkdir() + foreign.mkdir() + link = root / "linked" + link.symlink_to(original, target_is_directory=True) + foreign_destination = foreign / "output.xml" + foreign_destination.write_text("foreign bytes") + real_open = m._open_private + def retarget_open(path, accept_inherited): + link.unlink() + link.symlink_to(foreign, target_is_directory=True) + return real_open(path, accept_inherited) + m._open_private = retarget_open + try: + refuses(m, "output_path_changed", m.write_outputs, + [(str(link / "output.xml"), "new bytes")]) + finally: + m._open_private = real_open + assert not (original / "output.xml").exists() + assert foreign_destination.read_text() == "foreign bytes" + + +def test_new_output_changed_during_later_swap_rolls_back_existing_output(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + fresh, existing = root / "fresh.xml", root / "existing.xml" + existing.write_text("old bytes") + real_replace = m.os.replace + def replace_then_unlink(source, destination): + result = real_replace(source, destination) + if pathlib.Path(destination).resolve() == existing.resolve() and str(source).endswith(".part"): + fresh.unlink() + return result + m.os.replace = replace_then_unlink + try: + refuses(m, "output_path_changed", m.write_outputs, + [(str(fresh), "new fresh"), (str(existing), "new existing")]) + finally: + m.os.replace = real_replace + assert not fresh.exists() + assert existing.read_text() == "old bytes" + assert list(root.iterdir()) == [existing] + + def main(): module = load() for name, test in sorted(globals().items()): From 48ecad5c9f2ca3d4529fcf8d9a51594eae644e90 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 01:45:57 +0530 Subject: [PATCH 20/59] fix: retain output cleanup uncertainty --- scripts/bank_statement_import.py | 51 +++++++++++++++++++++-- scripts/bank_statement_import.test.py | 60 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 62136511b..b7daab4de 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1540,15 +1540,21 @@ def _unlink_for_cleanup(path, owned_identity, failures): # if it still exists without making a claim about foreign bytes. if os.path.lexists(path): failures.append(str(path)) - return + return True os.unlink(path) + return True except FileNotFoundError: - pass + # A caller holding a descriptor can distinguish this from a successful + # unlink. In particular, a parent-directory rename leaves the owned + # inode live at an unknown relative name rather than making it safe to + # call cleanup complete. + return False except OSError: # A filesystem call can report an error after taking effect. Only retain # the path when reconciliation shows bytes may still be present. if os.path.lexists(path): failures.append(str(path)) + return True def _open_regular_output(path, expected_identity): @@ -1656,7 +1662,23 @@ def _cleanup_owned_path(record, failures): _close_owned_path(record, failures) _unlink_for_cleanup(record.get("cleanup_path", record["path"]), record["identity"], failures) else: - _unlink_for_cleanup(record.get("cleanup_path", record["path"]), record["identity"], failures) + cleanup_path = record.get("cleanup_path", record["path"]) + located = _unlink_for_cleanup(cleanup_path, record["identity"], failures) + if not located and "cleanup_path" in record: + # The descriptor still proves this is our fresh output, but a + # stale parent pathname cannot say where it went. Do not turn a + # missing entry into a successful cleanup or invent a replacement + # path; a parent-directory rename is outside this CLI's namespace + # authority and needs an operator-visible recovery fact. + try: + if _fd_identity(record["pin"]) == record["identity"]: + failures.append( + f"owned output could not be located after cleanup: {record['path']}" + ) + except OSError: + failures.append( + f"owned output could not be located after cleanup: {record['path']}" + ) _close_owned_path(record, failures) @@ -1721,6 +1743,21 @@ def _pinned_original_still_has_one_link(record): ) +def _pinned_backup_still_has_one_link(record): + """Refuse a rollback copy that acquired an unlocatable hard-link alias.""" + stat_result = os.fstat(record["pin"]) + if (stat_result.st_dev, stat_result.st_ino) != record["identity"]: + raise Refusal( + "output_path_changed", + f"{record['path']} rollback copy changed before replacement", + ) + if stat_result.st_nlink != 1: + raise Refusal( + "rollback_backup_has_multiple_links", + f"{record['path']}: rollback copy gained a hard-link alias before replacement", + ) + + def _copy_private_backup(source_path, original_identity, backup_handle): """Copy the original inode into an owner-only backup already opened O_EXCL. @@ -2068,6 +2105,14 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} rollback copy changed before replacement", ) + try: + _pinned_backup_still_has_one_link(pending_swap["backup"]) + except Refusal as error: + _append_cleanup_detail( + error, + "rollback copy has an unknown hard-link alias; it may retain prior output bytes", + ) + raise # The first pin checked that the original was single-linked. A # backup hook can still add an alias before the commit boundary; # recheck this pinned inode so replacement never detaches a new diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 7446403bc..7e4f588f8 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2287,6 +2287,32 @@ def test_cleanup_keeps_a_reclaimed_owned_path(m): assert failures == [str(owned)] +def test_fresh_output_parent_rename_reports_an_unlocated_owned_descriptor(m): + """A parent rename preserves a newly created inode under a name cleanup + cannot discover. The failure must disclose that fact rather than calling + stale-path ENOENT a successful rollback.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) / "before" + moved = pathlib.Path(directory) / "after" + root.mkdir() + destination = root / "output.xml" + + def rename_parent(): + os.rename(root, moved) + + try: + m.write_outputs([(str(destination), "new bytes")], after_claim=rename_parent) + raise AssertionError("a moved fresh output must refuse before commit") + except m.Refusal as refusal: + assert refusal.category == "output_path_changed" + assert "owned output could not be located after cleanup" in str(refusal.code) + + retained = moved / "output.xml" + assert retained.read_text() == "new bytes" + + def test_writer_refuses_when_the_private_backup_path_is_reclaimed(m): """The commit boundary must still name this run's backup; if it does not, do not overwrite the destination without a recoverable owned copy.""" @@ -2674,6 +2700,40 @@ def add_link_after_backup(*args): assert sorted(path.name for path in root.iterdir()) == ["alias.xml", "output.xml"] +def test_write_outputs_refuses_a_hard_link_added_to_the_private_backup(m): + """The generated backup is ownership-only until commit. A link made after + its copy finishes leaves an unknown alias with old statement bytes, so the + run must refuse and describe the retention rather than report success.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "output.xml" + alias = root / "backup-alias.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def add_link_after_backup_copy(*args): + result = real_copy(*args) + backup, = root.glob("output.xml.*.bak") + os.link(backup, alias) + return result + + m._copy_private_backup = add_link_after_backup_copy + try: + refusal = refuses( + m, + "rollback_backup_has_multiple_links", + m.write_outputs, + [(str(destination), "new bytes")], + ) + finally: + m._copy_private_backup = real_copy + + assert "unknown hard-link alias" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert alias.read_text() == "old bytes" + assert not list(root.glob("output.xml.*.bak")) + + def test_committed_new_output_close_failure_is_not_a_retained_backup(m): """A failed ownership-pin close does not create a prior-output backup.""" with tempfile.TemporaryDirectory() as directory: From df2f2d169af83c5608a907f6a31d079603f7ecf2 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 01:52:08 +0530 Subject: [PATCH 21/59] fix: reconcile unlocated output cleanup --- scripts/bank_statement_import.py | 31 +++++++++++++++--------- scripts/bank_statement_import.test.py | 35 +++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index b7daab4de..e67c73357 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1540,21 +1540,21 @@ def _unlink_for_cleanup(path, owned_identity, failures): # if it still exists without making a claim about foreign bytes. if os.path.lexists(path): failures.append(str(path)) - return True + return "reclaimed" os.unlink(path) - return True + return "removed" except FileNotFoundError: # A caller holding a descriptor can distinguish this from a successful # unlink. In particular, a parent-directory rename leaves the owned # inode live at an unknown relative name rather than making it safe to # call cleanup complete. - return False + return "missing" except OSError: # A filesystem call can report an error after taking effect. Only retain # the path when reconciliation shows bytes may still be present. if os.path.lexists(path): failures.append(str(path)) - return True + return "uncertain" def _open_regular_output(path, expected_identity): @@ -1663,15 +1663,23 @@ def _cleanup_owned_path(record, failures): _unlink_for_cleanup(record.get("cleanup_path", record["path"]), record["identity"], failures) else: cleanup_path = record.get("cleanup_path", record["path"]) - located = _unlink_for_cleanup(cleanup_path, record["identity"], failures) - if not located and "cleanup_path" in record: + failure_start = len(failures) + outcome = _unlink_for_cleanup(cleanup_path, record["identity"], failures) + if outcome == "reclaimed": + # A foreign claimant of the stale name is not a retained path of + # this output. Keep any earlier diagnostics, but replace this + # pathname with the separate pinned-inode conclusion below. + del failures[failure_start:] + if outcome in ("missing", "reclaimed") and "cleanup_path" in record: # The descriptor still proves this is our fresh output, but a # stale parent pathname cannot say where it went. Do not turn a # missing entry into a successful cleanup or invent a replacement # path; a parent-directory rename is outside this CLI's namespace # authority and needs an operator-visible recovery fact. try: - if _fd_identity(record["pin"]) == record["identity"]: + stat_result = os.fstat(record["pin"]) + if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] + and stat_result.st_nlink > 0): failures.append( f"owned output could not be located after cleanup: {record['path']}" ) @@ -2108,10 +2116,11 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): try: _pinned_backup_still_has_one_link(pending_swap["backup"]) except Refusal as error: - _append_cleanup_detail( - error, - "rollback copy has an unknown hard-link alias; it may retain prior output bytes", - ) + if error.category == "rollback_backup_has_multiple_links": + _append_cleanup_detail( + error, + "rollback copy has an unknown hard-link alias; it may retain prior output bytes", + ) raise # The first pin checked that the original was single-linked. A # backup hook can still add an alias before the commit boundary; diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 7e4f588f8..c077b5d14 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2313,6 +2313,36 @@ def rename_parent(): assert retained.read_text() == "new bytes" +def test_fresh_output_parent_rename_with_foreign_replacement_reports_only_owned_uncertainty(m): + """A stale name can be reclaimed after its parent moves. Cleanup must not + remove or present that foreign file as the location of our pinned output.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) / "before" + moved = pathlib.Path(directory) / "after" + root.mkdir() + destination = root / "output.xml" + + def rename_parent_and_reclaim_old_name(): + os.rename(root, moved) + root.mkdir() + destination.write_text("foreign bytes") + + try: + m.write_outputs( + [(str(destination), "new bytes")], after_claim=rename_parent_and_reclaim_old_name) + raise AssertionError("a moved fresh output must refuse before commit") + except m.Refusal as refusal: + assert refusal.category == "output_path_changed" + detail = str(refusal.code) + assert "owned output could not be located after cleanup" in detail + assert "retained path(s): " + str(destination) not in detail + + assert destination.read_text() == "foreign bytes" + assert (moved / "output.xml").read_text() == "new bytes" + + def test_writer_refuses_when_the_private_backup_path_is_reclaimed(m): """The commit boundary must still name this run's backup; if it does not, do not overwrite the destination without a recoverable owned copy.""" @@ -2777,8 +2807,9 @@ def test_new_output_unlink_after_claim_refuses_and_cleans_owned_canonical_path(m destination = pathlib.Path(directory) / "output.xml" def unlink_after_claim(): destination.unlink() - refuses(m, "output_path_changed", m.write_outputs, - [(str(destination), "new bytes")], False, unlink_after_claim) + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")], False, unlink_after_claim) + assert "owned output could not be located after cleanup" not in str(refusal.code) assert not destination.exists() From 9d048d380aec48fd36a90c8e66dc63a996ac10cb Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 02:02:06 +0530 Subject: [PATCH 22/59] Preserve backup cleanup diagnostics and distinguish unlinked pins --- scripts/bank_statement_import.py | 5 ++-- scripts/bank_statement_import.test.py | 37 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index e67c73357..fb04f75dd 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1665,7 +1665,7 @@ def _cleanup_owned_path(record, failures): cleanup_path = record.get("cleanup_path", record["path"]) failure_start = len(failures) outcome = _unlink_for_cleanup(cleanup_path, record["identity"], failures) - if outcome == "reclaimed": + if outcome == "reclaimed" and "cleanup_path" in record: # A foreign claimant of the stale name is not a retained path of # this output. Keep any earlier diagnostics, but replace this # pathname with the separate pinned-inode conclusion below. @@ -1754,7 +1754,8 @@ def _pinned_original_still_has_one_link(record): def _pinned_backup_still_has_one_link(record): """Refuse a rollback copy that acquired an unlocatable hard-link alias.""" stat_result = os.fstat(record["pin"]) - if (stat_result.st_dev, stat_result.st_ino) != record["identity"]: + if ((stat_result.st_dev, stat_result.st_ino) != record["identity"] + or stat_result.st_nlink == 0): raise Refusal( "output_path_changed", f"{record['path']} rollback copy changed before replacement", diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index c077b5d14..822b861ee 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2287,6 +2287,43 @@ def test_cleanup_keeps_a_reclaimed_owned_path(m): assert failures == [str(owned)] +def test_pinned_backup_reclaimed_path_retains_cleanup_diagnostic(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "rollback.bak" + moved = pathlib.Path(directory) / "moved.bak" + path.write_text("prior output") + identity = m._entry_identity(path) + handle = os.open(path, os.O_RDONLY) + record = {"path": path, "identity": identity, "pin": handle} + path.rename(moved) + path.write_text("foreign bytes") + failures = [] + m._cleanup_owned_path(record, failures) + assert failures == [str(path)] + assert record["pin"] is None + assert path.read_text() == "foreign bytes" + assert moved.read_text() == "prior output" + + +def test_unlinked_pinned_backup_does_not_claim_a_hard_link_alias(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "rollback.bak" + path.write_text("prior output") + identity = m._entry_identity(path) + handle = os.open(path, os.O_RDONLY) + try: + path.unlink() + refusal = refuses(m, "output_path_changed", m._pinned_backup_still_has_one_link, + {"path": path, "identity": identity, "pin": handle}) + assert "hard-link alias" not in str(refusal.code) + finally: + os.close(handle) + + def test_fresh_output_parent_rename_reports_an_unlocated_owned_descriptor(m): """A parent rename preserves a newly created inode under a name cleanup cannot discover. The failure must disclose that fact rather than calling From 0674eccc7fc27a76fd0dc2ca9863fa4bbb7d7bc5 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 02:26:13 +0530 Subject: [PATCH 23/59] Revalidate every owned output and backup at commit --- scripts/bank_statement_import.py | 52 ++++++----- scripts/bank_statement_import.test.py | 125 ++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 20 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index fb04f75dd..e9bf30ffb 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1751,20 +1751,27 @@ def _pinned_original_still_has_one_link(record): ) -def _pinned_backup_still_has_one_link(record): - """Refuse a rollback copy that acquired an unlocatable hard-link alias.""" +def _require_single_owned_link(record, description, alias_category): + """Refuse an owned output that moved or gained an unlocatable alias.""" stat_result = os.fstat(record["pin"]) if ((stat_result.st_dev, stat_result.st_ino) != record["identity"] or stat_result.st_nlink == 0): raise Refusal( "output_path_changed", - f"{record['path']} rollback copy changed before replacement", + f"{record['path']} {description} changed before commit", ) if stat_result.st_nlink != 1: - raise Refusal( - "rollback_backup_has_multiple_links", - f"{record['path']}: rollback copy gained a hard-link alias before replacement", + error = Refusal( + alias_category, + f"{record['path']}: {description} gained a hard-link alias before commit", ) + _append_cleanup_detail( + error, f"{description} has an unknown hard-link alias; it may retain output bytes") + raise error + + +def _pinned_backup_still_has_one_link(record): + _require_single_owned_link(record, "rollback copy", "rollback_backup_has_multiple_links") def _copy_private_backup(source_path, original_identity, backup_handle): @@ -2008,7 +2015,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): changes the path again after that check remains outside this CLI's locking authority. """ - claimed, staged, replaced, new_outputs = [], [], [], [] + claimed, staged, replaced = [], [], [] # A record is the one ownership authority for a pathname: cleanup may # unlink it only while its identity still equals record["identity"]. pending_backup = None @@ -2029,6 +2036,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") record = _owned_path(temporary, handle, created=True) + record["supplied_path"] = path + record["canonical_path"] = real_path + record["cleanup_path"] = temporary claimed.append(record) staged.append({"temporary": record, "supplied_path": path, "real_path": real_path, @@ -2046,7 +2056,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): record["cleanup_path"] = canonical_path record["path"] = supplied_path claimed.append(record) - new_outputs.append(record) try: claimed_path_changed = ( str(pathlib.Path(supplied_path).resolve()) != canonical_path @@ -2108,21 +2117,15 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} staged output changed before replacement", ) + _require_single_owned_link( + temporary, "staged output", "staged_output_has_multiple_links") if _entry_identity(pending_swap["backup"]["path"]) != \ pending_swap["backup"]["identity"]: raise Refusal( "output_path_changed", f"{supplied_path} rollback copy changed before replacement", ) - try: - _pinned_backup_still_has_one_link(pending_swap["backup"]) - except Refusal as error: - if error.category == "rollback_backup_has_multiple_links": - _append_cleanup_detail( - error, - "rollback copy has an unknown hard-link alias; it may retain prior output bytes", - ) - raise + _pinned_backup_still_has_one_link(pending_swap["backup"]) # The first pin checked that the original was single-linked. A # backup hook can still add an alias before the commit boundary; # recheck this pinned inode so replacement never detaches a new @@ -2133,13 +2136,14 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): replaced.append(pending_swap) pending_swap = None # Keep this after every staged filesystem operation and before the - # committed boundary. Any changed new path still rolls back swaps. - for record in new_outputs: + # committed boundary. Revalidate every fresh and replaced destination: + # an earlier swap can be invalidated while a later backup is prepared. + for record in claimed: supplied_path = record["supplied_path"] canonical_path = record["canonical_path"] try: changed = (str(pathlib.Path(supplied_path).resolve()) != canonical_path - or _entry_identity(supplied_path) != record["identity"] + or _file_identity(supplied_path) != record["identity"] or _entry_identity(canonical_path) != record["identity"]) except (FileNotFoundError, OSError): changed = True @@ -2148,6 +2152,14 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} changed before commit; no output was committed", ) + _require_single_owned_link(record, "output", "output_has_multiple_links") + # Earlier rollback copies can also be changed during later swaps. + # A commit may retire them only while their ownership remains proved. + for swap in replaced: + backup = swap["backup"] + if _entry_identity(backup["path"]) != backup["identity"]: + raise Refusal("output_path_changed", "rollback copy changed before commit") + _pinned_backup_still_has_one_link(backup) # Final path validation is the boundary between rollback and committed # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 822b861ee..68522b77c 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2947,6 +2947,131 @@ def replace_then_unlink(source, destination): assert list(root.iterdir()) == [existing] +def test_staged_output_hard_link_before_commit_refuses_and_reports_alias(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, alias = root / "output.xml", root / "alias.xml" + destination.write_text("old bytes") + def link_staged(): + staged, = root.glob("output.xml.*.part") + os.link(staged, alias) + refusal = refuses(m, "staged_output_has_multiple_links", m.write_outputs, + [(str(destination), "new bytes")], False, link_staged) + assert "unknown hard-link alias" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert alias.read_text() == "new bytes" + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + +def test_fresh_output_hard_link_before_commit_refuses_and_reports_alias(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, alias = root / "output.xml", root / "alias.xml" + refusal = refuses(m, "output_has_multiple_links", m.write_outputs, + [(str(destination), "new bytes")], False, + lambda: os.link(destination, alias)) + assert "unknown hard-link alias" in str(refusal.code) + assert not destination.exists() + assert alias.read_text() == "new bytes" + + +def test_staged_parent_rename_reports_unlocated_output(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root, moved = pathlib.Path(directory) / "before", pathlib.Path(directory) / "after" + root.mkdir() + destination = root / "output.xml" + destination.write_text("old bytes") + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + try: + m.write_outputs([(str(destination), "new bytes")], + after_claim=lambda: root.rename(moved)) + raise AssertionError("missing backup parent must fail") + except FileNotFoundError as error: + detail = stderr.getvalue() + "\n".join(getattr(error, "__notes__", [])) + assert "owned output could not be located after cleanup" in detail + staged, = moved.glob("output.xml.*.part") + assert staged.read_text() == "new bytes" + assert (moved / "output.xml").read_text() == "old bytes" + + +def test_earlier_replacement_is_revalidated_after_later_swap(m): + for change in ("replace", "unlink", "symlink"): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, foreign = root / "first.xml", root / "second.xml", root / "foreign.xml" + first.write_text("old first") + second.write_text("old second") + foreign.write_text("foreign bytes") + supplied = root / "first-link.xml" if change == "symlink" else first + if change == "symlink": + supplied.symlink_to(first) + real_replace = m.os.replace + changed_first = [] + def replace_then_change_first(source, destination): + result = real_replace(source, destination) + if (str(source).endswith(".part") + and pathlib.Path(destination).resolve() == second.resolve()): + changed_first.append(change) + if change == "replace": + real_replace(foreign, first) + elif change == "unlink": + first.unlink() + else: + supplied.unlink() + supplied.symlink_to(foreign) + return result + m.os.replace = replace_then_change_first + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(supplied), "new first"), (str(second), "new second")]) + finally: + m.os.replace = real_replace + assert changed_first == [change], "interleave must run after the second swap" + assert second.read_text() == "old second" + assert "owned output could not be located after cleanup" not in str(refusal.code) + if change == "replace": + assert first.read_text() == "foreign bytes" + elif change == "unlink": + assert not first.exists() + else: + assert first.read_text() == "old first" + assert supplied.read_text() == "foreign bytes" + + +def test_earlier_backup_alias_is_revalidated_after_later_swap(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, alias = root / "first.xml", root / "second.xml", root / "alias.xml" + first.write_text("old first") + second.write_text("old second") + real_replace = m.os.replace + linked = [] + def replace_then_link_first_backup(source, destination): + result = real_replace(source, destination) + if (str(source).endswith(".part") + and pathlib.Path(destination).resolve() == second.resolve()): + backup, = root.glob("first.xml.*.bak") + os.link(backup, alias) + linked.append(True) + return result + m.os.replace = replace_then_link_first_backup + try: + refusal = refuses(m, "rollback_backup_has_multiple_links", m.write_outputs, + [(str(first), "new first"), (str(second), "new second")]) + finally: + m.os.replace = real_replace + assert linked == [True] + assert "unknown hard-link alias" in str(refusal.code) + assert first.read_text() == "old first" + assert second.read_text() == "old second" + assert alias.read_text() == "old first" + assert not list(root.glob("*.bak")) + + def main(): module = load() for name, test in sorted(globals().items()): From 78e5a826a1fa87a274da9aeb83e3fa902bdc591f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 02:38:44 +0530 Subject: [PATCH 24/59] fix(import): classify unresolved output paths during cleanup --- scripts/bank_statement_import.py | 26 +++++++++++++--- scripts/bank_statement_import.test.py | 45 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index e9bf30ffb..622c39a9b 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1531,6 +1531,22 @@ def _entry_identity(path): return stat_result.st_dev, stat_result.st_ino +def _resolve_output_path(path): + """Resolve an operator path into the spelling this run is authorised to touch. + + Python raises ``RuntimeError`` for a symlink loop on supported 3.10--3.12 + versions. That is a changed/unverifiable output path, not an implementation + traceback that should escape the writer. + """ + try: + return str(pathlib.Path(path).resolve()) + except RuntimeError as error: + raise Refusal( + "output_path_changed", + f"{path} could not be resolved safely before output was written", + ) from error + + def _unlink_for_cleanup(path, owned_identity, failures): """Remove a path only while it still names the inode this run created.""" try: @@ -1670,7 +1686,7 @@ def _cleanup_owned_path(record, failures): # this output. Keep any earlier diagnostics, but replace this # pathname with the separate pinned-inode conclusion below. del failures[failure_start:] - if outcome in ("missing", "reclaimed") and "cleanup_path" in record: + if outcome == "missing" or (outcome == "reclaimed" and "cleanup_path" in record): # The descriptor still proves this is our fresh output, but a # stale parent pathname cannot say where it went. Do not turn a # missing entry into a successful cleanup or invent a replacement @@ -2045,7 +2061,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "original_identity": _file_identity(real_path)}) else: supplied_path = path - canonical_path = str(pathlib.Path(supplied_path).resolve()) + canonical_path = _resolve_output_path(supplied_path) handle = _open_private(canonical_path, accept_inherited) # Keep cleanup on the canonical inode path captured before the # open. The supplied spelling remains an authority that must @@ -2058,7 +2074,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): claimed.append(record) try: claimed_path_changed = ( - str(pathlib.Path(supplied_path).resolve()) != canonical_path + _resolve_output_path(supplied_path) != canonical_path or _file_identity(canonical_path) != record["identity"]) except (FileNotFoundError, OSError): claimed_path_changed = True @@ -2142,7 +2158,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): supplied_path = record["supplied_path"] canonical_path = record["canonical_path"] try: - changed = (str(pathlib.Path(supplied_path).resolve()) != canonical_path + changed = (_resolve_output_path(supplied_path) != canonical_path or _file_identity(supplied_path) != record["identity"] or _entry_identity(canonical_path) != record["identity"]) except (FileNotFoundError, OSError): @@ -2223,7 +2239,7 @@ def _check_paths(args): `--out` and `--manifest` sharing a path leaves whichever was written second, with both success lines printed. """ - named = [(flag, pathlib.Path(value).expanduser().resolve()) + named = [(flag, pathlib.Path(_resolve_output_path(pathlib.Path(value).expanduser()))) for flag, value in (("--pdf", args.pdf), ("--mapping", args.mapping), ("--out", args.out), ("--manifest", args.manifest)) if value] diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 68522b77c..f4a5efde6 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2307,6 +2307,51 @@ def test_pinned_backup_reclaimed_path_retains_cleanup_diagnostic(m): assert moved.read_text() == "prior output" +def test_pinned_backup_parent_rename_reports_unlocated_copy(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + before, after = root / "before", root / "after" + before.mkdir() + path = before / "rollback.bak" + path.write_text("prior output") + identity = m._entry_identity(path) + handle = os.open(path, os.O_RDONLY) + record = {"path": path, "identity": identity, "pin": handle} + try: + before.rename(after) + failures = [] + m._cleanup_owned_path(record, failures) + finally: + if record["pin"] is not None: + os.close(record["pin"]) + assert failures == [ + f"owned output could not be located after cleanup: {path}" + ] + assert (after / "rollback.bak").read_text() == "prior output" + + +def test_new_output_symlink_loop_is_a_typed_path_refusal(m): + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + real_resolve = m.pathlib.Path.resolve + + def loop_resolve(path, *args, **kwargs): + if path == destination: + raise RuntimeError("controlled symlink loop") + return real_resolve(path, *args, **kwargs) + + m.pathlib.Path.resolve = loop_resolve + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m.pathlib.Path.resolve = real_resolve + assert "could not be resolved safely" in str(refusal.code) + assert not destination.exists() + + def test_unlinked_pinned_backup_does_not_claim_a_hard_link_alias(m): if os.name == "nt": return From bfd0f86e0c1b2c7886e2af756af5faf5bf668445 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 02:41:25 +0530 Subject: [PATCH 25/59] test(import): cover post-claim resolution failure --- scripts/bank_statement_import.test.py | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index f4a5efde6..2bee97832 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2352,6 +2352,41 @@ def loop_resolve(path, *args, **kwargs): assert not destination.exists() +def test_new_output_final_revalidation_loop_cleans_all_claimed_outputs(m): + """A loop discovered after claiming still rolls back every fresh output.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + real_resolve = m.pathlib.Path.resolve + resolve_calls = 0 + + def loop_after_claim(path): + nonlocal resolve_calls + if pathlib.Path(path) == first: + resolve_calls += 1 + # Claiming resolves once to choose the canonical path and once + # to verify the new inode. The third call is final + # revalidation, after both outputs have been written. + if resolve_calls == 3: + raise RuntimeError("controlled post-claim symlink loop") + return real_resolve(path) + + m.pathlib.Path.resolve = loop_after_claim + try: + refusal = refuses( + m, + "output_path_changed", + m.write_outputs, + [(str(first), "first bytes"), (str(second), "second bytes")], + ) + finally: + m.pathlib.Path.resolve = real_resolve + assert resolve_calls == 3, "RuntimeError must be injected during final revalidation" + assert "could not be resolved safely" in str(refusal.code) + assert not first.exists(), "the first claimed output must be cleaned" + assert not second.exists(), "the earlier claimed output must be cleaned too" + + def test_unlinked_pinned_backup_does_not_claim_a_hard_link_alias(m): if os.name == "nt": return From 8303492298e0fb0e47e2eefa25deaaa328f4f3c6 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:01:51 +0530 Subject: [PATCH 26/59] fix(import): pin existing outputs at claim --- scripts/bank_statement_import.py | 22 ++++++++++--- scripts/bank_statement_import.test.py | 46 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 622c39a9b..a564654a5 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2056,9 +2056,17 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): record["canonical_path"] = real_path record["cleanup_path"] = temporary claimed.append(record) + original_identity = _file_identity(real_path) + # Pin the original inode at first claim and retain this + # descriptor through payload generation and commit. A later + # path replacement must not be able to recycle the recorded + # identity and make the backup read from foreign bytes. + original_handle = _open_regular_output(real_path, original_identity) + original = _owned_path(real_path, original_handle, created=False) staged.append({"temporary": record, "supplied_path": path, "real_path": real_path, - "original_identity": _file_identity(real_path)}) + "original_identity": original_identity, + "original": original}) else: supplied_path = path canonical_path = _resolve_output_path(supplied_path) @@ -2110,11 +2118,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_backup = None # This ownership pin both prevents original-inode ABA reuse and # captures metadata before the backup read can update atime. - original_handle = _open_regular_output(real_path, original_identity) - pending_swap["original"] = _owned_path( - real_path, original_handle, created=False) + pending_swap["original"] = state["original"] pending_swap["metadata"] = _metadata_from_handle( - real_path, original_handle) + real_path, pending_swap["original"]["pin"]) _copy_private_backup(real_path, original_identity, backup_handle) # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after @@ -2227,6 +2233,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): _close_owned_path(swap["original"], cleanup_failures) for record in claimed: _cleanup_owned_path(record, cleanup_failures) + # Existing destinations are pinned at first claim, before they enter a + # swap record. Close any pin whose state never reached the rollback + # loops above; it is a descriptor-only ownership record and must never + # be unlinked as if it were a fresh output. + for state in staged: + _close_owned_path(state["original"], cleanup_failures) _note_cleanup_failures(error, cleanup_failures) _note_rollback_metadata_scope(error, metadata_scope_warnings) raise diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 2bee97832..f92a3a230 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2579,6 +2579,52 @@ def replace_then_conflict(src, dst): assert second.read_text() == "second old" +def test_original_inode_pin_blocks_after_claim_replacement_before_backup(m): + """A same-name replacement after claim cannot make backup copy foreign bytes.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "output.xml" + foreign = root / "foreign.xml" + destination.write_text("original bytes") + real_open = m._open_regular_output + original_handles = [] + + def observe_original(path, identity): + handle = real_open(path, identity) + if os.path.realpath(path) == os.path.realpath(destination): + original_handles.append(handle) + return handle + + def replace_after_claim(): + assert original_handles, "the original inode must be pinned before the hook" + assert os.pread(original_handles[0], 32, 0) == b"original bytes" + foreign.write_text("foreign bytes") + os.replace(foreign, destination) + assert os.pread(original_handles[0], 32, 0) == b"original bytes" + + m._open_regular_output = observe_original + try: + refusal = refuses( + m, + "output_path_changed", + m.write_outputs, + [(str(destination), "new bytes")], + False, + replace_after_claim, + ) + finally: + m._open_regular_output = real_open + assert "changed" in str(refusal.code) + assert destination.read_text() == "foreign bytes" + assert original_handles + try: + os.fstat(original_handles[0]) + except OSError: + pass + else: + raise AssertionError("the original pin must be closed during recovery") + + def test_committed_close_after_effect_does_not_report_missing_backup(m): with tempfile.TemporaryDirectory() as directory: destination = pathlib.Path(directory) / "output.xml" From ff67eb92a9fbdb63d8b791fbd18ef92756bf9bd3 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:02:51 +0530 Subject: [PATCH 27/59] Scope inode replacement control to supported POSIX writer --- scripts/bank_statement_import.test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index f92a3a230..2ac2a5257 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2581,6 +2581,8 @@ def replace_then_conflict(src, dst): def test_original_inode_pin_blocks_after_claim_replacement_before_backup(m): """A same-name replacement after claim cannot make backup copy foreign bytes.""" + if os.name == "nt": + return # Existing destinations are deliberately refused on Windows. with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) destination = root / "output.xml" From b3519c8badc4a65aca1cbe59743759e34fd112ea Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:19:35 +0530 Subject: [PATCH 28/59] fix(import): pin output identity before validation --- scripts/bank_statement_import.py | 54 +++++++++++++------ scripts/bank_statement_import.test.py | 75 +++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 15 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index a564654a5..22297283a 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1573,7 +1573,7 @@ def _unlink_for_cleanup(path, owned_identity, failures): return "uncertain" -def _open_regular_output(path, expected_identity): +def _open_regular_output(path, expected_identity=None): """Open and pin one existing regular output without waiting on a FIFO.""" handle = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) try: @@ -1589,7 +1589,8 @@ def _open_regular_output(path, expected_identity): f"{path}: replacement requires a single-link output; rollback " "cannot preserve hard-link topology", ) - if (stat_result.st_dev, stat_result.st_ino) != expected_identity: + if (expected_identity is not None + and (stat_result.st_dev, stat_result.st_ino) != expected_identity): raise Refusal( "output_path_changed", f"{path} changed while its rollback copy was prepared", @@ -1643,6 +1644,22 @@ def _owned_path(path, handle, *, created): return {"path": path, "identity": identity, "pin": handle} +def _claimed_output_changed(supplied_path, canonical_path, identity): + """Whether the pathname still names the descriptor-backed claimed inode. + + This is deliberately a helper rather than an inner ``try`` in + ``write_outputs``. A signal raised while this validation runs must unwind + through the transaction's outer recovery handler, which owns every staged + replacement and backup. + """ + try: + return (_resolve_output_path(supplied_path) != canonical_path + or _file_identity(supplied_path) != identity + or _entry_identity(canonical_path) != identity) + except (FileNotFoundError, OSError): + return True + + def _close_owned_path(record, failures): """Release an ownership pin only after its cleanup decision is complete.""" handle = record.get("pin") @@ -2056,17 +2073,29 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): record["canonical_path"] = real_path record["cleanup_path"] = temporary claimed.append(record) - original_identity = _file_identity(real_path) # Pin the original inode at first claim and retain this # descriptor through payload generation and commit. A later # path replacement must not be able to recycle the recorded # identity and make the backup read from foreign bytes. - original_handle = _open_regular_output(real_path, original_identity) + original_handle = _open_regular_output(real_path, None) original = _owned_path(real_path, original_handle, created=False) - staged.append({"temporary": record, "supplied_path": path, - "real_path": real_path, - "original_identity": original_identity, - "original": original}) + original_identity = original["identity"] + state = {"temporary": record, "supplied_path": path, + "real_path": real_path, + "original_identity": original_identity, + "original": original} + # The descriptor is the authoritative original identity. The + # path must still lead to that inode when the claim commits; + # if it changed after open, retain no authority to overwrite + # the replacement. A change before open is indistinguishable + # from the path state when this claim began; without a lock, + # no later operation can establish that it was foreign. + staged.append(state) + if _file_identity(real_path) != original_identity: + raise Refusal( + "output_path_changed", + f"{path} changed while it was being claimed", + ) else: supplied_path = path canonical_path = _resolve_output_path(supplied_path) @@ -2163,13 +2192,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): for record in claimed: supplied_path = record["supplied_path"] canonical_path = record["canonical_path"] - try: - changed = (_resolve_output_path(supplied_path) != canonical_path - or _file_identity(supplied_path) != record["identity"] - or _entry_identity(canonical_path) != record["identity"]) - except (FileNotFoundError, OSError): - changed = True - if changed: + if _claimed_output_changed( + supplied_path, canonical_path, record["identity"]): raise Refusal( "output_path_changed", f"{supplied_path} changed before commit; no output was committed", diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 2ac2a5257..29379ce5c 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2185,6 +2185,81 @@ def replace_before_copy(src, identity, backup_handle): assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] +def test_claim_refuses_a_foreign_replacement_after_committing_original_identity(m): + """The descriptor, rather than a pre-open stat, commits the old inode. + + An attacker can replace the path before this call opens it; without a + lock, that is the file the call is asked to replace. This regression covers + the actionable interval: a foreign replacement after the old descriptor + establishes identity but before the path is verified must remain untouched. + """ + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + foreign = root / "foreign.xml" + destination.write_text("old bytes") + foreign.write_text("foreign writer bytes") + real_owned_path = m._owned_path + real_replace = m.os.replace + + def replace_after_identity(path, handle, *, created): + record = real_owned_path(path, handle, created=created) + if not created and os.path.realpath(path) == os.path.realpath(destination): + real_replace(foreign, destination) + return record + + m._owned_path = replace_after_identity + try: + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._owned_path = real_owned_path + + assert destination.read_text() == "foreign writer bytes" + assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + +def test_line_interrupt_during_final_validation_rolls_back_replacements(m): + """A real line-traced SIGINT at the final validation stays recoverable.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first = root / "first.xml" + second = root / "second.csv" + first.write_text("first old") + second.write_text("second old") + _, start = inspect.getsourcelines(m.write_outputs) + validation_line = start + next( + index for index, line in enumerate( + inspect.getsource(m.write_outputs).splitlines()) + if line.strip() == "if _claimed_output_changed(") + old_trace = sys.gettrace() + fired = False + + def interrupt_final_validation(frame, event, _arg): + nonlocal fired + if (not fired and event == "line" and frame.f_code is m.write_outputs.__code__ + and frame.f_lineno == validation_line): + fired = True + raise KeyboardInterrupt("controlled final-validation interrupt") + return interrupt_final_validation + + sys.settrace(interrupt_final_validation) + try: + try: + m.write_outputs([(str(first), "new first"), + (str(second), "new second")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + sys.settrace(old_trace) + + assert fired + assert first.read_text() == "first old" + assert second.read_text() == "second old" + assert sorted(path.name for path in root.iterdir()) == ["first.xml", "second.csv"] + + def test_backup_copy_refuses_a_fifo_before_reading_it(m): """An existing output is data only when it is a regular file; opening a FIFO for its rollback copy would otherwise wait for an unrelated writer.""" From 499994d479cedbb1736a67527331c6af65b68e01 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:22:02 +0530 Subject: [PATCH 29/59] test(import): deliver SIGINT at final validation --- scripts/bank_statement_import.test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 29379ce5c..84c92971b 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -37,6 +37,7 @@ import inspect import os import pathlib +import signal import stat import subprocess import sys @@ -2193,6 +2194,8 @@ def test_claim_refuses_a_foreign_replacement_after_committing_original_identity( the actionable interval: a foreign replacement after the old descriptor establishes identity but before the path is verified must remain untouched. """ + if os.name == "nt": + return # Existing destinations are deliberately refused on Windows. with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) destination = root / "previous.xml" @@ -2221,6 +2224,8 @@ def replace_after_identity(path, handle, *, created): def test_line_interrupt_during_final_validation_rolls_back_replacements(m): """A real line-traced SIGINT at the final validation stays recoverable.""" + if os.name == "nt": + return # Existing destinations are deliberately refused on Windows. with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) first = root / "first.xml" @@ -2233,6 +2238,7 @@ def test_line_interrupt_during_final_validation_rolls_back_replacements(m): inspect.getsource(m.write_outputs).splitlines()) if line.strip() == "if _claimed_output_changed(") old_trace = sys.gettrace() + old_signal_handler = signal.getsignal(signal.SIGINT) fired = False def interrupt_final_validation(frame, event, _arg): @@ -2240,9 +2246,10 @@ def interrupt_final_validation(frame, event, _arg): if (not fired and event == "line" and frame.f_code is m.write_outputs.__code__ and frame.f_lineno == validation_line): fired = True - raise KeyboardInterrupt("controlled final-validation interrupt") + signal.raise_signal(signal.SIGINT) return interrupt_final_validation + signal.signal(signal.SIGINT, signal.default_int_handler) sys.settrace(interrupt_final_validation) try: try: @@ -2253,6 +2260,7 @@ def interrupt_final_validation(frame, event, _arg): pass finally: sys.settrace(old_trace) + signal.signal(signal.SIGINT, old_signal_handler) assert fired assert first.read_text() == "first old" From 2786e21b42a66e17b4d884c2caeea8677be2f9ca Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:46:43 +0530 Subject: [PATCH 30/59] Harden committed output cleanup reconciliation --- scripts/bank_statement_import.py | 20 ++++++++------------ scripts/bank_statement_import.test.py | 6 +++--- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 22297283a..9eaeb70ab 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1868,7 +1868,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings): destination, original_identity = swap["destination"], swap["original_identity"] staged_identity, metadata = swap["staged_identity"], swap["metadata"] try: - current_identity = _file_identity(destination) + current_identity = _entry_identity(destination) except OSError: current_identity = None if not swap["swap_started"] or current_identity == original_identity: @@ -1901,7 +1901,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings): restored = True except OSError: try: - restored = _file_identity(destination) == backup_identity + restored = _entry_identity(destination) == backup_identity except OSError: restored = False if not restored: @@ -1965,8 +1965,7 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ """Remove old private copies after every replacement has committed.""" for swap in replaced: backup = swap["backup"] - _unlink_for_cleanup(backup["path"], backup["identity"], retained_failures) - _close_owned_path(backup, retained_failures) + _cleanup_owned_path(backup, retained_failures) _close_owned_path(swap["original"], retained_failures) for record in claimed: # A claimed path did not exist before this run. Its close failure cannot @@ -1983,16 +1982,13 @@ def _reconcile_interrupted_committed_cleanup( try: if _entry_identity(backup["path"]) == backup["identity"]: retained_failures.append(str(backup["path"])) - elif os.path.lexists(backup["path"]): - retained_failures.append(str(backup["path"])) + else: + _cleanup_owned_path(backup, retained_failures) except FileNotFoundError: - pass - except OSError: - if os.path.lexists(backup["path"]): - retained_failures.append(str(backup["path"])) - finally: + _cleanup_owned_path(backup, retained_failures) + if backup.get("pin") is not None: _close_owned_path(backup, retained_failures) - _close_owned_path(swap["original"], retained_failures) + _close_owned_path(swap["original"], retained_failures) for record in claimed: _close_owned_path(record, descriptor_failures) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 84c92971b..b6c92a75a 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1789,9 +1789,9 @@ def test_original_pin_registration_failure_preserves_existing_output(m): def fail_original_pin(handle): nonlocal calls calls += 1 - # Staged output and private backup register first. The third pin - # is the old destination opened for backup and metadata capture. - if calls == 3: + # Staged output registers first; the second pin is the old + # destination opened for backup and metadata capture. + if calls == 2: raise OSError("controlled original pin fstat failure") return real_identity(handle) From 5cdc02984a80f975b386abcf3389fffaf77746ab Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:50:22 +0530 Subject: [PATCH 31/59] fix: preserve interrupted cleanup ownership recovery --- scripts/bank_statement_import.py | 50 +++++++--- scripts/bank_statement_import.test.py | 134 ++++++++++++++++++++++---- 2 files changed, 151 insertions(+), 33 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 9eaeb70ab..5ac0bab76 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1709,16 +1709,18 @@ def _cleanup_owned_path(record, failures): # missing entry into a successful cleanup or invent a replacement # path; a parent-directory rename is outside this CLI's namespace # authority and needs an operator-visible recovery fact. - try: - stat_result = os.fstat(record["pin"]) - if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] - and stat_result.st_nlink > 0): + pin = record.get("pin") + if pin is not None: + try: + stat_result = os.fstat(pin) + if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] + and stat_result.st_nlink > 0): + failures.append( + f"owned output could not be located after cleanup: {record['path']}" + ) + except OSError: failures.append( f"owned output could not be located after cleanup: {record['path']}" - ) - except OSError: - failures.append( - f"owned output could not be located after cleanup: {record['path']}" ) _close_owned_path(record, failures) @@ -1976,19 +1978,37 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ def _reconcile_interrupted_committed_cleanup( replaced, claimed, retained_failures, descriptor_failures): - """Close pins and disclose owned old copies without undoing a commit.""" + """Close every pin and disclose old copies without undoing a commit. + + Recovery runs while another exception is already escaping. Inspection or + cleanup failures are diagnostics here: they must never replace that + original exception or skip closure of later ownership descriptors. + """ for swap in replaced: backup = swap["backup"] try: - if _entry_identity(backup["path"]) == backup["identity"]: + try: + still_at_path = _entry_identity(backup["path"]) == backup["identity"] + except FileNotFoundError: + still_at_path = False + except OSError: + still_at_path = None + if still_at_path is True: retained_failures.append(str(backup["path"])) - else: + elif still_at_path is False: _cleanup_owned_path(backup, retained_failures) - except FileNotFoundError: - _cleanup_owned_path(backup, retained_failures) - if backup.get("pin") is not None: + else: + # We cannot identify an entry after an I/O/permission error. + # Preserve the original interruption and report no ownership + # claim about a possibly foreign pathname. + retained_failures.append( + "could not inspect committed rollback copy: " + str(backup["path"])) + except OSError: + retained_failures.append( + "could not reconcile committed rollback copy: " + str(backup["path"])) + finally: _close_owned_path(backup, retained_failures) - _close_owned_path(swap["original"], retained_failures) + _close_owned_path(swap["original"], retained_failures) for record in claimed: _close_owned_path(record, descriptor_failures) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index b6c92a75a..14208ec6c 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1778,35 +1778,75 @@ def fail_identity(_handle): def test_original_pin_registration_failure_preserves_existing_output(m): - """The original pin is not newly created cleanup authority.""" + """A created=False pin failure re-raises the same error and closes its FD.""" with tempfile.TemporaryDirectory() as directory: - root = pathlib.Path(directory) - destination = root / "previous.xml" + destination = pathlib.Path(directory) / "previous.xml" destination.write_text("old bytes") - real_identity = m._fd_identity - calls = 0 + handle = os.open(destination, os.O_RDONLY) + original = m._fd_identity + error = OSError("controlled original pin fstat failure") - def fail_original_pin(handle): - nonlocal calls - calls += 1 - # Staged output registers first; the second pin is the old - # destination opened for backup and metadata capture. - if calls == 2: - raise OSError("controlled original pin fstat failure") - return real_identity(handle) + def fail_original_pin(candidate): + assert candidate == handle + raise error m._fd_identity = fail_original_pin try: try: - m.write_outputs([(str(destination), "new bytes")]) + m._owned_path(destination, handle, created=False) raise AssertionError("the controlled original-pin failure must escape") - except OSError as error: - assert "controlled original pin fstat failure" in str(error) + except OSError as raised: + assert raised is error finally: - m._fd_identity = real_identity + m._fd_identity = original + try: + os.fstat(handle) + raise AssertionError("created=False registration failure must close its descriptor") + except OSError: + pass assert destination.read_text() == "old bytes" - assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] + + +def test_interrupted_committed_cleanup_preserves_interrupt_when_backup_inspection_fails(m): + """EACCES during reconciliation is diagnostic, not a replacement exception.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + backup_path, original_path, claimed_path = root / "old.bak", root / "old.xml", root / "new.xml" + for path in (backup_path, original_path, claimed_path): + path.write_text(path.name) + backup_fd, original_fd, claimed_fd = (os.open(path, os.O_RDONLY) for path in (backup_path, original_path, claimed_path)) + backup = {"path": backup_path, "identity": m._fd_identity(backup_fd), "pin": backup_fd} + original_record = {"path": original_path, "identity": m._fd_identity(original_fd), "pin": original_fd} + claimed = {"path": claimed_path, "identity": m._fd_identity(claimed_fd), "pin": claimed_fd} + real_entry = m._entry_identity + def deny_backup(path): + if pathlib.Path(path) == backup_path: + raise PermissionError("controlled EACCES") + return real_entry(path) + m._entry_identity = deny_backup + retained, closes = [], [] + real_close = m._close_owned_path + def record_close(record, failures): + closes.append(record["path"]) + return real_close(record, failures) + m._close_owned_path = record_close + try: + original_error = KeyboardInterrupt("controlled interrupt") + try: + raise original_error + except BaseException as caught: + m._reconcile_interrupted_committed_cleanup( + [{"backup": backup, "original": original_record}], [claimed], retained, []) + assert caught is original_error + finally: + m._entry_identity = real_entry + m._close_owned_path = real_close + assert backup_path in closes and original_path in closes and claimed_path in closes + assert any("could not inspect committed rollback copy" in value for value in retained) + assert backup_path.read_text() == "old.bak" def test_restore_reconciles_a_backup_replace_that_raised_after_effect(m): @@ -3292,6 +3332,64 @@ def main(): print("all offline contract tests passed") return 0 +def test_interrupted_committed_cleanup_parent_rename_retains_unlocated_backup(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) / "before"; moved = pathlib.Path(directory) / "after"; root.mkdir() + backup_path = root / "old.bak"; original_path = root / "old.xml"; claimed_path = root / "new.xml" + for path in (backup_path, original_path, claimed_path): path.write_text(path.name) + fds = [os.open(path, os.O_RDONLY) for path in (backup_path, original_path, claimed_path)] + records = [{"path": path, "identity": m._fd_identity(fd), "pin": fd} for path, fd in zip((backup_path, original_path, claimed_path), fds)] + root.rename(moved) + retained=[] + m._reconcile_interrupted_committed_cleanup([{"backup":records[0],"original":records[1]}], [records[2]], retained, []) + assert any("owned output could not be located after cleanup" in value for value in retained) + assert (moved / "old.bak").read_text() == "old.bak" + + +def test_restore_backup_preserves_foreign_symlink_entry(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root=pathlib.Path(directory); destination=root/"destination.xml"; backup=root/"backup.bak"; foreign=root/"foreign.xml" + destination.write_text("new bytes"); backup.write_text("old bytes"); foreign.write_text("foreign bytes") + backup_fd=os.open(backup, os.O_RDONLY); original_fd=os.open(destination, os.O_RDONLY) + swap={"destination":destination,"original_identity":m._fd_identity(original_fd),"staged_identity":m._entry_identity(destination),"metadata":None,"swap_started":True,"backup":{"path":backup,"identity":m._fd_identity(backup_fd),"pin":backup_fd}} + destination.unlink(); destination.symlink_to(foreign) + failures=[]; warnings=[] + m._restore_backup(swap, failures, warnings) + assert destination.is_symlink() and foreign.read_text() == "foreign bytes" + assert backup.exists() + os.close(backup_fd); os.close(original_fd) + + +def test_interrupted_cleanup_skips_closed_missing_backup_and_closes_later_pins(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + gone, later, original, claimed = (root / name for name in ("gone.bak", "later.bak", "old.xml", "new.xml")) + for path in (gone, later, original, claimed): path.write_text(path.name) + fds = [os.open(path, os.O_RDONLY) for path in (gone, later, original, claimed)] + records = [{"path": path, "identity": m._fd_identity(fd), "pin": fd} for path, fd in zip((gone, later, original, claimed), fds)] + m._cleanup_owned_path(records[0], []) + assert records[0]["pin"] is None and not gone.exists() + real_entry=m._entry_identity + def deny_later(path): + if pathlib.Path(path) == later: raise PermissionError("controlled EACCES") + return real_entry(path) + m._entry_identity=deny_later + retained=[] + try: + m._reconcile_interrupted_committed_cleanup( + [{"backup":records[0],"original":records[2]}, {"backup":records[1],"original":records[2]}], + [records[3]], retained, []) + finally: + m._entry_identity=real_entry + assert any("could not inspect committed rollback copy" in value for value in retained) + assert records[1]["pin"] is None and records[2]["pin"] is None and records[3]["pin"] is None + if __name__ == "__main__": raise SystemExit(main()) From fff1aa06c48fd03030227d4887f674cd6596473f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 03:56:43 +0530 Subject: [PATCH 32/59] test: distinguish foreign staged symlink rollback --- scripts/bank_statement_import.test.py | 51 +++++++++++++++++++++------ 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 14208ec6c..f9423b0f1 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -3349,19 +3349,50 @@ def test_interrupted_committed_cleanup_parent_rename_retains_unlocated_backup(m) def test_restore_backup_preserves_foreign_symlink_entry(m): + """A symlink to the staged-new inode is still a foreign directory entry.""" if os.name == "nt": return + def make_swap(root): + original, destination, staged, backup = (root / name for name in + ("original-old.xml", "destination.xml", "staged-new.xml", "backup.bak")) + original.write_text("old original bytes") + destination.write_text("new destination bytes") + staged.write_text("new staged bytes") + backup.write_text("old backup bytes") + original_fd, backup_fd = os.open(original, os.O_RDONLY), os.open(backup, os.O_RDONLY) + swap = {"destination": destination, "original_identity": m._fd_identity(original_fd), + "staged_identity": m._entry_identity(staged), "metadata": None, + "swap_started": True, "original": None, + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), "pin": backup_fd}} + destination.unlink(); destination.symlink_to(staged) + return destination, staged, backup, original_fd, backup_fd, swap with tempfile.TemporaryDirectory() as directory: - root=pathlib.Path(directory); destination=root/"destination.xml"; backup=root/"backup.bak"; foreign=root/"foreign.xml" - destination.write_text("new bytes"); backup.write_text("old bytes"); foreign.write_text("foreign bytes") - backup_fd=os.open(backup, os.O_RDONLY); original_fd=os.open(destination, os.O_RDONLY) - swap={"destination":destination,"original_identity":m._fd_identity(original_fd),"staged_identity":m._entry_identity(destination),"metadata":None,"swap_started":True,"backup":{"path":backup,"identity":m._fd_identity(backup_fd),"pin":backup_fd}} - destination.unlink(); destination.symlink_to(foreign) - failures=[]; warnings=[] - m._restore_backup(swap, failures, warnings) - assert destination.is_symlink() and foreign.read_text() == "foreign bytes" - assert backup.exists() - os.close(backup_fd); os.close(original_fd) + root = pathlib.Path(directory) + destination, staged, backup, original_fd, backup_fd, swap = make_swap(root) + try: + failures=[] + m._restore_backup(swap, failures, []) + assert destination.is_symlink() + assert staged.read_text() == "new staged bytes" + assert backup.read_text() == "old backup bytes" + assert failures == [backup] + finally: + os.close(original_fd); os.close(backup_fd) + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, staged, backup, original_fd, backup_fd, swap = make_swap(root) + original_entry = m._entry_identity + m._entry_identity = m._file_identity + try: + try: + m._restore_backup(swap, [], []) + except TypeError: + pass + assert not destination.is_symlink(), "old stat-following check replaces the foreign link" + assert destination.read_text() == "old backup bytes" + finally: + m._entry_identity = original_entry + os.close(original_fd); os.close(backup_fd) def test_interrupted_cleanup_skips_closed_missing_backup_and_closes_later_pins(m): From 81e887c9f2aa155d4a50667da77c26622a3dc501 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 04:26:40 +0530 Subject: [PATCH 33/59] fix: reconcile retained output cleanup --- scripts/bank_statement_import.py | 120 ++++++++---- scripts/bank_statement_import.test.py | 256 +++++++++++++++++++++++++- 2 files changed, 340 insertions(+), 36 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 5ac0bab76..e4d034ab0 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1531,6 +1531,28 @@ def _entry_identity(path): return stat_result.st_dev, stat_result.st_ino +def _cleanup_entry_state(path, owned_identity=None): + """Classify a cleanup name without following a replacement symlink. + + ``lexists`` maps permission errors to false, which would make an + inaccessible entry indistinguishable from one that an unlink removed. + Recovery needs that distinction before it can close the descriptor pin. + """ + try: + identity = _entry_identity(path) + except FileNotFoundError: + return "missing" + except OSError: + return "uninspectable" + if owned_identity is None: + return "present" + return "owned" if identity == owned_identity else "reclaimed" + + +def _record_uninspectable_cleanup(path, failures): + failures.append(f"could not inspect owned output during cleanup: {path}") + + def _resolve_output_path(path): """Resolve an operator path into the spelling this run is authorised to touch. @@ -1549,14 +1571,19 @@ def _resolve_output_path(path): def _unlink_for_cleanup(path, owned_identity, failures): """Remove a path only while it still names the inode this run created.""" + state = _cleanup_entry_state(path, owned_identity) + if state == "missing": + return "missing" + if state == "uninspectable": + _record_uninspectable_cleanup(path, failures) + return "uncertain" + if state == "reclaimed": + # The pathname has been reclaimed. It is not ours to delete, and + # reporting it gives the operator a chance to find the private copy + # if it still exists without making a claim about foreign bytes. + failures.append(str(path)) + return "reclaimed" try: - if _entry_identity(path) != owned_identity: - # The pathname has been reclaimed. It is not ours to delete, and - # reporting it gives the operator a chance to find the private copy - # if it still exists without making a claim about foreign bytes. - if os.path.lexists(path): - failures.append(str(path)) - return "reclaimed" os.unlink(path) return "removed" except FileNotFoundError: @@ -1567,9 +1594,14 @@ def _unlink_for_cleanup(path, owned_identity, failures): return "missing" except OSError: # A filesystem call can report an error after taking effect. Only retain - # the path when reconciliation shows bytes may still be present. - if os.path.lexists(path): + # the path when non-following reconciliation establishes that it remains. + state = _cleanup_entry_state(path, owned_identity) + if state == "missing": + return "missing" + if state in ("owned", "reclaimed"): failures.append(str(path)) + elif state == "uninspectable": + _record_uninspectable_cleanup(path, failures) return "uncertain" @@ -1624,16 +1656,27 @@ def _owned_path(path, handle, *, created): identity = None if created: if identity is None: - if os.path.lexists(path): + state = _cleanup_entry_state(path) + if state == "present": failures.append(str(path)) + elif state == "uninspectable": + _record_uninspectable_cleanup(path, failures) else: - _unlink_for_cleanup(path, identity, failures) + outcome = _unlink_for_cleanup(path, identity, failures) + _reconcile_owned_pin_after_cleanup( + {"path": path, "identity": identity, "pin": handle}, + outcome, + failures, + ) finally: try: os.close(handle) except OSError: - if created and os.path.lexists(path): + state = _cleanup_entry_state(path) if created else "missing" + if state == "present": failures.append(str(path)) + elif state == "uninspectable": + _record_uninspectable_cleanup(path, failures) if failures: _append_cleanup_detail( error, @@ -1678,8 +1721,34 @@ def _close_owned_path(record, failures): except FileNotFoundError: pass except OSError: - if os.path.lexists(cleanup_path): + state = _cleanup_entry_state(cleanup_path, record["identity"]) + if state in ("owned", "reclaimed"): failures.append(cleanup_path) + elif state == "uninspectable": + _record_uninspectable_cleanup(cleanup_path, failures) + + +def _reconcile_owned_pin_after_cleanup(record, outcome, failures): + """Disclose an owned inode whose descriptor proves it remains linked. + + A successful unlink only removes the claimed spelling. A hard-link alias or + parent-directory rename can leave the pinned inode linked elsewhere, where + this command has neither a pathname nor authority to remove it. + """ + if outcome not in ("removed", "missing", "reclaimed"): + return + pin = record.get("pin") + if pin is None: + return + try: + stat_result = os.fstat(pin) + except OSError: + _record_uninspectable_cleanup(record["path"], failures) + return + if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] + and stat_result.st_nlink > 0): + failures.append( + f"owned output could not be located after cleanup: {record['path']}") def _cleanup_owned_path(record, failures): @@ -1703,25 +1772,7 @@ def _cleanup_owned_path(record, failures): # this output. Keep any earlier diagnostics, but replace this # pathname with the separate pinned-inode conclusion below. del failures[failure_start:] - if outcome == "missing" or (outcome == "reclaimed" and "cleanup_path" in record): - # The descriptor still proves this is our fresh output, but a - # stale parent pathname cannot say where it went. Do not turn a - # missing entry into a successful cleanup or invent a replacement - # path; a parent-directory rename is outside this CLI's namespace - # authority and needs an operator-visible recovery fact. - pin = record.get("pin") - if pin is not None: - try: - stat_result = os.fstat(pin) - if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] - and stat_result.st_nlink > 0): - failures.append( - f"owned output could not be located after cleanup: {record['path']}" - ) - except OSError: - failures.append( - f"owned output could not be located after cleanup: {record['path']}" - ) + _reconcile_owned_pin_after_cleanup(record, outcome, failures) _close_owned_path(record, failures) @@ -1866,7 +1917,8 @@ def _restore_backup(swap, failures, metadata_scope_warnings): A different inode may be a foreign writer's success, so keep the private backup and report the conflict rather than overwriting it. """ - backup, backup_identity = swap["backup"]["path"], swap["backup"]["identity"] + backup_record = swap["backup"] + backup, backup_identity = backup_record["path"], backup_record["identity"] destination, original_identity = swap["destination"], swap["original_identity"] staged_identity, metadata = swap["staged_identity"], swap["metadata"] try: @@ -1886,7 +1938,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings): os.utime(handle, ns=(metadata["atime_ns"], current.st_mtime_ns)) except OSError: failures.append(destination) - _unlink_for_cleanup(backup, backup_identity, failures) + _cleanup_owned_path(backup_record, failures) return if current_identity != staged_identity: failures.append(backup) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index f9423b0f1..bb557948f 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1705,7 +1705,7 @@ def observe_cleanup(record, failures): assert fired assert destination.read_text() == "old bytes" - assert not any(path.endswith(".bak") for path in cleanup_paths) + assert sum(path.endswith(".bak") for path in cleanup_paths) == 1 assert sorted(path.name for path in root.iterdir()) == ["previous.xml"] @@ -1749,6 +1749,44 @@ def observe_close(candidate): assert handle in closed +def test_ownership_registration_recovery_discloses_a_retained_hard_link(m): + """Registration recovery has the same post-unlink alias obligation.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path, alias = root / "fresh.xml", root / "alias.xml" + handle = m._open_private(path) + os.link(path, alias) + real_identity = m._fd_identity + calls = 0 + + def fail_first_identity(candidate): + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("controlled registration identity failure") + return real_identity(candidate) + + m._fd_identity = fail_first_identity + try: + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + try: + m._owned_path(path, handle, created=True) + raise AssertionError("the registration failure must escape") + except OSError as error: + detail = (str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + + stderr.getvalue()) + assert "controlled registration identity failure" in detail + assert "owned output could not be located after cleanup" in detail + finally: + m._fd_identity = real_identity + + assert not path.exists() + assert alias.read_bytes() == b"" + + def test_ownership_registration_preserves_an_unproven_reclaimed_path(m): """When fstat cannot establish ownership, foreign bytes survive visibly.""" with tempfile.TemporaryDirectory() as directory: @@ -2424,7 +2462,10 @@ def test_pinned_backup_reclaimed_path_retains_cleanup_diagnostic(m): path.write_text("foreign bytes") failures = [] m._cleanup_owned_path(record, failures) - assert failures == [str(path)] + assert failures == [ + str(path), + f"owned output could not be located after cleanup: {path}", + ] assert record["pin"] is None assert path.read_text() == "foreign bytes" assert moved.read_text() == "prior output" @@ -2614,6 +2655,43 @@ def reclaim_backup_after_copy(src, identity, backup_handle): assert str(backups[0]) in str(refusal.code) +def test_parent_rename_after_backup_preparation_discloses_unlocated_backup(m): + """A no-swap rollback must reconcile the pinned backup before closing it.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) / "before" + moved = pathlib.Path(directory) / "after" + root.mkdir() + destination = root / "previous.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def rename_parent_after_backup(source, identity, backup_handle): + result = real_copy(source, identity, backup_handle) + root.rename(moved) + return result + + m._copy_private_backup = rename_parent_after_backup + detail = "" + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the renamed parent must prevent a swap") + except FileNotFoundError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert "owned output could not be located after cleanup" in detail + finally: + m._copy_private_backup = real_copy + + assert (moved / "previous.xml").read_text() == "old bytes" + backups = list(moved.glob("previous.xml.*.bak")) + assert len(backups) == 1 + assert backups[0].read_text() == "old bytes" + old_backup_path = root.resolve() / backups[0].name + assert f"owned output could not be located after cleanup: {old_backup_path}" in detail + + def test_rollback_keeps_a_foreign_destination_and_private_backup(m): """When an external writer replaces an already-swapped destination before another target fails, rollback must retain the owned backup rather than @@ -3101,6 +3179,180 @@ def unlink_after_claim(): assert not destination.exists() +def test_cleanup_unlinked_pinned_path_does_not_report_a_phantom_output(m): + """A descriptor with zero links is not an undisclosed cleanup location.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "output.xml" + path.write_text("new bytes") + handle = os.open(path, os.O_RDONLY) + record = {"path": path, "identity": m._fd_identity(handle), "pin": handle} + failures = [] + m._cleanup_owned_path(record, failures) + assert failures == [] + assert record["pin"] is None + assert not path.exists() + + +def test_cleanup_after_effect_unlink_without_alias_does_not_report_a_phantom(m): + """An unlink that removes its only link before raising has no retained path.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "output.xml" + path.write_text("new bytes") + handle = os.open(path, os.O_RDONLY) + record = {"path": path, "identity": m._fd_identity(handle), "pin": handle} + real_unlink = m.os.unlink + + def unlink_then_fail(candidate): + real_unlink(candidate) + raise OSError("controlled unlink after effect") + + m.os.unlink = unlink_then_fail + try: + failures = [] + m._cleanup_owned_path(record, failures) + finally: + m.os.unlink = real_unlink + + assert failures == [] + assert record["pin"] is None + assert not path.exists() + + +def test_write_outputs_after_effect_unlink_discloses_retained_hard_link(m): + """A cleanup EIO after unlink still reconciles a pinned hard-link alias.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, alias = root / "first.xml", root / "second.xml", root / "alias.xml" + real_dup, real_unlink = m.os.dup, m.os.unlink + dup_calls = 0 + + def fail_second_payload(handle): + nonlocal dup_calls + dup_calls += 1 + if dup_calls == 2: + raise OSError("controlled later payload failure") + return real_dup(handle) + + def unlink_after_effect(candidate): + real_unlink(candidate) + if pathlib.Path(candidate).resolve() == first.resolve(): + raise OSError("controlled cleanup EIO after unlink") + + m.os.dup, m.os.unlink = fail_second_payload, unlink_after_effect + try: + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + try: + m.write_outputs( + [(str(first), "first bytes"), (str(second), "second bytes")], + after_claim=lambda: os.link(first, alias), + ) + raise AssertionError("the controlled payload failure must escape") + except OSError as error: + detail = (str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + + stderr.getvalue()) + assert "controlled later payload failure" in detail + assert "owned output could not be located after cleanup" in detail + finally: + m.os.dup, m.os.unlink = real_dup, real_unlink + + assert not first.exists() + assert not second.exists() + assert alias.read_text() == "first bytes" + + +def test_new_output_alias_after_claim_and_later_payload_failure_is_disclosed(m): + """Cleanup may remove our name while an after-claim hard link stays live.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, alias = root / "first.xml", root / "second.xml", root / "alias.xml" + real_dup = m.os.dup + calls = 0 + + def fail_second_payload(handle): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("controlled later payload failure") + return real_dup(handle) + + m.os.dup = fail_second_payload + try: + try: + m.write_outputs( + [(str(first), "first bytes"), (str(second), "second bytes")], + after_claim=lambda: os.link(first, alias), + ) + raise AssertionError("the controlled payload failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert "controlled later payload failure" in detail + assert "owned output could not be located after cleanup" in detail + finally: + m.os.dup = real_dup + + assert not first.exists() + assert not second.exists() + assert alias.read_text() == "first bytes" + + +def test_write_outputs_inspection_eacces_is_not_reported_as_missing(m): + """A real payload rollback retains an inaccessible claimed output visibly.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + real_entry = m._entry_identity + real_dup = m.os.dup + denied = False + dup_calls = 0 + + def deny_entry(path): + if denied and pathlib.Path(path).resolve() == first.resolve(): + raise PermissionError("controlled EACCES") + return real_entry(path) + + def fail_second_payload(handle): + nonlocal dup_calls + dup_calls += 1 + if dup_calls == 2: + raise OSError("controlled later payload failure") + return real_dup(handle) + + def deny_after_claim(): + nonlocal denied + denied = True + + m._entry_identity, m.os.dup = deny_entry, fail_second_payload + try: + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + try: + m.write_outputs( + [(str(first), "first bytes"), (str(second), "second bytes")], + after_claim=deny_after_claim, + ) + raise AssertionError("the controlled payload failure must escape") + except OSError as error: + detail = (str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + + stderr.getvalue()) + assert "controlled later payload failure" in detail + assert ("could not inspect owned output during cleanup: " + + str(first.resolve())) in detail + finally: + m._entry_identity, m.os.dup = real_entry, real_dup + + assert first.read_text() == "first bytes" + assert not second.exists() + + def test_new_output_parent_retarget_refuses_and_preserves_foreign_path(m): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) From 614f82e48fa82ab3a175aec627fed0d768287d4f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 04:53:34 +0530 Subject: [PATCH 34/59] Rectify output recovery ownership boundaries --- scripts/bank_statement_import.py | 63 +++++++---- scripts/bank_statement_import.test.py | 144 +++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 27 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index e4d034ab0..9e9667b21 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1761,8 +1761,15 @@ def _cleanup_owned_path(record, failures): required for that platform-specific branch. """ if os.name == "nt": + # Windows requires closing before unlinking. A close can report an + # error after releasing the descriptor, so decide whether its + # pathname diagnostic remains only after the unlink outcome is known. + failure_start = len(failures) _close_owned_path(record, failures) - _unlink_for_cleanup(record.get("cleanup_path", record["path"]), record["identity"], failures) + outcome = _unlink_for_cleanup( + record.get("cleanup_path", record["path"]), record["identity"], failures) + if outcome == "removed": + del failures[failure_start:] else: cleanup_path = record.get("cleanup_path", record["path"]) failure_start = len(failures) @@ -1810,8 +1817,9 @@ def _restore_metadata(handle, metadata): current = os.fstat(handle) if (current.st_uid, current.st_gid) != (metadata["uid"], metadata["gid"]): os.fchown(handle, metadata["uid"], metadata["gid"]) - os.fchmod(handle, metadata["mode"]) - os.utime(handle, ns=(metadata["atime_ns"], metadata["mtime_ns"])) + # The private backup is writable. Restore extended attributes before the + # final mode: Linux requires write permission for xattr changes, including + # removal of attributes introduced by the staged output. original_xattrs = metadata["xattrs"] if original_xattrs is not None: for name in os.listxattr(handle): @@ -1819,6 +1827,8 @@ def _restore_metadata(handle, metadata): os.removexattr(handle, name) for name, value in original_xattrs.items(): os.setxattr(handle, name, value) + os.utime(handle, ns=(metadata["atime_ns"], metadata["mtime_ns"])) + os.fchmod(handle, metadata["mode"]) def _pinned_original_still_has_one_link(record): @@ -1944,6 +1954,18 @@ def _restore_backup(swap, failures, metadata_scope_warnings): failures.append(backup) return restored = False + try: + _pinned_backup_still_has_one_link(backup_record) + except Refusal: + # Keep the original failure escaping. The private backup is still + # named here, but its unknown alias may retain prior statement bytes; + # moving it back would make that alias a live copy of the destination. + failures.append( + f"unknown hard-link alias may retain rollback bytes: {backup}") + return + except OSError: + failures.append(backup) + return try: if _entry_identity(backup) != backup_identity: failures.append(backup) @@ -1998,7 +2020,7 @@ def _append_cleanup_detail(error, message): def _note_cleanup_failures(error, failures): if failures: - retained = ", ".join(sorted(set(failures))) + retained = ", ".join(sorted({str(failure) for failure in failures})) message = "output cleanup or rollback failed; retained path(s): " + retained _append_cleanup_detail(error, message) @@ -2133,37 +2155,32 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if refusal: raise refusal real_path = os.path.realpath(path) - handle, temporary = tempfile.mkstemp( - dir=os.path.dirname(real_path), - prefix=os.path.basename(real_path) + ".", suffix=".part") - record = _owned_path(temporary, handle, created=True) - record["supplied_path"] = path - record["canonical_path"] = real_path - record["cleanup_path"] = temporary - claimed.append(record) - # Pin the original inode at first claim and retain this - # descriptor through payload generation and commit. A later - # path replacement must not be able to recycle the recorded - # identity and make the backup read from foreign bytes. + # Commit the existing inode before any staging syscall makes + # a visible sibling. A replacement before this open is the + # requested current path; a replacement after it is detected + # before this run has authority to create or swap output. original_handle = _open_regular_output(real_path, None) original = _owned_path(real_path, original_handle, created=False) original_identity = original["identity"] - state = {"temporary": record, "supplied_path": path, + state = {"temporary": None, "supplied_path": path, "real_path": real_path, "original_identity": original_identity, "original": original} - # The descriptor is the authoritative original identity. The - # path must still lead to that inode when the claim commits; - # if it changed after open, retain no authority to overwrite - # the replacement. A change before open is indistinguishable - # from the path state when this claim began; without a lock, - # no later operation can establish that it was foreign. staged.append(state) if _file_identity(real_path) != original_identity: raise Refusal( "output_path_changed", f"{path} changed while it was being claimed", ) + handle, temporary = tempfile.mkstemp( + dir=os.path.dirname(real_path), + prefix=os.path.basename(real_path) + ".", suffix=".part") + record = _owned_path(temporary, handle, created=True) + record["supplied_path"] = path + record["canonical_path"] = real_path + record["cleanup_path"] = temporary + claimed.append(record) + state["temporary"] = record else: supplied_path = path canonical_path = _resolve_output_path(supplied_path) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index bb557948f..e66a33b50 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -3545,6 +3545,142 @@ def replace_then_change_first(source, destination): assert supplied.read_text() == "foreign bytes" +def test_rollback_keeps_an_aliased_earlier_backup_after_a_later_swap_failure(m): + """Recovery must not restore an old copy after it gained an unknown alias.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, alias = root / "first.xml", root / "second.xml", root / "alias.xml" + first.write_text("old first") + second.write_text("old second") + real_replace = m.os.replace + + def swap_first_alias_backup_then_fail_second(source, destination): + if (str(source).endswith(".part") + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + result = real_replace(source, destination) + if (str(source).endswith(".part") + and pathlib.Path(destination).resolve() == first.resolve()): + backup, = root.glob("first.xml.*.bak") + os.link(backup, alias) + return result + + m.os.replace = swap_first_alias_backup_then_fail_second + try: + try: + m.write_outputs([(str(first), "new first"), (str(second), "new second")]) + raise AssertionError("the controlled later failure must escape") + except OSError as error: + notes = "\n".join(getattr(error, "__notes__", [])) + assert "unknown hard-link alias may retain rollback bytes" in notes + finally: + m.os.replace = real_replace + + backup, = root.glob("first.xml.*.bak") + assert first.read_text() == "new first" + assert second.read_text() == "old second" + assert backup.read_text() == alias.read_text() == "old first" + + +def test_existing_destination_is_pinned_before_staging_side_effects(m): + """A mkstemp-time replacement is foreign because the original pin predates it.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, foreign = root / "output.xml", root / "foreign.xml" + destination.write_text("old bytes") + foreign.write_text("foreign bytes") + real_mkstemp, real_replace = m.tempfile.mkstemp, m.os.replace + fired = False + + def replace_from_part_mkstemp(*args, **kwargs): + nonlocal fired + handle, path = real_mkstemp(*args, **kwargs) + if kwargs.get("suffix") == ".part" and not fired: + fired = True + real_replace(foreign, destination) + return handle, path + + m.tempfile.mkstemp = replace_from_part_mkstemp + try: + refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m.tempfile.mkstemp = real_mkstemp + + assert fired + assert destination.read_text() == "foreign bytes" + assert sorted(path.name for path in root.iterdir()) == ["output.xml"] + + +def test_windows_modeled_close_after_effect_then_unlink_has_no_stale_retention(m): + """Model the Windows close-before-unlink order; Windows filesystem proof is separate.""" + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "fresh.xml" + handle = m._open_private(path) + record = m._owned_path(path, handle, created=True) + real_close, real_name = m.os.close, m.os.name + fired, failures = False, [] + + def close_then_error(candidate): + nonlocal fired + real_close(candidate) + if candidate == handle and not fired: + fired = True + raise OSError("controlled close after effect") + + m.os.close, m.os.name = close_then_error, "nt" + try: + m._cleanup_owned_path(record, failures) + finally: + m.os.close, m.os.name = real_close, real_name + + assert fired + assert failures == [] + assert not path.exists() + + +def test_rollback_restores_xattrs_before_a_readonly_final_mode(m): + """Linux xattrs need the backup's temporary write permission during rollback.""" + if os.name != "posix" or not hasattr(os, "setxattr"): + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("old first") + second.write_text("old second") + name, value = "user.bridge_readonly_rollback", b"original xattr" + try: + os.setxattr(first, name, value) + except OSError: + return + first.chmod(0o400) + real_replace = m.os.replace + + def fail_second_swap(source, destination): + if str(source).endswith(".part") and pathlib.Path(destination) == second: + raise OSError("controlled second swap failure") + return real_replace(source, destination) + + m.os.replace = fail_second_swap + try: + try: + m.write_outputs([(str(first), "new first"), (str(second), "new second")]) + raise AssertionError("the controlled second swap failure must escape") + except OSError as error: + assert "controlled second swap failure" in str(error) + finally: + m.os.replace = real_replace + + assert first.read_text() == "old first" + assert stat.S_IMODE(first.stat().st_mode) == 0o400 + assert os.getxattr(first, name) == value + assert second.read_text() == "old second" + + def test_earlier_backup_alias_is_revalidated_after_later_swap(m): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) @@ -3568,11 +3704,11 @@ def replace_then_link_first_backup(source, destination): finally: m.os.replace = real_replace assert linked == [True] - assert "unknown hard-link alias" in str(refusal.code) - assert first.read_text() == "old first" + assert "unknown hard-link alias may retain rollback bytes" in str(refusal.code) + backup, = root.glob("first.xml.*.bak") + assert first.read_text() == "new first" assert second.read_text() == "old second" - assert alias.read_text() == "old first" - assert not list(root.glob("*.bak")) + assert backup.read_text() == alias.read_text() == "old first" def main(): From e2ff1366d84ae400a9bb9e9c96232dd5cd89563d Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 04:58:31 +0530 Subject: [PATCH 35/59] Preserve rollback backup refusal categories --- scripts/bank_statement_import.py | 19 ++-- scripts/bank_statement_import.test.py | 126 ++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 6 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 9e9667b21..8d99e42e5 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1956,12 +1956,19 @@ def _restore_backup(swap, failures, metadata_scope_warnings): restored = False try: _pinned_backup_still_has_one_link(backup_record) - except Refusal: - # Keep the original failure escaping. The private backup is still - # named here, but its unknown alias may retain prior statement bytes; - # moving it back would make that alias a live copy of the destination. - failures.append( - f"unknown hard-link alias may retain rollback bytes: {backup}") + except Refusal as refusal: + if refusal.category == "rollback_backup_has_multiple_links": + # Keep the original failure escaping. The private backup is still + # named here, but its unknown alias may retain prior statement + # bytes; moving it back would make that alias a live copy of the + # destination. + failures.append( + f"unknown hard-link alias may retain rollback bytes: {backup}") + else: + # A missing or changed pinned backup is not an alias. Preserve + # the original error and disclose only the rollback path whose + # identity could no longer support a restore. + failures.append(backup) return except OSError: failures.append(backup) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index e66a33b50..2e0102518 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -3584,6 +3584,132 @@ def swap_first_alias_backup_then_fail_second(source, destination): assert backup.read_text() == alias.read_text() == "old first" + +def test_rollback_path_distinguishes_an_unlinked_backup_from_an_alias(m): + """A zero-link backup prevents restoration but must not claim an alias.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("old first") + second.write_text("old second") + real_replace = m.os.replace + + def swap_first_unlink_backup_then_fail_second(source, destination): + if (str(source).endswith(".part") + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + result = real_replace(source, destination) + if (str(source).endswith(".part") + and pathlib.Path(destination).resolve() == first.resolve()): + backup, = root.glob("first.xml.*.bak") + backup.unlink() + return result + + m.os.replace = swap_first_unlink_backup_then_fail_second + try: + try: + m.write_outputs([(str(first), "new first"), (str(second), "new second")]) + raise AssertionError("the controlled later failure must escape") + except OSError as error: + details = "\n".join(getattr(error, "__notes__", [])) + assert "controlled later swap failure" in str(error) + assert "unknown hard-link alias" not in details + finally: + m.os.replace = real_replace + + assert first.read_text() == "new first" + assert second.read_text() == "old second" + assert not list(root.glob("*.bak")) + + +def test_rollback_alias_diagnostic_reaches_stderr(m): + """The alias warning survives both exception-note and stderr render paths.""" + if os.name == "nt": + return + program = f'''\ +import importlib.util +import os +import pathlib +import tempfile + +script = {str(SCRIPT)!r} +spec = importlib.util.spec_from_file_location("bank_statement_import_subprocess", script) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, alias = root / "first.xml", root / "second.xml", root / "alias.xml" + first.write_text("old first") + second.write_text("old second") + real_replace = module.os.replace + def replace_first_alias_then_fail_second(source, destination): + if str(source).endswith(".part") and pathlib.Path(destination).resolve() == second.resolve(): + raise OSError("controlled later swap failure") + result = real_replace(source, destination) + if str(source).endswith(".part") and pathlib.Path(destination).resolve() == first.resolve(): + backup, = root.glob("first.xml.*.bak") + os.link(backup, alias) + return result + module.os.replace = replace_first_alias_then_fail_second + module.write_outputs([(str(first), "new first"), (str(second), "new second")]) +''' + done = subprocess.run([sys.executable, "-c", program], text=True, + capture_output=True, check=False) + assert done.returncode != 0 + assert "controlled later swap failure" in done.stderr, done.stderr + assert "unknown hard-link alias may retain rollback bytes" in done.stderr, done.stderr + + +def test_restore_metadata_orders_xattrs_before_final_mode(m): + """Portable ordering proof; Linux permission enforcement is tested separately.""" + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "output.xml" + path.write_text("bytes") + handle = os.open(path, os.O_RDWR) + current = os.fstat(handle) + metadata = { + "mode": 0o400, + "uid": current.st_uid, + "gid": current.st_gid, + "atime_ns": current.st_atime_ns, + "mtime_ns": current.st_mtime_ns, + "xattrs": {"user.bridge_order": b"value"}, + } + missing = object() + real_listxattr = getattr(m.os, "listxattr", missing) + real_setxattr = getattr(m.os, "setxattr", missing) + real_fchmod = m.os.fchmod + events = [] + + def list_no_xattrs(_): + return [] + + def record_setxattr(*args): + events.append("xattr") + + def record_fchmod(*args): + events.append("mode") + + m.os.listxattr, m.os.setxattr, m.os.fchmod = ( + list_no_xattrs, record_setxattr, record_fchmod) + try: + m._restore_metadata(handle, metadata) + finally: + for name, original in (("listxattr", real_listxattr), + ("setxattr", real_setxattr)): + if original is missing: + delattr(m.os, name) + else: + setattr(m.os, name, original) + m.os.fchmod = real_fchmod + os.close(handle) + + assert events == ["xattr", "mode"] + + + def test_existing_destination_is_pinned_before_staging_side_effects(m): """A mkstemp-time replacement is foreign because the original pin predates it.""" if os.name == "nt": From 8f922b994b2e2675293de7f4f1ecfa267f16e202 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 05:00:54 +0530 Subject: [PATCH 36/59] test: retain recovery diagnostics across supported Python versions --- scripts/bank_statement_import.test.py | 36 +++++++++++++++------------ 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 2e0102518..00be246d9 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -3568,13 +3568,15 @@ def swap_first_alias_backup_then_fail_second(source, destination): return result m.os.replace = swap_first_alias_backup_then_fail_second + stderr = io.StringIO() try: - try: - m.write_outputs([(str(first), "new first"), (str(second), "new second")]) - raise AssertionError("the controlled later failure must escape") - except OSError as error: - notes = "\n".join(getattr(error, "__notes__", [])) - assert "unknown hard-link alias may retain rollback bytes" in notes + with contextlib.redirect_stderr(stderr): + try: + m.write_outputs([(str(first), "new first"), (str(second), "new second")]) + raise AssertionError("the controlled later failure must escape") + except OSError as error: + notes = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + stderr.getvalue() + assert "unknown hard-link alias may retain rollback bytes" in notes finally: m.os.replace = real_replace @@ -3584,7 +3586,6 @@ def swap_first_alias_backup_then_fail_second(source, destination): assert backup.read_text() == alias.read_text() == "old first" - def test_rollback_path_distinguishes_an_unlinked_backup_from_an_alias(m): """A zero-link backup prevents restoration but must not claim an alias.""" if os.name == "nt": @@ -3608,14 +3609,16 @@ def swap_first_unlink_backup_then_fail_second(source, destination): return result m.os.replace = swap_first_unlink_backup_then_fail_second + stderr = io.StringIO() try: - try: - m.write_outputs([(str(first), "new first"), (str(second), "new second")]) - raise AssertionError("the controlled later failure must escape") - except OSError as error: - details = "\n".join(getattr(error, "__notes__", [])) - assert "controlled later swap failure" in str(error) - assert "unknown hard-link alias" not in details + with contextlib.redirect_stderr(stderr): + try: + m.write_outputs([(str(first), "new first"), (str(second), "new second")]) + raise AssertionError("the controlled later failure must escape") + except OSError as error: + details = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + stderr.getvalue() + assert "controlled later swap failure" in str(error) + assert "unknown hard-link alias" not in details finally: m.os.replace = real_replace @@ -3663,7 +3666,9 @@ def replace_first_alias_then_fail_second(source, destination): def test_restore_metadata_orders_xattrs_before_final_mode(m): - """Portable ordering proof; Linux permission enforcement is tested separately.""" + """POSIX call-order proof; Linux permission enforcement is tested separately.""" + if os.name != "posix": + return with tempfile.TemporaryDirectory() as directory: path = pathlib.Path(directory) / "output.xml" path.write_text("bytes") @@ -3709,7 +3714,6 @@ def record_fchmod(*args): assert events == ["xattr", "mode"] - def test_existing_destination_is_pinned_before_staging_side_effects(m): """A mkstemp-time replacement is foreign because the original pin predates it.""" if os.name == "nt": From ff82aa39e7eeb0a96c08996e51d1d15a2390dc82 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 05:29:06 +0530 Subject: [PATCH 37/59] Harden output creation recovery boundaries --- scripts/bank_statement_import.py | 107 ++++++++++++-- scripts/bank_statement_import.test.py | 202 ++++++++++++++++++++++++-- 2 files changed, 285 insertions(+), 24 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 8d99e42e5..b869c2274 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1510,9 +1510,36 @@ def _open_private(path, accept_inherited=False): if refusal: raise refusal try: - return os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + handle = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: raise _existing_target_on_windows(path) from None + try: + # Creation modes are filtered through the process umask. Reapply the + # owner-only contract to the descriptor before any caller writes. + os.fchmod(handle, 0o600) + return handle + except BaseException: + os.close(handle) + try: + os.unlink(path) + except OSError: + pass + raise + + +def _private_mkstemp(**kwargs): + """Create an owner-only sibling output despite a restrictive umask.""" + handle, path = tempfile.mkstemp(**kwargs) + try: + os.fchmod(handle, 0o600) + return handle, path + except BaseException: + os.close(handle) + try: + os.unlink(path) + except OSError: + pass + raise def _file_identity(path): @@ -1633,7 +1660,7 @@ def _open_regular_output(path, expected_identity=None): raise -def _owned_path(path, handle, *, created): +def _owned_path(path, handle, *, created, owned_records=None): """Record a pathname and retain the descriptor that pins its inode. `created` is explicit because a registration failure has opposite cleanup @@ -1684,7 +1711,12 @@ def _owned_path(path, handle, *, created): + ", ".join(sorted(set(failures))), ) raise - return {"path": path, "identity": identity, "pin": handle} + record = {"path": path, "identity": identity, "pin": handle} + if owned_records is not None: + # Insert before returning to the caller: an interrupt after successful + # registration still leaves one cleanup authority for this new inode. + owned_records.append(record) + return record def _claimed_output_changed(supplied_path, canonical_path, identity): @@ -2146,6 +2178,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): authority. """ claimed, staged, replaced = [], [], [] + # These hold pins which successfully registered but have not yet been + # transferred into a staged swap. They close or clean up an interrupt in + # the small caller-side handoff interval. + unpaired_originals, unpaired_backups = [], [] # A record is the one ownership authority for a pathname: cleanup may # unlink it only while its identity still equals record["identity"]. pending_backup = None @@ -2167,7 +2203,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # requested current path; a replacement after it is detected # before this run has authority to create or swap output. original_handle = _open_regular_output(real_path, None) - original = _owned_path(real_path, original_handle, created=False) + original = _owned_path( + real_path, original_handle, created=False, + owned_records=unpaired_originals) original_identity = original["identity"] state = {"temporary": None, "supplied_path": path, "real_path": real_path, @@ -2179,28 +2217,31 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{path} changed while it was being claimed", ) - handle, temporary = tempfile.mkstemp( + handle, temporary = _private_mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".part") - record = _owned_path(temporary, handle, created=True) + record = _owned_path(temporary, handle, created=True, + owned_records=claimed) record["supplied_path"] = path record["canonical_path"] = real_path record["cleanup_path"] = temporary - claimed.append(record) state["temporary"] = record else: supplied_path = path canonical_path = _resolve_output_path(supplied_path) - handle = _open_private(canonical_path, accept_inherited) + # Register this newly created inode before returning to an + # interruptible caller line. The record is the cleanup owner + # even if a signal arrives before its path metadata is filled. + record = _owned_path( + canonical_path, _open_private(canonical_path, accept_inherited), + created=True, owned_records=claimed) # Keep cleanup on the canonical inode path captured before the # open. The supplied spelling remains an authority that must # still resolve to that same inode at commit time. - record = _owned_path(canonical_path, handle, created=True) record["supplied_path"] = supplied_path record["canonical_path"] = canonical_path record["cleanup_path"] = canonical_path record["path"] = supplied_path - claimed.append(record) try: claimed_path_changed = ( _resolve_output_path(supplied_path) != canonical_path @@ -2227,10 +2268,11 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): temporary = state["temporary"] supplied_path, real_path = state["supplied_path"], state["real_path"] original_identity = state["original_identity"] - backup_handle, backup = tempfile.mkstemp( + backup_handle, backup = _private_mkstemp( dir=os.path.dirname(real_path), prefix=os.path.basename(real_path) + ".", suffix=".bak") - pending_backup = _owned_path(backup, backup_handle, created=True) + pending_backup = _owned_path( + backup, backup_handle, created=True, owned_records=unpaired_backups) pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, "staged_identity": temporary["identity"], @@ -2249,8 +2291,22 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # Revalidate *after* the backup operation: it is a filesystem call # an attacker can use to retarget the supplied symlink before this # commit. The pending backup lets the refusal cleanly undo itself. - if (os.path.realpath(supplied_path) != real_path - or _file_identity(real_path) != original_identity): + try: + existing_output_changed = ( + os.path.realpath(supplied_path) != real_path + or _file_identity(real_path) != original_identity) + except FileNotFoundError: + # A missing leaf under its original parent is a commit-boundary + # path change. A vanished parent can leave pinned private + # outputs under an unknown spelling, so preserve its existing + # recovery path and diagnostic rather than misclassifying it. + if os.path.isdir(os.path.dirname(real_path)): + existing_output_changed = True + else: + raise + except OSError: + existing_output_changed = True + if existing_output_changed: raise Refusal( "output_path_changed", f"{supplied_path} changed after it was claimed; no output was replaced", @@ -2302,6 +2358,11 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. committed = True + # The rollback handler still needs the old .part spelling until this + # boundary. Once committed, bind close diagnostics to the live output. + for state in staged: + state["temporary"]["path"] = state["real_path"] + state["temporary"]["cleanup_path"] = state["real_path"] _cleanup_committed_outputs( replaced, claimed, cleanup_failures, descriptor_close_failures) if cleanup_failures: @@ -2355,6 +2416,24 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # be unlinked as if it were a fresh output. for state in staged: _close_owned_path(state["original"], cleanup_failures) + # A signal can interrupt after _owned_path registered a backup or + # original pin but before its caller transferred it to pending_swap or + # staged. Reconcile only those unpaired records here; paired records + # were handled above with their transaction state. + paired_originals = {id(state["original"]) for state in staged} + if pending_swap is not None and pending_swap["original"] is not None: + paired_originals.add(id(pending_swap["original"])) + paired_backups = {id(swap["backup"]) for swap in replaced} + if pending_swap is not None: + paired_backups.add(id(pending_swap["backup"])) + if pending_backup is not None: + paired_backups.add(id(pending_backup)) + for record in unpaired_backups: + if id(record) not in paired_backups: + _cleanup_owned_path(record, cleanup_failures) + for record in unpaired_originals: + if id(record) not in paired_originals: + _close_owned_path(record, cleanup_failures) _note_cleanup_failures(error, cleanup_failures) _note_rollback_metadata_scope(error, metadata_scope_warnings) raise diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 00be246d9..b76c86175 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2042,7 +2042,7 @@ def observe_close(record, failures): assert destination.read_text() == "new bytes" assert os.path.realpath(destination) in closed_paths, closed_paths assert any(path.endswith(".bak") for path in closed_paths) - assert any(path.endswith(".part") for path in closed_paths) + assert not any(path.endswith(".part") for path in closed_paths) def test_interrupt_after_backup_unlink_does_not_report_a_phantom_path(m): @@ -2088,7 +2088,7 @@ def observe_close(record, failures): assert not list(root.glob("*.bak")) assert os.path.realpath(destination) in closed_paths, closed_paths assert any(path.endswith(".bak") for path in closed_paths) - assert any(path.endswith(".part") for path in closed_paths) + assert not any(path.endswith(".part") for path in closed_paths) def test_legacy_oserror_cleanup_diagnostic_reaches_stderr(m): @@ -2283,8 +2283,8 @@ def test_claim_refuses_a_foreign_replacement_after_committing_original_identity( real_owned_path = m._owned_path real_replace = m.os.replace - def replace_after_identity(path, handle, *, created): - record = real_owned_path(path, handle, created=created) + def replace_after_identity(path, handle, *, created, **kwargs): + record = real_owned_path(path, handle, created=created, **kwargs) if not created and os.path.realpath(path) == os.path.realpath(destination): real_replace(foreign, destination) return record @@ -2721,8 +2721,8 @@ def observe_closes(handle): closed_handles.append(handle) return real_close(handle) - def observe_owned_path(path, handle, *, created): - record = real_owned_path(path, handle, created=created) + def observe_owned_path(path, handle, *, created, **kwargs): + record = real_owned_path(path, handle, created=created, **kwargs) if os.path.basename(path).startswith("first.xml."): owned_handles[pathlib.Path(path).suffix] = record["pin"] return record @@ -2837,8 +2837,8 @@ def test_committed_close_after_effect_does_not_report_missing_backup(m): backup_handles = set() fired = False - def observe_owned(path, handle, *, created): - record = real_owned(path, handle, created=created) + def observe_owned(path, handle, *, created, **kwargs): + record = real_owned(path, handle, created=created, **kwargs) if str(path).endswith(".bak"): backup_handles.add(handle) return record @@ -3139,8 +3139,8 @@ def test_committed_new_output_close_failure_is_not_a_retained_backup(m): output_handles = set() fired = False - def observe_owned(path, handle, *, created): - record = real_owned(path, handle, created=created) + def observe_owned(path, handle, *, created, **kwargs): + record = real_owned(path, handle, created=created, **kwargs) if created and pathlib.Path(path).resolve() == destination.resolve(): output_handles.add(handle) return record @@ -3841,6 +3841,188 @@ def replace_then_link_first_backup(source, destination): assert backup.read_text() == alias.read_text() == "old first" +def test_interrupt_during_fresh_creation_closes_and_unlinks_the_output(m): + """A real SIGINT after open(2) still unwinds the private creation helper.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "fresh.xml" + _, start = inspect.getsourcelines(m._open_private) + fchmod_line = start + next( + offset for offset, line in enumerate(inspect.getsource(m._open_private).splitlines()) + if line.strip() == "os.fchmod(handle, 0o600)") + old_trace, old_handler = sys.gettrace(), signal.getsignal(signal.SIGINT) + fired = False + + def interrupt_after_open(frame, event, _arg): + nonlocal fired + if (not fired and event == "line" and frame.f_code is m._open_private.__code__ + and frame.f_lineno == fchmod_line): + fired = True + signal.raise_signal(signal.SIGINT) + return interrupt_after_open + + signal.signal(signal.SIGINT, signal.default_int_handler) + sys.settrace(interrupt_after_open) + try: + try: + m.write_outputs([(str(destination), "fresh bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + sys.settrace(old_trace) + signal.signal(signal.SIGINT, old_handler) + + assert fired + assert not destination.exists() + + +def test_interrupt_after_fresh_registration_cleans_the_requested_output(m): + """A SIGINT after registration has a claimed cleanup owner already.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "fresh.xml" + _, start = inspect.getsourcelines(m._owned_path) + return_line = start + next( + offset for offset, line in enumerate(inspect.getsource(m._owned_path).splitlines()) + if line.strip() == "return record") + old_trace, old_handler = sys.gettrace(), signal.getsignal(signal.SIGINT) + fired = False + + def interrupt_after_registration(frame, event, _arg): + nonlocal fired + if (not fired and event == "line" and frame.f_code is m._owned_path.__code__ + and frame.f_lineno == return_line + and frame.f_locals.get("created") + and frame.f_locals.get("owned_records") is not None): + fired = True + signal.raise_signal(signal.SIGINT) + return interrupt_after_registration + + signal.signal(signal.SIGINT, signal.default_int_handler) + sys.settrace(interrupt_after_registration) + try: + try: + m.write_outputs([(str(destination), "fresh bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + sys.settrace(old_trace) + signal.signal(signal.SIGINT, old_handler) + + assert fired + assert not destination.exists() + assert list(root.iterdir()) == [] + + +def test_existing_output_removed_after_backup_is_a_typed_path_refusal(m): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "output.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def copy_then_remove(*args): + result = real_copy(*args) + destination.unlink() + return result + + m._copy_private_backup = copy_then_remove + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + assert "changed after it was claimed" in str(refusal.code) + assert not destination.exists() + assert list(root.iterdir()) == [] + + +def test_committed_existing_output_close_failure_reports_destination(m): + """The post-rename staged pin must not inspect its stale .part spelling.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + destination.write_text("old bytes") + real_owned, real_close = m._owned_path, m.os.close + staged_handles, leaked = set(), [] + + def observe_owned(path, handle, *, created, **kwargs): + record = real_owned(path, handle, created=created, **kwargs) + if created and str(path).endswith(".part"): + staged_handles.add(handle) + return record + + def fail_before_close(handle): + if handle in staged_handles: + leaked.append(handle) + raise OSError("controlled staged close failure") + return real_close(handle) + + m._owned_path, m.os.close = observe_owned, fail_before_close + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the close failure must be reported") + except m.OutputDescriptorCloseFailure as failure: + assert failure.output_paths == (str(destination.resolve()),) + finally: + m._owned_path, m.os.close = real_owned, real_close + for handle in leaked: + try: + real_close(handle) + except OSError: + pass + + assert leaked + assert destination.read_text() == "new bytes" + + +def test_owner_only_outputs_survive_a_restrictive_umask(m): + """Use a child process so an extreme umask cannot affect this test process.""" + program = f'''\ +import importlib.util +import os +import pathlib +import stat +import tempfile + +script = {str(SCRIPT)!r} +spec = importlib.util.spec_from_file_location("bank_statement_import_subprocess", script) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + fresh = root / "fresh.xml" + existing = root / "existing.xml" + existing.write_text("old bytes") + observed = [] + real_copy = module._copy_private_backup + def observe_private_modes(*args): + result = real_copy(*args) + for path in root.glob("existing.xml.*"): + observed.append(stat.S_IMODE(path.stat().st_mode)) + return result + old_umask = os.umask(0o777) + try: + module.write_outputs([(str(fresh), "fresh bytes")]) + module._copy_private_backup = observe_private_modes + module.write_outputs([(str(existing), "new bytes")]) + finally: + os.umask(old_umask) + module._copy_private_backup = real_copy + assert stat.S_IMODE(fresh.stat().st_mode) == 0o600 + assert stat.S_IMODE(existing.stat().st_mode) == 0o600 + assert observed and all(mode == 0o600 for mode in observed), observed +''' + done = subprocess.run([sys.executable, "-c", program], text=True, + capture_output=True, check=False) + assert done.returncode == 0, done.stderr + + + def main(): module = load() for name, test in sorted(globals().items()): From 0fb542f692f64f0d225e09c19be9a9cac9b12eca Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 05:34:38 +0530 Subject: [PATCH 38/59] Make output registration recovery identity-safe --- scripts/bank_statement_import.py | 121 ++++++++++++++------------ scripts/bank_statement_import.test.py | 80 ++++++++++++----- 2 files changed, 122 insertions(+), 79 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index b869c2274..679ed22a5 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1495,10 +1495,6 @@ def windows_destination_refusal(path, accept_inherited): def _open_private(path, accept_inherited=False): """Create one output file readable only by its owner, and return the handle. - The XML and the manifest carry counterparty names, amounts, an account - label and every narration in the statement. On a shared host the default - 022 umask would publish all of it as mode 0644. - Always `O_EXCL`: this only ever creates a file that did not exist. On Windows that is the rule itself — an overwrite would keep the existing file's ACL — and deciding it at create time rather than after an @@ -1510,36 +1506,9 @@ def _open_private(path, accept_inherited=False): if refusal: raise refusal try: - handle = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + return os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: raise _existing_target_on_windows(path) from None - try: - # Creation modes are filtered through the process umask. Reapply the - # owner-only contract to the descriptor before any caller writes. - os.fchmod(handle, 0o600) - return handle - except BaseException: - os.close(handle) - try: - os.unlink(path) - except OSError: - pass - raise - - -def _private_mkstemp(**kwargs): - """Create an owner-only sibling output despite a restrictive umask.""" - handle, path = tempfile.mkstemp(**kwargs) - try: - os.fchmod(handle, 0o600) - return handle, path - except BaseException: - os.close(handle) - try: - os.unlink(path) - except OSError: - pass - raise def _file_identity(path): @@ -1719,6 +1688,46 @@ def _owned_path(path, handle, *, created, owned_records=None): return record +def _claim_owned_output(create, *, created, owned_records): + """Register a created or opened descriptor before a caller can own it. + + The factory returns ``(handle, path)``. If an interrupt lands after a + filesystem effect but before collection insertion, the still-pinned inode + is reconciled by identity; a replacement at that pathname is never + unlinked. On POSIX, apply the private mode through the descriptor because + creation modes are filtered by the process umask. Windows retains the + platform's existing ACL-based creation behavior. + """ + handle = None + record = None + try: + handle, path = create() + record = _owned_path(path, handle, created=created) + if created and os.name != "nt": + os.fchmod(handle, 0o600) + owned_records.append(record) + return record + except BaseException as error: + # If registration already reached its collection, the outer handler is + # its sole cleanup authority. Otherwise recover through the pin. + if handle is not None and not any(item.get("pin") == handle + for item in owned_records): + failures = [] + try: + identity = _fd_identity(handle) + except OSError: + # `_owned_path` may already have reconciled and closed it. + pass + else: + fallback = {"path": path, "identity": identity, "pin": handle} + if created: + _cleanup_owned_path(fallback, failures) + else: + _close_owned_path(fallback, failures) + _note_cleanup_failures(error, failures) + raise + + def _claimed_output_changed(supplied_path, canonical_path, identity): """Whether the pathname still names the descriptor-backed claimed inode. @@ -1735,7 +1744,7 @@ def _claimed_output_changed(supplied_path, canonical_path, identity): return True -def _close_owned_path(record, failures): +def _close_owned_path(record, failures, diagnostic_path=None): """Release an ownership pin only after its cleanup decision is complete.""" handle = record.get("pin") if handle is None: @@ -1747,7 +1756,7 @@ def _close_owned_path(record, failures): # close can fail after taking effect. Only an extant owned entry is a # retained-path failure; an already-unlinked backup has no such path. try: - cleanup_path = record.get("cleanup_path", record["path"]) + cleanup_path = diagnostic_path or record.get("cleanup_path", record["path"]) if _entry_identity(cleanup_path) == record["identity"]: failures.append(cleanup_path) except FileNotFoundError: @@ -2086,7 +2095,8 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ # A claimed path did not exist before this run. Its close failure cannot # retain a prior sensitive copy, so keep that diagnostic distinct from # a backup that an operator must protect or remove. - _close_owned_path(record, descriptor_failures) + _close_owned_path( + record, descriptor_failures, diagnostic_path=record.get("canonical_path")) def _reconcile_interrupted_committed_cleanup( @@ -2123,7 +2133,8 @@ def _reconcile_interrupted_committed_cleanup( _close_owned_path(backup, retained_failures) _close_owned_path(swap["original"], retained_failures) for record in claimed: - _close_owned_path(record, descriptor_failures) + _close_owned_path( + record, descriptor_failures, diagnostic_path=record.get("canonical_path")) def write_outputs(targets, accept_inherited=False, after_claim=None): @@ -2202,10 +2213,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # a visible sibling. A replacement before this open is the # requested current path; a replacement after it is detected # before this run has authority to create or swap output. - original_handle = _open_regular_output(real_path, None) - original = _owned_path( - real_path, original_handle, created=False, - owned_records=unpaired_originals) + original = _claim_owned_output( + lambda: (_open_regular_output(real_path, None), real_path), + created=False, owned_records=unpaired_originals) original_identity = original["identity"] state = {"temporary": None, "supplied_path": path, "real_path": real_path, @@ -2217,11 +2227,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{path} changed while it was being claimed", ) - handle, temporary = _private_mkstemp( - dir=os.path.dirname(real_path), - prefix=os.path.basename(real_path) + ".", suffix=".part") - record = _owned_path(temporary, handle, created=True, - owned_records=claimed) + record = _claim_owned_output( + lambda: tempfile.mkstemp( + dir=os.path.dirname(real_path), + prefix=os.path.basename(real_path) + ".", suffix=".part"), + created=True, owned_records=claimed) + temporary = record["path"] record["supplied_path"] = path record["canonical_path"] = real_path record["cleanup_path"] = temporary @@ -2232,8 +2243,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # Register this newly created inode before returning to an # interruptible caller line. The record is the cleanup owner # even if a signal arrives before its path metadata is filled. - record = _owned_path( - canonical_path, _open_private(canonical_path, accept_inherited), + record = _claim_owned_output( + lambda: (_open_private(canonical_path, accept_inherited), canonical_path), created=True, owned_records=claimed) # Keep cleanup on the canonical inode path captured before the # open. The supplied spelling remains an authority that must @@ -2268,11 +2279,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): temporary = state["temporary"] supplied_path, real_path = state["supplied_path"], state["real_path"] original_identity = state["original_identity"] - backup_handle, backup = _private_mkstemp( - dir=os.path.dirname(real_path), - prefix=os.path.basename(real_path) + ".", suffix=".bak") - pending_backup = _owned_path( - backup, backup_handle, created=True, owned_records=unpaired_backups) + pending_backup = _claim_owned_output( + lambda: tempfile.mkstemp( + dir=os.path.dirname(real_path), + prefix=os.path.basename(real_path) + ".", suffix=".bak"), + created=True, owned_records=unpaired_backups) + backup_handle, backup = pending_backup["pin"], pending_backup["path"] pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, "staged_identity": temporary["identity"], @@ -2358,11 +2370,6 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. committed = True - # The rollback handler still needs the old .part spelling until this - # boundary. Once committed, bind close diagnostics to the live output. - for state in staged: - state["temporary"]["path"] = state["real_path"] - state["temporary"]["cleanup_path"] = state["real_path"] _cleanup_committed_outputs( replaced, claimed, cleanup_failures, descriptor_close_failures) if cleanup_failures: diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index b76c86175..2027ea610 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -1867,9 +1867,9 @@ def deny_backup(path): m._entry_identity = deny_backup retained, closes = [], [] real_close = m._close_owned_path - def record_close(record, failures): + def record_close(record, failures, **kwargs): closes.append(record["path"]) - return real_close(record, failures) + return real_close(record, failures, **kwargs) m._close_owned_path = record_close try: original_error = KeyboardInterrupt("controlled interrupt") @@ -2014,10 +2014,10 @@ def interrupt_before_cleanup(frame, event, _arg): raise KeyboardInterrupt("controlled interrupt before cleanup") return interrupt_before_cleanup - def observe_close(record, failures): + def observe_close(record, failures, **kwargs): if record.get("pin") is not None: closed_paths.append(str(record["path"])) - return real_close(record, failures) + return real_close(record, failures, **kwargs) m._close_owned_path = observe_close sys.settrace(interrupt_before_cleanup) @@ -2042,7 +2042,7 @@ def observe_close(record, failures): assert destination.read_text() == "new bytes" assert os.path.realpath(destination) in closed_paths, closed_paths assert any(path.endswith(".bak") for path in closed_paths) - assert not any(path.endswith(".part") for path in closed_paths) + assert any(path.endswith(".part") for path in closed_paths) def test_interrupt_after_backup_unlink_does_not_report_a_phantom_path(m): @@ -2064,10 +2064,10 @@ def interrupt_after_backup_unlink(path): raise KeyboardInterrupt("controlled interrupt after backup unlink") return result - def observe_close(record, failures): + def observe_close(record, failures, **kwargs): if record.get("pin") is not None: closed_paths.append(str(record["path"])) - return real_close(record, failures) + return real_close(record, failures, **kwargs) m.os.unlink = interrupt_after_backup_unlink m._close_owned_path = observe_close @@ -2088,7 +2088,7 @@ def observe_close(record, failures): assert not list(root.glob("*.bak")) assert os.path.realpath(destination) in closed_paths, closed_paths assert any(path.endswith(".bak") for path in closed_paths) - assert not any(path.endswith(".part") for path in closed_paths) + assert any(path.endswith(".part") for path in closed_paths) def test_legacy_oserror_cleanup_diagnostic_reaches_stderr(m): @@ -3841,27 +3841,28 @@ def replace_then_link_first_backup(source, destination): assert backup.read_text() == alias.read_text() == "old first" -def test_interrupt_during_fresh_creation_closes_and_unlinks_the_output(m): - """A real SIGINT after open(2) still unwinds the private creation helper.""" +def test_interrupt_during_fresh_registration_reconciles_the_created_output(m): + """A real SIGINT at record construction leaves the helper's pin to clean.""" with tempfile.TemporaryDirectory() as directory: destination = pathlib.Path(directory) / "fresh.xml" - _, start = inspect.getsourcelines(m._open_private) - fchmod_line = start + next( - offset for offset, line in enumerate(inspect.getsource(m._open_private).splitlines()) - if line.strip() == "os.fchmod(handle, 0o600)") + _, start_line = inspect.getsourcelines(m._owned_path) + record_line = start_line + next( + offset for offset, line in enumerate(inspect.getsource(m._owned_path).splitlines()) + if line.strip() == 'record = {"path": path, "identity": identity, "pin": handle}') old_trace, old_handler = sys.gettrace(), signal.getsignal(signal.SIGINT) fired = False - def interrupt_after_open(frame, event, _arg): + def interrupt_at_record_construction(frame, event, _arg): nonlocal fired - if (not fired and event == "line" and frame.f_code is m._open_private.__code__ - and frame.f_lineno == fchmod_line): + if (not fired and event == "line" and frame.f_code is m._owned_path.__code__ + and frame.f_lineno == record_line + and frame.f_locals.get("created")): fired = True signal.raise_signal(signal.SIGINT) - return interrupt_after_open + return interrupt_at_record_construction signal.signal(signal.SIGINT, signal.default_int_handler) - sys.settrace(interrupt_after_open) + sys.settrace(interrupt_at_record_construction) try: try: m.write_outputs([(str(destination), "fresh bytes")]) @@ -3876,21 +3877,54 @@ def interrupt_after_open(frame, event, _arg): assert not destination.exists() +def test_created_output_fchmod_failure_preserves_a_foreign_replacement(m): + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, foreign = root / "fresh.xml", root / "foreign.xml" + foreign.write_text("foreign bytes") + real_fchmod, real_replace = m.os.fchmod, m.os.replace + fired = False + + def replace_then_fail(handle, mode): + nonlocal fired + if not fired: + fired = True + real_replace(foreign, destination) + raise OSError("controlled fchmod failure") + return real_fchmod(handle, mode) + + m.os.fchmod = replace_then_fail + try: + try: + m.write_outputs([(str(destination), "fresh bytes")]) + raise AssertionError("the controlled mode failure must escape") + except OSError as error: + assert "controlled fchmod failure" in str(error) + finally: + m.os.fchmod = real_fchmod + + assert fired + assert destination.read_text() == "foreign bytes" + + + def test_interrupt_after_fresh_registration_cleans_the_requested_output(m): """A SIGINT after registration has a claimed cleanup owner already.""" with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) destination = root / "fresh.xml" - _, start = inspect.getsourcelines(m._owned_path) + _, start = inspect.getsourcelines(m._claim_owned_output) return_line = start + next( - offset for offset, line in enumerate(inspect.getsource(m._owned_path).splitlines()) + offset for offset, line in enumerate(inspect.getsource(m._claim_owned_output).splitlines()) if line.strip() == "return record") old_trace, old_handler = sys.gettrace(), signal.getsignal(signal.SIGINT) fired = False def interrupt_after_registration(frame, event, _arg): nonlocal fired - if (not fired and event == "line" and frame.f_code is m._owned_path.__code__ + if (not fired and event == "line" and frame.f_code is m._claim_owned_output.__code__ and frame.f_lineno == return_line and frame.f_locals.get("created") and frame.f_locals.get("owned_records") is not None): @@ -3982,6 +4016,8 @@ def fail_before_close(handle): def test_owner_only_outputs_survive_a_restrictive_umask(m): """Use a child process so an extreme umask cannot affect this test process.""" + if os.name != "posix": + return program = f'''\ import importlib.util import os From 4fe945ad976655ec64896ffc701b4fde496b177c Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 05:54:31 +0530 Subject: [PATCH 39/59] Diagnose output pin closure and partial rollback --- scripts/bank_statement_import.py | 157 +++++++++++++++++++------- scripts/bank_statement_import.test.py | 141 +++++++++++++++++++++-- 2 files changed, 249 insertions(+), 49 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 679ed22a5..6bd0683df 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -92,6 +92,7 @@ import csv import datetime import decimal +import errno import getpass import hashlib import io @@ -1744,29 +1745,44 @@ def _claimed_output_changed(supplied_path, canonical_path, identity): return True -def _close_owned_path(record, failures, diagnostic_path=None): - """Release an ownership pin only after its cleanup decision is complete.""" +def _close_owned_path(record, failures, diagnostic_path=None, descriptor_failures=None): + """Release an ownership pin without guessing from a stale pathname. + + ``close`` may report an error after releasing a descriptor. EBADF from a + retained-descriptor ``fstat`` is the one supported indication of that + after-effect. Every other failed or mismatched inspection remains a + descriptor uncertainty: never retry a close that could target a reused fd. + """ handle = record.get("pin") if handle is None: return record["pin"] = None try: os.close(handle) + return except OSError: - # close can fail after taking effect. Only an extant owned entry is a - # retained-path failure; an already-unlinked backup has no such path. - try: - cleanup_path = diagnostic_path or record.get("cleanup_path", record["path"]) - if _entry_identity(cleanup_path) == record["identity"]: - failures.append(cleanup_path) - except FileNotFoundError: - pass - except OSError: - state = _cleanup_entry_state(cleanup_path, record["identity"]) - if state in ("owned", "reclaimed"): - failures.append(cleanup_path) - elif state == "uninspectable": - _record_uninspectable_cleanup(cleanup_path, failures) + pass + + diagnostic = str(diagnostic_path or record.get( + "cleanup_path", record["path"])) + destination = descriptor_failures if descriptor_failures is not None else failures + try: + stat_result = os.fstat(handle) + except OSError as error: + if error.errno == errno.EBADF: + # The close took effect (or the descriptor was independently made + # invalid). Do not retry it: a later fd could be foreign. + return + destination.append(diagnostic) + return + if (stat_result.st_dev, stat_result.st_ino) == record["identity"]: + # The original descriptor is still open. Its pathname may already + # name a staged replacement, so this is never a retained-path claim. + destination.append(diagnostic) + return + # A mocked or platform-specific close could leave a live but different fd. + # It is neither safe to close again nor evidence about a pathname. + destination.append(diagnostic) def _reconcile_owned_pin_after_cleanup(record, outcome, failures): @@ -1792,7 +1808,7 @@ def _reconcile_owned_pin_after_cleanup(record, outcome, failures): f"owned output could not be located after cleanup: {record['path']}") -def _cleanup_owned_path(record, failures): +def _cleanup_owned_path(record, failures, descriptor_failures=None): """Remove one owned pathname, releasing its pin first on Windows. POSIX keeps the descriptor open through the identity decision so an inode @@ -1806,7 +1822,7 @@ def _cleanup_owned_path(record, failures): # error after releasing the descriptor, so decide whether its # pathname diagnostic remains only after the unlink outcome is known. failure_start = len(failures) - _close_owned_path(record, failures) + _close_owned_path(record, failures, descriptor_failures=descriptor_failures) outcome = _unlink_for_cleanup( record.get("cleanup_path", record["path"]), record["identity"], failures) if outcome == "removed": @@ -1821,7 +1837,7 @@ def _cleanup_owned_path(record, failures): # pathname with the separate pinned-inode conclusion below. del failures[failure_start:] _reconcile_owned_pin_after_cleanup(record, outcome, failures) - _close_owned_path(record, failures) + _close_owned_path(record, failures, descriptor_failures=descriptor_failures) def _metadata_from_handle(path, handle): @@ -1960,7 +1976,7 @@ def _copy_private_backup(source_path, original_identity, backup_handle): os.close(source_handle) -def _restore_backup(swap, failures, metadata_scope_warnings): +def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures=None): """Restore an owned private backup after a caught swap failure. `os.replace` can report an exception after the filesystem call took effect. @@ -1989,7 +2005,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings): os.utime(handle, ns=(metadata["atime_ns"], current.st_mtime_ns)) except OSError: failures.append(destination) - _cleanup_owned_path(backup_record, failures) + _cleanup_owned_path(backup_record, failures, descriptor_failures=descriptor_failures) return if current_identity != staged_identity: failures.append(backup) @@ -2089,8 +2105,11 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ """Remove old private copies after every replacement has committed.""" for swap in replaced: backup = swap["backup"] - _cleanup_owned_path(backup, retained_failures) - _close_owned_path(swap["original"], retained_failures) + _cleanup_owned_path(backup, retained_failures, descriptor_failures=descriptor_failures) + _close_owned_path( + swap["original"], retained_failures, + diagnostic_path=swap.get("destination", swap["original"]["path"]), + descriptor_failures=descriptor_failures) for record in claimed: # A claimed path did not exist before this run. Its close failure cannot # retain a prior sensitive copy, so keep that diagnostic distinct from @@ -2119,7 +2138,7 @@ def _reconcile_interrupted_committed_cleanup( if still_at_path is True: retained_failures.append(str(backup["path"])) elif still_at_path is False: - _cleanup_owned_path(backup, retained_failures) + _cleanup_owned_path(backup, retained_failures, descriptor_failures=descriptor_failures) else: # We cannot identify an entry after an I/O/permission error. # Preserve the original interruption and report no ownership @@ -2130,8 +2149,12 @@ def _reconcile_interrupted_committed_cleanup( retained_failures.append( "could not reconcile committed rollback copy: " + str(backup["path"])) finally: - _close_owned_path(backup, retained_failures) - _close_owned_path(swap["original"], retained_failures) + _close_owned_path(backup, retained_failures, + descriptor_failures=descriptor_failures) + _close_owned_path( + swap["original"], retained_failures, + diagnostic_path=swap.get("destination", swap["original"]["path"]), + descriptor_failures=descriptor_failures) for record in claimed: _close_owned_path( record, descriptor_failures, diagnostic_path=record.get("canonical_path")) @@ -2200,6 +2223,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): cleanup_failures = [] descriptor_close_failures = [] metadata_scope_warnings = [] + partial_commit_failures = [] committed = False try: for path, _ in targets: @@ -2288,6 +2312,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_swap = {"backup": pending_backup, "destination": real_path, "original_identity": original_identity, "staged_identity": temporary["identity"], + "temporary": temporary, "original": None, "metadata": None, "swap_started": False} pending_backup = None @@ -2363,9 +2388,24 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # A commit may retire them only while their ownership remains proved. for swap in replaced: backup = swap["backup"] - if _entry_identity(backup["path"]) != backup["identity"]: - raise Refusal("output_path_changed", "rollback copy changed before commit") - _pinned_backup_still_has_one_link(backup) + try: + backup_unchanged = ( + _entry_identity(backup["path"]) == backup["identity"]) + except OSError: + backup_unchanged = False + if not backup_unchanged: + swap["rollback_unavailable"] = True + raise Refusal( + "output_path_changed", + f"{swap['destination']} rollback copy changed before commit; " + "this already replaced output could not be rolled back", + ) + try: + _pinned_backup_still_has_one_link(backup) + except Refusal as refusal: + if refusal.category == "output_path_changed": + swap["rollback_unavailable"] = True + raise # Final path validation is the boundary between rollback and committed # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. @@ -2397,13 +2437,19 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # original exception with their recoverable locations. if pending_swap is not None: backup = pending_swap["backup"] - _restore_backup(pending_swap, cleanup_failures, metadata_scope_warnings) - _close_owned_path(backup, cleanup_failures) + _restore_backup( + pending_swap, cleanup_failures, metadata_scope_warnings, + descriptor_close_failures) + _close_owned_path(backup, cleanup_failures, + descriptor_failures=descriptor_close_failures) if pending_swap["original"] is not None: - _close_owned_path(pending_swap["original"], cleanup_failures) + _close_owned_path( + pending_swap["original"], cleanup_failures, + diagnostic_path=pending_swap["destination"], + descriptor_failures=descriptor_close_failures) if pending_backup is not None and ( pending_swap is None or pending_backup is not pending_swap["backup"]): - _cleanup_owned_path(pending_backup, cleanup_failures) + _cleanup_owned_path(pending_backup, cleanup_failures, descriptor_failures=descriptor_close_failures) for swap in reversed(replaced): # An interrupt can arrive after `_record_replaced_swap` appends but # before its caller clears `pending_swap`. That one backup has @@ -2412,17 +2458,39 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if swap is pending_swap: continue backup = swap["backup"] - _restore_backup(swap, cleanup_failures, metadata_scope_warnings) - _close_owned_path(backup, cleanup_failures) - _close_owned_path(swap["original"], cleanup_failures) + if swap.get("rollback_unavailable"): + partial_commit_failures.append(str(swap["destination"])) + else: + _restore_backup( + swap, cleanup_failures, metadata_scope_warnings, + descriptor_close_failures) + _close_owned_path(backup, cleanup_failures, + descriptor_failures=descriptor_close_failures) + _close_owned_path( + swap["original"], cleanup_failures, + diagnostic_path=swap.get("destination", swap["original"]["path"]), + descriptor_failures=descriptor_close_failures) + unrollbackable_temporaries = { + id(swap["temporary"]) for swap in replaced + if swap.get("rollback_unavailable") + } for record in claimed: - _cleanup_owned_path(record, cleanup_failures) + if id(record) in unrollbackable_temporaries: + _close_owned_path( + record, cleanup_failures, + diagnostic_path=record.get("canonical_path"), + descriptor_failures=descriptor_close_failures) + else: + _cleanup_owned_path(record, cleanup_failures, + descriptor_failures=descriptor_close_failures) # Existing destinations are pinned at first claim, before they enter a # swap record. Close any pin whose state never reached the rollback # loops above; it is a descriptor-only ownership record and must never # be unlinked as if it were a fresh output. for state in staged: - _close_owned_path(state["original"], cleanup_failures) + _close_owned_path(state["original"], cleanup_failures, + diagnostic_path=state["real_path"], + descriptor_failures=descriptor_close_failures) # A signal can interrupt after _owned_path registered a backup or # original pin but before its caller transferred it to pending_swap or # staged. Reconcile only those unpaired records here; paired records @@ -2437,11 +2505,20 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): paired_backups.add(id(pending_backup)) for record in unpaired_backups: if id(record) not in paired_backups: - _cleanup_owned_path(record, cleanup_failures) + _cleanup_owned_path(record, cleanup_failures, descriptor_failures=descriptor_close_failures) for record in unpaired_originals: if id(record) not in paired_originals: - _close_owned_path(record, cleanup_failures) + _close_owned_path(record, cleanup_failures, + descriptor_failures=descriptor_close_failures) _note_cleanup_failures(error, cleanup_failures) + if partial_commit_failures: + _append_cleanup_detail( + error, "partially committed output could not be rolled back: " + + ", ".join(sorted(set(partial_commit_failures)))) + if descriptor_close_failures: + _append_cleanup_detail( + error, "ownership descriptor close failed or could not be verified for: " + + ", ".join(sorted(set(descriptor_close_failures)))) _note_rollback_metadata_scope(error, metadata_scope_warnings) raise diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 2027ea610..7d754f224 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -31,6 +31,7 @@ import contextlib import datetime import decimal +import errno import hashlib import io import importlib.util @@ -322,7 +323,7 @@ def test_real_hdfc_capture_binds_the_account_no_geometry_only(m): (369.261, 149.001, 379.037, 156.201, "No"), ]], selected account = m.require_account_match(pages, bank, "xx1111111") - assert account == "11111111111111" + assert account == "1" * 14 # Mutation controls select existing captured header geometry. They prove # that the production Account/No selector excludes phone, customer-id, @@ -1687,9 +1688,9 @@ def interrupt_before_clear(frame, event, _arg): raise KeyboardInterrupt("controlled pending-backup interrupt") return interrupt_before_clear - def observe_cleanup(record, failures): + def observe_cleanup(record, failures, **kwargs): cleanup_paths.append(str(record["path"])) - return real_cleanup(record, failures) + return real_cleanup(record, failures, **kwargs) m._cleanup_owned_path = observe_cleanup sys.settrace(interrupt_before_clear) @@ -3154,15 +3155,12 @@ def close_then_error(handle): m._owned_path, m.os.close = observe_owned, close_then_error try: - try: - m.write_outputs([(str(destination), "new bytes")]) - raise AssertionError("the controlled close failure must escape") - except m.OutputDescriptorCloseFailure as failure: - assert failure.output_paths == (str(destination.resolve()),) - assert "prior output retained" not in str(failure) + m.write_outputs([(str(destination), "new bytes")]) finally: m._owned_path, m.os.close = real_owned, real_close + # EBADF from the retained-descriptor probe proves this close took + # effect, so it is not a leaked-pin failure. assert fired assert destination.read_text() == "new bytes" assert list(pathlib.Path(directory).iterdir()) == [destination] @@ -4014,6 +4012,131 @@ def fail_before_close(handle): assert destination.read_text() == "new bytes" + +def test_committed_original_pin_close_failure_reports_destination(m): + """The old inode pin can stay open after its pathname names staged bytes.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + destination.write_text("old bytes") + real_owned, real_close = m._owned_path, m.os.close + original_handles, leaked = set(), [] + + def observe_owned(path, handle, *, created, **kwargs): + record = real_owned(path, handle, created=created, **kwargs) + if not created and pathlib.Path(path).resolve() == destination.resolve(): + original_handles.add(handle) + return record + + def fail_before_close(handle): + if handle in original_handles: + leaked.append(handle) + raise OSError("controlled original-pin close failure") + return real_close(handle) + + m._owned_path, m.os.close = observe_owned, fail_before_close + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the old descriptor leak must be reported") + except m.OutputDescriptorCloseFailure as failure: + assert failure.output_paths == (str(destination.resolve()),) + finally: + m._owned_path, m.os.close = real_owned, real_close + for handle in leaked: + try: + real_close(handle) + except OSError: + pass + + assert leaked + assert destination.read_text() == "new bytes" + + +def test_committed_close_unknown_descriptor_state_is_not_treated_as_after_effect(m): + """Only EBADF proves a failed close already released its descriptor.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + destination.write_text("old bytes") + real_owned, real_close, real_fstat = m._owned_path, m.os.close, m.os.fstat + original_handles, leaked = set(), [] + close_attempted = set() + + def observe_owned(path, handle, *, created, **kwargs): + record = real_owned(path, handle, created=created, **kwargs) + if not created and pathlib.Path(path).resolve() == destination.resolve(): + original_handles.add(handle) + return record + + def fail_before_close(handle): + if handle in original_handles: + close_attempted.add(handle) + leaked.append(handle) + raise OSError("controlled original-pin close failure") + return real_close(handle) + + def unreadable_pin(handle): + if handle in close_attempted: + raise OSError(errno.EIO, "controlled descriptor inspection failure") + return real_fstat(handle) + + m._owned_path, m.os.close, m.os.fstat = observe_owned, fail_before_close, unreadable_pin + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("unknown descriptor state must be reported") + except m.OutputDescriptorCloseFailure as failure: + assert failure.output_paths == (str(destination.resolve()),) + finally: + m._owned_path, m.os.close, m.os.fstat = real_owned, real_close, real_fstat + for handle in leaked: + try: + real_close(handle) + except OSError: + pass + + assert destination.read_text() == "new bytes" + + +def test_missing_earlier_backup_reports_the_partial_committed_destination(m): + """A vanished first rollback copy leaves its new destination explicit.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_copy = m._copy_private_backup + + def remove_first_backup_during_second_prepare(source_path, *args): + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == second.resolve(): + first_backup, = root.glob("first.xml.*.bak") + first_backup.unlink() + return result + + m._copy_private_backup = remove_first_backup_during_second_prepare + try: + refusal = refuses( + m, "output_path_changed", m.write_outputs, + [(str(first), "first new"), (str(second), "second new")]) + finally: + m._copy_private_backup = real_copy + + message = str(refusal.code) + assert str(first.resolve()) in message + assert "partially committed output could not be rolled back" in message + assert ".bak" not in message + assert ".part" not in message + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert not list(root.glob("*.bak")) + assert not list(root.glob("*.part")) + def test_owner_only_outputs_survive_a_restrictive_umask(m): """Use a child process so an extreme umask cannot affect this test process.""" if os.name != "posix": From 09ebbaedd775fe7346c888adf06adec9c1b7673a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 06:01:45 +0530 Subject: [PATCH 40/59] Reconcile every unavailable rollback backup --- scripts/bank_statement_import.py | 45 ++++++++------ scripts/bank_statement_import.test.py | 85 +++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 18 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 6bd0683df..f9e90d1d7 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2010,6 +2010,8 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures if current_identity != staged_identity: failures.append(backup) return + if swap.get("rollback_unavailable"): + return "unrollbackable" restored = False try: _pinned_backup_still_has_one_link(backup_record) @@ -2021,19 +2023,19 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures # destination. failures.append( f"unknown hard-link alias may retain rollback bytes: {backup}") - else: - # A missing or changed pinned backup is not an alias. Preserve - # the original error and disclose only the rollback path whose - # identity could no longer support a restore. - failures.append(backup) - return + return None + # A missing or changed backup cannot restore a destination that still + # names this run's staged inode. Report that partial result by its + # actual destination, never by the vanished private spelling. + swap["rollback_unavailable"] = True + return "unrollbackable" except OSError: failures.append(backup) - return + return None try: if _entry_identity(backup) != backup_identity: - failures.append(backup) - return + swap["rollback_unavailable"] = True + return "unrollbackable" # This observes ownership immediately before the replace. POSIX has no # compare-and-swap rename, so a hostile concurrent rename after this # check is still outside the CLI's locking authority. @@ -2049,7 +2051,13 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures backup_retained = _entry_identity(backup) == backup_identity except OSError: backup_retained = False - failures.append(backup if backup_retained else destination) + if backup_retained: + failures.append(backup) + elif current_identity == staged_identity: + swap["rollback_unavailable"] = True + return "unrollbackable" + else: + failures.append(destination) if restored: try: restore_handle = _open_regular_output(destination, backup_identity) @@ -2437,9 +2445,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # original exception with their recoverable locations. if pending_swap is not None: backup = pending_swap["backup"] - _restore_backup( - pending_swap, cleanup_failures, metadata_scope_warnings, - descriptor_close_failures) + if _restore_backup( + pending_swap, cleanup_failures, metadata_scope_warnings, + descriptor_close_failures) == "unrollbackable": + partial_commit_failures.append(str(pending_swap["destination"])) _close_owned_path(backup, cleanup_failures, descriptor_failures=descriptor_close_failures) if pending_swap["original"] is not None: @@ -2458,12 +2467,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): if swap is pending_swap: continue backup = swap["backup"] - if swap.get("rollback_unavailable"): - partial_commit_failures.append(str(swap["destination"])) - else: - _restore_backup( + if _restore_backup( swap, cleanup_failures, metadata_scope_warnings, - descriptor_close_failures) + descriptor_close_failures) == "unrollbackable": + partial_commit_failures.append(str(swap["destination"])) _close_owned_path(backup, cleanup_failures, descriptor_failures=descriptor_close_failures) _close_owned_path( @@ -2474,6 +2481,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): id(swap["temporary"]) for swap in replaced if swap.get("rollback_unavailable") } + if pending_swap is not None and pending_swap.get("rollback_unavailable"): + unrollbackable_temporaries.add(id(pending_swap["temporary"])) for record in claimed: if id(record) in unrollbackable_temporaries: _close_owned_path( diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 7d754f224..d7cac8fa6 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4137,6 +4137,91 @@ def remove_first_backup_during_second_prepare(source_path, *args): assert not list(root.glob("*.bak")) assert not list(root.glob("*.part")) + +def test_later_backup_prepare_failure_reports_an_earlier_partial_destination(m): + """Rollback recovery, not just final validation, owns this diagnosis.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_copy = m._copy_private_backup + fired = False + + def remove_first_backup_then_fail_second_prepare(source_path, *args): + nonlocal fired + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == second.resolve(): + first_backup, = root.glob("first.xml.*.bak") + first_backup.unlink() + fired = True + raise OSError("controlled later backup preparation failure") + return result + + m._copy_private_backup = remove_first_backup_then_fail_second_prepare + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the later preparation failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert "controlled later backup preparation failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert ".bak" not in detail + assert ".part" not in detail + finally: + m._copy_private_backup = real_copy + + assert fired + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert not list(root.glob("*.bak")) + assert not list(root.glob("*.part")) + + +def test_all_missing_backups_report_every_partial_committed_destination(m): + """One final-check refusal must still reconcile every already-swapped row.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_changed = m._claimed_output_changed + fired = False + + def remove_backups_before_final_checks(*args): + nonlocal fired + if not fired: + fired = True + for backup in root.glob("*.bak"): + backup.unlink() + return real_changed(*args) + + m._claimed_output_changed = remove_backups_before_final_checks + try: + refusal = refuses( + m, "output_path_changed", m.write_outputs, + [(str(first), "first new"), (str(second), "second new")]) + finally: + m._claimed_output_changed = real_changed + + detail = str(refusal.code) + assert fired + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert str(second.resolve()) in detail + assert ".bak" not in detail + assert ".part" not in detail + assert first.read_text() == "first new" + assert second.read_text() == "second new" + assert not list(root.glob("*.bak")) + assert not list(root.glob("*.part")) + def test_owner_only_outputs_survive_a_restrictive_umask(m): """Use a child process so an extreme umask cannot affect this test process.""" if os.name != "posix": From 04161e1e5656cfe53440267d4dacad1bb87add9c Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 06:29:05 +0530 Subject: [PATCH 41/59] Verify rollback backup authority --- scripts/bank_statement_import.py | 108 +++++++++++++-- scripts/bank_statement_import.test.py | 184 +++++++++++++++++++++++++- 2 files changed, 281 insertions(+), 11 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index f9e90d1d7..2593f39da 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1927,6 +1927,75 @@ def _pinned_backup_still_has_one_link(record): _require_single_owned_link(record, "rollback copy", "rollback_backup_has_multiple_links") + +def _digest_pinned_bytes(handle): + """Hash a retained descriptor without changing its caller's file offset.""" + digest = hashlib.sha256() + if hasattr(os, "pread"): + offset = 0 + while True: + chunk = os.pread(handle, 1024 * 1024, offset) + if not chunk: + return digest.digest() + digest.update(chunk) + offset += len(chunk) + # Existing-output replacement is refused on Windows, where ``pread`` is + # not universally available. Keep that fallback ownership-local as well. + original_offset = os.lseek(handle, 0, os.SEEK_CUR) + try: + os.lseek(handle, 0, os.SEEK_SET) + while True: + chunk = os.read(handle, 1024 * 1024) + if not chunk: + return digest.digest() + digest.update(chunk) + finally: + os.lseek(handle, original_offset, os.SEEK_SET) + + +def _mark_rollback_unavailable(swap, failures, *, retain_named=False): + """Record a partial destination and reconcile its still-pinned backup. + + A missing pathname does not prove the backup inode vanished: its descriptor + can still reveal a moved or aliased private copy. Inspect before the outer + recovery releases that descriptor; zero links are the only no-path case. + """ + swap["rollback_unavailable"] = True + if swap.get("rollback_unavailable_reported"): + return "unrollbackable" + swap["rollback_unavailable_reported"] = True + record = swap["backup"] + pin = record.get("pin") + if pin is None: + _record_uninspectable_cleanup(record["path"], failures) + return "unrollbackable" + try: + stat_result = os.fstat(pin) + except OSError: + _record_uninspectable_cleanup(record["path"], failures) + return "unrollbackable" + if (stat_result.st_dev, stat_result.st_ino) != record["identity"]: + _record_uninspectable_cleanup(record["path"], failures) + return "unrollbackable" + if stat_result.st_nlink == 0: + return "unrollbackable" + try: + still_named = _entry_identity(record["path"]) == record["identity"] + except FileNotFoundError: + still_named = False + except OSError: + _record_uninspectable_cleanup(record["path"], failures) + return "unrollbackable" + if stat_result.st_nlink > 1: + failures.append( + f"unknown hard-link alias may retain rollback bytes: {record['path']}") + elif still_named and retain_named: + failures.append(record["path"]) + elif not still_named: + failures.append( + f"owned rollback copy could not be located after cleanup: {record['path']}") + return "unrollbackable" + def _copy_private_backup(source_path, original_identity, backup_handle): """Copy the original inode into an owner-only backup already opened O_EXCL. @@ -1966,12 +2035,14 @@ def _copy_private_backup(source_path, original_identity, backup_handle): verified.update(chunk) if copied.digest() != verified.digest(): raise OSError("private backup did not retain the copied bytes") + backup_digest = verified.digest() if (_fd_identity(source_handle) != original_identity or _file_identity(source_path) != original_identity): raise Refusal( "output_path_changed", f"{source_path} changed while its rollback copy was prepared", ) + return backup_digest finally: os.close(source_handle) @@ -2011,7 +2082,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures failures.append(backup) return if swap.get("rollback_unavailable"): - return "unrollbackable" + return _mark_rollback_unavailable(swap, failures) restored = False try: _pinned_backup_still_has_one_link(backup_record) @@ -2027,15 +2098,20 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures # A missing or changed backup cannot restore a destination that still # names this run's staged inode. Report that partial result by its # actual destination, never by the vanished private spelling. - swap["rollback_unavailable"] = True - return "unrollbackable" + return _mark_rollback_unavailable(swap, failures) except OSError: failures.append(backup) return None try: if _entry_identity(backup) != backup_identity: - swap["rollback_unavailable"] = True - return "unrollbackable" + return _mark_rollback_unavailable(swap, failures) + try: + digest_matches = ( + _digest_pinned_bytes(backup_record["pin"]) == backup_record["digest"]) + except OSError: + return _mark_rollback_unavailable(swap, failures, retain_named=True) + if not digest_matches: + return _mark_rollback_unavailable(swap, failures, retain_named=True) # This observes ownership immediately before the replace. POSIX has no # compare-and-swap rename, so a hostile concurrent rename after this # check is still outside the CLI's locking authority. @@ -2054,8 +2130,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures if backup_retained: failures.append(backup) elif current_identity == staged_identity: - swap["rollback_unavailable"] = True - return "unrollbackable" + return _mark_rollback_unavailable(swap, failures) else: failures.append(destination) if restored: @@ -2329,7 +2404,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_swap["original"] = state["original"] pending_swap["metadata"] = _metadata_from_handle( real_path, pending_swap["original"]["pin"]) - _copy_private_backup(real_path, original_identity, backup_handle) + pending_swap["backup"]["digest"] = _copy_private_backup( + real_path, original_identity, backup_handle) # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after # the filesystem call has taken effect but before it returns. @@ -2402,7 +2478,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): except OSError: backup_unchanged = False if not backup_unchanged: - swap["rollback_unavailable"] = True + _mark_rollback_unavailable(swap, cleanup_failures) raise Refusal( "output_path_changed", f"{swap['destination']} rollback copy changed before commit; " @@ -2412,8 +2488,20 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): _pinned_backup_still_has_one_link(backup) except Refusal as refusal: if refusal.category == "output_path_changed": - swap["rollback_unavailable"] = True + _mark_rollback_unavailable(swap, cleanup_failures) raise + try: + digest_matches = ( + _digest_pinned_bytes(backup["pin"]) == backup["digest"]) + except OSError: + digest_matches = False + if not digest_matches: + _mark_rollback_unavailable(swap, cleanup_failures, retain_named=True) + raise Refusal( + "output_path_changed", + f"{swap['destination']} rollback copy content could not be verified before commit; " + "this already replaced output could not be rolled back", + ) # Final path validation is the boundary between rollback and committed # cleanup. Keep it in this same handler so an interrupt before cleanup # starts cannot skip both recovery paths. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index d7cac8fa6..97e99628e 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4222,6 +4222,187 @@ def remove_backups_before_final_checks(*args): assert not list(root.glob("*.bak")) assert not list(root.glob("*.part")) + + +def test_digest_pinned_bytes_preserves_the_owned_pin_position(m): + """The integrity check must not disturb later ownership operations.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "backup.bak" + path.write_bytes(b"rollback bytes") + handle = os.open(path, os.O_RDONLY) + try: + os.lseek(handle, 3, os.SEEK_SET) + assert m._digest_pinned_bytes(handle) == hashlib.sha256(b"rollback bytes").digest() + assert os.lseek(handle, 0, os.SEEK_CUR) == 3 + finally: + os.close(handle) + + +def test_backup_digest_read_failure_preserves_original_error_and_untrusted_copy(m): + """An unreadable pin is untrusted, never permission to restore from it.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_copy, real_digest = m._copy_private_backup, m._digest_pinned_bytes + first_handles = set() + + def remember_first_backup_then_fail_second_prepare(source_path, identity, handle): + result = real_copy(source_path, identity, handle) + if pathlib.Path(source_path).resolve() == first.resolve(): + first_handles.add(handle) + if pathlib.Path(source_path).resolve() == second.resolve(): + raise OSError("controlled later preparation failure") + return result + + def fail_first_backup_read(handle): + if handle in first_handles: + raise OSError("controlled backup digest read failure") + return real_digest(handle) + + m._copy_private_backup, m._digest_pinned_bytes = ( + remember_first_backup_then_fail_second_prepare, fail_first_backup_read) + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the later failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert "controlled later preparation failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + backup, = root.glob("first.xml.*.bak") + assert str(backup) in detail + finally: + m._copy_private_backup, m._digest_pinned_bytes = real_copy, real_digest + + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert not list(root.glob("*.part")) + +def test_mutated_backup_during_later_prepare_preserves_original_error_and_bytes(m): + """An inode-stable backup needs its verified content before restoration.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_copy = m._copy_private_backup + changed = [] + + def mutate_first_backup_then_fail_second_prepare(source_path, *args): + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == second.resolve(): + backup, = root.glob("first.xml.*.bak") + backup.write_text("injected bytes") + changed.append(backup) + raise OSError("controlled later preparation failure") + return result + + m._copy_private_backup = mutate_first_backup_then_fail_second_prepare + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the later failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert "controlled later preparation failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert str(changed[0]) in detail + finally: + m._copy_private_backup = real_copy + + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert changed[0].read_text() == "injected bytes" + assert not list(root.glob("*.part")) + + +def test_mutated_backup_before_final_check_is_not_restored(m): + """The final authority check uses the same pinned-byte digest.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_changed = m._claimed_output_changed + mutated = [] + + def mutate_first_backup_before_final_checks(*args): + if not mutated: + backup, = root.glob("first.xml.*.bak") + backup.write_text("injected bytes") + mutated.append(backup) + return real_changed(*args) + + m._claimed_output_changed = mutate_first_backup_before_final_checks + try: + refusal = refuses( + m, "output_path_changed", m.write_outputs, + [(str(first), "first new"), (str(second), "second new")]) + finally: + m._claimed_output_changed = real_changed + + detail = str(refusal.code) + assert "rollback copy content could not be verified before commit" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert str(mutated[0]) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert mutated[0].read_text() == "injected bytes" + assert not list(root.glob("*.part")) + + +def test_moved_backup_during_later_prepare_is_disclosed_before_pin_close(m): + """A linked but renamed backup remains an unlocated retained old copy.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, moved = root / "first.xml", root / "second.xml", root / "moved.xml" + first.write_text("first old") + second.write_text("second old") + real_copy = m._copy_private_backup + + def move_first_backup_then_fail_second_prepare(source_path, *args): + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == second.resolve(): + backup, = root.glob("first.xml.*.bak") + backup.rename(moved) + raise OSError("controlled later preparation failure") + return result + + m._copy_private_backup = move_first_backup_then_fail_second_prepare + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the later failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert "controlled later preparation failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert "owned rollback copy could not be located after cleanup" in detail + finally: + m._copy_private_backup = real_copy + + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert moved.read_text() == "first old" + assert not list(root.glob("*.bak")) + assert not list(root.glob("*.part")) + def test_owner_only_outputs_survive_a_restrictive_umask(m): """Use a child process so an extreme umask cannot affect this test process.""" if os.name != "posix": @@ -4307,7 +4488,8 @@ def make_swap(root): swap = {"destination": destination, "original_identity": m._fd_identity(original_fd), "staged_identity": m._entry_identity(staged), "metadata": None, "swap_started": True, "original": None, - "backup": {"path": backup, "identity": m._fd_identity(backup_fd), "pin": backup_fd}} + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}} destination.unlink(); destination.symlink_to(staged) return destination, staged, backup, original_fd, backup_fd, swap with tempfile.TemporaryDirectory() as directory: From 14d84f7c9bf04a336a85e10db9ef89e5925dd1d0 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 12:20:46 +0530 Subject: [PATCH 42/59] Rectify bank output rollback recovery --- scripts/bank_statement_import.py | 63 +++++++++++-- scripts/bank_statement_import.test.py | 126 ++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 7 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 2593f39da..4f4d4cd64 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -98,6 +98,7 @@ import io import os import pathlib +import signal import re import shutil import stat @@ -1630,6 +1631,23 @@ def _open_regular_output(path, expected_identity=None): raise +@contextlib.contextmanager +def _defer_sigint_during_claim(): + """Pair a newly-created inode with its cleanup owner before SIGINT.""" + if not hasattr(signal, "pthread_sigmask"): + yield + return + try: + previous = signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT}) + except (OSError, ValueError): + yield + return + try: + yield + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous) + + def _owned_path(path, handle, *, created, owned_records=None): """Record a pathname and retain the descriptor that pins its inode. @@ -1702,12 +1720,15 @@ def _claim_owned_output(create, *, created, owned_records): handle = None record = None try: - handle, path = create() - record = _owned_path(path, handle, created=created) - if created and os.name != "nt": - os.fchmod(handle, 0o600) - owned_records.append(record) - return record + # A private create can finish before its factory returns. Defer Ctrl-C + # until the descriptor and its cleanup owner have been registered. + with _defer_sigint_during_claim(): + handle, path = create() + record = _owned_path(path, handle, created=created) + if created and os.name != "nt": + os.fchmod(handle, 0o600) + owned_records.append(record) + return record except BaseException as error: # If registration already reached its collection, the outer handler is # its sole cleanup authority. Otherwise recover through the pin. @@ -2129,6 +2150,13 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures backup_retained = False if backup_retained: failures.append(backup) + try: + destination_still_staged = ( + _entry_identity(destination) == staged_identity) + except OSError: + destination_still_staged = False + if destination_still_staged: + return _mark_rollback_unavailable(swap, failures) elif current_identity == staged_identity: return _mark_rollback_unavailable(swap, failures) else: @@ -2220,6 +2248,22 @@ def _reconcile_interrupted_committed_cleanup( still_at_path = None if still_at_path is True: retained_failures.append(str(backup["path"])) + # A named backup can also have gained an alias after final + # validation. Inspect its pin before release so the operator + # does not remove the .bak and miss a second private copy. + try: + stat_result = os.fstat(backup["pin"]) + except OSError: + retained_failures.append( + "could not inspect committed rollback copy: " + str(backup["path"])) + else: + if (stat_result.st_dev, stat_result.st_ino) != backup["identity"]: + retained_failures.append( + "could not inspect committed rollback copy: " + str(backup["path"])) + elif stat_result.st_nlink > 1: + retained_failures.append( + "unknown hard-link alias may retain rollback bytes: " + + str(backup["path"])) elif still_at_path is False: _cleanup_owned_path(backup, retained_failures, descriptor_failures=descriptor_failures) else: @@ -2329,7 +2373,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "original_identity": original_identity, "original": original} staged.append(state) - if _file_identity(real_path) != original_identity: + try: + claimed_existing_output_changed = ( + _file_identity(real_path) != original_identity) + except (FileNotFoundError, OSError): + claimed_existing_output_changed = True + if claimed_existing_output_changed: raise Refusal( "output_path_changed", f"{path} changed while it was being claimed", diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 97e99628e..3d31c656a 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -3947,6 +3947,67 @@ def interrupt_after_registration(frame, event, _arg): assert list(root.iterdir()) == [] +def test_sigint_during_created_factory_return_waits_for_ownership_handoff(m): + """A factory-side SIGINT cannot strand a file before registration.""" + if not hasattr(signal, "pthread_sigmask"): + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "fresh.xml" + real_open = m._open_private + old_handler = signal.getsignal(signal.SIGINT) + fired = False + + def create_then_interrupt(*args, **kwargs): + nonlocal fired + handle = real_open(*args, **kwargs) + fired = True + signal.raise_signal(signal.SIGINT) + return handle + + m._open_private = create_then_interrupt + signal.signal(signal.SIGINT, signal.default_int_handler) + try: + try: + m.write_outputs([(str(destination), "fresh bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + m._open_private = real_open + signal.signal(signal.SIGINT, old_handler) + + assert fired + assert not destination.exists() + assert list(root.iterdir()) == [] + + +def test_initial_existing_path_revalidation_is_a_typed_refusal(m): + """The first post-pin check cannot leak a raw filesystem exception.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "output.xml" + destination.write_text("old bytes") + real_open = m._open_regular_output + + def open_then_remove(path, *args): + handle = real_open(path, *args) + if pathlib.Path(path).resolve() == destination.resolve(): + destination.unlink() + return handle + + m._open_regular_output = open_then_remove + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._open_regular_output = real_open + + assert "changed while it was being claimed" in str(refusal.code) + assert not destination.exists() + assert list(root.iterdir()) == [] + + def test_existing_output_removed_after_backup_is_a_typed_path_refusal(m): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) @@ -4473,6 +4534,71 @@ def test_interrupted_committed_cleanup_parent_rename_retains_unlocated_backup(m) assert (moved / "old.bak").read_text() == "old.bak" +def test_interrupted_committed_cleanup_reports_a_retained_backup_alias(m): + """A known .bak name cannot conceal a later hard-link alias.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + backup_path, alias = root / "old.bak", root / "old-alias.bak" + original_path, claimed_path = root / "old.xml", root / "new.xml" + for path in (backup_path, original_path, claimed_path): + path.write_text(path.name) + os.link(backup_path, alias) + fds = [os.open(path, os.O_RDONLY) for path in + (backup_path, original_path, claimed_path)] + backup, original, claimed = [ + {"path": path, "identity": m._fd_identity(fd), "pin": fd} + for path, fd in zip((backup_path, original_path, claimed_path), fds)] + retained = [] + m._reconcile_interrupted_committed_cleanup( + [{"backup": backup, "original": original}], [claimed], retained, []) + + assert str(backup_path) in retained + assert any("unknown hard-link alias may retain rollback bytes" in value + for value in retained) + assert backup["pin"] is None and original["pin"] is None and claimed["pin"] is None + + +def test_failed_restore_before_effect_reports_the_partial_destination(m): + """A retained backup does not make an unchanged staged destination safe.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + + def fail_second_swap_and_first_restore(source, destination): + source = pathlib.Path(source) + destination = pathlib.Path(destination) + if source.suffix == ".part" and destination.resolve() == second.resolve(): + raise OSError("controlled later swap failure") + if source.suffix == ".bak" and destination.resolve() == first.resolve(): + raise OSError("controlled restore failure before effect") + return real_replace(source, destination) + + m.os.replace = fail_second_swap_and_first_restore + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m.os.replace = real_replace + + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + backup, = root.glob("first.xml.*.bak") + assert backup.read_text() == "first old" + + def test_restore_backup_preserves_foreign_symlink_entry(m): """A symlink to the staged-new inode is still a foreign directory entry.""" if os.name == "nt": From b589dc383e868626988586af8a11633873174149 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 12:23:43 +0530 Subject: [PATCH 43/59] Handle closed rollback pins during cleanup recovery --- scripts/bank_statement_import.py | 24 +++++++++++++----------- scripts/bank_statement_import.test.py | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 4f4d4cd64..db0bbb935 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2251,19 +2251,21 @@ def _reconcile_interrupted_committed_cleanup( # A named backup can also have gained an alias after final # validation. Inspect its pin before release so the operator # does not remove the .bak and miss a second private copy. - try: - stat_result = os.fstat(backup["pin"]) - except OSError: - retained_failures.append( - "could not inspect committed rollback copy: " + str(backup["path"])) - else: - if (stat_result.st_dev, stat_result.st_ino) != backup["identity"]: + pin = backup.get("pin") + if pin is not None: + try: + stat_result = os.fstat(pin) + except OSError: retained_failures.append( "could not inspect committed rollback copy: " + str(backup["path"])) - elif stat_result.st_nlink > 1: - retained_failures.append( - "unknown hard-link alias may retain rollback bytes: " - + str(backup["path"])) + else: + if (stat_result.st_dev, stat_result.st_ino) != backup["identity"]: + retained_failures.append( + "could not inspect committed rollback copy: " + str(backup["path"])) + elif stat_result.st_nlink > 1: + retained_failures.append( + "unknown hard-link alias may retain rollback bytes: " + + str(backup["path"])) elif still_at_path is False: _cleanup_owned_path(backup, retained_failures, descriptor_failures=descriptor_failures) else: diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 3d31c656a..03e2a6378 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4560,6 +4560,30 @@ def test_interrupted_committed_cleanup_reports_a_retained_backup_alias(m): assert backup["pin"] is None and original["pin"] is None and claimed["pin"] is None +def test_interrupted_committed_cleanup_keeps_a_named_backup_after_prior_close(m): + """A prior cleanup may close its pin while leaving the known backup named.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + backup_path, original_path, claimed_path = ( + root / "old.bak", root / "old.xml", root / "new.xml") + for path in (backup_path, original_path, claimed_path): + path.write_text(path.name) + backup_fd, original_fd, claimed_fd = ( + os.open(path, os.O_RDONLY) for path in + (backup_path, original_path, claimed_path)) + backup = {"path": backup_path, "identity": m._fd_identity(backup_fd), "pin": backup_fd} + original = {"path": original_path, "identity": m._fd_identity(original_fd), "pin": original_fd} + claimed = {"path": claimed_path, "identity": m._fd_identity(claimed_fd), "pin": claimed_fd} + os.close(backup_fd) + backup["pin"] = None + retained = [] + m._reconcile_interrupted_committed_cleanup( + [{"backup": backup, "original": original}], [claimed], retained, []) + + assert retained == [str(backup_path)] + assert original["pin"] is None and claimed["pin"] is None + + def test_failed_restore_before_effect_reports_the_partial_destination(m): """A retained backup does not make an unchanged staged destination safe.""" if os.name == "nt": From 3e8c4a2aab20cd52af8533f3c7dd2b0f398e9ce4 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 12:53:33 +0530 Subject: [PATCH 44/59] Harden bank output cleanup recovery --- scripts/bank_statement_import.py | 61 +++++++++--- scripts/bank_statement_import.test.py | 136 ++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 11 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index db0bbb935..232a0ba54 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1596,7 +1596,10 @@ def _unlink_for_cleanup(path, owned_identity, failures): state = _cleanup_entry_state(path, owned_identity) if state == "missing": return "missing" - if state in ("owned", "reclaimed"): + if state == "reclaimed": + failures.append(str(path)) + return "reclaimed" + if state == "owned": failures.append(str(path)) elif state == "uninspectable": _record_uninspectable_cleanup(path, failures) @@ -1634,18 +1637,35 @@ def _open_regular_output(path, expected_identity=None): @contextlib.contextmanager def _defer_sigint_during_claim(): """Pair a newly-created inode with its cleanup owner before SIGINT.""" - if not hasattr(signal, "pthread_sigmask"): - yield - return + if hasattr(signal, "pthread_sigmask"): + try: + previous = signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT}) + except (OSError, ValueError): + previous = None + if previous is not None: + try: + yield + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous) + return + # Windows has no pthread signal mask. Temporarily retain Ctrl-C as a + # pending event, restore the caller's handler after registration, and then + # deliver it through that original handler. A non-main-thread claim cannot + # install a signal handler, so its existing exception recovery remains the + # authority in that unsupported execution context. + pending = [] try: - previous = signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT}) + previous = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, lambda _signum, _frame: pending.append(True)) except (OSError, ValueError): yield return try: yield finally: - signal.pthread_sigmask(signal.SIG_SETMASK, previous) + signal.signal(signal.SIGINT, previous) + if pending: + signal.raise_signal(signal.SIGINT) def _owned_path(path, handle, *, created, owned_records=None): @@ -1829,6 +1849,21 @@ def _reconcile_owned_pin_after_cleanup(record, outcome, failures): f"owned output could not be located after cleanup: {record['path']}") +def _record_windows_cleanup_alias(record, failures): + """Disclose a hard-link alias while Windows still permits pin inspection.""" + pin = record.get("pin") + if pin is None: + return + try: + stat_result = os.fstat(pin) + except OSError: + return + if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] + and stat_result.st_nlink > 1): + failures.append( + f"unknown hard-link alias may retain output bytes: {record['path']}") + + def _cleanup_owned_path(record, failures, descriptor_failures=None): """Remove one owned pathname, releasing its pin first on Windows. @@ -1842,6 +1877,7 @@ def _cleanup_owned_path(record, failures, descriptor_failures=None): # Windows requires closing before unlinking. A close can report an # error after releasing the descriptor, so decide whether its # pathname diagnostic remains only after the unlink outcome is known. + _record_windows_cleanup_alias(record, failures) failure_start = len(failures) _close_owned_path(record, failures, descriptor_failures=descriptor_failures) outcome = _unlink_for_cleanup( @@ -2057,8 +2093,13 @@ def _copy_private_backup(source_path, original_identity, backup_handle): if copied.digest() != verified.digest(): raise OSError("private backup did not retain the copied bytes") backup_digest = verified.digest() - if (_fd_identity(source_handle) != original_identity - or _file_identity(source_path) != original_identity): + try: + source_unchanged = ( + _fd_identity(source_handle) == original_identity + and _file_identity(source_path) == original_identity) + except (FileNotFoundError, OSError): + source_unchanged = False + if not source_unchanged: raise Refusal( "output_path_changed", f"{source_path} changed while its rollback copy was prepared", @@ -2113,9 +2154,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures # named here, but its unknown alias may retain prior statement # bytes; moving it back would make that alias a live copy of the # destination. - failures.append( - f"unknown hard-link alias may retain rollback bytes: {backup}") - return None + return _mark_rollback_unavailable(swap, failures) # A missing or changed backup cannot restore a destination that still # names this run's staged inode. Report that partial result by its # actual destination, never by the vanished private spelling. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 03e2a6378..13d7c2d3f 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2449,6 +2449,42 @@ def test_cleanup_keeps_a_reclaimed_owned_path(m): assert failures == [str(owned)] +def test_cleanup_reconciles_a_reclaimed_path_after_unlink_error(m): + """An unlink error after a move must retain the moved inode diagnostic.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path, moved, foreign = ( + root / "output.xml.pending", root / "moved.xml.pending", root / "foreign.xml") + handle = m._open_private(path) + os.write(handle, b"generated statement") + record = m._owned_path(path, handle, created=True) + record["cleanup_path"] = path + foreign.write_text("foreign writer bytes") + real_unlink = m.os.unlink + + def move_reclaim_then_fail(candidate): + if pathlib.Path(candidate) == path: + os.replace(path, moved) + os.replace(foreign, path) + raise OSError("controlled unlink failure before effect") + return real_unlink(candidate) + + m.os.unlink = move_reclaim_then_fail + failures = [] + try: + m._cleanup_owned_path(record, failures) + finally: + m.os.unlink = real_unlink + + assert failures == [ + f"owned output could not be located after cleanup: {path}"] + assert record["pin"] is None + assert path.read_text() == "foreign writer bytes" + assert moved.read_bytes() == b"generated statement" + + def test_pinned_backup_reclaimed_path_retains_cleanup_diagnostic(m): if os.name == "nt": return @@ -3575,6 +3611,8 @@ def swap_first_alias_backup_then_fail_second(source, destination): except OSError as error: notes = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + stderr.getvalue() assert "unknown hard-link alias may retain rollback bytes" in notes + assert "partially committed output could not be rolled back" in notes + assert str(first.resolve()) in notes finally: m.os.replace = real_replace @@ -3771,6 +3809,29 @@ def close_then_error(candidate): assert not path.exists() +def test_windows_modeled_cleanup_reports_an_alias_before_closing_the_pin(m): + """The Windows close-before-unlink branch still discloses linked output.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path, alias = root / "fresh.xml", root / "fresh-alias.xml" + handle = m._open_private(path) + os.write(handle, b"generated statement") + record = m._owned_path(path, handle, created=True) + os.link(path, alias) + real_name, failures = m.os.name, [] + + m.os.name = "nt" + try: + m._cleanup_owned_path(record, failures) + finally: + m.os.name = real_name + + assert not path.exists() + assert alias.read_bytes() == b"generated statement" + assert failures == [ + f"unknown hard-link alias may retain output bytes: {path}"] + + def test_rollback_restores_xattrs_before_a_readonly_final_mode(m): """Linux xattrs need the backup's temporary write permission during rollback.""" if os.name != "posix" or not hasattr(os, "setxattr"): @@ -3982,6 +4043,45 @@ def create_then_interrupt(*args, **kwargs): assert list(root.iterdir()) == [] +def test_factory_sigint_without_pthread_mask_waits_for_ownership_handoff(m): + """The signal-handler fallback keeps no-pthread creation recoverable.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "fresh.xml" + real_open, real_signal = m._open_private, m.signal + old_handler = signal.getsignal(signal.SIGINT) + fired = False + + def create_then_interrupt(*args, **kwargs): + nonlocal fired + handle = real_open(*args, **kwargs) + fired = True + signal.raise_signal(signal.SIGINT) + return handle + + m.signal = types.SimpleNamespace( + SIGINT=signal.SIGINT, + getsignal=signal.getsignal, + signal=signal.signal, + raise_signal=signal.raise_signal, + ) + m._open_private = create_then_interrupt + signal.signal(signal.SIGINT, signal.default_int_handler) + try: + try: + m.write_outputs([(str(destination), "fresh bytes")]) + raise AssertionError("the controlled interrupt must escape") + except KeyboardInterrupt: + pass + finally: + m._open_private, m.signal = real_open, real_signal + signal.signal(signal.SIGINT, old_handler) + + assert fired + assert not destination.exists() + assert list(root.iterdir()) == [] + + def test_initial_existing_path_revalidation_is_a_typed_refusal(m): """The first post-pin check cannot leak a raw filesystem exception.""" with tempfile.TemporaryDirectory() as directory: @@ -4008,6 +4108,42 @@ def open_then_remove(path, *args): assert list(root.iterdir()) == [] +def test_backup_copy_removal_during_final_source_check_is_a_typed_refusal(m): + """A removed source during post-copy validation cannot leak a traceback.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "output.xml" + destination.write_text("old bytes") + real_copy, real_identity = m._copy_private_backup, m._fd_identity + + def copy_then_remove_at_final_identity(*args): + calls = 0 + + def remove_on_final_identity(handle): + nonlocal calls + calls += 1 + if calls == 2: + destination.unlink() + return real_identity(handle) + + m._fd_identity = remove_on_final_identity + try: + return real_copy(*args) + finally: + m._fd_identity = real_identity + + m._copy_private_backup = copy_then_remove_at_final_identity + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + assert "changed while its rollback copy was prepared" in str(refusal.code) + assert not destination.exists() + assert list(root.iterdir()) == [] + + def test_existing_output_removed_after_backup_is_a_typed_path_refusal(m): with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) From 95a101cb637352f5f814ad1fa39e15325d00e8c4 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 13:17:14 +0530 Subject: [PATCH 45/59] Harden bank output claim verification --- scripts/bank_statement_import.py | 37 ++++++++- scripts/bank_statement_import.test.py | 107 ++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 232a0ba54..ac505c24b 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2160,8 +2160,11 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures # actual destination, never by the vanished private spelling. return _mark_rollback_unavailable(swap, failures) except OSError: + # An uninspectable pin leaves the named backup insufficient evidence + # that rollback remains safe. Preserve its known name, but report the + # actual staged destination as a partial commit for the caller. failures.append(backup) - return None + return _mark_rollback_unavailable(swap, failures) try: if _entry_identity(backup) != backup_identity: return _mark_rollback_unavailable(swap, failures) @@ -2405,9 +2408,19 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # a visible sibling. A replacement before this open is the # requested current path; a replacement after it is detected # before this run has authority to create or swap output. - original = _claim_owned_output( - lambda: (_open_regular_output(real_path, None), real_path), - created=False, owned_records=unpaired_originals) + try: + original = _claim_owned_output( + lambda: (_open_regular_output(real_path, None), real_path), + created=False, owned_records=unpaired_originals) + except OSError: + # A path which disappears between exists() and the pinned + # open is a claim-time retarget, not an untyped filesystem + # failure. Nothing has been staged yet, so refuse it in + # the same typed channel as the later revalidation. + raise Refusal( + "output_path_changed", + f"{path} changed while it was being claimed", + ) from None original_identity = original["identity"] state = {"temporary": None, "supplied_path": path, "real_path": real_path, @@ -2470,6 +2483,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle = os.dup(record["pin"]) with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) + # ``_open_private`` retains a write-only ownership pin, so retain + # the exact UTF-8/no-translation payload digest here. A later + # in-place edit of a staged .part preserves its entry identity and + # link count, so those checks alone cannot prove it is still this + # run's output. + record["digest"] = hashlib.sha256(text.encode("utf-8")).digest() # Every payload is on disk. A private copy preserves the old bytes while # the requested destination stays present until the atomic replacement. for state in staged: @@ -2527,6 +2546,16 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} staged output changed before replacement", ) + try: + staged_digest_matches = ( + _digest_pinned_bytes(temporary["pin"]) == temporary["digest"]) + except OSError: + staged_digest_matches = False + if not staged_digest_matches: + raise Refusal( + "output_path_changed", + f"{supplied_path} staged output changed before replacement", + ) _require_single_owned_link( temporary, "staged output", "staged_output_has_multiple_links") if _entry_identity(pending_swap["backup"]["path"]) != \ diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 13d7c2d3f..78b01f6a4 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4834,5 +4834,112 @@ def deny_later(path): assert records[1]["pin"] is None and records[2]["pin"] is None and records[3]["pin"] is None +def test_initial_existing_claim_disappearance_is_a_typed_refusal(m): + """A vanished existing leaf must not escape as an untyped open failure.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "previous.xml" + destination.write_text("old bytes") + real_open = m._open_regular_output + + def remove_before_open(path, *args): + if pathlib.Path(path).resolve() == destination.resolve(): + destination.unlink() + return real_open(path, *args) + + m._open_regular_output = remove_before_open + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._open_regular_output = real_open + + assert "changed while it was being claimed" in str(refusal.code) + assert not destination.exists() + assert not list(pathlib.Path(directory).glob("*.part")) + assert not list(pathlib.Path(directory).glob("*.bak")) + + +def test_uninspectable_backup_pin_reports_partial_destination(m): + """Rollback pin errors retain the original failure and name the new destination.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + real_pin_check = m._pinned_backup_still_has_one_link + first_backup_checks = 0 + + def fail_second_swap(source, destination): + if (pathlib.Path(source).suffix == ".part" + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + return real_replace(source, destination) + + def fail_first_backup_pin_during_rollback(record): + nonlocal first_backup_checks + if pathlib.Path(record["path"]).name.startswith("first.xml."): + first_backup_checks += 1 + if first_backup_checks == 2: + raise OSError("controlled backup pin inspection failure") + return real_pin_check(record) + + m.os.replace = fail_second_swap + m._pinned_backup_still_has_one_link = fail_first_backup_pin_during_rollback + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m.os.replace = real_replace + m._pinned_backup_still_has_one_link = real_pin_check + + assert first_backup_checks == 2 + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + backup, = root.glob("first.xml.*.bak") + assert backup.read_text() == "first old" + assert not list(root.glob("*.part")) + + +def test_staged_output_in_place_mutation_refuses_before_replacement(m): + """A stable staged inode still needs byte-for-byte ownership at commit.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def mutate_staged_after_backup(source_path, *args): + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == destination.resolve(): + staged, = root.glob("previous.xml.*.part") + staged.write_text("foreign staged bytes") + return result + + m._copy_private_backup = mutate_staged_after_backup + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + assert "staged output changed before replacement" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + if __name__ == "__main__": raise SystemExit(main()) From 829a73d05b1c7e4ea652a735dda6946a36e6dd0c Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 13:22:41 +0530 Subject: [PATCH 46/59] Verify fresh bank output bytes before commit --- scripts/bank_statement_import.py | 24 ++++++++++++++++++------ scripts/bank_statement_import.test.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index ac505c24b..a2e44b1bd 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1508,7 +1508,10 @@ def _open_private(path, accept_inherited=False): if refusal: raise refusal try: - return os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + # The retained ownership pin is also the final byte-verification + # authority. It must be readable without reopening the pathname, + # which could have been reclaimed by another writer. + return os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: raise _existing_target_on_windows(path) from None @@ -2483,11 +2486,9 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): handle = os.dup(record["pin"]) with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: stream.write(text) - # ``_open_private`` retains a write-only ownership pin, so retain - # the exact UTF-8/no-translation payload digest here. A later - # in-place edit of a staged .part preserves its entry identity and - # link count, so those checks alone cannot prove it is still this - # run's output. + # Retain the exact UTF-8/no-translation payload digest. A later + # in-place edit preserves the entry identity and link count, so + # those checks alone cannot prove it is still this run's output. record["digest"] = hashlib.sha256(text.encode("utf-8")).digest() # Every payload is on disk. A private copy preserves the old bytes while # the requested destination stays present until the atomic replacement. @@ -2586,6 +2587,17 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} changed before commit; no output was committed", ) + try: + payload_digest_matches = ( + _digest_pinned_bytes(record["pin"]) == record["digest"]) + except OSError: + payload_digest_matches = False + if not payload_digest_matches: + raise Refusal( + "output_path_changed", + f"{supplied_path} output contents changed before commit; " + "no output was committed", + ) _require_single_owned_link(record, "output", "output_has_multiple_links") # Earlier rollback copies can also be changed during later swaps. # A commit may retire them only while their ownership remains proved. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 78b01f6a4..bbe9502c7 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4941,5 +4941,29 @@ def mutate_staged_after_backup(source_path, *args): assert not list(root.glob("*.bak")) +def test_fresh_output_in_place_mutation_refuses_at_final_authority_boundary(m): + """Fresh output bytes need the same pinned-content proof as staged bytes.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "fresh.xml" + real_changed = m._claimed_output_changed + + def mutate_before_final_validation(supplied_path, canonical_path, identity): + if pathlib.Path(canonical_path).resolve() == destination.resolve(): + destination.write_text("foreign output bytes") + return real_changed(supplied_path, canonical_path, identity) + + m._claimed_output_changed = mutate_before_final_validation + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "owned output bytes")]) + finally: + m._claimed_output_changed = real_changed + + assert "output contents changed before commit" in str(refusal.code) + assert not destination.exists() + assert not list(root.iterdir()) + + if __name__ == "__main__": raise SystemExit(main()) From 24d9aa3e38fb574c65d5c778bedb1b87c386fdbd Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 14:16:13 +0530 Subject: [PATCH 47/59] Harden bank output recovery refusals --- scripts/bank_statement_import.py | 13 ++++- scripts/bank_statement_import.test.py | 75 +++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index a2e44b1bd..d28390bcf 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2127,6 +2127,12 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures try: current_identity = _entry_identity(destination) except OSError: + # After a replacement started, an uninspectable destination might + # still name this run's staged inode. It is not evidence of a foreign + # writer, so keep the private backup and report the real destination + # as a partial result. + if swap["swap_started"]: + return _mark_rollback_unavailable(swap, failures) current_identity = None if not swap["swap_started"] or current_identity == original_identity: # Backup reads can update atime before any swap. Restore that effect @@ -2542,7 +2548,12 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} changed after it was claimed; no output was replaced", ) - if _entry_identity(temporary["path"]) != temporary["identity"]: + try: + staged_entry_unchanged = ( + _entry_identity(temporary["path"]) == temporary["identity"]) + except OSError: + staged_entry_unchanged = False + if not staged_entry_unchanged: raise Refusal( "output_path_changed", f"{supplied_path} staged output changed before replacement", diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index bbe9502c7..3fa64a19d 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4965,5 +4965,80 @@ def mutate_before_final_validation(supplied_path, canonical_path, identity): assert not list(root.iterdir()) +def test_uninspectable_rollback_destination_reports_partial_output(m): + """An unknown post-swap destination cannot be treated as a foreign inode.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + real_entry = m._entry_identity + + def fail_second_swap(source, destination): + if (pathlib.Path(source).suffix == ".part" + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + return real_replace(source, destination) + + def deny_first_destination(path): + if pathlib.Path(path).resolve() == first.resolve(): + raise PermissionError("controlled rollback destination inspection failure") + return real_entry(path) + + m.os.replace = fail_second_swap + m._entry_identity = deny_first_destination + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m.os.replace = real_replace + m._entry_identity = real_entry + + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + backup, = root.glob("first.xml.*.bak") + assert backup.read_text() == "first old" + assert not list(root.glob("*.part")) + + +def test_vanished_staged_output_is_a_typed_refusal(m): + """A removed .part after backup creation must not leak FileNotFoundError.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def remove_staged_after_backup(source_path, *args): + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == destination.resolve(): + staged, = root.glob("previous.xml.*.part") + staged.unlink() + return result + + m._copy_private_backup = remove_staged_after_backup + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + assert "staged output changed before replacement" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + if __name__ == "__main__": raise SystemExit(main()) From eaccd861d036f0acb41091207e52c5807b82defb Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 14:52:11 +0530 Subject: [PATCH 48/59] Preserve bank output recovery uncertainty --- scripts/bank_statement_import.py | 20 ++++- scripts/bank_statement_import.test.py | 105 ++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index d28390bcf..0023ff1e3 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1860,6 +1860,11 @@ def _record_windows_cleanup_alias(record, failures): try: stat_result = os.fstat(pin) except OSError: + # Windows must close before it can unlink. If this pre-close + # inspection cannot determine whether a hard-link alias exists, the + # later unlink of the known spelling cannot make that uncertainty go + # away. + _record_uninspectable_cleanup(record["path"], failures) return if ((stat_result.st_dev, stat_result.st_ino) == record["identity"] and stat_result.st_nlink > 1): @@ -2150,8 +2155,10 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures _cleanup_owned_path(backup_record, failures, descriptor_failures=descriptor_failures) return if current_identity != staged_identity: - failures.append(backup) - return + # Keep the foreign destination untouched, but reconcile the pinned + # rollback inode before releasing it. A moved backup has no longer + # been disclosed merely by its stale private spelling. + return _mark_rollback_unavailable(swap, failures, retain_named=True) if swap.get("rollback_unavailable"): return _mark_rollback_unavailable(swap, failures) restored = False @@ -2570,8 +2577,13 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): ) _require_single_owned_link( temporary, "staged output", "staged_output_has_multiple_links") - if _entry_identity(pending_swap["backup"]["path"]) != \ - pending_swap["backup"]["identity"]: + try: + pending_backup_unchanged = ( + _entry_identity(pending_swap["backup"]["path"]) + == pending_swap["backup"]["identity"]) + except OSError: + pending_backup_unchanged = False + if not pending_backup_unchanged: raise Refusal( "output_path_changed", f"{supplied_path} rollback copy changed before replacement", diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 3fa64a19d..b86a7425e 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -5040,5 +5040,110 @@ def remove_staged_after_backup(source_path, *args): assert not list(root.glob("*.bak")) +def test_windows_modeled_alias_inspection_failure_is_retained_before_close(m): + """Windows cleanup cannot erase uncertainty about an uninspectable pin.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path, alias = root / "fresh.xml", root / "fresh-alias.xml" + handle = m._open_private(path) + os.write(handle, b"generated statement") + record = m._owned_path(path, handle, created=True) + os.link(path, alias) + real_name, real_fstat = m.os.name, m.os.fstat + + def deny_owned_pin(candidate): + if candidate == handle: + raise OSError("controlled Windows alias inspection failure") + return real_fstat(candidate) + + failures = [] + m.os.name, m.os.fstat = "nt", deny_owned_pin + try: + m._cleanup_owned_path(record, failures) + finally: + m.os.name, m.os.fstat = real_name, real_fstat + + assert not path.exists() + assert alias.read_bytes() == b"generated statement" + assert failures == [ + f"could not inspect owned output during cleanup: {path}"] + + +def test_vanished_pending_backup_is_a_typed_refusal(m): + """A removed .bak after copy creation must not leak FileNotFoundError.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination = root / "previous.xml" + destination.write_text("old bytes") + real_copy = m._copy_private_backup + + def remove_backup_after_copy(source_path, *args): + result = real_copy(source_path, *args) + if pathlib.Path(source_path).resolve() == destination.resolve(): + backup, = root.glob("previous.xml.*.bak") + backup.unlink() + return result + + m._copy_private_backup = remove_backup_after_copy + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + + assert "rollback copy changed before replacement" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + +def test_moved_backup_with_foreign_destination_is_reported_unlocated(m): + """Foreign bytes stay intact while the pinned moved rollback copy is named.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, foreign, moved = ( + root / "first.xml", root / "second.xml", root / "foreign.xml", root / "moved.bak") + first.write_text("first old") + second.write_text("second old") + real_replace = m.os.replace + + def replace_first_then_move_backup_and_fail_second(source, destination): + source, destination = pathlib.Path(source), pathlib.Path(destination) + if source.suffix == ".part" and destination.resolve() == first.resolve(): + result = real_replace(source, destination) + foreign.write_text("foreign writer bytes") + real_replace(foreign, first) + backup, = root.glob("first.xml.*.bak") + backup.rename(moved) + return result + if source.suffix == ".part" and destination.resolve() == second.resolve(): + raise OSError("controlled later swap failure") + return real_replace(source, destination) + + m.os.replace = replace_first_then_move_backup_and_fail_second + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m.os.replace = real_replace + + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert "owned rollback copy could not be located after cleanup" in detail + assert first.read_text() == "foreign writer bytes" + assert second.read_text() == "second old" + assert moved.read_text() == "first old" + assert not list(root.glob("*.part")) + assert not list(root.glob("first.xml.*.bak")) + + if __name__ == "__main__": raise SystemExit(main()) From 075b0b8a6f1ecac856e5ac7d7c126c901fe87ed4 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 15:33:41 +0530 Subject: [PATCH 49/59] Revalidate bank output replacement boundaries --- scripts/bank_statement_import.py | 41 ++++++- scripts/bank_statement_import.test.py | 150 +++++++++++++++++++++++++- 2 files changed, 189 insertions(+), 2 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 0023ff1e3..1a76d19fe 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2114,7 +2114,20 @@ def _copy_private_backup(source_path, original_identity, backup_handle): ) return backup_digest finally: - os.close(source_handle) + active_error = sys.exc_info()[1] + try: + os.close(source_handle) + except OSError: + if active_error is None: + raise + # The typed copy refusal is the transaction outcome. A failed + # source-pin close is separately actionable, but must not replace + # that refusal with an untyped descriptor exception. + _append_cleanup_detail( + active_error, + "ownership descriptor close failed or could not be verified for: " + + str(source_path), + ) def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures=None): @@ -2191,6 +2204,16 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures return _mark_rollback_unavailable(swap, failures, retain_named=True) if not digest_matches: return _mark_rollback_unavailable(swap, failures, retain_named=True) + try: + destination_still_staged = ( + _entry_identity(destination) == staged_identity) + except OSError: + destination_still_staged = False + if not destination_still_staged: + # A concurrent writer now owns the destination. Keep its bytes + # intact and reconcile this run's private rollback copy instead + # of restoring over the foreign inode. + return _mark_rollback_unavailable(swap, failures, retain_named=True) # This observes ownership immediately before the replace. POSIX has no # compare-and-swap rename, so a hostile concurrent rename after this # check is still outside the CLI's locking authority. @@ -2594,6 +2617,22 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): # recheck this pinned inode so replacement never detaches a new # hard link while reporting a successful overwrite. _pinned_original_still_has_one_link(pending_swap["original"]) + try: + original_still_current = ( + _entry_identity(real_path) == original_identity) + except OSError: + original_still_current = False + if not original_still_current: + # The pre-claim original remains pinned, but its pathname no + # longer gives this run authority to replace it. Reconcile the + # pin before recovery closes it so a moved prior statement is + # not silently lost from the diagnostic. + _reconcile_owned_pin_after_cleanup( + pending_swap["original"], "missing", cleanup_failures) + raise Refusal( + "output_path_changed", + f"{supplied_path} changed before replacement; no output was replaced", + ) pending_swap["swap_started"] = True os.replace(temporary["path"], real_path) replaced.append(pending_swap) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index b86a7425e..dcca5a2a8 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -4976,6 +4976,7 @@ def test_uninspectable_rollback_destination_reports_partial_output(m): second.write_text("second old") real_replace = m.os.replace real_entry = m._entry_identity + first_entry_checks = 0 def fail_second_swap(source, destination): if (pathlib.Path(source).suffix == ".part" @@ -4984,8 +4985,12 @@ def fail_second_swap(source, destination): return real_replace(source, destination) def deny_first_destination(path): + nonlocal first_entry_checks if pathlib.Path(path).resolve() == first.resolve(): - raise PermissionError("controlled rollback destination inspection failure") + first_entry_checks += 1 + if first_entry_checks == 2: + raise PermissionError( + "controlled rollback destination inspection failure") return real_entry(path) m.os.replace = fail_second_swap @@ -5001,6 +5006,7 @@ def deny_first_destination(path): m._entry_identity = real_entry assert "controlled later swap failure" in detail + assert first_entry_checks == 2 assert "partially committed output could not be rolled back" in detail assert str(first.resolve()) in detail assert first.read_text() == "first new" @@ -5145,5 +5151,147 @@ def replace_first_then_move_backup_and_fail_second(source, destination): assert not list(root.glob("first.xml.*.bak")) +def test_rollback_rechecks_destination_before_restoring_backup(m): + """A foreign writer between backup digest and restore keeps its own bytes.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, foreign = ( + root / "first.xml", root / "second.xml", root / "foreign.xml") + first.write_text("first old") + second.write_text("second old") + real_replace, real_copy, real_digest = ( + m.os.replace, m._copy_private_backup, m._digest_pinned_bytes) + first_backup_handles, replaced_foreign = set(), [] + + def remember_first_backup(source_path, identity, backup_handle): + result = real_copy(source_path, identity, backup_handle) + if pathlib.Path(source_path).resolve() == first.resolve(): + first_backup_handles.add(backup_handle) + return result + + def replace_first_then_fail_second(source, destination): + if (pathlib.Path(source).suffix == ".part" + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + return real_replace(source, destination) + + def replace_foreign_during_rollback_digest(handle): + if handle in first_backup_handles and not replaced_foreign: + foreign.write_text("foreign writer bytes") + real_replace(foreign, first) + replaced_foreign.append(True) + return real_digest(handle) + + m._copy_private_backup = remember_first_backup + m.os.replace = replace_first_then_fail_second + m._digest_pinned_bytes = replace_foreign_during_rollback_digest + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m._copy_private_backup = real_copy + m.os.replace = real_replace + m._digest_pinned_bytes = real_digest + + assert replaced_foreign == [True] + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert first.read_text() == "foreign writer bytes" + assert second.read_text() == "second old" + backup, = root.glob("first.xml.*.bak") + assert backup.read_text() == "first old" + assert not list(root.glob("*.part")) + + +def test_existing_commit_rechecks_moved_original_before_replacement(m): + """A renamed prior statement is disclosed instead of silently stranded.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, moved, foreign = ( + root / "previous.xml", root / "moved.xml", root / "foreign.xml") + destination.write_text("old bytes") + real_pin_check, real_replace = m._pinned_original_still_has_one_link, m.os.replace + + def move_after_original_pin(record): + result = real_pin_check(record) + destination.rename(moved) + foreign.write_text("foreign writer bytes") + real_replace(foreign, destination) + return result + + m._pinned_original_still_has_one_link = move_after_original_pin + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._pinned_original_still_has_one_link = real_pin_check + + detail = str(refusal.code) + assert "changed before replacement" in detail + assert "owned output could not be located after cleanup" in detail + assert destination.read_text() == "foreign writer bytes" + assert moved.read_text() == "old bytes" + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + +def test_backup_refusal_survives_source_close_failure(m): + """A path-race refusal remains typed even when its local source pin fails close.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source, backup = root / "previous.xml", root / "rollback.bak" + source.write_text("old bytes") + backup_handle = os.open(backup, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + identity = m._file_identity(source) + real_open, real_identity, real_close = ( + m._open_regular_output, m._file_identity, m.os.close) + source_handles = [] + + def remember_source_handle(path, expected_identity): + handle = real_open(path, expected_identity) + source_handles.append(handle) + return handle + + def reject_final_source_identity(path): + if pathlib.Path(path).resolve() == source.resolve(): + return (0, 0) + return real_identity(path) + + def fail_source_close(handle): + if handle in source_handles: + raise OSError("controlled source close failure") + return real_close(handle) + + m._open_regular_output = remember_source_handle + m._file_identity = reject_final_source_identity + m.os.close = fail_source_close + try: + refusal = refuses( + m, "output_path_changed", m._copy_private_backup, + str(source), identity, backup_handle) + finally: + m._open_regular_output = real_open + m._file_identity = real_identity + m.os.close = real_close + for handle in source_handles: + try: + real_close(handle) + except OSError: + pass + real_close(backup_handle) + + detail = str(refusal.code) + assert "changed while its rollback copy was prepared" in detail + assert "ownership descriptor close failed or could not be verified" in detail + + if __name__ == "__main__": raise SystemExit(main()) From 6094269b58e3896c2bf02887daef34e115a24f37 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 15:52:52 +0530 Subject: [PATCH 50/59] Retain bank output recovery diagnostics --- scripts/bank_statement_import.py | 23 ++++- scripts/bank_statement_import.test.py | 128 ++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 1a76d19fe..1bf07cf9d 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1633,7 +1633,18 @@ def _open_regular_output(path, expected_identity=None): ) return handle except BaseException: - os.close(handle) + active_error = sys.exc_info()[1] + try: + os.close(handle) + except OSError: + # The validation refusal identifies the unsafe output shape. A + # failed pin close is separately actionable, but must not replace + # that typed outcome or be silently forgotten. + _append_cleanup_detail( + active_error, + "ownership descriptor close failed or could not be verified for: " + + str(path), + ) raise @@ -2235,7 +2246,12 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures destination_still_staged = ( _entry_identity(destination) == staged_identity) except OSError: - destination_still_staged = False + # The failed replace may have taken effect. An unknown + # destination cannot be classified as a safe foreign + # inode, so disclose this actual partial result while the + # named backup remains inspectable. + return _mark_rollback_unavailable( + swap, failures, retain_named=True) if destination_still_staged: return _mark_rollback_unavailable(swap, failures) elif current_identity == staged_identity: @@ -2671,7 +2687,8 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): except OSError: backup_unchanged = False if not backup_unchanged: - _mark_rollback_unavailable(swap, cleanup_failures) + _mark_rollback_unavailable( + swap, cleanup_failures, retain_named=True) raise Refusal( "output_path_changed", f"{swap['destination']} rollback copy changed before commit; " diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index dcca5a2a8..b064ac556 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -5293,5 +5293,133 @@ def fail_source_close(handle): assert "ownership descriptor close failed or could not be verified" in detail +def test_regular_output_refusal_survives_pin_close_failure(m): + """Validation keeps its category when releasing the rejected pin fails.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path, alias = root / "previous.xml", root / "previous-alias.xml" + path.write_text("old bytes") + os.link(path, alias) + real_open, real_close = m.os.open, m.os.close + opened = [] + + def remember_open(*args): + handle = real_open(*args) + opened.append(handle) + return handle + + def fail_rejected_pin_close(handle): + if handle in opened: + raise OSError("controlled rejected-pin close failure") + return real_close(handle) + + m.os.open, m.os.close = remember_open, fail_rejected_pin_close + try: + refusal = refuses(m, "output_has_multiple_links", m._open_regular_output, path) + finally: + m.os.open, m.os.close = real_open, real_close + for handle in opened: + try: + real_close(handle) + except OSError: + pass + + detail = str(refusal.code) + assert "replacement requires a single-link output" in detail + assert "ownership descriptor close failed or could not be verified" in detail + + +def test_uninspectable_post_failed_restore_reports_partial_destination(m): + """A failed restore with an unknown destination remains an in-band partial.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_replace, real_entry = m.os.replace, m._entry_identity + deny_after_restore_failure = False + + def fail_second_swap_and_first_restore(source, destination): + nonlocal deny_after_restore_failure + source, destination = pathlib.Path(source), pathlib.Path(destination) + if source.suffix == ".part" and destination.resolve() == second.resolve(): + raise OSError("controlled later swap failure") + if source.suffix == ".bak" and destination.resolve() == first.resolve(): + deny_after_restore_failure = True + raise OSError("controlled restore failure before effect") + return real_replace(source, destination) + + def deny_first_after_failed_restore(path): + if deny_after_restore_failure and pathlib.Path(path).resolve() == first.resolve(): + raise OSError("controlled post-restore destination inspection failure") + return real_entry(path) + + m.os.replace, m._entry_identity = ( + fail_second_swap_and_first_restore, deny_first_after_failed_restore) + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m.os.replace, m._entry_identity = real_replace, real_entry + + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + backup, = root.glob("first.xml.*.bak") + assert backup.read_text() == "first old" + assert not list(root.glob("*.part")) + + +def test_transient_final_backup_inspection_reports_named_backup(m): + """One transient backup lstat failure still leaves its private name visible.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_entry = m._entry_identity + first_backup_checks = 0 + + def fail_only_final_first_backup_check(path): + nonlocal first_backup_checks + name = pathlib.Path(path).name + if name.startswith("first.xml.") and name.endswith(".bak"): + first_backup_checks += 1 + if first_backup_checks == 2: + raise OSError("controlled transient final backup inspection failure") + return real_entry(path) + + m._entry_identity = fail_only_final_first_backup_check + try: + refusal = refuses( + m, "output_path_changed", m.write_outputs, + [(str(first), "first new"), (str(second), "second new")]) + finally: + m._entry_identity = real_entry + + detail = str(refusal.code) + backup, = root.glob("first.xml.*.bak") + assert first_backup_checks == 3 + assert "rollback copy changed before commit" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert str(backup) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert backup.read_text() == "first old" + assert not list(root.glob("*.part")) + + if __name__ == "__main__": raise SystemExit(main()) From 6f1e53fa062178c905762648dbf598dcb2d85f0a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 16:24:53 +0530 Subject: [PATCH 51/59] Revalidate bank replacement sources --- scripts/bank_statement_import.py | 18 ++++++ scripts/bank_statement_import.test.py | 88 +++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 1bf07cf9d..61fdca301 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2225,6 +2225,14 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures # intact and reconcile this run's private rollback copy instead # of restoring over the foreign inode. return _mark_rollback_unavailable(swap, failures, retain_named=True) + try: + backup_still_current = _entry_identity(backup) == backup_identity + except OSError: + backup_still_current = False + if not backup_still_current: + # The named source of the restore can be retargeted after its + # digest check. Do not move a foreign inode over the destination. + return _mark_rollback_unavailable(swap, failures, retain_named=True) # This observes ownership immediately before the replace. POSIX has no # compare-and-swap rename, so a hostile concurrent rename after this # check is still outside the CLI's locking authority. @@ -2649,6 +2657,16 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} changed before replacement; no output was replaced", ) + try: + staged_still_current = ( + _entry_identity(temporary["path"]) == temporary["identity"]) + except OSError: + staged_still_current = False + if not staged_still_current: + raise Refusal( + "output_path_changed", + f"{supplied_path} staged output changed before replacement", + ) pending_swap["swap_started"] = True os.replace(temporary["path"], real_path) replaced.append(pending_swap) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index b064ac556..2768fe7f9 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -5421,5 +5421,93 @@ def fail_only_final_first_backup_check(path): assert not list(root.glob("*.part")) +def test_rollback_rechecks_backup_path_before_restoring(m): + """A retargeted backup source never overwrites the staged destination.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, foreign, moved = ( + root / "first.xml", root / "second.xml", root / "foreign.bak", root / "moved.bak") + first.write_text("first old") + second.write_text("second old") + real_replace, real_entry = m.os.replace, m._entry_identity + first_destination_checks, retargeted = 0, [] + + def fail_second_swap(source, destination): + if (pathlib.Path(source).suffix == ".part" + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + return real_replace(source, destination) + + def retarget_backup_after_rollback_destination_check(path): + nonlocal first_destination_checks + if pathlib.Path(path).resolve() == first.resolve(): + first_destination_checks += 1 + if first_destination_checks == 3: + backup, = root.glob("first.xml.*.bak") + backup.rename(moved) + foreign.write_text("foreign backup bytes") + real_replace(foreign, backup) + retargeted.append(backup) + return real_entry(path) + + m.os.replace, m._entry_identity = fail_second_swap, retarget_backup_after_rollback_destination_check + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m.os.replace, m._entry_identity = real_replace, real_entry + + assert first_destination_checks == 3 and retargeted + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert moved.read_text() == "first old" + assert retargeted[0].read_text() == "foreign backup bytes" + assert not list(root.glob("*.part")) + + +def test_commit_rechecks_staged_path_before_replacement(m): + """A retargeted .part is refused instead of being installed as output.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, moved, foreign = ( + root / "previous.xml", root / "moved.part", root / "foreign.part") + destination.write_text("old bytes") + real_pin_check, real_replace = m._pinned_original_still_has_one_link, m.os.replace + retargeted = [] + + def retarget_staged_after_original_check(record): + result = real_pin_check(record) + staged, = root.glob("previous.xml.*.part") + staged.rename(moved) + foreign.write_text("foreign staged bytes") + real_replace(foreign, staged) + retargeted.append(staged) + return result + + m._pinned_original_still_has_one_link = retarget_staged_after_original_check + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._pinned_original_still_has_one_link = real_pin_check + + assert retargeted + assert "staged output changed before replacement" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert moved.read_text() == "new bytes" + assert retargeted[0].read_text() == "foreign staged bytes" + assert not list(root.glob("*.bak")) + + if __name__ == "__main__": raise SystemExit(main()) From c7b9d2eaa053edf291bfd712e6eb71f7b7378d74 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 16:56:36 +0530 Subject: [PATCH 52/59] Rectify bank output recovery ownership --- scripts/bank_statement_import.py | 74 +++++++++-- scripts/bank_statement_import.test.py | 184 +++++++++++++++++++++++++- 2 files changed, 245 insertions(+), 13 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 61fdca301..0ffa6539b 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1883,7 +1883,8 @@ def _record_windows_cleanup_alias(record, failures): f"unknown hard-link alias may retain output bytes: {record['path']}") -def _cleanup_owned_path(record, failures, descriptor_failures=None): +def _cleanup_owned_path( + record, failures, descriptor_failures=None, suppress_reclaimed_name=False): """Remove one owned pathname, releasing its pin first on Windows. POSIX keeps the descriptor open through the identity decision so an inode @@ -1901,13 +1902,19 @@ def _cleanup_owned_path(record, failures, descriptor_failures=None): _close_owned_path(record, failures, descriptor_failures=descriptor_failures) outcome = _unlink_for_cleanup( record.get("cleanup_path", record["path"]), record["identity"], failures) - if outcome == "removed": + if outcome == "removed" or ( + outcome == "reclaimed" and suppress_reclaimed_name): del failures[failure_start:] + if outcome == "reclaimed": + failures.append( + "owned output could not be located after cleanup: " + + str(record["path"])) else: cleanup_path = record.get("cleanup_path", record["path"]) failure_start = len(failures) outcome = _unlink_for_cleanup(cleanup_path, record["identity"], failures) - if outcome == "reclaimed" and "cleanup_path" in record: + if outcome == "reclaimed" and ( + suppress_reclaimed_name or "cleanup_path" in record): # A foreign claimant of the stale name is not a retained path of # this output. Keep any earlier diagnostics, but replace this # pathname with the separate pinned-inode conclusion below. @@ -1967,7 +1974,8 @@ def _restore_metadata(handle, metadata): def _pinned_original_still_has_one_link(record): """Refuse if the original gained a hard link after its first pin.""" stat_result = os.fstat(record["pin"]) - if (stat_result.st_dev, stat_result.st_ino) != record["identity"]: + if ((stat_result.st_dev, stat_result.st_ino) != record["identity"] + or stat_result.st_nlink == 0): raise Refusal( "output_path_changed", f"{record['path']} changed while its rollback copy was prepared", @@ -2161,7 +2169,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures # writer, so keep the private backup and report the real destination # as a partial result. if swap["swap_started"]: - return _mark_rollback_unavailable(swap, failures) + return _mark_rollback_unavailable(swap, failures, retain_named=True) current_identity = None if not swap["swap_started"] or current_identity == original_identity: # Backup reads can update atime before any swap. Restore that effect @@ -2176,7 +2184,9 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures os.utime(handle, ns=(metadata["atime_ns"], current.st_mtime_ns)) except OSError: failures.append(destination) - _cleanup_owned_path(backup_record, failures, descriptor_failures=descriptor_failures) + _cleanup_owned_path( + backup_record, failures, descriptor_failures=descriptor_failures, + suppress_reclaimed_name=True) return if current_identity != staged_identity: # Keep the foreign destination untouched, but reconcile the pinned @@ -2272,7 +2282,11 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures try: _restore_metadata(restore_handle, metadata) finally: - os.close(restore_handle) + _close_owned_path( + {"path": destination, "identity": backup_identity, + "pin": restore_handle}, + failures, diagnostic_path=destination, + descriptor_failures=descriptor_failures) metadata_scope_warnings.append(destination) except (OSError, Refusal): failures.append(destination) @@ -2321,7 +2335,9 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ """Remove old private copies after every replacement has committed.""" for swap in replaced: backup = swap["backup"] - _cleanup_owned_path(backup, retained_failures, descriptor_failures=descriptor_failures) + _cleanup_owned_path( + backup, retained_failures, descriptor_failures=descriptor_failures, + suppress_reclaimed_name=True) _close_owned_path( swap["original"], retained_failures, diagnostic_path=swap.get("destination", swap["original"]["path"]), @@ -2334,6 +2350,33 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ record, descriptor_failures, diagnostic_path=record.get("canonical_path")) +def _reconcile_committed_staged_output_pin(record, failures): + """Check a staged descriptor at its committed name before releasing it.""" + if record.get("cleanup_path") == record.get("canonical_path"): + return + pin = record.get("pin") + if pin is None: + return + try: + stat_result = os.fstat(pin) + except OSError: + _record_uninspectable_cleanup(record["canonical_path"], failures) + return + if ((stat_result.st_dev, stat_result.st_ino) != record["identity"] + or stat_result.st_nlink == 0): + _record_uninspectable_cleanup(record["canonical_path"], failures) + return + try: + committed_here = ( + _entry_identity(record["canonical_path"]) == record["identity"]) + except OSError: + committed_here = False + if not committed_here: + failures.append( + "committed staged output could not be located after cleanup: " + + str(record["canonical_path"])) + + def _reconcile_interrupted_committed_cleanup( replaced, claimed, retained_failures, descriptor_failures): """Close every pin and disclose old copies without undoing a commit. @@ -2372,7 +2415,9 @@ def _reconcile_interrupted_committed_cleanup( "unknown hard-link alias may retain rollback bytes: " + str(backup["path"])) elif still_at_path is False: - _cleanup_owned_path(backup, retained_failures, descriptor_failures=descriptor_failures) + _cleanup_owned_path( + backup, retained_failures, descriptor_failures=descriptor_failures, + suppress_reclaimed_name=True) else: # We cannot identify an entry after an I/O/permission error. # Preserve the original interruption and report no ownership @@ -2390,6 +2435,7 @@ def _reconcile_interrupted_committed_cleanup( diagnostic_path=swap.get("destination", swap["original"]["path"]), descriptor_failures=descriptor_failures) for record in claimed: + _reconcile_committed_staged_output_pin(record, retained_failures) _close_owned_path( record, descriptor_failures, diagnostic_path=record.get("canonical_path")) @@ -2774,7 +2820,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): descriptor_failures=descriptor_close_failures) if pending_backup is not None and ( pending_swap is None or pending_backup is not pending_swap["backup"]): - _cleanup_owned_path(pending_backup, cleanup_failures, descriptor_failures=descriptor_close_failures) + _cleanup_owned_path( + pending_backup, cleanup_failures, + descriptor_failures=descriptor_close_failures, + suppress_reclaimed_name=True) for swap in reversed(replaced): # An interrupt can arrive after `_record_replaced_swap` appends but # before its caller clears `pending_swap`. That one backup has @@ -2830,7 +2879,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): paired_backups.add(id(pending_backup)) for record in unpaired_backups: if id(record) not in paired_backups: - _cleanup_owned_path(record, cleanup_failures, descriptor_failures=descriptor_close_failures) + _cleanup_owned_path( + record, cleanup_failures, + descriptor_failures=descriptor_close_failures, + suppress_reclaimed_name=True) for record in unpaired_originals: if id(record) not in paired_originals: _close_owned_path(record, cleanup_failures, diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 2768fe7f9..a0616e36b 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2663,7 +2663,7 @@ def rename_parent_and_reclaim_old_name(): def test_writer_refuses_when_the_private_backup_path_is_reclaimed(m): """The commit boundary must still name this run's backup; if it does not, - do not overwrite the destination without a recoverable owned copy.""" + do not overwrite the destination or claim a foreign backup as owned.""" with tempfile.TemporaryDirectory() as directory: root = pathlib.Path(directory) destination = root / "previous.xml" @@ -2689,7 +2689,8 @@ def reclaim_backup_after_copy(src, identity, backup_handle): assert destination.read_text() == "old bytes" assert len(backups) == 1 assert backups[0].read_text() == "foreign writer bytes" - assert str(backups[0]) in str(refusal.code) + detail = str(refusal.code) + assert "retained path(s): " + str(backups[0]) not in detail def test_parent_rename_after_backup_preparation_discloses_unlocated_backup(m): @@ -5509,5 +5510,184 @@ def retarget_staged_after_original_check(record): assert not list(root.glob("*.bak")) +def test_reclaimed_committed_backup_is_not_reported_as_its_old_path(m): + """A foreign claimant of a backup name is never an owned retained output.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + backup_path, moved, foreign = ( + root / "old.bak", root / "moved.bak", root / "foreign.bak") + original_path, claimed_path = root / "old.xml", root / "new.xml" + for path in (backup_path, original_path, claimed_path): + path.write_text(path.name) + backup_fd, original_fd, claimed_fd = ( + os.open(path, os.O_RDONLY) for path in + (backup_path, original_path, claimed_path)) + backup, original, claimed = [ + {"path": path, "identity": m._fd_identity(fd), "pin": fd} + for path, fd in zip((backup_path, original_path, claimed_path), + (backup_fd, original_fd, claimed_fd))] + backup_path.rename(moved) + foreign.write_text("foreign backup bytes") + os.replace(foreign, backup_path) + retained = [] + + m._reconcile_interrupted_committed_cleanup( + [{"backup": backup, "original": original}], [claimed], retained, []) + + assert str(backup_path) not in retained + assert retained == [ + f"owned output could not be located after cleanup: {backup_path}"] + assert backup_path.read_text() == "foreign backup bytes" + assert moved.read_text() == "old.bak" + + +def test_uninspectable_post_swap_destination_retains_named_backup(m): + """An unknown post-swap destination keeps the inspectable backup visible.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak") + original.write_text("old bytes") + destination.write_text("new bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": None, + "swap_started": True, + "original": None, + } + real_entry = m._entry_identity + + def deny_destination(path): + if pathlib.Path(path) == destination: + raise PermissionError("controlled post-swap inspection failure") + return real_entry(path) + + m._entry_identity = deny_destination + try: + failures = [] + assert m._restore_backup(swap, failures, []) == "unrollbackable" + finally: + m._entry_identity = real_entry + os.close(backup_fd) + + assert failures == [backup] + assert backup.read_text() == "old bytes" + assert destination.read_text() == "new bytes" + + +def test_interrupted_committed_cleanup_reconciles_moved_staged_output_pin(m): + """A committed .part pin is checked at its destination before close.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + staged, destination, moved = ( + root / "output.xml.owned.part", root / "output.xml", root / "moved.xml") + staged.write_text("committed bytes") + pin = os.open(staged, os.O_RDONLY) + record = { + "path": staged, + "identity": m._fd_identity(pin), + "pin": pin, + "cleanup_path": staged, + "canonical_path": destination, + } + os.replace(staged, destination) + destination.rename(moved) + retained = [] + + m._reconcile_interrupted_committed_cleanup([], [record], retained, []) + + assert retained == [ + "committed staged output could not be located after cleanup: " + + str(destination)] + assert record["pin"] is None + assert moved.read_text() == "committed bytes" + + +def test_zero_link_original_is_a_typed_path_change(m): + """An unlinked original is a move, never a multiple-link topology.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "previous.xml" + path.write_text("old bytes") + pin = os.open(path, os.O_RDONLY) + record = {"path": path, "identity": m._fd_identity(pin), "pin": pin} + try: + path.unlink() + refusal = refuses(m, "output_path_changed", m._pinned_original_still_has_one_link, + record) + finally: + os.close(pin) + + assert "changed while its rollback copy was prepared" in str(refusal.code) + + +def test_restored_output_close_recovers_after_effect_without_a_false_retention(m): + """A restored descriptor close may report after release without failing rollback.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak") + original.write_text("old bytes") + destination.write_text("new bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": {}, + "swap_started": True, + "original": None, + } + real_open, real_close, real_restore = ( + m._open_regular_output, m.os.close, m._restore_metadata) + restored_handles = [] + + def remember_restored_handle(*args): + handle = real_open(*args) + restored_handles.append(handle) + return handle + + def close_then_report_failure(handle): + if handle in restored_handles: + real_close(handle) + raise OSError("controlled close failure after effect") + return real_close(handle) + + m._open_regular_output = remember_restored_handle + m.os.close = close_then_report_failure + m._restore_metadata = lambda *_: None + try: + failures, descriptor_failures, restored = [], [], [] + m._restore_backup(swap, failures, restored, descriptor_failures) + finally: + m._open_regular_output = real_open + m.os.close = real_close + m._restore_metadata = real_restore + os.close(backup_fd) + + assert destination.read_text() == "old bytes" + assert failures == [] + assert descriptor_failures == [] + assert restored == [destination] + + if __name__ == "__main__": raise SystemExit(main()) From fcf9d780dbc392aecfd05d939cc461ba69664bac Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 17:18:19 +0530 Subject: [PATCH 53/59] Cover Windows reclaimed backup cleanup --- scripts/bank_statement_import.test.py | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index a0616e36b..b16188902 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -3810,6 +3810,39 @@ def close_then_error(candidate): assert not path.exists() +def test_windows_modeled_reclaimed_backup_name_is_not_retained(m): + """A foreign replacement after close is disclosed only as an unlocated copy.""" + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + path, moved, foreign = ( + root / "rollback.bak", root / "moved.bak", root / "foreign.bak") + handle = m._open_private(path) + os.write(handle, b"prior statement bytes") + record = m._owned_path(path, handle, created=True) + real_close, real_name = m.os.close, m.os.name + + def close_then_reclaim(candidate): + result = real_close(candidate) + if candidate == handle: + path.rename(moved) + foreign.write_text("foreign backup bytes") + os.replace(foreign, path) + return result + + failures = [] + m.os.close, m.os.name = close_then_reclaim, "nt" + try: + m._cleanup_owned_path(record, failures, suppress_reclaimed_name=True) + finally: + m.os.close, m.os.name = real_close, real_name + + assert record["pin"] is None + assert failures == [ + f"owned output could not be located after cleanup: {path}"] + assert path.read_text() == "foreign backup bytes" + assert moved.read_bytes() == b"prior statement bytes" + + def test_windows_modeled_cleanup_reports_an_alias_before_closing_the_pin(m): """The Windows close-before-unlink branch still discloses linked output.""" with tempfile.TemporaryDirectory() as directory: From 8a96fe7089636dcfe38cedab3e93ac6bc97e4691 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 18:03:05 +0530 Subject: [PATCH 54/59] Harden bank recovery pin handoffs --- scripts/bank_statement_import.py | 49 ++++--- scripts/bank_statement_import.test.py | 189 +++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 24 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 0ffa6539b..6a6fcaafc 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1650,7 +1650,7 @@ def _open_regular_output(path, expected_identity=None): @contextlib.contextmanager def _defer_sigint_during_claim(): - """Pair a newly-created inode with its cleanup owner before SIGINT.""" + """Keep an ownership handoff whole until a pending SIGINT can unwind it.""" if hasattr(signal, "pthread_sigmask"): try: previous = signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT}) @@ -1808,15 +1808,18 @@ def _close_owned_path(record, failures, diagnostic_path=None, descriptor_failure after-effect. Every other failed or mismatched inspection remains a descriptor uncertainty: never retry a close that could target a reused fd. """ - handle = record.get("pin") - if handle is None: - return - record["pin"] = None - try: - os.close(handle) + close_failed = False + with _defer_sigint_during_claim(): + handle = record.get("pin") + if handle is None: + return + record["pin"] = None + try: + os.close(handle) + except OSError: + close_failed = True + if not close_failed: return - except OSError: - pass diagnostic = str(diagnostic_path or record.get( "cleanup_path", record["path"])) @@ -2281,6 +2284,13 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures restore_handle = _open_regular_output(destination, backup_identity) try: _restore_metadata(restore_handle, metadata) + try: + restored_still_current = ( + _entry_identity(destination) == backup_identity) + except OSError: + return _mark_rollback_unavailable(swap, failures) + if not restored_still_current: + return _mark_rollback_unavailable(swap, failures) finally: _close_owned_path( {"path": destination, "identity": backup_identity, @@ -2350,8 +2360,8 @@ def _cleanup_committed_outputs(replaced, claimed, retained_failures, descriptor_ record, descriptor_failures, diagnostic_path=record.get("canonical_path")) -def _reconcile_committed_staged_output_pin(record, failures): - """Check a staged descriptor at its committed name before releasing it.""" +def _reconcile_staged_output_pin(record, failures): + """Check a moved staged descriptor at its destination before release.""" if record.get("cleanup_path") == record.get("canonical_path"): return pin = record.get("pin") @@ -2373,7 +2383,7 @@ def _reconcile_committed_staged_output_pin(record, failures): committed_here = False if not committed_here: failures.append( - "committed staged output could not be located after cleanup: " + "staged output could not be located after cleanup: " + str(record["canonical_path"])) @@ -2435,7 +2445,7 @@ def _reconcile_interrupted_committed_cleanup( diagnostic_path=swap.get("destination", swap["original"]["path"]), descriptor_failures=descriptor_failures) for record in claimed: - _reconcile_committed_staged_output_pin(record, retained_failures) + _reconcile_staged_output_pin(record, retained_failures) _close_owned_path( record, descriptor_failures, diagnostic_path=record.get("canonical_path")) @@ -2633,14 +2643,10 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): os.path.realpath(supplied_path) != real_path or _file_identity(real_path) != original_identity) except FileNotFoundError: - # A missing leaf under its original parent is a commit-boundary - # path change. A vanished parent can leave pinned private - # outputs under an unknown spelling, so preserve its existing - # recovery path and diagnostic rather than misclassifying it. - if os.path.isdir(os.path.dirname(real_path)): - existing_output_changed = True - else: - raise + # A missing leaf or parent is a commit-boundary path change. + # Recovery retains any pinned inode it cannot locate, but the + # operator still receives the typed path outcome. + existing_output_changed = True except OSError: existing_output_changed = True if existing_output_changed: @@ -2850,6 +2856,7 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): unrollbackable_temporaries.add(id(pending_swap["temporary"])) for record in claimed: if id(record) in unrollbackable_temporaries: + _reconcile_staged_output_pin(record, cleanup_failures) _close_owned_path( record, cleanup_failures, diagnostic_path=record.get("canonical_path"), diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index b16188902..584738978 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -2716,8 +2716,9 @@ def rename_parent_after_backup(source, identity, backup_handle): try: m.write_outputs([(str(destination), "new bytes")]) raise AssertionError("the renamed parent must prevent a swap") - except FileNotFoundError as error: - detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + except m.Refusal as error: + detail = str(error.code) + "\n" + "\n".join(getattr(error, "__notes__", [])) + assert error.category == "output_path_changed" assert "owned output could not be located after cleanup" in detail finally: m._copy_private_backup = real_copy @@ -5642,7 +5643,7 @@ def test_interrupted_committed_cleanup_reconciles_moved_staged_output_pin(m): m._reconcile_interrupted_committed_cleanup([], [record], retained, []) assert retained == [ - "committed staged output could not be located after cleanup: " + "staged output could not be located after cleanup: " + str(destination)] assert record["pin"] is None assert moved.read_text() == "committed bytes" @@ -5722,5 +5723,187 @@ def close_then_report_failure(handle): assert restored == [destination] +def test_precommit_unrollbackable_staged_pin_is_reconciled_before_close(m): + """A partial staged output moved during recovery stays visible before close.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second, moved, foreign = ( + root / "first.xml", root / "second.xml", root / "moved.xml", root / "foreign.xml") + first.write_text("first old") + second.write_text("second old") + real_copy, real_digest, real_reconcile, real_replace = ( + m._copy_private_backup, m._digest_pinned_bytes, + m._reconcile_staged_output_pin, m.os.replace) + first_backup_pins, reconciled = set(), [] + + def fail_second_prepare(source_path, identity, backup_handle): + if pathlib.Path(source_path).resolve() == second.resolve(): + raise OSError("controlled later preparation failure") + result = real_copy(source_path, identity, backup_handle) + first_backup_pins.add(backup_handle) + return result + + def fail_first_backup_digest(handle): + if handle in first_backup_pins: + raise OSError("controlled rollback digest failure") + return real_digest(handle) + + def move_partial_before_reconciliation(record, failures): + if pathlib.Path(record.get("canonical_path", "")).resolve() == first.resolve(): + first.rename(moved) + foreign.write_text("foreign bytes") + real_replace(foreign, first) + reconciled.append(record) + return real_reconcile(record, failures) + + m._copy_private_backup = fail_second_prepare + m._digest_pinned_bytes = fail_first_backup_digest + m._reconcile_staged_output_pin = move_partial_before_reconciliation + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled preparation failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m._copy_private_backup = real_copy + m._digest_pinned_bytes = real_digest + m._reconcile_staged_output_pin = real_reconcile + + assert reconciled + assert "controlled later preparation failure" in detail + assert "staged output could not be located after cleanup" in detail + assert first.read_text() == "foreign bytes" + assert moved.read_text() == "first new" + assert second.read_text() == "second old" + + +def test_close_owned_path_defers_sigint_until_the_pin_is_closed(m): + """The record cannot lose its fd ownership between clearing and close.""" + if not hasattr(signal, "pthread_sigmask"): + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "output.xml" + path.write_text("bytes") + handle = os.open(path, os.O_RDONLY) + + class InterruptOnClear(dict): + def __setitem__(self, key, value): + result = super().__setitem__(key, value) + if key == "pin" and value is None: + signal.raise_signal(signal.SIGINT) + return result + + record = InterruptOnClear(path=path, identity=m._fd_identity(handle), pin=handle) + old_handler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.default_int_handler) + try: + try: + m._close_owned_path(record, []) + raise AssertionError("the deferred SIGINT must escape after close") + except KeyboardInterrupt: + pass + finally: + signal.signal(signal.SIGINT, old_handler) + + assert record["pin"] is None + try: + os.fstat(handle) + raise AssertionError("the descriptor must close before the deferred interrupt") + except OSError as error: + assert error.errno == errno.EBADF + + +def test_restore_revalidates_destination_after_metadata_before_reporting_success(m): + """Metadata work cannot turn a retargeted restored destination into success.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup, moved, foreign = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak", + root / "moved.xml", root / "foreign.xml") + original.write_text("old bytes") + destination.write_text("new bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": {}, "swap_started": True, "original": None, + } + real_restore = m._restore_metadata + + def retarget_after_metadata(*_): + destination.rename(moved) + foreign.write_text("foreign bytes") + os.replace(foreign, destination) + + m._restore_metadata = retarget_after_metadata + try: + failures, restored = [], [] + assert m._restore_backup(swap, failures, restored) == "unrollbackable" + finally: + m._restore_metadata = real_restore + os.close(backup_fd) + + assert restored == [] + assert failures == [ + f"owned rollback copy could not be located after cleanup: {backup}"] + assert destination.read_text() == "foreign bytes" + assert moved.read_text() == "old bytes" + + +def test_restore_reconciles_backup_when_post_metadata_destination_is_uninspectable(m): + """A failed post-metadata inspection is not evidence that restore held.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak") + original.write_text("old bytes") + destination.write_text("new bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": {}, "swap_started": True, "original": None, + } + real_restore, real_entry = m._restore_metadata, m._entry_identity + metadata_finished = False + + def finish_metadata(*_): + nonlocal metadata_finished + metadata_finished = True + + def deny_only_post_metadata_destination(path): + if metadata_finished and pathlib.Path(path) == destination: + raise PermissionError("controlled post-metadata inspection failure") + return real_entry(path) + + m._restore_metadata, m._entry_identity = ( + finish_metadata, deny_only_post_metadata_destination) + try: + failures = [] + assert m._restore_backup(swap, failures, []) == "unrollbackable" + finally: + m._restore_metadata, m._entry_identity = real_restore, real_entry + os.close(backup_fd) + + assert failures == [ + f"owned rollback copy could not be located after cleanup: {backup}"] + assert destination.read_text() == "old bytes" + + if __name__ == "__main__": raise SystemExit(main()) From 508fcdcac061f709cf9a2c58b2edf63abbcc9f2f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 18:05:06 +0530 Subject: [PATCH 55/59] Update rollback recovery observation count --- scripts/bank_statement_import.test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 584738978..223dbb694 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -5497,7 +5497,7 @@ def retarget_backup_after_rollback_destination_check(path): finally: m.os.replace, m._entry_identity = real_replace, real_entry - assert first_destination_checks == 3 and retargeted + assert first_destination_checks == 4 and retargeted assert "controlled later swap failure" in detail assert "partially committed output could not be rolled back" in detail assert str(first.resolve()) in detail From 3e20092b71bdc50df82f01592b5490dfad002e12 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 18:06:10 +0530 Subject: [PATCH 56/59] Update rollback inspection control count --- scripts/bank_statement_import.test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 223dbb694..7b20368b1 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -5041,7 +5041,7 @@ def deny_first_destination(path): m._entry_identity = real_entry assert "controlled later swap failure" in detail - assert first_entry_checks == 2 + assert first_entry_checks == 3, first_entry_checks assert "partially committed output could not be rolled back" in detail assert str(first.resolve()) in detail assert first.read_text() == "first new" From 969c6e273f376d2eb98ebd219c488b1304a18eaf Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 19:30:46 +0530 Subject: [PATCH 57/59] Harden restored output recovery authority --- scripts/bank_statement_import.py | 57 ++++++---- scripts/bank_statement_import.test.py | 155 ++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 22 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 6a6fcaafc..c47761c40 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -1818,29 +1818,30 @@ def _close_owned_path(record, failures, diagnostic_path=None, descriptor_failure os.close(handle) except OSError: close_failed = True - if not close_failed: - return + if not close_failed: + return - diagnostic = str(diagnostic_path or record.get( - "cleanup_path", record["path"])) - destination = descriptor_failures if descriptor_failures is not None else failures - try: - stat_result = os.fstat(handle) - except OSError as error: - if error.errno == errno.EBADF: - # The close took effect (or the descriptor was independently made - # invalid). Do not retry it: a later fd could be foreign. + diagnostic = str(diagnostic_path or record.get( + "cleanup_path", record["path"])) + destination = (descriptor_failures if descriptor_failures is not None + else failures) + try: + stat_result = os.fstat(handle) + except OSError as error: + if error.errno == errno.EBADF: + # The close took effect (or the descriptor was independently + # made invalid). Do not retry it: a later fd could be foreign. + return + destination.append(diagnostic) return + if (stat_result.st_dev, stat_result.st_ino) == record["identity"]: + # The original descriptor is still open. Its pathname may already + # name a staged replacement, so this is never a retained-path claim. + destination.append(diagnostic) + return + # A mocked or platform-specific close could leave a live but different fd. + # It is neither safe to close again nor evidence about a pathname. destination.append(diagnostic) - return - if (stat_result.st_dev, stat_result.st_ino) == record["identity"]: - # The original descriptor is still open. Its pathname may already - # name a staged replacement, so this is never a retained-path claim. - destination.append(diagnostic) - return - # A mocked or platform-specific close could leave a live but different fd. - # It is neither safe to close again nor evidence about a pathname. - destination.append(diagnostic) def _reconcile_owned_pin_after_cleanup(record, outcome, failures): @@ -2291,6 +2292,10 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures return _mark_rollback_unavailable(swap, failures) if not restored_still_current: return _mark_rollback_unavailable(swap, failures) + try: + _pinned_backup_still_has_one_link(swap["backup"]) + except (OSError, Refusal): + return _mark_rollback_unavailable(swap, failures) finally: _close_owned_path( {"path": destination, "identity": backup_identity, @@ -2299,7 +2304,7 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures descriptor_failures=descriptor_failures) metadata_scope_warnings.append(destination) except (OSError, Refusal): - failures.append(destination) + return _mark_rollback_unavailable(swap, failures) def _append_cleanup_detail(error, message): @@ -2746,7 +2751,15 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): f"{supplied_path} output contents changed before commit; " "no output was committed", ) - _require_single_owned_link(record, "output", "output_has_multiple_links") + try: + _require_single_owned_link( + record, "output", "output_has_multiple_links") + except OSError: + raise Refusal( + "output_path_changed", + f"{supplied_path} output ownership could not be verified before " + "commit; no output was committed", + ) from None # Earlier rollback copies can also be changed during later swaps. # A commit may retire them only while their ownership remains proved. for swap in replaced: diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 7b20368b1..3962ed429 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -5905,5 +5905,160 @@ def deny_only_post_metadata_destination(path): assert destination.read_text() == "old bytes" +def test_restore_open_refusal_reconciles_the_pinned_backup_once(m): + """A restored-open refusal reports the pinned backup, without a second path claim.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak") + original.write_text("old bytes") + destination.write_text("new bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": {}, "swap_started": True, "original": None, + } + real_open = m._open_regular_output + + def refuse_restored_open(path, expected_identity=None): + if (pathlib.Path(path) == destination + and expected_identity == swap["backup"]["identity"]): + raise m.Refusal("output_path_changed", "controlled restored-open refusal") + return real_open(path, expected_identity) + + m._open_regular_output = refuse_restored_open + try: + failures = [] + assert m._restore_backup(swap, failures, []) == "unrollbackable" + finally: + m._open_regular_output = real_open + os.close(backup_fd) + + assert failures == [ + f"owned rollback copy could not be located after cleanup: {backup}"] + assert destination.read_text() == "old bytes" + + +def test_restore_reports_alias_added_during_metadata(m): + """Metadata cannot add a restored-output alias without recovery disclosure.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup, alias = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak", + root / "alias.xml") + original.write_text("old bytes") + destination.write_text("new bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": {}, "swap_started": True, "original": None, + } + real_restore = m._restore_metadata + + def add_alias(*_): + os.link(destination, alias) + + m._restore_metadata = add_alias + try: + failures = [] + assert m._restore_backup(swap, failures, []) == "unrollbackable" + finally: + m._restore_metadata = real_restore + os.close(backup_fd) + + assert failures == [ + f"unknown hard-link alias may retain rollback bytes: {backup}"] + assert destination.read_text() == alias.read_text() == "old bytes" + + +def test_close_owned_path_reconciles_failed_close_before_deferred_sigint(m): + """A deferred interrupt cannot preempt failed-close descriptor disclosure.""" + if not hasattr(signal, "pthread_sigmask"): + return + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "output.xml" + path.write_text("bytes") + handle = os.open(path, os.O_RDONLY) + + class InterruptOnClear(dict): + def __setitem__(self, key, value): + result = super().__setitem__(key, value) + if key == "pin" and value is None: + signal.raise_signal(signal.SIGINT) + return result + + record = InterruptOnClear(path=path, identity=m._fd_identity(handle), pin=handle) + real_close = m.os.close + + def fail_close(candidate): + if candidate == handle: + raise OSError("controlled close failure") + return real_close(candidate) + + old_handler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.default_int_handler) + m.os.close = fail_close + try: + descriptor_failures = [] + try: + m._close_owned_path(record, [], descriptor_failures=descriptor_failures) + raise AssertionError("the deferred SIGINT must escape after reconciliation") + except KeyboardInterrupt: + pass + finally: + m.os.close = real_close + signal.signal(signal.SIGINT, old_handler) + os.close(handle) + + assert record["pin"] is None + assert descriptor_failures == [str(path)] + + +def test_final_output_pin_fstat_failure_is_a_typed_path_change(m): + """The final output ownership check cannot leak a raw descriptor error.""" + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "output.xml" + real_owned_path, real_fstat = m._owned_path, m.os.fstat + owned_pins = set() + + def arm_final_pin_failure(*args, **kwargs): + record = real_owned_path(*args, **kwargs) + owned_pins.add(record["pin"]) + m.os.fstat = fail_final_pin_fstat + return record + + def fail_final_pin_fstat(handle): + if handle in owned_pins: + m.os.fstat = real_fstat + raise OSError("controlled final ownership fstat failure") + return real_fstat(handle) + + m._owned_path = arm_final_pin_failure + try: + refusal = refuses( + m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._owned_path = real_owned_path + m.os.fstat = real_fstat + + assert "ownership could not be verified" in str(refusal.code) + assert not destination.exists() + + if __name__ == "__main__": raise SystemExit(main()) From 5d899304e20d9d91dd4632142515be9976ad2975 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 20:59:36 +0530 Subject: [PATCH 58/59] Type pre-replacement ownership failures --- scripts/bank_statement_import.py | 34 +++++++- scripts/bank_statement_import.test.py | 113 ++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index c47761c40..36c482902 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2635,8 +2635,20 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): pending_swap["original"] = state["original"] pending_swap["metadata"] = _metadata_from_handle( real_path, pending_swap["original"]["pin"]) - pending_swap["backup"]["digest"] = _copy_private_backup( - real_path, original_identity, backup_handle) + try: + pending_swap["backup"]["digest"] = _copy_private_backup( + real_path, original_identity, backup_handle) + except FileNotFoundError: + # The existing destination remains pinned even when its path + # disappears before the backup source can open. Reconcile + # that inode before recovery releases it, then expose this as + # the same typed authority failure as later path races. + _reconcile_owned_pin_after_cleanup( + pending_swap["original"], "missing", cleanup_failures) + raise Refusal( + "output_path_changed", + f"{supplied_path} disappeared before its rollback source could be opened", + ) from None # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after # the filesystem call has taken effect but before it returns. @@ -2692,12 +2704,26 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} rollback copy changed before replacement", ) - _pinned_backup_still_has_one_link(pending_swap["backup"]) + try: + _pinned_backup_still_has_one_link(pending_swap["backup"]) + except OSError: + raise Refusal( + "output_path_changed", + f"{supplied_path} rollback copy ownership could not be verified " + "before replacement", + ) from None # The first pin checked that the original was single-linked. A # backup hook can still add an alias before the commit boundary; # recheck this pinned inode so replacement never detaches a new # hard link while reporting a successful overwrite. - _pinned_original_still_has_one_link(pending_swap["original"]) + try: + _pinned_original_still_has_one_link(pending_swap["original"]) + except OSError: + raise Refusal( + "output_path_changed", + f"{supplied_path} original ownership could not be verified " + "before replacement", + ) from None try: original_still_current = ( _entry_identity(real_path) == original_identity) diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 3962ed429..41b01d4af 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -6060,5 +6060,118 @@ def fail_final_pin_fstat(handle): assert not destination.exists() +def test_backup_source_disappearance_is_typed_and_reconciles_the_original_pin(m): + """A source lost before backup open keeps its moved original visible.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, moved = root / "previous.xml", root / "moved.xml" + destination.write_text("old bytes") + real_open = m._open_regular_output + + def remove_before_backup_source_open(path, expected_identity=None): + if (expected_identity is not None + and pathlib.Path(path).resolve() == destination.resolve()): + destination.rename(moved) + return real_open(path, expected_identity) + + m._open_regular_output = remove_before_backup_source_open + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._open_regular_output = real_open + + detail = str(refusal.code) + assert "disappeared before its rollback source could be opened" in detail + assert "owned output could not be located after cleanup" in detail + assert not destination.exists() + assert moved.read_text() == "old bytes" + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + +def test_pre_replacement_backup_pin_fstat_failure_is_a_typed_path_change(m): + """A backup descriptor inspection error cannot escape as an OSError.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "previous.xml" + destination.write_text("old bytes") + real_copy, real_fstat = m._copy_private_backup, m.os.fstat + backup_pins = set() + + def arm_backup_pin_failure(source_path, identity, backup_handle): + result = real_copy(source_path, identity, backup_handle) + backup_pins.add(backup_handle) + m.os.fstat = fail_backup_pin_fstat + return result + + def fail_backup_pin_fstat(handle): + if handle in backup_pins: + m.os.fstat = real_fstat + raise OSError("controlled backup pin fstat failure") + return real_fstat(handle) + + m._copy_private_backup = arm_backup_pin_failure + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._copy_private_backup = real_copy + m.os.fstat = real_fstat + + assert "rollback copy ownership could not be verified" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert not list(pathlib.Path(directory).glob("*.part")) + assert not list(pathlib.Path(directory).glob("*.bak")) + + +def test_pre_replacement_original_pin_fstat_failure_is_a_typed_path_change(m): + """An original descriptor inspection error cannot escape as an OSError.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "previous.xml" + destination.write_text("old bytes") + real_open, real_copy, real_fstat = ( + m._open_regular_output, m._copy_private_backup, m.os.fstat) + original_pins = set() + + def remember_original_pin(path, expected_identity=None): + handle = real_open(path, expected_identity) + if (expected_identity is None + and pathlib.Path(path).resolve() == destination.resolve()): + original_pins.add(handle) + return handle + + def arm_original_pin_failure(*args): + result = real_copy(*args) + m.os.fstat = fail_original_pin_fstat + return result + + def fail_original_pin_fstat(handle): + if handle in original_pins: + m.os.fstat = real_fstat + raise OSError("controlled original pin fstat failure") + return real_fstat(handle) + + m._open_regular_output = remember_original_pin + m._copy_private_backup = arm_original_pin_failure + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._open_regular_output = real_open + m._copy_private_backup = real_copy + m.os.fstat = real_fstat + + assert "original ownership could not be verified" in str(refusal.code) + assert destination.read_text() == "old bytes" + assert not list(pathlib.Path(directory).glob("*.part")) + assert not list(pathlib.Path(directory).glob("*.bak")) + + if __name__ == "__main__": raise SystemExit(main()) From 3759f245c3969f2508e8a30426189e7e64d709e3 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 13 Sep 2026 22:00:46 +0530 Subject: [PATCH 59/59] fix(bank): preserve rollback recovery evidence --- scripts/bank_statement_import.py | 60 ++++++-- scripts/bank_statement_import.test.py | 202 ++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 13 deletions(-) diff --git a/scripts/bank_statement_import.py b/scripts/bank_statement_import.py index 36c482902..37b91aa3d 100644 --- a/scripts/bank_statement_import.py +++ b/scripts/bank_statement_import.py @@ -2094,7 +2094,22 @@ def _copy_private_backup(source_path, original_identity, backup_handle): the same inode while it is being read remains outside this command's authority, so this does not promise a crash transaction. """ - source_handle = _open_regular_output(source_path, original_identity) + try: + source_handle = _open_regular_output(source_path, original_identity) + except FileNotFoundError: + # The caller retains the established missing-source diagnostic and + # reconciliation path for this specific authority outcome. + raise + except OSError as error: + # This is the only I/O boundary translated here. Later read, write, + # sync, and verification errors describe a partially prepared private + # copy and must keep their original exception for recovery to handle. + refusal = Refusal( + "output_path_changed", + f"{source_path} could not be opened as its rollback source", + ) + refusal.backup_source_open_failed = True + raise refusal from error try: if _fd_identity(source_handle) != original_identity: raise Refusal( @@ -2253,30 +2268,36 @@ def _restore_backup(swap, failures, metadata_scope_warnings, descriptor_failures os.replace(backup, destination) restored = True except OSError: + # A failed rename can still take effect. Re-inspect the destination + # before reporting a partial result; `current_identity` predates the + # restore attempt and cannot classify its outcome. try: - restored = _entry_identity(destination) == backup_identity + destination_after_restore = _entry_identity(destination) except OSError: - restored = False + destination_after_restore = None + restored = destination_after_restore == backup_identity if not restored: try: backup_retained = _entry_identity(backup) == backup_identity except OSError: backup_retained = False if backup_retained: - failures.append(backup) - try: - destination_still_staged = ( - _entry_identity(destination) == staged_identity) - except OSError: + if destination_after_restore is None: # The failed replace may have taken effect. An unknown # destination cannot be classified as a safe foreign # inode, so disclose this actual partial result while the # named backup remains inspectable. return _mark_rollback_unavailable( swap, failures, retain_named=True) - if destination_still_staged: - return _mark_rollback_unavailable(swap, failures) - elif current_identity == staged_identity: + # Whether the failed restore left the staged inode in place + # or a foreign inode at the destination, the still-named + # private backup is the only recoverable old copy. Report it + # through the rollback path and let the caller disclose the + # destination as partial; never clean it as a successful + # rollback based on the pre-restore identity. + return _mark_rollback_unavailable( + swap, failures, retain_named=True) + elif destination_after_restore == staged_identity: return _mark_rollback_unavailable(swap, failures) else: failures.append(destination) @@ -2377,10 +2398,14 @@ def _reconcile_staged_output_pin(record, failures): except OSError: _record_uninspectable_cleanup(record["canonical_path"], failures) return - if ((stat_result.st_dev, stat_result.st_ino) != record["identity"] - or stat_result.st_nlink == 0): + if (stat_result.st_dev, stat_result.st_ino) != record["identity"]: _record_uninspectable_cleanup(record["canonical_path"], failures) return + # An unlinked inode cannot be a retained foreign destination. Its pin is + # still useful to prove that cleanup removed the staged output, but there + # is no pathname for an operator to recover. + if stat_result.st_nlink == 0: + return try: committed_here = ( _entry_identity(record["canonical_path"]) == record["identity"]) @@ -2649,6 +2674,15 @@ def write_outputs(targets, accept_inherited=False, after_claim=None): "output_path_changed", f"{supplied_path} disappeared before its rollback source could be opened", ) from None + except Refusal as refusal: + if getattr(refusal, "backup_source_open_failed", False): + # The original descriptor is still the only trustworthy + # evidence after a source-open denial or I/O failure. + # Reconcile it before recovery closes the pin, while + # leaving later copy I/O errors untyped and untouched. + _reconcile_owned_pin_after_cleanup( + pending_swap["original"], "missing", cleanup_failures) + raise # The destination stays present until this one atomic replacement. # `pending_swap` is set first because an interrupt may arrive after # the filesystem call has taken effect but before it returns. diff --git a/scripts/bank_statement_import.test.py b/scripts/bank_statement_import.test.py index 41b01d4af..4d93227f1 100644 --- a/scripts/bank_statement_import.test.py +++ b/scripts/bank_statement_import.test.py @@ -6092,6 +6092,73 @@ def remove_before_backup_source_open(path, expected_identity=None): assert not list(root.glob("*.bak")) +def test_backup_source_open_permission_is_typed_and_reconciles_the_original_pin(m): + """A denied rollback-source open remains typed and retains a moved old file.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + destination, moved = root / "previous.xml", root / "moved.xml" + destination.write_text("old bytes") + real_open = m._open_regular_output + + def deny_after_moving_source(path, expected_identity=None): + if (expected_identity is not None + and pathlib.Path(path).resolve() == destination.resolve()): + destination.rename(moved) + raise PermissionError("controlled rollback-source open denial") + return real_open(path, expected_identity) + + m._open_regular_output = deny_after_moving_source + try: + refusal = refuses(m, "output_path_changed", m.write_outputs, + [(str(destination), "new bytes")]) + finally: + m._open_regular_output = real_open + + detail = str(refusal.code) + assert "could not be opened as its rollback source" in detail + assert "owned output could not be located after cleanup" in detail + assert moved.read_text() == "old bytes" + assert not destination.exists() + assert not list(root.glob("*.part")) + assert not list(root.glob("*.bak")) + + +def test_backup_source_read_error_is_not_retyped_as_an_open_failure(m): + """Only source open errors are typed; later copy I/O retains its OSError.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + destination = pathlib.Path(directory) / "previous.xml" + destination.write_text("old bytes") + real_open, real_read = m._open_regular_output, m.os.read + source_pins = set() + + def remember_source_pin(path, expected_identity=None): + handle = real_open(path, expected_identity) + if expected_identity is not None: + source_pins.add(handle) + return handle + + def fail_copy_read(handle, size): + if handle in source_pins: + raise OSError("controlled rollback-source read failure") + return real_read(handle, size) + + m._open_regular_output, m.os.read = remember_source_pin, fail_copy_read + try: + try: + m.write_outputs([(str(destination), "new bytes")]) + raise AssertionError("the controlled copy read failure must escape") + except OSError as error: + assert str(error) == "controlled rollback-source read failure" + finally: + m._open_regular_output, m.os.read = real_open, real_read + + assert destination.read_text() == "old bytes" + + def test_pre_replacement_backup_pin_fstat_failure_is_a_typed_path_change(m): """A backup descriptor inspection error cannot escape as an OSError.""" if os.name == "nt": @@ -6173,5 +6240,140 @@ def fail_original_pin_fstat(handle): assert not list(pathlib.Path(directory).glob("*.bak")) +def test_failed_restore_rechecks_destination_before_partial_report(m): + """A foreign destination installed during failed restore is never overwritten.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + original, destination, backup, foreign = ( + root / "original.xml", root / "destination.xml", root / "rollback.bak", + root / "foreign.xml") + original.write_text("old bytes") + destination.write_text("staged bytes") + backup.write_text("old bytes") + backup_fd = os.open(backup, os.O_RDONLY) + swap = { + "backup": {"path": backup, "identity": m._fd_identity(backup_fd), + "pin": backup_fd, "digest": m._digest_pinned_bytes(backup_fd)}, + "destination": destination, + "original_identity": m._entry_identity(original), + "staged_identity": m._entry_identity(destination), + "metadata": None, "swap_started": True, "original": None, + } + real_replace = m.os.replace + + def replace_destination_then_fail(source, target): + if pathlib.Path(source) == backup and pathlib.Path(target) == destination: + foreign.write_text("foreign bytes") + real_replace(foreign, destination) + raise OSError("controlled restore failure after foreign replacement") + return real_replace(source, target) + + m.os.replace = replace_destination_then_fail + try: + failures = [] + assert m._restore_backup(swap, failures, []) == "unrollbackable" + finally: + m.os.replace = real_replace + os.close(backup_fd) + + assert failures == [backup] + assert destination.read_text() == "foreign bytes" + assert backup.read_text() == "old bytes" + + +def test_final_rollback_backup_pin_fstat_failure_is_typed_and_conservative(m): + """The post-swap rollback pin check cannot replace the original failure.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + first, second = root / "first.xml", root / "second.xml" + first.write_text("first old") + second.write_text("second old") + real_copy, real_check, real_fstat, real_replace = ( + m._copy_private_backup, m._pinned_backup_still_has_one_link, + m.os.fstat, m.os.replace) + first_backup_pins, armed, failed = set(), False, [] + + def remember_first_backup(source_path, identity, backup_handle): + result = real_copy(source_path, identity, backup_handle) + if pathlib.Path(source_path).resolve() == first.resolve(): + first_backup_pins.add(backup_handle) + return result + + def fail_final_backup_fstat(handle): + if handle in first_backup_pins: + failed.append(handle) + m.os.fstat = real_fstat + raise OSError("controlled final rollback backup-pin fstat failure") + return real_fstat(handle) + + def arm_after_pre_replacement_check(record): + nonlocal armed + result = real_check(record) + if record.get("pin") in first_backup_pins and not armed: + armed = True + m.os.fstat = fail_final_backup_fstat + return result + + def fail_second_swap(source, destination): + if (pathlib.Path(source).suffix == ".part" + and pathlib.Path(destination).resolve() == second.resolve()): + raise OSError("controlled later swap failure") + return real_replace(source, destination) + + m._copy_private_backup = remember_first_backup + m._pinned_backup_still_has_one_link = arm_after_pre_replacement_check + m.os.replace = fail_second_swap + try: + try: + m.write_outputs([(str(first), "first new"), (str(second), "second new")]) + raise AssertionError("the controlled later swap failure must escape") + except OSError as error: + detail = str(error) + "\n" + "\n".join(getattr(error, "__notes__", [])) + finally: + m._copy_private_backup = real_copy + m._pinned_backup_still_has_one_link = real_check + m.os.fstat = real_fstat + m.os.replace = real_replace + + backup, = root.glob("first.xml.*.bak") + assert armed and failed + assert "controlled later swap failure" in detail + assert "partially committed output could not be rolled back" in detail + assert str(first.resolve()) in detail + assert str(backup) in detail + assert first.read_text() == "first new" + assert second.read_text() == "second old" + assert backup.read_text() == "first old" + assert not list(root.glob("*.part")) + + +def test_zero_link_unrollbackable_staged_pin_is_removed_not_retained(m): + """An unlinked staged inode has no foreign destination to disclose.""" + if os.name == "nt": + return + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + staged, destination = root / "staged.part", root / "output.xml" + staged.write_text("staged bytes") + pin = os.open(staged, os.O_RDONLY) + record = { + "path": staged, "cleanup_path": root / "other.part", + "canonical_path": destination, "identity": m._fd_identity(pin), "pin": pin, + } + staged.unlink() + try: + failures = [] + m._reconcile_staged_output_pin(record, failures) + finally: + os.close(pin) + + assert failures == [] + assert not destination.exists() + + if __name__ == "__main__": raise SystemExit(main())