From a1ff8c91f749d7f7045cbf97d0c543c09ee41e6c Mon Sep 17 00:00:00 2001 From: Paulo Cabral Sanz Date: Sat, 15 Aug 2026 00:56:40 -0300 Subject: [PATCH 1/3] test: sealed mid-file length inflate must not drop the tail Inflating val_size so the record does not fit never reaches CRC. scan_sealed used to return the prefix. This fails on current main. --- crates/engine/src/log/record.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/engine/src/log/record.rs b/crates/engine/src/log/record.rs index 594b27b..f2b5a6c 100644 --- a/crates/engine/src/log/record.rs +++ b/crates/engine/src/log/record.rs @@ -303,4 +303,24 @@ mod tests { let offs = scan_sealed(&buf).expect("torn tail is not corruption"); assert_eq!(offs, vec![0]); } + + /// A mid-file val_size inflate that does not fit in the file never + /// reaches CRC. Treating it as a torn tail drops every later record. + #[test] + fn sealed_inflated_mid_file_length_is_not_a_silent_tail() { + let mut buf = Vec::new(); + encode_into(&mut buf, 1, flags::NO_EXPIRY, 0, b"keep", b"one", b"").unwrap(); + let second_at = buf.len(); + encode_into(&mut buf, 2, flags::NO_EXPIRY, 0, b"drop-me", b"two", b"").unwrap(); + encode_into(&mut buf, 3, flags::NO_EXPIRY, 0, b"also-keep", b"three", b"").unwrap(); + + buf[second_at + 29..second_at + 33].copy_from_slice(&0x00FF_FFFFu32.to_le_bytes()); + + let err = scan_sealed(&buf) + .expect_err("inflated mid-file length in a sealed image is corruption"); + assert!( + matches!(err, EngineError::CrcMismatch { offset } if offset == second_at as u64), + "got {err:?}" + ); + } } From aec04998e9cef3dc95d0955285bc75cc6c0eceac Mon Sep 17 00:00:00 2001 From: Paulo Cabral Sanz Date: Sat, 15 Aug 2026 00:57:21 -0300 Subject: [PATCH 2/3] fix: sealed length inflate past EOF is fail-stop A complete header that claims a body bigger than the file is bitrot of the CRC'd length fields, not a torn tail. scan_sealed and rebuild_from_records now CrcMismatch instead of returning a silent prefix. --- crates/engine/src/log/record.rs | 10 ++++++++++ crates/engine/src/log/recover.rs | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/log/record.rs b/crates/engine/src/log/record.rs index f2b5a6c..0e8c127 100644 --- a/crates/engine/src/log/record.rs +++ b/crates/engine/src/log/record.rs @@ -205,6 +205,16 @@ pub fn scan_sealed(bytes: &[u8]) -> Result> { }; let rec_len = hdr.record_len(); if offset + rec_len > bytes.len() { + // A complete header that claims a body past EOF is length + // bitrot, not a torn last write: the length fields sit + // inside the CRC'd header, so a genuine torn tail is a + // short header (`< HEADER_LEN` above). Fail-stop so a + // mid-file inflate cannot drop later sealed records. + if offset + HEADER_LEN <= bytes.len() { + return Err(EngineError::CrcMismatch { + offset: offset as u64, + }); + } break; } let body = &bytes[offset + HEADER_LEN..offset + rec_len]; diff --git a/crates/engine/src/log/recover.rs b/crates/engine/src/log/recover.rs index 11a4e75..df84b19 100644 --- a/crates/engine/src/log/recover.rs +++ b/crates/engine/src/log/recover.rs @@ -144,7 +144,9 @@ async fn rebuild_from_records(file: &LogFile, file_id: u32, index: &mut NsIndex) }; let body_size = hdr.body_len(); let body = match file.read_at(offset + HEADER_LEN as u64, body_size).await { - Ok(b) if b.len() < body_size => break, + Ok(b) if b.len() < body_size => { + return Err(EngineError::CrcMismatch { offset }); + } Ok(b) => b, Err(e) => { warn!(file_id, offset, error = %e, "I/O error reading sealed file body; stopping scan at this offset"); From f192dcbdfd1ad647c157c8faa1f7b404b5c1d962 Mon Sep 17 00:00:00 2001 From: Paulo Cabral Sanz Date: Sat, 15 Aug 2026 01:05:36 -0300 Subject: [PATCH 3/3] fix: length inflate fail-stop must not reject a torn last record A complete header whose body never landed is a torn tail. Fail-stop only when a later CRC-ok record is still in the file (mid-file val_size inflate). Restores torn_footer_falls_back_to_scan_across_files. --- crates/engine/src/log/record.rs | 65 ++++++++++++++++++++++++++------ crates/engine/src/log/recover.rs | 33 +++++++++++++++- 2 files changed, 85 insertions(+), 13 deletions(-) diff --git a/crates/engine/src/log/record.rs b/crates/engine/src/log/record.rs index 0e8c127..cc69af9 100644 --- a/crates/engine/src/log/record.rs +++ b/crates/engine/src/log/record.rs @@ -181,13 +181,40 @@ pub fn encode( Ok(buf) } +/// True if `bytes` contains at least one fully framed, CRC-valid record. +/// +/// Distinguishes a torn last write (header landed, body did not; nothing +/// after it verifies) from mid-file length bitrot (a later record still +/// sits in the file, hidden behind an inflated `val_size`). +pub fn has_later_valid_record(bytes: &[u8]) -> bool { + let mut i = 0; + while i + HEADER_LEN <= bytes.len() { + if let Ok(hdr) = parse_header(&bytes[i..i + HEADER_LEN], i as u64) { + let rec_len = hdr.record_len(); + if rec_len >= HEADER_LEN && i + rec_len <= bytes.len() { + let header_buf = &bytes[i..i + HEADER_LEN]; + let body = &bytes[i + HEADER_LEN..i + rec_len]; + if verify_crc(&hdr, header_buf, body, i as u64).is_ok() { + return true; + } + } + } + i += 1; + } + false +} + /// Walk a sealed log image. /// /// A short last record at EOF is a torn tail and is dropped (`Ok` with the -/// prefix). A CRC mismatch on a *fully framed* mid-file record is bitrot: -/// the rest of the sealed file is still there and must not be silently -/// discarded. Callers treat that as `CrcMismatch` (fail-stop) rather than -/// "this is the torn active tail". +/// prefix). That includes a *complete* header whose body never landed — +/// the last write flushed 37 bytes and then crashed. A CRC mismatch on a +/// fully framed mid-file record is bitrot: the rest of the sealed file is +/// still there and must not be silently discarded. +/// +/// The same applies to a mid-file `val_size` inflate that does not fit: +/// later records are still in the file, so this is `CrcMismatch`, not a +/// torn tail. [`has_later_valid_record`] is the distinguisher. /// /// [`crate::log::recover`] uses the same rule when a sealed footer is /// missing and the file is rebuilt from records. @@ -205,12 +232,7 @@ pub fn scan_sealed(bytes: &[u8]) -> Result> { }; let rec_len = hdr.record_len(); if offset + rec_len > bytes.len() { - // A complete header that claims a body past EOF is length - // bitrot, not a torn last write: the length fields sit - // inside the CRC'd header, so a genuine torn tail is a - // short header (`< HEADER_LEN` above). Fail-stop so a - // mid-file inflate cannot drop later sealed records. - if offset + HEADER_LEN <= bytes.len() { + if has_later_valid_record(&bytes[offset + HEADER_LEN..]) { return Err(EngineError::CrcMismatch { offset: offset as u64, }); @@ -314,6 +336,18 @@ mod tests { assert_eq!(offs, vec![0]); } + /// Last write flushed a complete header and only part of the body. + /// Nothing after it verifies — this is a torn tail, not bitrot. + #[test] + fn sealed_incomplete_last_record_is_torn_tail() { + let mut buf = Vec::new(); + encode_into(&mut buf, 1, flags::NO_EXPIRY, 0, b"keep", b"one", b"").unwrap(); + encode_into(&mut buf, 2, flags::NO_EXPIRY, 0, b"tail", b"partial", b"").unwrap(); + buf.truncate(buf.len() - 5); + let offs = scan_sealed(&buf).expect("complete header + short body at EOF is torn"); + assert_eq!(offs, vec![0]); + } + /// A mid-file val_size inflate that does not fit in the file never /// reaches CRC. Treating it as a torn tail drops every later record. #[test] @@ -322,7 +356,16 @@ mod tests { encode_into(&mut buf, 1, flags::NO_EXPIRY, 0, b"keep", b"one", b"").unwrap(); let second_at = buf.len(); encode_into(&mut buf, 2, flags::NO_EXPIRY, 0, b"drop-me", b"two", b"").unwrap(); - encode_into(&mut buf, 3, flags::NO_EXPIRY, 0, b"also-keep", b"three", b"").unwrap(); + encode_into( + &mut buf, + 3, + flags::NO_EXPIRY, + 0, + b"also-keep", + b"three", + b"", + ) + .unwrap(); buf[second_at + 29..second_at + 33].copy_from_slice(&0x00FF_FFFFu32.to_le_bytes()); diff --git a/crates/engine/src/log/recover.rs b/crates/engine/src/log/recover.rs index df84b19..f0bcda1 100644 --- a/crates/engine/src/log/recover.rs +++ b/crates/engine/src/log/recover.rs @@ -6,7 +6,9 @@ use tracing::warn; use crate::error::{EngineError, Result}; use crate::log::file::{FooterEntry, LogFile, list_data_files}; use crate::log::index::{IndexEntry, NsIndex}; -use crate::log::record::{HEADER_LEN, flags as rflags, parse_header, verify_crc}; +use crate::log::record::{ + HEADER_LEN, flags as rflags, has_later_valid_record, parse_header, verify_crc, +}; /// Result of opening a namespace directory. pub struct OpenedFiles { @@ -143,9 +145,36 @@ async fn rebuild_from_records(file: &LogFile, file_id: u32, index: &mut NsIndex) Err(_) => break, }; let body_size = hdr.body_len(); + // Don't ask the pool for an inflated val_size (up to 4 GiB). A body + // that cannot fit in the remaining file is either a torn last write + // or mid-file length bitrot — distinguisher is a later CRC-ok record. + let remaining = total.saturating_sub(offset.saturating_add(HEADER_LEN as u64)); + if (body_size as u64) > remaining { + let rest = if remaining == 0 { + Vec::new() + } else { + match file + .read_at(offset + HEADER_LEN as u64, remaining as usize) + .await + { + Ok(b) => b.to_vec(), + Err(e) => { + warn!(file_id, offset, error = %e, "I/O error peeking sealed short body; stopping scan at this offset"); + break; + } + } + }; + if has_later_valid_record(&rest) { + return Err(EngineError::CrcMismatch { offset }); + } + break; + } let body = match file.read_at(offset + HEADER_LEN as u64, body_size).await { Ok(b) if b.len() < body_size => { - return Err(EngineError::CrcMismatch { offset }); + if has_later_valid_record(&b) { + return Err(EngineError::CrcMismatch { offset }); + } + break; } Ok(b) => b, Err(e) => {