diff --git a/Cargo.lock b/Cargo.lock index 848cb7320..6def48327 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,7 @@ name = "originweave-evidence" version = "0.1.0" dependencies = [ "originweave-core", + "sha2", ] [[package]] diff --git a/crates/originweave-evidence/Cargo.toml b/crates/originweave-evidence/Cargo.toml index a69386c38..35c21a7fb 100644 --- a/crates/originweave-evidence/Cargo.toml +++ b/crates/originweave-evidence/Cargo.toml @@ -12,6 +12,7 @@ publish = false [dependencies] originweave-core = { path = "../originweave-core" } +sha2 = "=0.10.9" [lints] workspace = true diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 8474b3618..d57dc524f 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -9,6 +9,7 @@ mod extraction_schema; mod sensitive_access; +mod warc_resource_record; pub use extraction_schema::{ ExtractionCardinality, ExtractionField, ExtractionNormalizationRule, ExtractionSchema, @@ -20,6 +21,11 @@ pub use sensitive_access::{ SensitiveAccessEvidence, SensitiveAccessEvidenceInput, SensitiveAccessOutcome, SensitiveEvidenceError, }; +pub use warc_resource_record::{ + MAX_WARC_CONTENT_TYPE_BYTES, MAX_WARC_DATE_BYTES, MAX_WARC_PAYLOAD_BYTES, + MAX_WARC_RECORD_ID_BYTES, WarcPayloadCompleteness, WarcResourceRecord, WarcResourceRecordError, + WarcTruncationReason, +}; use std::collections::BTreeMap; diff --git a/crates/originweave-evidence/src/warc_resource_record.rs b/crates/originweave-evidence/src/warc_resource_record.rs new file mode 100644 index 000000000..c5ba2d003 --- /dev/null +++ b/crates/originweave-evidence/src/warc_resource_record.rs @@ -0,0 +1,559 @@ +use std::fmt; + +use sha2::{Digest, Sha256}; + +use crate::{ProvenanceRecord, VerificationResult}; + +/// Maximum encoded size of the UUID-based WARC record identifier. +pub const MAX_WARC_RECORD_ID_BYTES: usize = 45; +/// Maximum encoded size accepted for a UTC WARC date. +pub const MAX_WARC_DATE_BYTES: usize = 30; +/// Maximum encoded size retained for a WARC content type. +pub const MAX_WARC_CONTENT_TYPE_BYTES: usize = 256; +/// Maximum resource payload retained by one immutable WARC record. +pub const MAX_WARC_PAYLOAD_BYTES: usize = 1_048_576; + +/// Standard WARC 1.1 reason for a deliberately or unexpectedly truncated record block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WarcTruncationReason { + /// Capture stopped because the configured maximum length was reached. + Length, + /// Capture stopped because the configured maximum capture time was reached. + Time, + /// Capture stopped because the network connection disconnected. + Disconnect, + /// Capture stopped for another or unknown reason. + Unspecified, +} + +impl WarcTruncationReason { + const fn warc_token(self) -> &'static str { + match self { + Self::Length => "length", + Self::Time => "time", + Self::Disconnect => "disconnect", + Self::Unspecified => "unspecified", + } + } +} + +/// Whether the retained WARC record block is complete or explicitly truncated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WarcPayloadCompleteness { + /// The retained record block is complete for the caller-authorized capture. + Complete, + /// The retained record block is partial for the stated WARC 1.1 truncation reason. + Truncated(WarcTruncationReason), +} + +/// A validation failure while constructing an immutable WARC resource record. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WarcResourceRecordError { + /// The record identifier was not a bounded non-nil UUID URN. + InvalidRecordId, + /// The date was not a bounded UTC RFC 3339 timestamp. + InvalidDate, + /// The content type was not a bounded, syntactically valid MIME media type. + InvalidContentType, + /// A record field or payload exceeded its retention limit. + LimitExceeded, + /// The WARC target URI contained octets outside RFC 3986 URI syntax. + InvalidTargetUri, + /// The WARC target URI differed from its provenance source URL. + TargetUriMismatch, + /// The source provenance was not independently verified. + UnverifiedProvenance, +} + +impl fmt::Display for WarcResourceRecordError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidRecordId => "invalid WARC record identifier", + Self::InvalidDate => "invalid WARC date", + Self::InvalidContentType => "invalid WARC content type", + Self::LimitExceeded => "WARC resource record limit exceeded", + Self::InvalidTargetUri => "invalid WARC target URI", + Self::TargetUriMismatch => "WARC target URI does not match provenance", + Self::UnverifiedProvenance => "WARC provenance is not independently verified", + }) + } +} + +impl std::error::Error for WarcResourceRecordError {} + +/// An immutable, bounded WARC `resource` record over already-authorized bytes. +#[derive(Clone, PartialEq, Eq)] +pub struct WarcResourceRecord { + record_id: String, + warc_date: String, + target_uri: String, + content_type: String, + payload: Vec, + block_digest: String, + provenance: ProvenanceRecord, + completeness: WarcPayloadCompleteness, +} + +impl fmt::Debug for WarcResourceRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WarcResourceRecord") + .field("record_id", &self.record_id) + .field("warc_date", &self.warc_date) + .field("content_type", &self.content_type) + .field("payload_byte_count", &self.payload.len()) + .field("block_digest", &self.block_digest) + .field("completeness", &self.completeness) + .field( + "provenance_verification_result", + &self.provenance.verification_result(), + ) + .finish() + } +} + +impl WarcResourceRecord { + /// Validate and construct one complete resource record without contacting a live origin. + pub fn new( + record_id: &str, + warc_date: &str, + target_uri: &str, + content_type: &str, + payload: Vec, + provenance: ProvenanceRecord, + ) -> Result { + Self::new_with_completeness( + record_id, + warc_date, + target_uri, + content_type, + payload, + provenance, + WarcPayloadCompleteness::Complete, + ) + } + + /// Validate and construct one resource record with explicit capture completeness. + /// + /// Truncated records retain the caller-provided partial block and emit the standard WARC 1.1 + /// `WARC-Truncated` reason. This constructor never truncates an oversized payload implicitly; + /// the retained block must still satisfy [`MAX_WARC_PAYLOAD_BYTES`]. + pub fn new_with_completeness( + record_id: &str, + warc_date: &str, + target_uri: &str, + content_type: &str, + payload: Vec, + provenance: ProvenanceRecord, + completeness: WarcPayloadCompleteness, + ) -> Result { + if record_id.len() > MAX_WARC_RECORD_ID_BYTES { + return Err(WarcResourceRecordError::LimitExceeded); + } + if warc_date.len() > MAX_WARC_DATE_BYTES { + return Err(WarcResourceRecordError::LimitExceeded); + } + if !valid_record_id(record_id) { + return Err(WarcResourceRecordError::InvalidRecordId); + } + if !valid_utc_date(warc_date) { + return Err(WarcResourceRecordError::InvalidDate); + } + if !valid_content_type(content_type) { + return Err(if content_type.len() > MAX_WARC_CONTENT_TYPE_BYTES { + WarcResourceRecordError::LimitExceeded + } else { + WarcResourceRecordError::InvalidContentType + }); + } + if !valid_target_uri_presentation(target_uri) { + return Err(WarcResourceRecordError::InvalidTargetUri); + } + if target_uri != provenance.source_url() { + return Err(WarcResourceRecordError::TargetUriMismatch); + } + if provenance.verification_result() != VerificationResult::Verified { + return Err(WarcResourceRecordError::UnverifiedProvenance); + } + if payload.len() > MAX_WARC_PAYLOAD_BYTES { + return Err(WarcResourceRecordError::LimitExceeded); + } + + Ok(Self { + record_id: record_id.to_owned(), + warc_date: warc_date.to_owned(), + target_uri: target_uri.to_owned(), + content_type: content_type.to_owned(), + block_digest: sha256_digest(&payload), + payload, + provenance, + completeness, + }) + } + + /// Return the UUID URN used as the WARC record identity. + #[must_use] + pub fn record_id(&self) -> &str { + &self.record_id + } + + /// Return the normalized UTC capture timestamp. + #[must_use] + pub fn warc_date(&self) -> &str { + &self.warc_date + } + + /// Return the provenance-bound target URI. + #[must_use] + pub fn target_uri(&self) -> &str { + &self.target_uri + } + + /// Return the payload media type retained in the WARC record. + #[must_use] + pub fn content_type(&self) -> &str { + &self.content_type + } + + /// Return the immutable resource bytes. + #[must_use] + pub fn payload(&self) -> &[u8] { + &self.payload + } + + /// Return the lowercase SHA-256 block digest. + #[must_use] + pub fn block_digest(&self) -> &str { + &self.block_digest + } + + /// Return the independently verified provenance bound to this record. + #[must_use] + pub const fn provenance(&self) -> &ProvenanceRecord { + &self.provenance + } + + /// Return whether the retained block is complete or explicitly truncated. + #[must_use] + pub const fn completeness(&self) -> WarcPayloadCompleteness { + self.completeness + } + + /// Serialize this bounded resource record as deterministic WARC 1.1 bytes. + #[must_use] + pub fn to_warc_bytes(&self) -> Vec { + let truncated_header = match self.completeness { + WarcPayloadCompleteness::Complete => String::new(), + WarcPayloadCompleteness::Truncated(reason) => { + format!("WARC-Truncated: {}\r\n", reason.warc_token()) + } + }; + let header = format!( + "WARC/1.1\r\nWARC-Type: resource\r\nWARC-Record-ID: <{}>\r\nWARC-Date: {}\r\nWARC-Target-URI: {}\r\n{}Content-Type: {}\r\nWARC-Block-Digest: {}\r\nContent-Length: {}\r\n\r\n", + self.record_id, + self.warc_date, + self.target_uri, + truncated_header, + self.content_type, + self.block_digest, + self.payload.len() + ); + let mut bytes = header.into_bytes(); + bytes.extend_from_slice(&self.payload); + bytes.extend_from_slice(b"\r\n\r\n"); + bytes + } +} + +fn valid_record_id(record_id: &str) -> bool { + let bytes = record_id.as_bytes(); + if bytes.len() != MAX_WARC_RECORD_ID_BYTES || !record_id.starts_with("urn:uuid:") { + return false; + } + let mut has_nonzero_hex = false; + for (index, byte) in bytes[9..].iter().copied().enumerate() { + if matches!(index, 8 | 13 | 18 | 23) { + if byte != b'-' { + return false; + } + } else if !byte.is_ascii_hexdigit() { + return false; + } else if byte != b'0' { + has_nonzero_hex = true; + } + } + has_nonzero_hex +} + +fn valid_utc_date(date: &str) -> bool { + let bytes = date.as_bytes(); + if !(20..=MAX_WARC_DATE_BYTES).contains(&bytes.len()) + || bytes.get(4) != Some(&b'-') + || bytes.get(7) != Some(&b'-') + || bytes.get(10) != Some(&b'T') + || bytes.get(13) != Some(&b':') + || bytes.get(16) != Some(&b':') + { + return false; + } + let has_fraction = bytes[19] == b'.'; + if has_fraction { + if bytes.last() != Some(&b'Z') || bytes.len() < 22 { + return false; + } + let fraction = &bytes[20..bytes.len() - 1]; + if fraction.iter().any(|byte| !byte.is_ascii_digit()) { + return false; + } + } else if bytes.len() != 20 || bytes[19] != b'Z' { + return false; + } + if !bytes[..19] + .iter() + .enumerate() + .all(|(index, byte)| matches!(index, 4 | 7 | 10 | 13 | 16) || byte.is_ascii_digit()) + { + return false; + } + let year = four_digits(bytes[0], bytes[1], bytes[2], bytes[3]); + let month = two_digits(bytes[5], bytes[6]); + let day = two_digits(bytes[8], bytes[9]); + let hour = two_digits(bytes[11], bytes[12]); + let minute = two_digits(bytes[14], bytes[15]); + let second = two_digits(bytes[17], bytes[18]); + valid_calendar_date(year, month, day) && hour < 24 && minute < 60 && second < 60 +} + +fn four_digits(first: u8, second: u8, third: u8, fourth: u8) -> u16 { + u16::from(first - b'0') * 1000 + + u16::from(second - b'0') * 100 + + u16::from(third - b'0') * 10 + + u16::from(fourth - b'0') +} + +fn two_digits(high: u8, low: u8) -> u8 { + (high - b'0') * 10 + (low - b'0') +} + +fn valid_calendar_date(year: u16, month: u8, day: u8) -> bool { + if !(1..=12).contains(&month) { + return false; + } + let days_in_month = if month == 2 { + if is_leap_year(year) { 29 } else { 28 } + } else { + 30 + ((month + month / 8) % 2) + }; + (1..=days_in_month).contains(&day) +} + +fn is_leap_year(year: u16) -> bool { + year.is_multiple_of(400) || (year.is_multiple_of(4) && !year.is_multiple_of(100)) +} + +fn valid_target_uri_presentation(target_uri: &str) -> bool { + let bytes = target_uri.as_bytes(); + let mut index = 0_usize; + let mut slash_count = 0_usize; + while index < bytes.len() { + let byte = bytes[index]; + if !is_rfc3986_uri_byte(byte) { + return false; + } + if matches!(byte, b'[' | b']') && slash_count > 2 { + return false; + } + if byte == b'%' { + if index + 2 >= bytes.len() + || !bytes[index + 1].is_ascii_hexdigit() + || !bytes[index + 2].is_ascii_hexdigit() + { + return false; + } + index += 3; + } else { + if byte == b'/' { + slash_count += 1; + } + index += 1; + } + } + true +} + +const fn is_rfc3986_uri_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'-' | b'.' + | b'_' + | b'~' + | b':' + | b'/' + | b'?' + | b'#' + | b'[' + | b']' + | b'@' + | b'!' + | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b',' + | b';' + | b'=' + | b'%' + ) +} + +fn valid_content_type(content_type: &str) -> bool { + if content_type.is_empty() || content_type.len() > MAX_WARC_CONTENT_TYPE_BYTES { + return false; + } + + let (essence, mut parameters) = content_type + .split_once(';') + .map_or((content_type, None), |(essence, parameters)| { + (essence, Some(parameters)) + }); + let essence = trim_ows_end(essence); + let Some((media_type, media_subtype)) = essence.split_once('/') else { + return false; + }; + if !valid_mime_token(media_type) || !valid_mime_token(media_subtype) { + return false; + } + + while let Some(parameter_text) = parameters { + let (parameter, remaining) = split_mime_parameter(parameter_text); + if !valid_mime_parameter(parameter) { + return false; + } + parameters = remaining; + } + true +} + +fn split_mime_parameter(value: &str) -> (&str, Option<&str>) { + let mut quoted = false; + let mut escaped = false; + for (index, byte) in value.bytes().enumerate() { + if quoted { + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + quoted = false; + } + } else if byte == b'"' { + quoted = true; + } else if byte == b';' { + return (&value[..index], Some(&value[index + 1..])); + } + } + (value, None) +} + +fn valid_mime_parameter(parameter: &str) -> bool { + let parameter = trim_ows(parameter); + let Some((attribute, value)) = parameter.split_once('=') else { + return false; + }; + valid_mime_token(attribute) && valid_mime_parameter_value(value) +} + +fn valid_mime_parameter_value(value: &str) -> bool { + if valid_mime_token(value) { + return true; + } + + let bytes = value.as_bytes(); + if bytes.len() < 2 || bytes.first() != Some(&b'"') || bytes.last() != Some(&b'"') { + return false; + } + + let mut escaped = false; + for byte in bytes[1..bytes.len() - 1].iter().copied() { + if escaped { + if !valid_quoted_pair_byte(byte) { + return false; + } + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if !valid_quoted_text_byte(byte) { + return false; + } + } + !escaped +} + +const fn valid_mime_token(value: &str) -> bool { + let bytes = value.as_bytes(); + if bytes.is_empty() { + return false; + } + let mut index = 0; + while index < bytes.len() { + if !is_mime_token_byte(bytes[index]) { + return false; + } + index += 1; + } + true +} + +const fn is_mime_token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) +} + +const fn valid_quoted_text_byte(byte: u8) -> bool { + byte == b'\t' + || byte == b' ' + || byte == b'!' + || (byte >= 0x23 && byte <= 0x5b) + || (byte >= 0x5d && byte <= 0x7e) +} + +const fn valid_quoted_pair_byte(byte: u8) -> bool { + byte == b'\t' || byte == b' ' || (byte >= 0x21 && byte <= 0x7e) +} + +fn trim_ows(value: &str) -> &str { + value.trim_matches([' ', '\t']) +} + +fn trim_ows_end(value: &str) -> &str { + value.trim_end_matches([' ', '\t']) +} + +fn sha256_digest(payload: &[u8]) -> String { + let digest = Sha256::digest(payload); + let mut encoded = String::from("sha256:"); + for byte in digest { + encoded.push_str(&format!("{byte:02x}")); + } + encoded +} diff --git a/crates/originweave-evidence/tests/warc_debug_redaction.rs b/crates/originweave-evidence/tests/warc_debug_redaction.rs new file mode 100644 index 000000000..b51d8d604 --- /dev/null +++ b/crates/originweave-evidence/tests/warc_debug_redaction.rs @@ -0,0 +1,31 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcResourceRecord, +}; + +#[test] +fn warc_record_debug_does_not_disclose_payload_or_provenance_locator() { + let provenance = ProvenanceRecord::new( + "https://example.com/resource", + "private-selector-marker", + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + let record = WarcResourceRecord::new( + "urn:uuid:123e4567-e89b-12d3-a456-426614174000", + "2026-08-21T09:00:00Z", + "https://example.com/resource", + "application/octet-stream", + vec![254, 237, 250, 206], + provenance, + ) + .expect("WARC resource record"); + + let debug = format!("{record:?}"); + assert!(debug.contains("payload_byte_count")); + assert!(!debug.contains("254, 237, 250, 206")); + assert!(!debug.contains("private-selector-marker")); +} diff --git a/crates/originweave-evidence/tests/warc_field_limit_errors.rs b/crates/originweave-evidence/tests/warc_field_limit_errors.rs new file mode 100644 index 000000000..2a07adb81 --- /dev/null +++ b/crates/originweave-evidence/tests/warc_field_limit_errors.rs @@ -0,0 +1,51 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcResourceRecord, + WarcResourceRecordError, +}; + +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-21T00:00:00Z"; +const TARGET_URI: &str = "https://example.com/item"; + +fn provenance() -> ProvenanceRecord { + ProvenanceRecord::new( + TARGET_URI, + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("valid provenance") +} + +#[test] +fn oversized_record_fields_report_the_bounded_limit_error() { + let oversized_record_id = format!("{RECORD_ID}x"); + assert_eq!( + WarcResourceRecord::new( + &oversized_record_id, + DATE, + TARGET_URI, + "text/plain", + Vec::new(), + provenance(), + ), + Err(WarcResourceRecordError::LimitExceeded), + ); + + let oversized_date = "2026-08-21T00:00:00.1234567890Z"; + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + oversized_date, + TARGET_URI, + "text/plain", + Vec::new(), + provenance(), + ), + Err(WarcResourceRecordError::LimitExceeded), + ); +} diff --git a/crates/originweave-evidence/tests/warc_resource_record.rs b/crates/originweave-evidence/tests/warc_resource_record.rs new file mode 100644 index 000000000..d360169f5 --- /dev/null +++ b/crates/originweave-evidence/tests/warc_resource_record.rs @@ -0,0 +1,273 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, MAX_WARC_PAYLOAD_BYTES, ProvenanceRecord, VerificationResult, + WarcResourceRecord, WarcResourceRecordError, +}; + +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-21T00:00:00Z"; + +fn provenance(source_url: &str, verification: VerificationResult) -> ProvenanceRecord { + ProvenanceRecord::new( + source_url, + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + verification, + ) + .expect("provenance") +} + +fn assert_standard_error_contract() {} + +#[test] +fn warc_resource_record_error_implements_standard_error_contract() { + assert_standard_error_contract::(); + + for (error, message) in [ + ( + WarcResourceRecordError::InvalidRecordId, + "invalid WARC record identifier", + ), + (WarcResourceRecordError::InvalidDate, "invalid WARC date"), + ( + WarcResourceRecordError::InvalidContentType, + "invalid WARC content type", + ), + ( + WarcResourceRecordError::LimitExceeded, + "WARC resource record limit exceeded", + ), + ( + WarcResourceRecordError::InvalidTargetUri, + "invalid WARC target URI", + ), + ( + WarcResourceRecordError::TargetUriMismatch, + "WARC target URI does not match provenance", + ), + ( + WarcResourceRecordError::UnverifiedProvenance, + "WARC provenance is not independently verified", + ), + ] { + assert_eq!(error.to_string(), message); + } +} + +#[test] +fn resource_record_binds_verified_provenance_and_emits_deterministic_warc_bytes() { + let record = WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + b"hello".to_vec(), + provenance("https://example.com/item", VerificationResult::Verified), + ) + .expect("resource record"); + + assert_eq!(record.record_id(), RECORD_ID); + assert_eq!(record.warc_date(), DATE); + assert_eq!(record.target_uri(), "https://example.com/item"); + assert_eq!(record.content_type(), "text/plain"); + assert_eq!(record.payload(), b"hello"); + assert_eq!( + record.block_digest(), + "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + assert_eq!(record.provenance().source_url(), record.target_uri()); + assert!(record.provenance().verification_result() == VerificationResult::Verified); + assert_eq!( + record.to_warc_bytes(), + b"WARC/1.1\r\nWARC-Type: resource\r\nWARC-Record-ID: \r\nWARC-Date: 2026-08-21T00:00:00Z\r\nWARC-Target-URI: https://example.com/item\r\nContent-Type: text/plain\r\nWARC-Block-Digest: sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\r\nContent-Length: 5\r\n\r\nhello\r\n\r\n" + ); + for date in [ + "2026-08-21T00:00:00.123Z", + "2024-02-29T00:00:00Z", + "2000-02-29T00:00:00Z", + ] { + WarcResourceRecord::new( + RECORD_ID, + date, + "https://example.com/item", + "text/plain", + Vec::new(), + provenance("https://example.com/item", VerificationResult::Verified), + ) + .expect("valid UTC date"); + } +} + +#[test] +fn resource_record_rejects_invalid_identifiers_dates_content_and_limits() { + let valid = |record_id, date, content_type, payload| { + WarcResourceRecord::new( + record_id, + date, + "https://example.com/item", + content_type, + payload, + provenance("https://example.com/item", VerificationResult::Verified), + ) + }; + + for record_id in [ + "", + "http://example.com/record", + "urn:uuid:00000000-0000-0000-0000-000000000000", + "urn:uuid:123e4567-e89b-12d3-a456-42661417400", + "urn:uuid:123e4567_e89b-12d3-a456-426614174000", + "urn:uuid:123e4567-e89b-12d3-a456-42661417400z", + "xrn:uuid:123e4567-e89b-12d3-a456-426614174000", + ] { + assert_eq!( + valid(record_id, DATE, "text/plain", Vec::new()), + Err(WarcResourceRecordError::InvalidRecordId), + "record_id={record_id:?}" + ); + } + + for date in [ + "", + "2026-08-21 00:00:00Z", + "2026-13-21T00:00:00Z", + "2026-08-32T00:00:00Z", + "2026-02-29T00:00:00Z", + "2024-02-30T00:00:00Z", + "2026-04-31T00:00:00Z", + "1900-02-29T00:00:00Z", + "2026-08-21T24:00:00Z", + "2026-0x-21T00:00:00Z", + "2026-08-21T00:00:00+00:00", + "2026-08-21T00:00:00.123", + "2026-08-21T00:00:00.XZ", + "2026-08-21T00:00:00.Z", + "2026x08-21T00:00:00Z", + "2026-08x21T00:00:00Z", + "2026-08-21T00x00:00Z", + "2026-08-21T00:00x00Z", + "2026-08-21T00:00:00X", + "2026-08-21T00:61:00Z", + "2026-08-21T00:00:61Z", + "2026-08-21T12:34:60Z", + "2026-06-30T23:58:60Z", + ] { + assert_eq!( + valid(RECORD_ID, date, "text/plain", Vec::new()), + Err(WarcResourceRecordError::InvalidDate), + "date={date:?}" + ); + } + + for content_type in ["", "text plain", "text\nplain"] { + assert_eq!( + valid(RECORD_ID, DATE, content_type, Vec::new()), + Err(WarcResourceRecordError::InvalidContentType), + "content_type={content_type:?}" + ); + } + + assert_eq!( + valid( + RECORD_ID, + DATE, + "text/plain", + vec![b'x'; MAX_WARC_PAYLOAD_BYTES + 1], + ), + Err(WarcResourceRecordError::LimitExceeded) + ); + assert_eq!( + valid( + RECORD_ID, + DATE, + &"x".repeat(originweave_evidence::MAX_WARC_CONTENT_TYPE_BYTES + 1), + Vec::new(), + ), + Err(WarcResourceRecordError::LimitExceeded) + ); +} + +#[test] +fn resource_record_accepts_valid_mime_parameters_and_rejects_malformed_media_types() { + let build = |content_type| { + WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/item", + content_type, + Vec::new(), + provenance("https://example.com/item", VerificationResult::Verified), + ) + }; + + for content_type in [ + "text/plain; charset=utf-8", + "application/http; msgtype=response", + "multipart/form-data; boundary=example-boundary", + "text/plain; note=\"a;b\"", + "text/plain; note=\"a;b\"; charset=utf-8", + "text/plain; note=\"a\\;b\"", + "text/plain; note=\"\t !\"", + "text/plain; note=\"\\\t\"", + "text/plain; note=\"\\ \"", + "text/plain; note=\"\\!\"", + ] { + let record = build(content_type).expect("valid WARC MIME media type"); + assert_eq!(record.content_type(), content_type); + } + + for content_type in [ + "plain", + "/plain", + "text/", + "text//plain", + "text/plain;", + "text/plain; charset", + "text/plain; =utf-8", + "text/plain; charset=", + "text/(plain)", + "text/plain; note=(x", + "text/plain; note=\"unterminated", + "text/plain; note=\"a\"b\"", + "text/plain; note=\"a\nb\"", + "text/plain; note=\"\\\nb\"", + "text/plain; note=\"\\\u{7f}\"", + ] { + assert_eq!( + build(content_type), + Err(WarcResourceRecordError::InvalidContentType), + "content_type={content_type:?}" + ); + } +} + +#[test] +fn resource_record_rejects_provenance_drift_and_unverified_sources() { + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/other", + "text/plain", + Vec::new(), + provenance("https://example.com/item", VerificationResult::Verified), + ), + Err(WarcResourceRecordError::TargetUriMismatch) + ); + for verification in [VerificationResult::Unverified, VerificationResult::Rejected] { + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + Vec::new(), + provenance("https://example.com/item", verification), + ), + Err(WarcResourceRecordError::UnverifiedProvenance) + ); + } +} diff --git a/crates/originweave-evidence/tests/warc_target_uri_presentation.rs b/crates/originweave-evidence/tests/warc_target_uri_presentation.rs new file mode 100644 index 000000000..12dfc0c36 --- /dev/null +++ b/crates/originweave-evidence/tests/warc_target_uri_presentation.rs @@ -0,0 +1,183 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcResourceRecord, + WarcResourceRecordError, +}; + +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-21T00:00:00Z"; + +fn provenance(source_url: &str) -> ProvenanceRecord { + ProvenanceRecord::new( + source_url, + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("provenance") +} + +#[test] +fn warc_target_uri_rejects_invisible_formatting_characters_before_serialization() { + let source_provenance = provenance("https://example.com/valid"); + for formatting_character in [ + '\u{00ad}', '\u{061c}', '\u{200b}', '\u{200e}', '\u{202e}', '\u{2066}', '\u{2060}', + '\u{feff}', + ] { + let target_uri = format!("https://example.com/item{formatting_character}shadow"); + + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + &target_uri, + "text/plain", + Vec::new(), + source_provenance.clone(), + ), + Err(WarcResourceRecordError::InvalidTargetUri), + "target_uri={target_uri:?}" + ); + } +} + +#[test] +fn warc_target_uri_rejects_control_and_whitespace_before_provenance_comparison() { + let source_provenance = provenance("https://example.com/item"); + for target_uri in [ + "https://example.com/item\rshadow", + "https://example.com/item shadow", + ] { + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + target_uri, + "text/plain", + Vec::new(), + source_provenance.clone(), + ), + Err(WarcResourceRecordError::InvalidTargetUri), + "target_uri={target_uri:?}" + ); + } +} + +#[test] +fn warc_target_uri_rejects_raw_unicode_because_warc_uses_rfc3986_uri_syntax() { + let target_uri = "https://example.com/상품/상세"; + + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + target_uri, + "text/plain", + Vec::new(), + provenance("https://example.com/valid"), + ), + Err(WarcResourceRecordError::InvalidTargetUri), + ); +} + +#[test] +fn warc_target_uri_rejects_ascii_characters_outside_rfc3986_uri_syntax() { + let source_provenance = provenance("https://example.com/valid"); + for invalid_character in ['<', '>', '"', '{', '}', '|', '^', '`'] { + let target_uri = format!("https://example.com/item{invalid_character}shadow"); + + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + &target_uri, + "text/plain", + Vec::new(), + source_provenance.clone(), + ), + Err(WarcResourceRecordError::InvalidTargetUri), + "target_uri={target_uri:?}" + ); + } +} + +#[test] +fn warc_target_uri_rejects_general_delimiters_in_path_segments() { + let source_provenance = provenance("https://example.com/valid"); + for target_uri in [ + "https://example.com/[segment]", + "https://example.com/item]shadow", + ] { + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + target_uri, + "text/plain", + Vec::new(), + source_provenance.clone(), + ), + Err(WarcResourceRecordError::InvalidTargetUri), + "target_uri={target_uri:?}" + ); + } +} + +#[test] +fn warc_target_uri_preserves_brackets_in_ipv6_authority() { + let target_uri = "https://[::1]:8443/path"; + let record = WarcResourceRecord::new( + RECORD_ID, + DATE, + target_uri, + "text/plain", + Vec::new(), + provenance(target_uri), + ) + .expect("RFC 3986 bracketed IPv6 authority"); + + assert_eq!(record.target_uri(), target_uri); +} + +#[test] +fn warc_target_uri_rejects_malformed_percent_encoding() { + let source_provenance = provenance("https://example.com/valid"); + for target_uri in [ + "https://example.com/%", + "https://example.com/%2", + "https://example.com/%GG", + "https://example.com/%0G", + ] { + assert_eq!( + WarcResourceRecord::new( + RECORD_ID, + DATE, + target_uri, + "text/plain", + Vec::new(), + source_provenance.clone(), + ), + Err(WarcResourceRecordError::InvalidTargetUri), + "target_uri={target_uri:?}" + ); + } +} + +#[test] +fn warc_target_uri_accepts_percent_encoded_utf8_path_octets() { + let target_uri = "https://example.com/%EC%83%81%ED%92%88/%EC%83%81%EC%84%B8"; + let record = WarcResourceRecord::new( + RECORD_ID, + DATE, + target_uri, + "text/plain", + Vec::new(), + provenance(target_uri), + ) + .expect("RFC 3986 percent-encoded target URI"); + + assert_eq!(record.target_uri(), target_uri); +} diff --git a/crates/originweave-evidence/tests/warc_truncation_state.rs b/crates/originweave-evidence/tests/warc_truncation_state.rs new file mode 100644 index 000000000..e09d01ae0 --- /dev/null +++ b/crates/originweave-evidence/tests/warc_truncation_state.rs @@ -0,0 +1,65 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcPayloadCompleteness, + WarcResourceRecord, WarcTruncationReason, +}; + +const RECORD_ID: &str = "urn:uuid:01234567-89ab-cdef-0123-456789abcdef"; +const DATE: &str = "2026-08-22T00:00:00Z"; +const SOURCE_URL: &str = "https://example.com/resource"; +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +fn verified_provenance() -> ProvenanceRecord { + ProvenanceRecord::new( + SOURCE_URL, + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("reviewed provenance fixture must be valid") +} + +#[test] +fn warc_resource_records_preserve_explicit_completeness_and_truncation_reason() { + let complete = WarcResourceRecord::new_with_completeness( + RECORD_ID, + DATE, + SOURCE_URL, + "text/plain", + b"hello".to_vec(), + verified_provenance(), + WarcPayloadCompleteness::Complete, + ) + .expect("complete capture must be admitted"); + assert_eq!(complete.completeness(), WarcPayloadCompleteness::Complete); + let complete_bytes = String::from_utf8(complete.to_warc_bytes()) + .expect("text fixture must serialize as UTF-8 WARC bytes"); + assert!(!complete_bytes.contains("WARC-Truncated:")); + + for (reason, token) in [ + (WarcTruncationReason::Length, "length"), + (WarcTruncationReason::Time, "time"), + (WarcTruncationReason::Disconnect, "disconnect"), + (WarcTruncationReason::Unspecified, "unspecified"), + ] { + let completeness = WarcPayloadCompleteness::Truncated(reason); + let truncated = WarcResourceRecord::new_with_completeness( + RECORD_ID, + DATE, + SOURCE_URL, + "text/plain", + b"hello".to_vec(), + verified_provenance(), + completeness, + ) + .expect("typed truncated capture must be admitted"); + + assert_eq!(truncated.completeness(), completeness); + let truncated_bytes = String::from_utf8(truncated.to_warc_bytes()) + .expect("text fixture must serialize as UTF-8 WARC bytes"); + assert!(truncated_bytes.contains(&format!("WARC-Truncated: {token}\r\n"))); + assert!(truncated_bytes.contains("Content-Length: 5\r\n")); + } +} diff --git a/docs/adr/0106-provenance-evidence-model.md b/docs/adr/0106-provenance-evidence-model.md index 09cb0d7ca..d2494ad29 100644 --- a/docs/adr/0106-provenance-evidence-model.md +++ b/docs/adr/0106-provenance-evidence-model.md @@ -9,6 +9,14 @@ OriginWeave must not merely act; it must let a user, operator, auditor, or downstream system establish what was observed, authorized, executed, and verified. Browser logs alone are not sufficient because they collapse observation, policy, action, network identity, approvals, and post-conditions. At the same time, evidence can itself contain sensitive or attacker-controlled content. The product needs a durable model compatible with web-archive and provenance concepts without claiming that every conceptual record is already persisted. +## Active implementation boundary + +The extraction lane currently implements one bounded, verified in-memory WARC 1.1 +`resource` record contract over already-authorized bytes. It binds the WARC target +URI to independently verified provenance, computes a SHA-256 block digest, and +emits deterministic record bytes. This is active-PR evidence rather than a claim +of durable storage, tenant retention, request/response capture, or PROV export. + ## Decision drivers - `Browse. Act. Prove.` requires evidence as a first-class product output.