diff --git a/CHANGELOG.md b/CHANGELOG.md index 405cf7659..d0fb49aeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ All notable changes to OriginWeave are documented in this file. The format follo - Authoritative product documentation graph spanning PRD, TRD, ADR lifecycle/index, product-wide UML, conceptual ERD, requirement/decision traceability, threat modeling, product-wide test strategy, operability, API/protocol, release/rollback, and current primary-source standards doctoring, with machine-checkable repository contracts that keep conversation-derived future work distinct from protected-main implementation claims. - Purpose-bound data-governance and privacy baseline that rejects both blanket masking and ambient raw-value propagation, defines field-scoped just-in-time disclosure, opaque-handle/trusted-broker boundaries, model/provider/region policy, retention/deletion/residency/break-glass controls, truthful CSAP/SOC 2 readiness language, and machine-checkable documentation contracts without inventing an OriginWeave-owned production database. - Proposed product-wide target-architecture ADRs for the Rust control plane, isolated execution modes, typed actions, semantic observation/stale-node authority, prompt-injection and secret separation, resource-governor priority, provenance evidence, browser/protocol adapters, crawler policy, and hourly automation operational closure; these remain Proposed rather than shipped claims until protected review and merge. +- Active PR #217 adds a bounded in-memory `WarcProvBundle` for deterministic W3C PROV-O JSON-LD projection and offline verification of exact WARC records; durable persistence, retention, and transport-specific export remain planned rather than shipped. +- Connected the active WARC/PROV bundle evidence to the primary standards doctoring record, preserving the in-memory-only and non-shipped durable-adapter boundary. +- Made WARC/PROV offline verification use the complete deterministic WARC digest as the authoritative block-digest binding, preserving fail-closed exact-record checks while keeping the public verifier surface minimal. ### Changed diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index b3cca4154..e2e4c40ff 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -10,6 +10,7 @@ mod extraction_schema; mod sensitive_access; mod sensitive_handle_lifecycle; +mod warc_prov_bundle; mod warc_resource_record; pub use extraction_schema::{ @@ -25,6 +26,10 @@ pub use sensitive_access::{ pub use sensitive_handle_lifecycle::{ SensitiveHandleLifecycleEvidence, SensitiveHandleLifecycleEvidenceInput, }; +pub use warc_prov_bundle::{ + MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES, WarcProvBundle, WarcProvBundleError, + WarcProvBundleVerificationError, +}; pub use warc_resource_record::{ MAX_WARC_CONTENT_TYPE_BYTES, MAX_WARC_DATE_BYTES, MAX_WARC_PAYLOAD_BYTES, MAX_WARC_RECORD_ID_BYTES, MAX_WARC_TARGET_URI_BYTES, WarcPayloadCompleteness, diff --git a/crates/originweave-evidence/src/warc_prov_bundle.rs b/crates/originweave-evidence/src/warc_prov_bundle.rs new file mode 100644 index 000000000..11788e771 --- /dev/null +++ b/crates/originweave-evidence/src/warc_prov_bundle.rs @@ -0,0 +1,273 @@ +use std::fmt; + +use sha2::{Digest, Sha256}; + +use crate::{ProvenanceRecord, WarcPayloadCompleteness, WarcResourceRecord}; + +const ORIGINWEAVE_COMMIT_URL_PREFIX: &str = + "https://github.com/ContextualWisdomLab/OriginWeave/commit/"; +const WARC_RECORD_DIGEST_IRI: &str = + "tag:contextualwisdomlab.github.io,2026:OriginWeave/warcRecordDigest"; +const WARC_PAYLOAD_COMPLETENESS_IRI: &str = + "tag:contextualwisdomlab.github.io,2026:OriginWeave/warcPayloadCompleteness"; +const WARC_TRUNCATION_REASON_IRI: &str = + "tag:contextualwisdomlab.github.io,2026:OriginWeave/warcTruncationReason"; + +/// Exact byte length accepted for a canonical Git SHA-1 software revision. +pub const MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES: usize = 40; + +/// A validation failure while constructing a deterministic WARC provenance bundle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WarcProvBundleError { + /// The software revision was not one canonical lower-case 40-byte Git SHA-1. + InvalidSoftwareCommitSha, + /// A bounded provenance field exceeded its allowed size. + LimitExceeded, +} + +impl fmt::Display for WarcProvBundleError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidSoftwareCommitSha => "invalid OriginWeave software commit SHA", + Self::LimitExceeded => "WARC PROV bundle limit exceeded", + }) + } +} + +impl std::error::Error for WarcProvBundleError {} + +/// A deterministic offline verification failure between a PROV bundle and a WARC record. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WarcProvBundleVerificationError { + /// The WARC record identifier differs from the identifier bound into the PROV bundle. + RecordIdentityMismatch, + /// The independently verified source provenance differs from the provenance bound into the bundle. + SourceEvidenceMismatch, + /// The WARC capture timestamp differs from the timestamp bound into the PROV bundle. + CaptureTimeMismatch, + /// The WARC complete-versus-truncated state differs from the state bound into the bundle. + PayloadCompletenessMismatch, + /// The digest of the deterministic WARC serialization differs from the bundle binding. + WarcRecordDigestMismatch, +} + +impl fmt::Display for WarcProvBundleVerificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::RecordIdentityMismatch => "WARC PROV record identity does not match", + Self::SourceEvidenceMismatch => "WARC PROV source evidence does not match", + Self::CaptureTimeMismatch => "WARC PROV capture time does not match", + Self::PayloadCompletenessMismatch => "WARC PROV payload completeness does not match", + Self::WarcRecordDigestMismatch => "WARC PROV serialized record digest does not match", + }) + } +} + +impl std::error::Error for WarcProvBundleVerificationError {} + +/// A deterministic PROV-O JSON-LD projection over one validated WARC resource record. +/// +/// The bundle contains identifiers, exact validated source provenance, record hashes, source +/// location, capture time, explicit WARC payload completeness, and the exact OriginWeave software +/// revision. It deliberately does not retain or emit the WARC payload. +#[derive(Clone, PartialEq, Eq)] +pub struct WarcProvBundle { + record_entity_id: String, + source_entity_id: String, + capture_activity_id: String, + software_agent_id: String, + software_commit_sha: String, + source_provenance: ProvenanceRecord, + warc_date: String, + block_digest: String, + warc_record_digest: String, + payload_completeness: WarcPayloadCompleteness, +} + +impl fmt::Debug for WarcProvBundle { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WarcProvBundle") + .field("record_entity_id", &self.record_entity_id) + .field("source_entity_id", &self.source_entity_id) + .field("capture_activity_id", &self.capture_activity_id) + .field("software_agent_id", &self.software_agent_id) + .field("payload_completeness", &self.payload_completeness) + .finish_non_exhaustive() + } +} + +impl WarcProvBundle { + /// Construct a provenance bundle from one already-validated WARC resource record. + /// + /// `software_commit_sha` is an immutable canonical Git SHA-1 identifier. This constructor + /// does not contact GitHub and does not treat the identifier as authentication or authority. + pub fn new( + record: &WarcResourceRecord, + software_commit_sha: &str, + ) -> Result { + if software_commit_sha.len() > MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES { + return Err(WarcProvBundleError::LimitExceeded); + } + if !valid_software_commit_sha(software_commit_sha) { + return Err(WarcProvBundleError::InvalidSoftwareCommitSha); + } + + let record_entity_id = record.record_id().to_owned(); + let source_entity_id = format!("{}#source", record.record_id()); + let capture_activity_id = format!("{}#capture", record.record_id()); + let software_agent_id = format!("{ORIGINWEAVE_COMMIT_URL_PREFIX}{software_commit_sha}"); + let warc_record_digest = sha256_digest(&record.to_warc_bytes()); + + Ok(Self { + record_entity_id, + source_entity_id, + capture_activity_id, + software_agent_id, + software_commit_sha: software_commit_sha.to_owned(), + source_provenance: record.provenance().clone(), + warc_date: record.warc_date().to_owned(), + block_digest: record.block_digest().to_owned(), + warc_record_digest, + payload_completeness: record.completeness(), + }) + } + + /// Return the PROV entity identifier for the WARC record. + #[must_use] + pub fn record_entity_id(&self) -> &str { + &self.record_entity_id + } + + /// Return the PROV entity identifier for the independently verified source. + #[must_use] + pub fn source_entity_id(&self) -> &str { + &self.source_entity_id + } + + /// Return the PROV activity identifier for this capture. + #[must_use] + pub fn capture_activity_id(&self) -> &str { + &self.capture_activity_id + } + + /// Return the immutable OriginWeave commit URL used as the PROV software-agent identifier. + #[must_use] + pub fn software_agent_id(&self) -> &str { + &self.software_agent_id + } + + /// Return the canonical lower-case Git SHA-1 of the OriginWeave revision. + #[must_use] + pub fn software_commit_sha(&self) -> &str { + &self.software_commit_sha + } + + /// Verify offline that one validated WARC record is exactly the record bound by this bundle. + /// + /// Verification includes the complete validated [`ProvenanceRecord`] rather than only the + /// source URL and digest, so locator or evidence-channel drift cannot collapse into a match. + /// It is deterministic and performs no network, DNS, browser, model, persistence, or authority + /// operation. A matching digest proves byte identity only; it does not authenticate the actor + /// that produced either value or establish factual correctness. + pub fn verify_record( + &self, + record: &WarcResourceRecord, + ) -> Result<(), WarcProvBundleVerificationError> { + if self.record_entity_id != record.record_id() { + return Err(WarcProvBundleVerificationError::RecordIdentityMismatch); + } + if self.source_provenance != *record.provenance() { + return Err(WarcProvBundleVerificationError::SourceEvidenceMismatch); + } + if self.warc_date != record.warc_date() { + return Err(WarcProvBundleVerificationError::CaptureTimeMismatch); + } + // `warc_record_digest` covers the complete deterministic serialization, including the + // WARC-Block-Digest header, so a block-digest drift is reported by that binding below. + if self.payload_completeness != record.completeness() { + return Err(WarcProvBundleVerificationError::PayloadCompletenessMismatch); + } + if self.warc_record_digest != sha256_digest(&record.to_warc_bytes()) { + return Err(WarcProvBundleVerificationError::WarcRecordDigestMismatch); + } + Ok(()) + } + + /// Serialize the bundle as deterministic compact W3C PROV-O JSON-LD. + /// + /// All interpolated values originate from the validated WARC record or the canonical + /// lower-case software commit identifier, so no raw payload bytes enter this document. The + /// payload block digest binds the retained resource bytes while `warcRecordDigest` binds the + /// complete deterministic WARC serialization, including its headers. WARC payload completeness + /// is retained as an OriginWeave-owned absolute-IRI attribute; truncated records also retain + /// the exact WARC truncation token. The JSON-LD projection exposes source URL and digest while + /// offline verification additionally preserves the exact validated source locator and channel. + #[must_use] + pub fn to_json_ld(&self) -> String { + let completeness_attributes = + warc_payload_completeness_attributes(self.payload_completeness); + format!( + "{{\"@context\":{{\"prov\":\"http://www.w3.org/ns/prov#\",\"xsd\":\"http://www.w3.org/2001/XMLSchema#\"}},\"@graph\":[{{\"@id\":\"{}\",\"@type\":\"prov:Entity\",\"prov:atLocation\":{{\"@id\":\"{}\"}},\"prov:value\":\"{}\"}},{{\"@id\":\"{}\",\"@type\":\"prov:Activity\",\"prov:startedAtTime\":{{\"@value\":\"{}\",\"@type\":\"xsd:dateTime\"}},\"prov:used\":{{\"@id\":\"{}\"}},\"prov:wasAssociatedWith\":{{\"@id\":\"{}\"}}}},{{\"@id\":\"{}\",\"@type\":\"prov:SoftwareAgent\"}},{{\"@id\":\"{}\",\"@type\":\"prov:Entity\",\"prov:value\":\"{}\",\"{WARC_RECORD_DIGEST_IRI}\":\"{}\",{},\"prov:wasDerivedFrom\":{{\"@id\":\"{}\"}},\"prov:wasGeneratedBy\":{{\"@id\":\"{}\"}}}}]}}", + self.source_entity_id, + self.source_provenance.source_url(), + self.source_provenance.source_hash(), + self.capture_activity_id, + self.warc_date, + self.source_entity_id, + self.software_agent_id, + self.software_agent_id, + self.record_entity_id, + self.block_digest, + self.warc_record_digest, + completeness_attributes, + self.source_entity_id, + self.capture_activity_id, + ) + } +} + +fn warc_payload_completeness_attributes(completeness: WarcPayloadCompleteness) -> String { + match completeness { + WarcPayloadCompleteness::Complete => { + format!("\"{WARC_PAYLOAD_COMPLETENESS_IRI}\":\"complete\"") + } + WarcPayloadCompleteness::Truncated(reason) => format!( + "\"{WARC_PAYLOAD_COMPLETENESS_IRI}\":\"truncated\",\"{WARC_TRUNCATION_REASON_IRI}\":\"{}\"", + reason.warc_token() + ), + } +} + +fn sha256_digest(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut encoded = String::from("sha256:"); + for byte in digest { + encoded.push_str(&format!("{byte:02x}")); + } + encoded +} + +fn valid_software_commit_sha(software_commit_sha: &str) -> bool { + software_commit_sha.len() == MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES + && software_commit_sha + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + && software_commit_sha.bytes().any(|byte| byte != b'0') +} + +#[cfg(test)] +mod tests { + use crate::WarcTruncationReason; + + #[test] + fn truncation_tokens_use_the_shared_warc_mapping() { + assert_eq!(WarcTruncationReason::Length.warc_token(), "length"); + assert_eq!(WarcTruncationReason::Time.warc_token(), "time"); + assert_eq!(WarcTruncationReason::Disconnect.warc_token(), "disconnect"); + assert_eq!( + WarcTruncationReason::Unspecified.warc_token(), + "unspecified" + ); + } +} diff --git a/crates/originweave-evidence/src/warc_resource_record.rs b/crates/originweave-evidence/src/warc_resource_record.rs index 150fafe87..4fdc7a475 100644 --- a/crates/originweave-evidence/src/warc_resource_record.rs +++ b/crates/originweave-evidence/src/warc_resource_record.rs @@ -29,6 +29,7 @@ pub enum WarcTruncationReason { } impl WarcTruncationReason { + /// Return the canonical WARC 1.1 truncation token for this reason. pub(crate) const fn warc_token(self) -> &'static str { match self { Self::Length => "length", @@ -522,3 +523,40 @@ fn sha256_digest(payload: &[u8]) -> String { } encoded } + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use super::*; + use crate::{WarcProvBundle, WarcProvBundleVerificationError}; + + #[test] + fn provenance_bundle_rejects_a_tampered_block_digest() { + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", + crate::EvidenceSourceKind::NetworkResponse, + crate::VerificationResult::Verified, + ) + .expect("verified provenance"); + let mut record = WarcResourceRecord::new( + "urn:uuid:123e4567-e89b-12d3-a456-426614174000", + "2026-08-22T12:00:00Z", + "https://example.com/item", + "text/plain", + b"hello".to_vec(), + provenance, + ) + .expect("WARC resource record"); + let bundle = WarcProvBundle::new(&record, "0123456789abcdef0123456789abcdef01234567") + .expect("PROV bundle"); + + record.block_digest = "sha256:tampered".to_owned(); + + assert_eq!( + bundle.verify_record(&record), + Err(WarcProvBundleVerificationError::WarcRecordDigestMismatch) + ); + } +} diff --git a/crates/originweave-evidence/tests/warc_prov_jsonld.rs b/crates/originweave-evidence/tests/warc_prov_jsonld.rs new file mode 100644 index 000000000..bec8b803c --- /dev/null +++ b/crates/originweave-evidence/tests/warc_prov_jsonld.rs @@ -0,0 +1,218 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES, ProvenanceRecord, VerificationResult, + WarcPayloadCompleteness, WarcProvBundle, WarcProvBundleError, WarcResourceRecord, + WarcTruncationReason, +}; + +const SOURCE_HASH: &str = "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-22T12:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; +const WARC_RECORD_DIGEST_IRI: &str = + "tag:contextualwisdomlab.github.io,2026:OriginWeave/warcRecordDigest"; +const PAYLOAD_COMPLETENESS_IRI: &str = + "tag:contextualwisdomlab.github.io,2026:OriginWeave/warcPayloadCompleteness"; +const TRUNCATION_REASON_IRI: &str = + "tag:contextualwisdomlab.github.io,2026:OriginWeave/warcTruncationReason"; + +fn resource_record() -> WarcResourceRecord { + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + b"hello".to_vec(), + provenance, + ) + .expect("WARC resource record") +} + +fn resource_record_with_completeness(completeness: WarcPayloadCompleteness) -> WarcResourceRecord { + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new_with_completeness( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + b"hello".to_vec(), + provenance, + completeness, + ) + .expect("WARC resource record") +} + +fn assert_standard_error_contract() {} + +#[test] +fn warc_prov_bundle_exposes_stable_capture_identities_and_standard_errors() { + assert_standard_error_contract::(); + assert_eq!( + WarcProvBundleError::InvalidSoftwareCommitSha.to_string(), + "invalid OriginWeave software commit SHA" + ); + assert_eq!( + WarcProvBundleError::LimitExceeded.to_string(), + "WARC PROV bundle limit exceeded" + ); + + let bundle = WarcProvBundle::new(&resource_record(), SOFTWARE_COMMIT_SHA) + .expect("PROV bundle over a validated WARC record"); + assert_eq!(bundle.record_entity_id(), RECORD_ID); + assert_eq!( + bundle.source_entity_id(), + "urn:uuid:123e4567-e89b-12d3-a456-426614174000#source" + ); + assert_eq!( + bundle.capture_activity_id(), + "urn:uuid:123e4567-e89b-12d3-a456-426614174000#capture" + ); + assert_eq!( + bundle.software_agent_id(), + "https://github.com/ContextualWisdomLab/OriginWeave/commit/0123456789abcdef0123456789abcdef01234567" + ); + assert_eq!(bundle.software_commit_sha(), SOFTWARE_COMMIT_SHA); + + let debug = format!("{bundle:?}"); + assert!(debug.contains(RECORD_ID)); + assert!(!debug.contains("https://example.com/item")); + assert!(!debug.contains(SOURCE_HASH)); + assert!(!debug.contains("hello")); +} + +#[test] +fn warc_prov_bundle_emits_deterministic_prov_o_json_ld_without_raw_payload() { + let bundle = WarcProvBundle::new(&resource_record(), SOFTWARE_COMMIT_SHA) + .expect("PROV bundle over a validated WARC record"); + let json_ld = bundle.to_json_ld(); + + assert_eq!( + json_ld, + concat!( + "{\"@context\":{\"prov\":\"http://www.w3.org/ns/prov#\",\"xsd\":\"http://www.w3.org/2001/XMLSchema#\"},\"@graph\":[", + "{\"@id\":\"urn:uuid:123e4567-e89b-12d3-a456-426614174000#source\",\"@type\":\"prov:Entity\",\"prov:atLocation\":{\"@id\":\"https://example.com/item\"},\"prov:value\":\"sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\"},", + "{\"@id\":\"urn:uuid:123e4567-e89b-12d3-a456-426614174000#capture\",\"@type\":\"prov:Activity\",\"prov:startedAtTime\":{\"@value\":\"2026-08-22T12:00:00Z\",\"@type\":\"xsd:dateTime\"},\"prov:used\":{\"@id\":\"urn:uuid:123e4567-e89b-12d3-a456-426614174000#source\"},\"prov:wasAssociatedWith\":{\"@id\":\"https://github.com/ContextualWisdomLab/OriginWeave/commit/0123456789abcdef0123456789abcdef01234567\"}},", + "{\"@id\":\"https://github.com/ContextualWisdomLab/OriginWeave/commit/0123456789abcdef0123456789abcdef01234567\",\"@type\":\"prov:SoftwareAgent\"},", + "{\"@id\":\"urn:uuid:123e4567-e89b-12d3-a456-426614174000\",\"@type\":\"prov:Entity\",\"prov:value\":\"sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\",\"tag:contextualwisdomlab.github.io,2026:OriginWeave/warcRecordDigest\":\"sha256:b6ea360a1ec548527ff5ed9c03966b05c8afd5c2b882bee259e362effb0fe0a8\",\"tag:contextualwisdomlab.github.io,2026:OriginWeave/warcPayloadCompleteness\":\"complete\",\"prov:wasDerivedFrom\":{\"@id\":\"urn:uuid:123e4567-e89b-12d3-a456-426614174000#source\"},\"prov:wasGeneratedBy\":{\"@id\":\"urn:uuid:123e4567-e89b-12d3-a456-426614174000#capture\"}}", + "]}" + ) + ); + assert!(!json_ld.contains("hello")); +} + +#[test] +fn warc_prov_bundle_rejects_noncanonical_or_oversized_software_revisions() { + let record = resource_record(); + for software_commit_sha in [ + "", + "0000000000000000000000000000000000000000", + "0123456789abcdef0123456789abcdef0123456", + "0123456789abcdef0123456789abcdef0123456G", + "0123456789ABCDEF0123456789ABCDEF01234567", + "0123456789abcdef0123456789abcdef0123456 ", + ] { + assert_eq!( + WarcProvBundle::new(&record, software_commit_sha), + Err(WarcProvBundleError::InvalidSoftwareCommitSha), + "software_commit_sha={software_commit_sha:?}" + ); + } + + assert_eq!( + MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES, + SOFTWARE_COMMIT_SHA.len() + ); + assert_eq!( + WarcProvBundle::new(&record, &"a".repeat(MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES + 1)), + Err(WarcProvBundleError::LimitExceeded) + ); +} + +#[test] +fn warc_prov_bundle_preserves_warc_payload_completeness_for_replay() { + let complete = WarcProvBundle::new( + &resource_record_with_completeness(WarcPayloadCompleteness::Complete), + SOFTWARE_COMMIT_SHA, + ) + .expect("complete PROV bundle"); + let complete_json = complete.to_json_ld(); + assert!(complete_json.contains(&format!("\"{PAYLOAD_COMPLETENESS_IRI}\":\"complete\""))); + assert!(!complete_json.contains(TRUNCATION_REASON_IRI)); + + for (reason, token) in [ + (WarcTruncationReason::Length, "length"), + (WarcTruncationReason::Time, "time"), + (WarcTruncationReason::Disconnect, "disconnect"), + (WarcTruncationReason::Unspecified, "unspecified"), + ] { + let truncated = WarcProvBundle::new( + &resource_record_with_completeness(WarcPayloadCompleteness::Truncated(reason)), + SOFTWARE_COMMIT_SHA, + ) + .expect("truncated PROV bundle"); + let truncated_json = truncated.to_json_ld(); + assert!(truncated_json.contains(&format!("\"{PAYLOAD_COMPLETENESS_IRI}\":\"truncated\""))); + assert!(truncated_json.contains(&format!("\"{TRUNCATION_REASON_IRI}\":\"{token}\""))); + } +} + +#[test] +fn warc_prov_bundle_distinguishes_distinct_warc_serializations() { + let record_with_content_type = |content_type: &str| { + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/item", + content_type, + b"hello".to_vec(), + provenance, + ) + .expect("WARC resource record") + }; + + let text_record = record_with_content_type("text/plain"); + let binary_record = record_with_content_type("application/octet-stream"); + assert_ne!(text_record.to_warc_bytes(), binary_record.to_warc_bytes()); + + let text_prov = WarcProvBundle::new(&text_record, SOFTWARE_COMMIT_SHA) + .expect("text PROV bundle") + .to_json_ld(); + let binary_prov = WarcProvBundle::new(&binary_record, SOFTWARE_COMMIT_SHA) + .expect("binary PROV bundle") + .to_json_ld(); + + assert_ne!( + text_prov, binary_prov, + "provenance must distinguish WARC records whose serialized headers differ" + ); + assert!(text_prov.contains(&format!( + "\"{WARC_RECORD_DIGEST_IRI}\":\"sha256:b6ea360a1ec548527ff5ed9c03966b05c8afd5c2b882bee259e362effb0fe0a8\"" + ))); + assert!(binary_prov.contains(&format!( + "\"{WARC_RECORD_DIGEST_IRI}\":\"sha256:9c59979535d4a1b3589c0fe2d17837c4ddb0e4cf911854d9aae362903ff83db9\"" + ))); +} diff --git a/crates/originweave-evidence/tests/warc_prov_offline_verification.rs b/crates/originweave-evidence/tests/warc_prov_offline_verification.rs new file mode 100644 index 000000000..7222cd956 --- /dev/null +++ b/crates/originweave-evidence/tests/warc_prov_offline_verification.rs @@ -0,0 +1,203 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcPayloadCompleteness, + WarcProvBundle, WarcProvBundleVerificationError, WarcResourceRecord, WarcTruncationReason, +}; + +const SOURCE_HASH: &str = "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"; +const OTHER_SOURCE_HASH: &str = + "sha256:486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65260e9cb8a7"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const OTHER_RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; +const DATE: &str = "2026-08-22T12:00:00Z"; +const OTHER_DATE: &str = "2026-08-22T12:00:01Z"; +const SOURCE_URL: &str = "https://example.com/item"; +const OTHER_SOURCE_URL: &str = "https://example.com/other"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn record( + record_id: &str, + date: &str, + source_url: &str, + source_hash: &str, + content_type: &str, + payload: &[u8], + completeness: WarcPayloadCompleteness, +) -> WarcResourceRecord { + let provenance = ProvenanceRecord::new( + source_url, + "body", + source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new_with_completeness( + record_id, + date, + source_url, + content_type, + payload.to_vec(), + provenance, + completeness, + ) + .expect("WARC resource record") +} + +fn baseline_record_with_source_evidence( + source_locator: &str, + source_kind: EvidenceSourceKind, +) -> WarcResourceRecord { + let provenance = ProvenanceRecord::new( + SOURCE_URL, + source_locator, + SOURCE_HASH, + source_kind, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new_with_completeness( + RECORD_ID, + DATE, + SOURCE_URL, + "text/plain", + b"hello".to_vec(), + provenance, + WarcPayloadCompleteness::Complete, + ) + .expect("WARC resource record") +} + +fn baseline_record() -> WarcResourceRecord { + record( + RECORD_ID, + DATE, + SOURCE_URL, + SOURCE_HASH, + "text/plain", + b"hello", + WarcPayloadCompleteness::Complete, + ) +} + +fn assert_standard_error_contract() {} + +#[test] +fn warc_prov_bundle_offline_verification_accepts_only_the_exact_bound_record() { + assert_standard_error_contract::(); + assert_eq!( + WarcProvBundleVerificationError::RecordIdentityMismatch.to_string(), + "WARC PROV record identity does not match" + ); + assert_eq!( + WarcProvBundleVerificationError::SourceEvidenceMismatch.to_string(), + "WARC PROV source evidence does not match" + ); + assert_eq!( + WarcProvBundleVerificationError::CaptureTimeMismatch.to_string(), + "WARC PROV capture time does not match" + ); + assert_eq!( + WarcProvBundleVerificationError::PayloadCompletenessMismatch.to_string(), + "WARC PROV payload completeness does not match" + ); + assert_eq!( + WarcProvBundleVerificationError::WarcRecordDigestMismatch.to_string(), + "WARC PROV serialized record digest does not match" + ); + + let exact = baseline_record(); + let bundle = WarcProvBundle::new(&exact, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + assert_eq!(bundle.verify_record(&exact), Ok(())); + + let mismatches = [ + ( + record( + OTHER_RECORD_ID, + DATE, + SOURCE_URL, + SOURCE_HASH, + "text/plain", + b"hello", + WarcPayloadCompleteness::Complete, + ), + WarcProvBundleVerificationError::RecordIdentityMismatch, + ), + ( + record( + RECORD_ID, + DATE, + OTHER_SOURCE_URL, + SOURCE_HASH, + "text/plain", + b"hello", + WarcPayloadCompleteness::Complete, + ), + WarcProvBundleVerificationError::SourceEvidenceMismatch, + ), + ( + record( + RECORD_ID, + DATE, + SOURCE_URL, + OTHER_SOURCE_HASH, + "text/plain", + b"world", + WarcPayloadCompleteness::Complete, + ), + WarcProvBundleVerificationError::SourceEvidenceMismatch, + ), + ( + baseline_record_with_source_evidence( + "different-body", + EvidenceSourceKind::NetworkResponse, + ), + WarcProvBundleVerificationError::SourceEvidenceMismatch, + ), + ( + baseline_record_with_source_evidence("body", EvidenceSourceKind::StructuredData), + WarcProvBundleVerificationError::SourceEvidenceMismatch, + ), + ( + record( + RECORD_ID, + OTHER_DATE, + SOURCE_URL, + SOURCE_HASH, + "text/plain", + b"hello", + WarcPayloadCompleteness::Complete, + ), + WarcProvBundleVerificationError::CaptureTimeMismatch, + ), + ( + record( + RECORD_ID, + DATE, + SOURCE_URL, + SOURCE_HASH, + "text/plain", + b"hello", + WarcPayloadCompleteness::Truncated(WarcTruncationReason::Length), + ), + WarcProvBundleVerificationError::PayloadCompletenessMismatch, + ), + ( + record( + RECORD_ID, + DATE, + SOURCE_URL, + SOURCE_HASH, + "application/octet-stream", + b"hello", + WarcPayloadCompleteness::Complete, + ), + WarcProvBundleVerificationError::WarcRecordDigestMismatch, + ), + ]; + + for (candidate, expected) in mismatches { + assert_eq!(bundle.verify_record(&candidate), Err(expected)); + } +} diff --git a/docs/adr/0106-provenance-evidence-model.md b/docs/adr/0106-provenance-evidence-model.md index d2494ad29..5bc6147a9 100644 --- a/docs/adr/0106-provenance-evidence-model.md +++ b/docs/adr/0106-provenance-evidence-model.md @@ -14,8 +14,11 @@ OriginWeave must not merely act; it must let a user, operator, auditor, or downs The extraction lane currently implements one bounded, verified in-memory WARC 1.1 `resource` record contract over already-authorized bytes. It binds the WARC target URI to independently verified provenance, computes a SHA-256 block digest, and -emits deterministic record bytes. This is active-PR evidence rather than a claim -of durable storage, tenant retention, request/response capture, or PROV export. +emits deterministic record bytes. The active-PR `WarcProvBundle` projection adds +deterministic W3C PROV-O JSON-LD and offline verification of the exact bound WARC +record without retaining the payload in the bundle. This is active-PR evidence; +durable persistence remains planned, as do tenant retention, request/response +capture, and transport-specific export adapters. ## Decision drivers diff --git a/docs/doctoring.md b/docs/doctoring.md index 7866e2759..47392f03c 100644 --- a/docs/doctoring.md +++ b/docs/doctoring.md @@ -90,6 +90,8 @@ RFC 9309 standardizes robots parsing, matching, error handling, and caching. It W3C PROV-O supplies interoperable Entity, Activity, Agent, derivation, attribution, and responsibility concepts. ISO 28500:2017, confirmed in 2023, defines WARC storage for protocol payloads, control information, metadata, transformations, duplicate detection, integrity, and segmentation. OriginWeave uses source hashes and locators in the safety kernel, then adds WARC and PROV adapters as separately testable modules. +Active PR #217 adds a bounded in-memory `WarcProvBundle` for deterministic W3C PROV-O JSON-LD projection and offline verification of exact WARC records. The bundle emits credential-free identifiers, source location, capture time, completeness, and digests without retaining the WARC payload. Offline verification uses the complete deterministic WARC serialization digest as the authoritative record binding, so block-digest drift cannot bypass the exact-record check. This is active-PR evidence only until merged; durable persistence, retention, and transport-specific export remain planned and are not implied by the in-memory projection. + The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. RFC 3986 permits `?` and `/` within the query component, so OriginWeave keeps safe query-bearing URLs. RFC 9700 specifically requires clients not to pass access tokens in URI query parameters. Independently, OriginWeave applies a broader evidence-retention policy: decoded credential-like field names are rejected at the top level and inside nested query-like values, and residual nested percent-encoding in a field name fails closed rather than hiding another encoded credential name. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. The versioned `ExtractionSchema` is an admission and interpretation contract for typed extracted fields: each field is bounded, declares a value type, cardinality, normalization rule, and a canonical duplicate-free set of reviewed source-channel classes. That declaration does not create browser, network, model, secret, storage, retention, disclosure, or governance authority. PROV/WARC interoperability is therefore layered after the schema contract rather than inferred from it. diff --git a/tests/test_doctoring_reference_contract.py b/tests/test_doctoring_reference_contract.py index bdeded44f..144f0c1e0 100644 --- a/tests/test_doctoring_reference_contract.py +++ b/tests/test_doctoring_reference_contract.py @@ -23,6 +23,18 @@ def test_rfc_5280_reference_uses_canonical_author_initials(self) -> None: ) self.assertIn(expected, text) + def test_warc_prov_decision_trace_distinguishes_in_memory_evidence(self) -> None: + """The standards record must bound the public WARC/PROV projection honestly.""" + text = DOCTORING.read_text(encoding="utf-8") + for expected in ( + "WarcProvBundle", + "deterministic W3C PROV-O JSON-LD projection", + "offline verification of exact WARC records", + "durable persistence, retention, and transport-specific export remain planned", + ): + with self.subTest(expected=expected): + self.assertIn(expected, text) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_product_documentation_contract.py b/tests/test_product_documentation_contract.py index f192aaa4d..412aaeba4 100644 --- a/tests/test_product_documentation_contract.py +++ b/tests/test_product_documentation_contract.py @@ -77,6 +77,19 @@ def test_product_technical_gap_baseline_records_live_delivery_state(self) -> Non vpn_status, ) + def test_warc_prov_bundle_boundary_is_documented_as_in_memory_evidence(self) -> None: + """The public WARC/PROV projection must not be mistaken for durable persistence.""" + adr = (ROOT / "docs/adr/0106-provenance-evidence-model.md").read_text(encoding="utf-8") + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + for text in (adr, changelog): + with self.subTest(document=text[:40]): + self.assertIn("WarcProvBundle", text) + self.assertIn("deterministic", text) + self.assertIn("JSON-LD", text) + self.assertIn("offline verification", text) + self.assertIn("in-memory", text) + self.assertIn("durable persistence remains planned", adr) + def test_root_architecture_links_the_authoritative_product_graph(self) -> None: """Architecture readers must be able to reach requirements, decisions, diagrams, and data.""" architecture = (ROOT / "ARCHITECTURE.md").read_text(encoding="utf-8")