diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs new file mode 100644 index 00000000..0e593ce6 --- /dev/null +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -0,0 +1,608 @@ +//! Deterministic identity manifests for schema-bound WARC/PROV capture evidence. +//! +//! A capture manifest binds one reviewed extraction-schema contract to exact WARC and PROV +//! serialization identities plus one immutable OriginWeave software revision. It intentionally +//! contains no captured payload, source URL, source locator, credential, browser authority, +//! persistence authority, retention decision, signature, or release authority. + +use std::{collections::BTreeMap, collections::BTreeSet, fmt}; + +use sha2::{Digest, Sha256}; + +use crate::{ + ExtractionCardinality, ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, + ExtractionValueType, MAX_EXTRACTION_IDENTIFIER_BYTES, WarcProvBundle, + WarcProvBundleVerificationError, WarcResourceRecord, +}; + +/// Version of the deterministic OriginWeave capture-manifest serialization contract. +pub const CAPTURE_MANIFEST_VERSION: u16 = 1; +/// Maximum number of WARC/PROV pairs admitted by one capture manifest. +pub const MAX_CAPTURE_MANIFEST_RECORDS: usize = 256; +/// Maximum number of structured-value digest bindings admitted by one capture manifest. +pub const MAX_CAPTURE_MANIFEST_VALUES: usize = 1_024; + +/// A validation failure while constructing one deterministic capture manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureManifestError { + /// A capture manifest must bind at least one WARC/PROV pair. + MissingRecord, + /// The number of supplied records exceeded the bounded manifest limit. + LimitExceeded, + /// More than one supplied pair used the same WARC record identifier. + DuplicateRecord, + /// A supplied PROV bundle did not exactly verify its paired WARC record. + BundleMismatch(WarcProvBundleVerificationError), + /// Supplied PROV bundles referred to different OriginWeave software revisions. + SoftwareRevisionMismatch, + /// The number of structured values exceeded the bounded manifest limit. + ValueLimitExceeded, + /// A structured-value field identifier did not use the extraction-schema identifier grammar. + InvalidValueField, + /// A structured-value digest was not a canonical lowercase SHA-256 identifier. + InvalidValueDigest, + /// A structured-value field was not declared by the bound extraction schema. + UnknownValueField, + /// A structured value referenced a WARC record absent from the manifest. + ValueSourceRecordMissing, + /// A structured value used WARC evidence for a field that did not admit network-response evidence. + ValueSourceChannelMismatch, + /// The same field, value digest, and source WARC record were supplied more than once. + DuplicateValue, + /// A structured-value field exceeded its declared extraction cardinality. + ValueCardinalityExceeded, + /// A required extraction field had no structured-value binding. + RequiredValueMissing, +} + +impl fmt::Display for CaptureManifestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingRecord => { + formatter.write_str("capture manifest requires at least one record") + } + Self::LimitExceeded => formatter.write_str("capture manifest record limit exceeded"), + Self::DuplicateRecord => { + formatter.write_str("capture manifest contains a duplicate WARC record") + } + Self::BundleMismatch(error) => { + write!(formatter, "capture manifest WARC/PROV mismatch: {error}") + } + Self::SoftwareRevisionMismatch => formatter.write_str( + "capture manifest records do not share one OriginWeave software revision", + ), + Self::ValueLimitExceeded => { + formatter.write_str("capture manifest structured-value limit exceeded") + } + Self::InvalidValueField => { + formatter.write_str("capture manifest structured-value field is invalid") + } + Self::InvalidValueDigest => formatter + .write_str("capture manifest structured-value digest is not canonical SHA-256"), + Self::UnknownValueField => formatter + .write_str("capture manifest structured-value field is absent from the schema"), + Self::ValueSourceRecordMissing => formatter + .write_str("capture manifest structured value references an absent WARC record"), + Self::ValueSourceChannelMismatch => formatter.write_str( + "capture manifest structured value is not admitted by the field source channels", + ), + Self::DuplicateValue => { + formatter.write_str("capture manifest contains a duplicate structured value") + } + Self::ValueCardinalityExceeded => { + formatter.write_str("capture manifest structured value exceeds field cardinality") + } + Self::RequiredValueMissing => { + formatter.write_str("capture manifest is missing a required structured value") + } + } + } +} + +impl std::error::Error for CaptureManifestError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::BundleMismatch(error) => Some(error), + Self::MissingRecord + | Self::LimitExceeded + | Self::DuplicateRecord + | Self::SoftwareRevisionMismatch + | Self::ValueLimitExceeded + | Self::InvalidValueField + | Self::InvalidValueDigest + | Self::UnknownValueField + | Self::ValueSourceRecordMissing + | Self::ValueSourceChannelMismatch + | Self::DuplicateValue + | Self::ValueCardinalityExceeded + | Self::RequiredValueMissing => None, + } + } +} + +/// A deterministic offline verification failure for a previously constructed capture manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureManifestVerificationError { + /// A valid candidate manifest did not have the same immutable identity as the expected manifest. + IdentityMismatch, + /// The candidate inputs did not form a valid manifest at all. + InvalidCandidate(CaptureManifestError), +} + +impl fmt::Display for CaptureManifestVerificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::IdentityMismatch => { + formatter.write_str("capture manifest identity does not match") + } + Self::InvalidCandidate(error) => { + write!( + formatter, + "invalid capture manifest verification candidate: {error}" + ) + } + } + } +} + +impl std::error::Error for CaptureManifestVerificationError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::IdentityMismatch => None, + Self::InvalidCandidate(error) => Some(error), + } + } +} + +/// Payload-free immutable identity for one WARC/PROV pair in a [`CaptureManifest`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CaptureManifestRecord { + warc_record_id: String, + warc_record_digest: String, + prov_json_ld_digest: String, +} + +impl CaptureManifestRecord { + /// Return the UUID URN of the WARC record bound by this entry. + #[must_use] + pub fn warc_record_id(&self) -> &str { + &self.warc_record_id + } + + /// Return the SHA-256 digest of the complete deterministic WARC serialization. + #[must_use] + pub fn warc_record_digest(&self) -> &str { + &self.warc_record_digest + } + + /// Return the SHA-256 digest of the deterministic PROV JSON-LD serialization. + #[must_use] + pub fn prov_json_ld_digest(&self) -> &str { + &self.prov_json_ld_digest + } +} + +/// Payload-free binding from one schema field digest to its exact source WARC record. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct CaptureManifestValueBinding { + field_name: String, + value_digest: String, + source_warc_record_id: String, +} + +impl CaptureManifestValueBinding { + /// Validate one credential-safe structured-value identity binding. + /// + /// The binding carries only a schema field identifier, canonical value digest, and WARC record + /// identifier. The referenced record and field authority are validated when the binding is + /// admitted into a [`CaptureManifest`]. Raw extracted values are never stored here. + pub fn new( + field_name: &str, + value_digest: &str, + source_warc_record_id: &str, + ) -> Result { + if !valid_value_field_name(field_name) { + return Err(CaptureManifestError::InvalidValueField); + } + if !valid_sha256(value_digest) { + return Err(CaptureManifestError::InvalidValueDigest); + } + Ok(Self { + field_name: field_name.to_owned(), + value_digest: value_digest.to_owned(), + source_warc_record_id: source_warc_record_id.to_owned(), + }) + } + + /// Return the extraction-schema field identifier. + #[must_use] + pub fn field_name(&self) -> &str { + &self.field_name + } + + /// Return the canonical lowercase SHA-256 digest of the extracted value bytes. + #[must_use] + pub fn value_digest(&self) -> &str { + &self.value_digest + } + + /// Return the exact WARC record identifier that supplied this value. + #[must_use] + pub fn source_warc_record_id(&self) -> &str { + &self.source_warc_record_id + } +} + +/// Deterministic payload-free identity binding a schema to exact WARC/PROV capture evidence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CaptureManifest { + version: u16, + schema_version: String, + schema_digest: String, + software_commit_sha: String, + records: Vec, + values: Vec, +} + +impl CaptureManifest { + /// Construct one bounded deterministic evidence-only capture manifest. + /// + /// Every supplied PROV bundle must verify the paired WARC record exactly and all pairs must + /// name the same canonical OriginWeave software revision. Caller order is not identity: + /// records are canonicalized by WARC record identifier before serialization and comparison. + /// This low-level constructor does not assert that required extraction fields have values; + /// use [`Self::new_with_warc_values`] for a schema-conforming structured-result manifest. + /// It performs no capture, network, browser, persistence, model, signing, or authorization + /// operation. + pub fn new( + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + ) -> Result { + if records.is_empty() { + return Err(CaptureManifestError::MissingRecord); + } + if records.len() > MAX_CAPTURE_MANIFEST_RECORDS { + return Err(CaptureManifestError::LimitExceeded); + } + + let software_commit_sha = records[0].1.software_commit_sha(); + let mut canonical_records = BTreeMap::new(); + for (record, bundle) in records { + bundle + .verify_record(record) + .map_err(CaptureManifestError::BundleMismatch)?; + if bundle.software_commit_sha() != software_commit_sha { + return Err(CaptureManifestError::SoftwareRevisionMismatch); + } + + let manifest_record = CaptureManifestRecord { + warc_record_id: record.record_id().to_owned(), + warc_record_digest: sha256_digest(&record.to_warc_bytes()), + prov_json_ld_digest: sha256_digest(bundle.to_json_ld().as_bytes()), + }; + if canonical_records + .insert(record.record_id().to_owned(), manifest_record) + .is_some() + { + return Err(CaptureManifestError::DuplicateRecord); + } + } + + Ok(Self { + version: CAPTURE_MANIFEST_VERSION, + schema_version: schema.version().to_owned(), + schema_digest: extraction_schema_digest(schema), + software_commit_sha: software_commit_sha.to_owned(), + records: canonical_records.into_values().collect(), + values: Vec::new(), + }) + } + + /// Construct a schema-conforming manifest with WARC-backed structured-value identities. + /// + /// Every value must name a declared schema field that admits network-response evidence and an + /// exact WARC record present in this manifest. Required fields must be present; `One` and + /// `ZeroOrOne` fields admit at most one value. Duplicate bindings and over-limit collections + /// fail closed. Values are canonicalized independently of caller order. No raw extracted value + /// is retained and no browser, network, persistence, secret, model, or authorization operation + /// is performed. + pub fn new_with_warc_values( + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + values: &[CaptureManifestValueBinding], + ) -> Result { + let mut manifest = Self::new(schema, records)?; + if values.len() > MAX_CAPTURE_MANIFEST_VALUES { + return Err(CaptureManifestError::ValueLimitExceeded); + } + + let mut seen_values = BTreeSet::new(); + let mut field_counts: BTreeMap<&str, usize> = BTreeMap::new(); + for value in values { + let Some(field) = schema.field(value.field_name()) else { + return Err(CaptureManifestError::UnknownValueField); + }; + if !manifest + .records + .iter() + .any(|record| record.warc_record_id() == value.source_warc_record_id()) + { + return Err(CaptureManifestError::ValueSourceRecordMissing); + } + if !field + .source_channels() + .contains(&ExtractionSourceChannel::NetworkResponse) + { + return Err(CaptureManifestError::ValueSourceChannelMismatch); + } + if !seen_values.insert(( + value.field_name(), + value.value_digest(), + value.source_warc_record_id(), + )) { + return Err(CaptureManifestError::DuplicateValue); + } + + let count = field_counts.entry(value.field_name()).or_default(); + *count += 1; + if *count > 1 + && matches!( + field.cardinality(), + ExtractionCardinality::One | ExtractionCardinality::ZeroOrOne + ) + { + return Err(CaptureManifestError::ValueCardinalityExceeded); + } + } + + if schema.fields().iter().any(|field| { + field.required() + && field_counts + .get(field.identifier()) + .copied() + .unwrap_or_default() + == 0 + }) { + return Err(CaptureManifestError::RequiredValueMissing); + } + + manifest.values = values.to_vec(); + manifest.values.sort(); + Ok(manifest) + } + + /// Return the capture-manifest serialization-contract version. + #[must_use] + pub const fn version(&self) -> u16 { + self.version + } + + /// Return the extraction-schema version identifier bound by this manifest. + #[must_use] + pub fn schema_version(&self) -> &str { + &self.schema_version + } + + /// Return a SHA-256 digest of the complete ordered extraction-schema contract. + #[must_use] + pub fn schema_digest(&self) -> &str { + &self.schema_digest + } + + /// Return the canonical lower-case OriginWeave Git SHA-1 shared by every PROV bundle. + #[must_use] + pub fn software_commit_sha(&self) -> &str { + &self.software_commit_sha + } + + /// Return manifest entries in canonical WARC-record-identifier order. + #[must_use] + pub fn records(&self) -> &[CaptureManifestRecord] { + &self.records + } + + /// Return structured-value identities in canonical field/digest/source-record order. + #[must_use] + pub fn values(&self) -> &[CaptureManifestValueBinding] { + &self.values + } + + /// Serialize the manifest deterministically without captured payloads or source locations. + #[must_use] + pub fn to_json(&self) -> String { + let records = self + .records + .iter() + .map(|record| { + format!( + "{{\"warcRecordId\":\"{}\",\"warcRecordDigest\":\"{}\",\"provJsonLdDigest\":\"{}\"}}", + record.warc_record_id, record.warc_record_digest, record.prov_json_ld_digest + ) + }) + .collect::>() + .join(","); + let values = self + .values + .iter() + .map(|value| { + format!( + "{{\"fieldName\":\"{}\",\"valueDigest\":\"{}\",\"sourceWarcRecordId\":\"{}\"}}", + value.field_name, value.value_digest, value.source_warc_record_id + ) + }) + .collect::>() + .join(","); + format!( + "{{\"manifestVersion\":{},\"schemaVersion\":\"{}\",\"schemaDigest\":\"{}\",\"softwareCommitSha\":\"{}\",\"records\":[{}],\"values\":[{}]}}", + self.version, + self.schema_version, + self.schema_digest, + self.software_commit_sha, + records, + values + ) + } + + /// Return the SHA-256 identity of the deterministic manifest serialization. + #[must_use] + pub fn manifest_digest(&self) -> String { + sha256_digest(self.to_json().as_bytes()) + } + + /// Require candidate bytes to be the exact deterministic serialization of this manifest. + /// + /// This verification deliberately does not parse or normalize JSON: whitespace, member-order, + /// encoding, or other serialization drift is an identity mismatch even when a generic JSON + /// parser could assign equivalent data semantics. It performs no network, browser, persistence, + /// model, signing, or authority action and does not authenticate the producer of either value. + pub fn verify_serialized_json( + &self, + candidate: &[u8], + ) -> Result<(), CaptureManifestVerificationError> { + let expected = self.to_json(); + if candidate == expected.as_bytes() { + Ok(()) + } else { + Err(CaptureManifestVerificationError::IdentityMismatch) + } + } + + /// Reconstruct a candidate evidence-only manifest and require exact immutable identity equality. + /// + /// Malformed candidate inputs remain distinguishable from a valid-but-different manifest. + /// Verification performs no network, browser, persistence, model, signing, or authority action. + pub fn verify( + &self, + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + ) -> Result<(), CaptureManifestVerificationError> { + let candidate = Self::new(schema, records) + .map_err(CaptureManifestVerificationError::InvalidCandidate)?; + if candidate == *self { + Ok(()) + } else { + Err(CaptureManifestVerificationError::IdentityMismatch) + } + } + + /// Reconstruct a candidate WARC-backed structured-result manifest and require exact identity. + /// + /// Candidate schema, WARC/PROV evidence, field admission, cardinality, requiredness, source + /// identity, and value digests are all revalidated before immutable manifest equality is tested. + pub fn verify_with_warc_values( + &self, + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + values: &[CaptureManifestValueBinding], + ) -> Result<(), CaptureManifestVerificationError> { + let candidate = Self::new_with_warc_values(schema, records, values) + .map_err(CaptureManifestVerificationError::InvalidCandidate)?; + if candidate == *self { + Ok(()) + } else { + Err(CaptureManifestVerificationError::IdentityMismatch) + } + } +} + +fn valid_value_field_name(field_name: &str) -> bool { + if field_name.len() > MAX_EXTRACTION_IDENTIFIER_BYTES { + return false; + } + let mut bytes = field_name.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + first.is_ascii_lowercase() + && bytes.all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'_' | b'-')) +} + +fn valid_sha256(value_digest: &str) -> bool { + let Some(hex_digest) = value_digest.strip_prefix("sha256:") else { + return false; + }; + hex_digest.len() == 64 + && hex_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn extraction_schema_digest(schema: &ExtractionSchema) -> String { + let mut hasher = Sha256::new(); + update_length_prefixed(&mut hasher, b"originweave:capture-schema:v1"); + update_length_prefixed(&mut hasher, schema.version().as_bytes()); + hasher.update((schema.fields().len() as u64).to_be_bytes()); + for field in schema.fields() { + update_length_prefixed(&mut hasher, field.identifier().as_bytes()); + update_length_prefixed(&mut hasher, extraction_value_type_token(field.value_type())); + update_length_prefixed( + &mut hasher, + extraction_cardinality_token(field.cardinality()), + ); + hasher.update([u8::from(field.required())]); + update_length_prefixed( + &mut hasher, + extraction_normalization_token(field.normalization_rule()), + ); + hasher.update((field.source_channels().len() as u64).to_be_bytes()); + for channel in field.source_channels() { + update_length_prefixed(&mut hasher, extraction_source_channel_token(*channel)); + } + } + encode_sha256(hasher.finalize()) +} + +fn update_length_prefixed(hasher: &mut Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +const fn extraction_value_type_token(value_type: ExtractionValueType) -> &'static [u8] { + match value_type { + ExtractionValueType::Text => b"text", + ExtractionValueType::Integer => b"integer", + ExtractionValueType::Decimal => b"decimal", + ExtractionValueType::Boolean => b"boolean", + ExtractionValueType::Timestamp => b"timestamp", + } +} + +const fn extraction_cardinality_token(cardinality: ExtractionCardinality) -> &'static [u8] { + match cardinality { + ExtractionCardinality::One => b"one", + ExtractionCardinality::ZeroOrOne => b"zero_or_one", + ExtractionCardinality::Many => b"many", + } +} + +const fn extraction_normalization_token( + normalization_rule: ExtractionNormalizationRule, +) -> &'static [u8] { + match normalization_rule { + ExtractionNormalizationRule::Verbatim => b"verbatim", + ExtractionNormalizationRule::TrimTextWhitespace => b"trim_text_whitespace", + ExtractionNormalizationRule::Rfc3339Utc => b"rfc3339_utc", + } +} + +const fn extraction_source_channel_token(channel: ExtractionSourceChannel) -> &'static [u8] { + match channel { + ExtractionSourceChannel::SemanticNode => b"semantic_node", + ExtractionSourceChannel::StructuredData => b"structured_data", + ExtractionSourceChannel::TableCell => b"table_cell", + ExtractionSourceChannel::NetworkResponse => b"network_response", + ExtractionSourceChannel::ModelInterpretation => b"model_interpretation", + } +} + +fn sha256_digest(bytes: &[u8]) -> String { + encode_sha256(Sha256::digest(bytes)) +} + +fn encode_sha256(digest: impl IntoIterator) -> String { + let mut encoded = String::from("sha256:"); + for byte in digest { + encoded.push_str(&format!("{byte:02x}")); + } + encoded +} diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 75f4587a..a4e19634 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -7,12 +7,18 @@ #![forbid(unsafe_code)] #![deny(missing_docs)] +mod capture_manifest; mod extraction_schema; mod sensitive_access; mod sensitive_handle_lifecycle; mod warc_prov_bundle; mod warc_resource_record; +pub use capture_manifest::{ + CAPTURE_MANIFEST_VERSION, CaptureManifest, CaptureManifestError, CaptureManifestRecord, + CaptureManifestValueBinding, CaptureManifestVerificationError, MAX_CAPTURE_MANIFEST_RECORDS, + MAX_CAPTURE_MANIFEST_VALUES, +}; pub use extraction_schema::{ ExtractionCardinality, ExtractionField, ExtractionNormalizationRule, ExtractionSchema, ExtractionSchemaError, ExtractionSourceChannel, ExtractionValueType, diff --git a/crates/originweave-evidence/tests/capture_manifest.rs b/crates/originweave-evidence/tests/capture_manifest.rs new file mode 100644 index 00000000..5ca144f3 --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest.rs @@ -0,0 +1,295 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CAPTURE_MANIFEST_VERSION, CaptureManifest, CaptureManifestError, + CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, ExtractionField, + ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, + MAX_CAPTURE_MANIFEST_RECORDS, ProvenanceRecord, VerificationResult, WarcProvBundle, + WarcProvBundleVerificationError, WarcResourceRecord, +}; +use sha2::{Digest, Sha256}; + +const RECORD_ID_A: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const RECORD_ID_B: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; +const DATE: &str = "2026-08-24T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; +const OTHER_SOFTWARE_COMMIT_SHA: &str = "1123456789abcdef0123456789abcdef01234567"; + +fn schema(version: &str) -> ExtractionSchema { + let field = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("field contract"); + ExtractionSchema::new(version, vec![field]).expect("schema contract") +} + +fn schema_covering_all_semantics(version: &str) -> ExtractionSchema { + let text = ExtractionField::new_with_normalization( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + ExtractionNormalizationRule::TrimTextWhitespace, + &[ + ExtractionSourceChannel::SemanticNode, + ExtractionSourceChannel::StructuredData, + ExtractionSourceChannel::TableCell, + ExtractionSourceChannel::NetworkResponse, + ExtractionSourceChannel::ModelInterpretation, + ], + ) + .expect("text field"); + let integer = ExtractionField::new( + "quantity", + ExtractionValueType::Integer, + ExtractionCardinality::ZeroOrOne, + false, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("integer field"); + let decimal = ExtractionField::new( + "price", + ExtractionValueType::Decimal, + ExtractionCardinality::Many, + true, + &[ExtractionSourceChannel::StructuredData], + ) + .expect("decimal field"); + let boolean = ExtractionField::new( + "available", + ExtractionValueType::Boolean, + ExtractionCardinality::Many, + false, + &[ExtractionSourceChannel::TableCell], + ) + .expect("boolean field"); + let timestamp = ExtractionField::new_with_normalization( + "captured_at", + ExtractionValueType::Timestamp, + ExtractionCardinality::One, + true, + ExtractionNormalizationRule::Rfc3339Utc, + &[ExtractionSourceChannel::SemanticNode], + ) + .expect("timestamp field"); + ExtractionSchema::new(version, vec![text, integer, decimal, boolean, timestamp]) + .expect("complete semantic schema") +} + +fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + &source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + record_id, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + ) + .expect("WARC resource record") +} + +fn assert_standard_error_contract() {} + +#[test] +fn capture_manifest_binds_schema_warc_prov_and_software_identity_without_payload() { + assert_standard_error_contract::(); + assert_standard_error_contract::(); + + let schema = schema("catalog-v1"); + let record = resource_record(RECORD_ID_A, b"secret-like-captured-payload"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let manifest = CaptureManifest::new(&schema, &[(&record, &bundle)]).expect("capture manifest"); + + assert_eq!(manifest.version(), CAPTURE_MANIFEST_VERSION); + assert_eq!(manifest.schema_version(), "catalog-v1"); + assert!(manifest.schema_digest().starts_with("sha256:")); + assert_eq!(manifest.software_commit_sha(), SOFTWARE_COMMIT_SHA); + assert_eq!(manifest.records().len(), 1); + assert_eq!(manifest.records()[0].warc_record_id(), RECORD_ID_A); + assert!( + manifest.records()[0] + .warc_record_digest() + .starts_with("sha256:") + ); + assert!( + manifest.records()[0] + .prov_json_ld_digest() + .starts_with("sha256:") + ); + assert!(manifest.manifest_digest().starts_with("sha256:")); + assert_eq!(manifest.verify(&schema, &[(&record, &bundle)]), Ok(())); + + let json = manifest.to_json(); + assert!(json.contains("\"manifestVersion\":1")); + assert!(json.contains("\"schemaVersion\":\"catalog-v1\"")); + assert!(json.contains(RECORD_ID_A)); + assert!(!json.contains("secret-like-captured-payload")); + assert!(!json.contains("https://example.com/item")); + assert!(!json.contains(record.provenance().source_hash())); +} + +#[test] +fn capture_manifest_schema_digest_binds_every_typed_schema_dimension() { + let record = resource_record(RECORD_ID_A, b"schema-sensitive"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let complete_schema = schema_covering_all_semantics("catalog-semantic-v1"); + let minimal_schema = schema("catalog-semantic-v1"); + + let complete = + CaptureManifest::new(&complete_schema, &[(&record, &bundle)]).expect("complete manifest"); + let minimal = + CaptureManifest::new(&minimal_schema, &[(&record, &bundle)]).expect("minimal manifest"); + + assert_ne!(complete.schema_digest(), minimal.schema_digest()); + assert_ne!(complete, minimal); +} + +#[test] +fn capture_manifest_error_contracts_preserve_typed_causes() { + let missing = CaptureManifestError::MissingRecord; + assert_eq!( + missing.to_string(), + "capture manifest requires at least one record" + ); + assert!(std::error::Error::source(&missing).is_none()); + + let limit = CaptureManifestError::LimitExceeded; + assert_eq!(limit.to_string(), "capture manifest record limit exceeded"); + assert!(std::error::Error::source(&limit).is_none()); + + let duplicate = CaptureManifestError::DuplicateRecord; + assert_eq!( + duplicate.to_string(), + "capture manifest contains a duplicate WARC record" + ); + assert!(std::error::Error::source(&duplicate).is_none()); + + let software = CaptureManifestError::SoftwareRevisionMismatch; + assert_eq!( + software.to_string(), + "capture manifest records do not share one OriginWeave software revision" + ); + assert!(std::error::Error::source(&software).is_none()); + + let mismatch = CaptureManifestError::BundleMismatch( + WarcProvBundleVerificationError::RecordIdentityMismatch, + ); + assert!(mismatch.to_string().contains("WARC/PROV mismatch")); + assert!(std::error::Error::source(&mismatch).is_some()); + + let identity = CaptureManifestVerificationError::IdentityMismatch; + assert_eq!( + identity.to_string(), + "capture manifest identity does not match" + ); + assert!(std::error::Error::source(&identity).is_none()); + + let invalid = CaptureManifestVerificationError::InvalidCandidate(mismatch); + assert!( + invalid + .to_string() + .starts_with("invalid capture manifest verification candidate:") + ); + assert!(std::error::Error::source(&invalid).is_some()); +} + +#[test] +fn capture_manifest_is_order_independent_and_rejects_empty_duplicate_or_oversized_sets() { + let schema = schema("catalog-v1"); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); + + let first = CaptureManifest::new(&schema, &[(&record_b, &bundle_b), (&record_a, &bundle_a)]) + .expect("first manifest"); + let second = CaptureManifest::new(&schema, &[(&record_a, &bundle_a), (&record_b, &bundle_b)]) + .expect("second manifest"); + assert_eq!(first, second); + assert_eq!(first.records()[0].warc_record_id(), RECORD_ID_A); + assert_eq!(first.records()[1].warc_record_id(), RECORD_ID_B); + + assert_eq!( + CaptureManifest::new(&schema, &[]), + Err(CaptureManifestError::MissingRecord) + ); + assert_eq!( + CaptureManifest::new(&schema, &[(&record_a, &bundle_a), (&record_a, &bundle_a)],), + Err(CaptureManifestError::DuplicateRecord) + ); + + let oversized = vec![(&record_a, &bundle_a); MAX_CAPTURE_MANIFEST_RECORDS + 1]; + assert_eq!( + CaptureManifest::new(&schema, &oversized), + Err(CaptureManifestError::LimitExceeded) + ); +} + +#[test] +fn capture_manifest_rejects_mismatched_bundle_or_mixed_software_revision() { + let schema = schema("catalog-v1"); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b_other = WarcProvBundle::new(&record_b, OTHER_SOFTWARE_COMMIT_SHA) + .expect("PROV B with other software"); + + assert_eq!( + CaptureManifest::new(&schema, &[(&record_b, &bundle_a)]), + Err(CaptureManifestError::BundleMismatch( + WarcProvBundleVerificationError::RecordIdentityMismatch, + )) + ); + assert_eq!( + CaptureManifest::new( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b_other)], + ), + Err(CaptureManifestError::SoftwareRevisionMismatch) + ); +} + +#[test] +fn capture_manifest_verification_fails_closed_on_schema_or_record_drift() { + let original_schema = schema("catalog-v1"); + let record = resource_record(RECORD_ID_A, b"original"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let manifest = + CaptureManifest::new(&original_schema, &[(&record, &bundle)]).expect("capture manifest"); + + assert_eq!( + manifest.verify(&schema("catalog-v2"), &[(&record, &bundle)]), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); + + let changed_record = resource_record(RECORD_ID_A, b"changed"); + let changed_bundle = + WarcProvBundle::new(&changed_record, SOFTWARE_COMMIT_SHA).expect("changed PROV bundle"); + assert_eq!( + manifest.verify(&original_schema, &[(&changed_record, &changed_bundle)]), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); + + let other_record = resource_record(RECORD_ID_B, b"other"); + assert_eq!( + manifest.verify(&original_schema, &[(&other_record, &bundle)]), + Err(CaptureManifestVerificationError::InvalidCandidate( + CaptureManifestError::BundleMismatch( + WarcProvBundleVerificationError::RecordIdentityMismatch, + ), + )) + ); +} diff --git a/crates/originweave-evidence/tests/capture_manifest_serialized.rs b/crates/originweave-evidence/tests/capture_manifest_serialized.rs new file mode 100644 index 00000000..451dcdfa --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_serialized.rs @@ -0,0 +1,59 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, + ExtractionField, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, + ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, +}; +use sha2::{Digest, Sha256}; + +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn manifest() -> CaptureManifest { + let field = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("field contract"); + let schema = ExtractionSchema::new("catalog-v1", vec![field]).expect("schema contract"); + let payload = b"captured-payload"; + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + &source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + let record = WarcResourceRecord::new( + RECORD_ID, + "2026-08-24T00:00:00Z", + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + ) + .expect("WARC resource record"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + CaptureManifest::new(&schema, &[(&record, &bundle)]).expect("capture manifest") +} + +#[test] +fn persisted_capture_manifest_requires_exact_deterministic_serialization() { + let manifest = manifest(); + let exact = manifest.to_json().into_bytes(); + + assert_eq!(manifest.verify_serialized_json(&exact), Ok(())); + + let mut drifted = exact.clone(); + drifted.push(b'\n'); + assert_eq!( + manifest.verify_serialized_json(&drifted), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); +} diff --git a/crates/originweave-evidence/tests/capture_manifest_value_edges.rs b/crates/originweave-evidence/tests/capture_manifest_value_edges.rs new file mode 100644 index 00000000..66cbe07f --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_value_edges.rs @@ -0,0 +1,78 @@ +use originweave_evidence::{ + CaptureManifestError, CaptureManifestValueBinding, MAX_EXTRACTION_IDENTIFIER_BYTES, +}; + +const VALUE_HASH: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; + +#[test] +fn structured_value_errors_have_deterministic_standard_error_contracts() { + let cases = [ + ( + CaptureManifestError::ValueLimitExceeded, + "capture manifest structured-value limit exceeded", + ), + ( + CaptureManifestError::InvalidValueField, + "capture manifest structured-value field is invalid", + ), + ( + CaptureManifestError::InvalidValueDigest, + "capture manifest structured-value digest is not canonical SHA-256", + ), + ( + CaptureManifestError::UnknownValueField, + "capture manifest structured-value field is absent from the schema", + ), + ( + CaptureManifestError::ValueSourceRecordMissing, + "capture manifest structured value references an absent WARC record", + ), + ( + CaptureManifestError::ValueSourceChannelMismatch, + "capture manifest structured value is not admitted by the field source channels", + ), + ( + CaptureManifestError::DuplicateValue, + "capture manifest contains a duplicate structured value", + ), + ( + CaptureManifestError::ValueCardinalityExceeded, + "capture manifest structured value exceeds field cardinality", + ), + ( + CaptureManifestError::RequiredValueMissing, + "capture manifest is missing a required structured value", + ), + ]; + + for (error, expected_message) in cases { + assert_eq!(error.to_string(), expected_message); + assert!(std::error::Error::source(&error).is_none()); + } +} + +#[test] +fn value_binding_rejects_every_identifier_and_digest_shape_boundary() { + let overlong_field = "a".repeat(MAX_EXTRACTION_IDENTIFIER_BYTES + 1); + for invalid_field in ["", "Title", "title.value", overlong_field.as_str()] { + assert_eq!( + CaptureManifestValueBinding::new(invalid_field, VALUE_HASH, RECORD_ID), + Err(CaptureManifestError::InvalidValueField) + ); + } + + assert_eq!( + CaptureManifestValueBinding::new("title", "not-a-sha256", RECORD_ID), + Err(CaptureManifestError::InvalidValueDigest) + ); + let invalid_hex_digest = format!("sha256:{}", "g".repeat(64)); + assert_eq!( + CaptureManifestValueBinding::new("title", &invalid_hex_digest, RECORD_ID), + Err(CaptureManifestError::InvalidValueDigest) + ); + + let numeric_hex_digest = format!("sha256:{}", "0".repeat(64)); + assert!(CaptureManifestValueBinding::new("title", &numeric_hex_digest, RECORD_ID).is_ok()); + assert!(CaptureManifestValueBinding::new("title_1-tag", VALUE_HASH, RECORD_ID).is_ok()); +} diff --git a/crates/originweave-evidence/tests/capture_manifest_values.rs b/crates/originweave-evidence/tests/capture_manifest_values.rs new file mode 100644 index 00000000..792d9dbc --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_values.rs @@ -0,0 +1,232 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, + CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, ExtractionField, + ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, MAX_CAPTURE_MANIFEST_VALUES, + ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, +}; +use sha2::{Digest, Sha256}; + +const VALUE_HASH_A: &str = + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const VALUE_HASH_B: &str = + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const RECORD_ID_A: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const RECORD_ID_B: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; +const DATE: &str = "2026-08-25T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn schema() -> ExtractionSchema { + let title = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("title field"); + let tags = ExtractionField::new( + "tags", + ExtractionValueType::Text, + ExtractionCardinality::Many, + false, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("tags field"); + ExtractionSchema::new("catalog-v2", vec![title, tags]).expect("schema") +} + +fn semantic_only_schema() -> ExtractionSchema { + let title = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::SemanticNode], + ) + .expect("semantic-only field"); + ExtractionSchema::new("semantic-v1", vec![title]).expect("schema") +} + +fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + &source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + record_id, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + ) + .expect("WARC record") +} + +#[test] +fn capture_manifest_binds_required_warc_backed_structured_values() { + let schema = schema(); + let record = resource_record(RECORD_ID_A, b"captured-payload"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let title = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A) + .expect("value binding"); + + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&title), + ) + .expect("manifest with value"); + + assert_eq!(manifest.values(), std::slice::from_ref(&title)); + assert_eq!(manifest.values()[0].field_name(), "title"); + assert_eq!(manifest.values()[0].value_digest(), VALUE_HASH_A); + assert_eq!(manifest.values()[0].source_warc_record_id(), RECORD_ID_A); + assert_eq!( + manifest.verify_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&title), + ), + Ok(()) + ); + assert_eq!( + manifest.verify_with_warc_values(&schema, &[], std::slice::from_ref(&title)), + Err(CaptureManifestVerificationError::InvalidCandidate( + CaptureManifestError::MissingRecord, + )) + ); + + let json = manifest.to_json(); + assert!(json.contains("\"values\":[")); + assert!(json.contains("\"fieldName\":\"title\"")); + assert!(json.contains(VALUE_HASH_A)); + assert!(json.contains(RECORD_ID_A)); + assert!(!json.contains("captured-payload")); +} + +#[test] +fn capture_manifest_value_admission_fails_closed() { + assert_eq!( + CaptureManifestValueBinding::new("Title", VALUE_HASH_A, RECORD_ID_A), + Err(CaptureManifestError::InvalidValueField) + ); + assert_eq!( + CaptureManifestValueBinding::new("title", "sha256:ABC", RECORD_ID_A), + Err(CaptureManifestError::InvalidValueDigest) + ); + + let schema = schema(); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); + let title_a = + CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A).expect("title A"); + let title_b = + CaptureManifestValueBinding::new("title", VALUE_HASH_B, RECORD_ID_B).expect("title B"); + let unknown = CaptureManifestValueBinding::new("missing", VALUE_HASH_A, RECORD_ID_A) + .expect("syntactically valid unknown field"); + let missing_record = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_B) + .expect("syntactically valid missing record"); + + assert_eq!( + CaptureManifest::new_with_warc_values(&schema, &[(&record_a, &bundle_a)], &[]), + Err(CaptureManifestError::RequiredValueMissing) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a)], + std::slice::from_ref(&unknown), + ), + Err(CaptureManifestError::UnknownValueField) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a)], + std::slice::from_ref(&missing_record), + ), + Err(CaptureManifestError::ValueSourceRecordMissing) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &semantic_only_schema(), + &[(&record_a, &bundle_a)], + std::slice::from_ref(&title_a), + ), + Err(CaptureManifestError::ValueSourceChannelMismatch) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + &[title_a.clone(), title_b], + ), + Err(CaptureManifestError::ValueCardinalityExceeded) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a)], + &[title_a.clone(), title_a.clone()], + ), + Err(CaptureManifestError::DuplicateValue) + ); + + let oversized = vec![title_a; MAX_CAPTURE_MANIFEST_VALUES + 1]; + assert_eq!( + CaptureManifest::new_with_warc_values(&schema, &[(&record_a, &bundle_a)], &oversized), + Err(CaptureManifestError::ValueLimitExceeded) + ); +} + +#[test] +fn capture_manifest_value_order_is_canonical_and_verification_detects_drift() { + let schema = schema(); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); + let title = + CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A).expect("title"); + let tag_a = CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_A).expect("tag A"); + let tag_b = CaptureManifestValueBinding::new("tags", VALUE_HASH_B, RECORD_ID_B).expect("tag B"); + + let first = CaptureManifest::new_with_warc_values( + &schema, + &[(&record_b, &bundle_b), (&record_a, &bundle_a)], + &[tag_b.clone(), title.clone(), tag_a.clone()], + ) + .expect("first manifest"); + let second = CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + &[tag_a, tag_b.clone(), title.clone()], + ) + .expect("second manifest"); + + assert_eq!(first, second); + assert_eq!(first.values()[0].field_name(), "tags"); + assert_eq!(first.values()[1].field_name(), "tags"); + assert_eq!(first.values()[2].field_name(), "title"); + + let drifted_tag = + CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_B).expect("drifted tag"); + assert_eq!( + first.verify_with_warc_values( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + &[drifted_tag, tag_b, title], + ), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); +}