From b5d1cdae44fd370b997b3e6b4dc44c0c51e428d8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:04:09 -0700 Subject: [PATCH 01/44] test(evidence): require deterministic capture manifest binding --- .../tests/capture_manifest.rs | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 crates/originweave-evidence/tests/capture_manifest.rs diff --git a/crates/originweave-evidence/tests/capture_manifest.rs b/crates/originweave-evidence/tests/capture_manifest.rs new file mode 100644 index 000000000..15e13a75a --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest.rs @@ -0,0 +1,177 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestError, CaptureManifestVerificationError, EvidenceSourceKind, + ExtractionCardinality, ExtractionField, ExtractionSchema, ExtractionSourceChannel, + ExtractionValueType, ProvenanceRecord, VerificationResult, WarcProvBundle, + WarcProvBundleVerificationError, WarcResourceRecord, CAPTURE_MANIFEST_VERSION, + MAX_CAPTURE_MANIFEST_RECORDS, +}; + +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const RECORD_ID_A: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const RECORD_ID_B: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; +const DATE: &str = "2026-08-24T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; +const OTHER_SOFTWARE_COMMIT_SHA: &str = "1123456789abcdef0123456789abcdef01234567"; + +fn schema(version: &str) -> ExtractionSchema { + let field = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("field contract"); + ExtractionSchema::new(version, vec![field]).expect("schema contract") +} + +fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + record_id, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + ) + .expect("WARC resource record") +} + +fn assert_standard_error_contract() {} + +#[test] +fn capture_manifest_binds_schema_warc_prov_and_software_identity_without_payload() { + assert_standard_error_contract::(); + assert_standard_error_contract::(); + + let schema = schema("catalog-v1"); + let record = resource_record(RECORD_ID_A, b"secret-like-captured-payload"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let manifest = CaptureManifest::new(&schema, &[(&record, &bundle)]).expect("capture manifest"); + + assert_eq!(manifest.version(), CAPTURE_MANIFEST_VERSION); + assert_eq!(manifest.schema_version(), "catalog-v1"); + assert!(manifest.schema_digest().starts_with("sha256:")); + assert_eq!(manifest.software_commit_sha(), SOFTWARE_COMMIT_SHA); + assert_eq!(manifest.records().len(), 1); + assert_eq!(manifest.records()[0].warc_record_id(), RECORD_ID_A); + assert!(manifest.records()[0].warc_record_digest().starts_with("sha256:")); + assert!(manifest.records()[0].prov_json_ld_digest().starts_with("sha256:")); + assert!(manifest.manifest_digest().starts_with("sha256:")); + assert_eq!(manifest.verify(&schema, &[(&record, &bundle)]), Ok(())); + + let json = manifest.to_json(); + assert!(json.contains("\"manifestVersion\":1")); + assert!(json.contains("\"schemaVersion\":\"catalog-v1\"")); + assert!(json.contains(RECORD_ID_A)); + assert!(!json.contains("secret-like-captured-payload")); + assert!(!json.contains("https://example.com/item")); + assert!(!json.contains(SOURCE_HASH)); +} + +#[test] +fn capture_manifest_is_order_independent_and_rejects_empty_duplicate_or_oversized_sets() { + let schema = schema("catalog-v1"); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); + + let first = CaptureManifest::new( + &schema, + &[(&record_b, &bundle_b), (&record_a, &bundle_a)], + ) + .expect("first manifest"); + let second = CaptureManifest::new( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + ) + .expect("second manifest"); + assert_eq!(first, second); + assert_eq!(first.records()[0].warc_record_id(), RECORD_ID_A); + assert_eq!(first.records()[1].warc_record_id(), RECORD_ID_B); + + assert_eq!( + CaptureManifest::new(&schema, &[]), + Err(CaptureManifestError::MissingRecord) + ); + assert_eq!( + CaptureManifest::new( + &schema, + &[(&record_a, &bundle_a), (&record_a, &bundle_a)], + ), + Err(CaptureManifestError::DuplicateRecord) + ); + + let oversized = vec![(&record_a, &bundle_a); MAX_CAPTURE_MANIFEST_RECORDS + 1]; + assert_eq!( + CaptureManifest::new(&schema, &oversized), + Err(CaptureManifestError::LimitExceeded) + ); +} + +#[test] +fn capture_manifest_rejects_mismatched_bundle_or_mixed_software_revision() { + let schema = schema("catalog-v1"); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b_other = WarcProvBundle::new(&record_b, OTHER_SOFTWARE_COMMIT_SHA) + .expect("PROV B with other software"); + + assert_eq!( + CaptureManifest::new(&schema, &[(&record_b, &bundle_a)]), + Err(CaptureManifestError::BundleMismatch( + WarcProvBundleVerificationError::RecordIdentityMismatch, + )) + ); + assert_eq!( + CaptureManifest::new( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b_other)], + ), + Err(CaptureManifestError::SoftwareRevisionMismatch) + ); +} + +#[test] +fn capture_manifest_verification_fails_closed_on_schema_or_record_drift() { + let original_schema = schema("catalog-v1"); + let record = resource_record(RECORD_ID_A, b"original"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let manifest = CaptureManifest::new(&original_schema, &[(&record, &bundle)]) + .expect("capture manifest"); + + assert_eq!( + manifest.verify(&schema("catalog-v2"), &[(&record, &bundle)]), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); + + let changed_record = resource_record(RECORD_ID_A, b"changed"); + let changed_bundle = WarcProvBundle::new(&changed_record, SOFTWARE_COMMIT_SHA) + .expect("changed PROV bundle"); + assert_eq!( + manifest.verify(&original_schema, &[(&changed_record, &changed_bundle)]), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); + + let other_record = resource_record(RECORD_ID_B, b"other"); + assert_eq!( + manifest.verify(&original_schema, &[(&other_record, &bundle)]), + Err(CaptureManifestVerificationError::InvalidCandidate( + CaptureManifestError::BundleMismatch( + WarcProvBundleVerificationError::RecordIdentityMismatch, + ), + )) + ); +} From 0e0234e4c5017efc42290b995013203b738c2bdf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:06:47 -0700 Subject: [PATCH 02/44] test(evidence): format capture manifest regression --- .../tests/capture_manifest.rs | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest.rs b/crates/originweave-evidence/tests/capture_manifest.rs index 15e13a75a..af380582f 100644 --- a/crates/originweave-evidence/tests/capture_manifest.rs +++ b/crates/originweave-evidence/tests/capture_manifest.rs @@ -1,11 +1,11 @@ #![allow(clippy::expect_used)] use originweave_evidence::{ - CaptureManifest, CaptureManifestError, CaptureManifestVerificationError, EvidenceSourceKind, - ExtractionCardinality, ExtractionField, ExtractionSchema, ExtractionSourceChannel, - ExtractionValueType, ProvenanceRecord, VerificationResult, WarcProvBundle, - WarcProvBundleVerificationError, WarcResourceRecord, CAPTURE_MANIFEST_VERSION, - MAX_CAPTURE_MANIFEST_RECORDS, + CAPTURE_MANIFEST_VERSION, CaptureManifest, CaptureManifestError, + CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, ExtractionField, + ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, MAX_CAPTURE_MANIFEST_RECORDS, + ProvenanceRecord, VerificationResult, WarcProvBundle, WarcProvBundleVerificationError, + WarcResourceRecord, }; const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -65,8 +65,16 @@ fn capture_manifest_binds_schema_warc_prov_and_software_identity_without_payload assert_eq!(manifest.software_commit_sha(), SOFTWARE_COMMIT_SHA); assert_eq!(manifest.records().len(), 1); assert_eq!(manifest.records()[0].warc_record_id(), RECORD_ID_A); - assert!(manifest.records()[0].warc_record_digest().starts_with("sha256:")); - assert!(manifest.records()[0].prov_json_ld_digest().starts_with("sha256:")); + assert!( + manifest.records()[0] + .warc_record_digest() + .starts_with("sha256:") + ); + assert!( + manifest.records()[0] + .prov_json_ld_digest() + .starts_with("sha256:") + ); assert!(manifest.manifest_digest().starts_with("sha256:")); assert_eq!(manifest.verify(&schema, &[(&record, &bundle)]), Ok(())); @@ -87,16 +95,10 @@ fn capture_manifest_is_order_independent_and_rejects_empty_duplicate_or_oversize let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); - let first = CaptureManifest::new( - &schema, - &[(&record_b, &bundle_b), (&record_a, &bundle_a)], - ) - .expect("first manifest"); - let second = CaptureManifest::new( - &schema, - &[(&record_a, &bundle_a), (&record_b, &bundle_b)], - ) - .expect("second manifest"); + let first = CaptureManifest::new(&schema, &[(&record_b, &bundle_b), (&record_a, &bundle_a)]) + .expect("first manifest"); + let second = CaptureManifest::new(&schema, &[(&record_a, &bundle_a), (&record_b, &bundle_b)]) + .expect("second manifest"); assert_eq!(first, second); assert_eq!(first.records()[0].warc_record_id(), RECORD_ID_A); assert_eq!(first.records()[1].warc_record_id(), RECORD_ID_B); @@ -106,10 +108,7 @@ fn capture_manifest_is_order_independent_and_rejects_empty_duplicate_or_oversize Err(CaptureManifestError::MissingRecord) ); assert_eq!( - CaptureManifest::new( - &schema, - &[(&record_a, &bundle_a), (&record_a, &bundle_a)], - ), + CaptureManifest::new(&schema, &[(&record_a, &bundle_a), (&record_a, &bundle_a)],), Err(CaptureManifestError::DuplicateRecord) ); @@ -149,8 +148,8 @@ fn capture_manifest_verification_fails_closed_on_schema_or_record_drift() { let original_schema = schema("catalog-v1"); let record = resource_record(RECORD_ID_A, b"original"); let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); - let manifest = CaptureManifest::new(&original_schema, &[(&record, &bundle)]) - .expect("capture manifest"); + let manifest = + CaptureManifest::new(&original_schema, &[(&record, &bundle)]).expect("capture manifest"); assert_eq!( manifest.verify(&schema("catalog-v2"), &[(&record, &bundle)]), @@ -158,8 +157,8 @@ fn capture_manifest_verification_fails_closed_on_schema_or_record_drift() { ); let changed_record = resource_record(RECORD_ID_A, b"changed"); - let changed_bundle = WarcProvBundle::new(&changed_record, SOFTWARE_COMMIT_SHA) - .expect("changed PROV bundle"); + let changed_bundle = + WarcProvBundle::new(&changed_record, SOFTWARE_COMMIT_SHA).expect("changed PROV bundle"); assert_eq!( manifest.verify(&original_schema, &[(&changed_record, &changed_bundle)]), Err(CaptureManifestVerificationError::IdentityMismatch) From 81abac8f492aba4dc46b075411be04aa24d59b55 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:11:30 -0700 Subject: [PATCH 03/44] feat(evidence): bind deterministic capture manifest identity --- .../src/capture_manifest.rs | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 crates/originweave-evidence/src/capture_manifest.rs diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs new file mode 100644 index 000000000..a3e3164b4 --- /dev/null +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -0,0 +1,335 @@ +//! Deterministic identity manifests for schema-bound WARC/PROV capture evidence. +//! +//! A capture manifest binds one reviewed extraction-schema contract to exact WARC and PROV +//! serialization identities plus one immutable OriginWeave software revision. It intentionally +//! contains no captured payload, source URL, source locator, credential, browser authority, +//! persistence authority, retention decision, signature, or release authority. + +use std::{collections::BTreeMap, fmt}; + +use sha2::{Digest, Sha256}; + +use crate::{ + ExtractionCardinality, ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, + ExtractionValueType, WarcProvBundle, WarcProvBundleVerificationError, WarcResourceRecord, +}; + +/// Version of the deterministic OriginWeave capture-manifest serialization contract. +pub const CAPTURE_MANIFEST_VERSION: u16 = 1; +/// Maximum number of WARC/PROV pairs admitted by one capture manifest. +pub const MAX_CAPTURE_MANIFEST_RECORDS: usize = 256; + +/// A validation failure while constructing one deterministic capture manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureManifestError { + /// A capture manifest must bind at least one WARC/PROV pair. + MissingRecord, + /// The number of supplied records exceeded the bounded manifest limit. + LimitExceeded, + /// More than one supplied pair used the same WARC record identifier. + DuplicateRecord, + /// A supplied PROV bundle did not exactly verify its paired WARC record. + BundleMismatch(WarcProvBundleVerificationError), + /// Supplied PROV bundles referred to different OriginWeave software revisions. + SoftwareRevisionMismatch, +} + +impl fmt::Display for CaptureManifestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingRecord => formatter.write_str("capture manifest requires at least one record"), + Self::LimitExceeded => formatter.write_str("capture manifest record limit exceeded"), + Self::DuplicateRecord => formatter.write_str("capture manifest contains a duplicate WARC record"), + Self::BundleMismatch(error) => write!(formatter, "capture manifest WARC/PROV mismatch: {error}"), + Self::SoftwareRevisionMismatch => formatter.write_str( + "capture manifest records do not share one OriginWeave software revision", + ), + } + } +} + +impl std::error::Error for CaptureManifestError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::BundleMismatch(error) => Some(error), + Self::MissingRecord + | Self::LimitExceeded + | Self::DuplicateRecord + | Self::SoftwareRevisionMismatch => None, + } + } +} + +/// A deterministic offline verification failure for a previously constructed capture manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureManifestVerificationError { + /// A valid candidate manifest did not have the same immutable identity as the expected manifest. + IdentityMismatch, + /// The candidate inputs did not form a valid manifest at all. + InvalidCandidate(CaptureManifestError), +} + +impl fmt::Display for CaptureManifestVerificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::IdentityMismatch => formatter.write_str("capture manifest identity does not match"), + Self::InvalidCandidate(error) => { + write!(formatter, "invalid capture manifest verification candidate: {error}") + } + } + } +} + +impl std::error::Error for CaptureManifestVerificationError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::IdentityMismatch => None, + Self::InvalidCandidate(error) => Some(error), + } + } +} + +/// Payload-free immutable identity for one WARC/PROV pair in a [`CaptureManifest`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CaptureManifestRecord { + warc_record_id: String, + warc_record_digest: String, + prov_json_ld_digest: String, +} + +impl CaptureManifestRecord { + /// Return the UUID URN of the WARC record bound by this entry. + #[must_use] + pub fn warc_record_id(&self) -> &str { + &self.warc_record_id + } + + /// Return the SHA-256 digest of the complete deterministic WARC serialization. + #[must_use] + pub fn warc_record_digest(&self) -> &str { + &self.warc_record_digest + } + + /// Return the SHA-256 digest of the deterministic PROV JSON-LD serialization. + #[must_use] + pub fn prov_json_ld_digest(&self) -> &str { + &self.prov_json_ld_digest + } +} + +/// Deterministic payload-free identity binding a schema to exact WARC/PROV capture evidence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CaptureManifest { + version: u16, + schema_version: String, + schema_digest: String, + software_commit_sha: String, + records: Vec, +} + +impl CaptureManifest { + /// Construct one bounded deterministic capture manifest. + /// + /// Every supplied PROV bundle must verify the paired WARC record exactly and all pairs must + /// name the same canonical OriginWeave software revision. Caller order is not identity: + /// records are canonicalized by WARC record identifier before serialization and comparison. + /// This constructor performs no capture, network, browser, persistence, model, signing, or + /// authorization operation. + pub fn new( + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + ) -> Result { + if records.is_empty() { + return Err(CaptureManifestError::MissingRecord); + } + if records.len() > MAX_CAPTURE_MANIFEST_RECORDS { + return Err(CaptureManifestError::LimitExceeded); + } + + let software_commit_sha = records[0].1.software_commit_sha(); + let mut canonical_records = BTreeMap::new(); + for (record, bundle) in records { + bundle + .verify_record(record) + .map_err(CaptureManifestError::BundleMismatch)?; + if bundle.software_commit_sha() != software_commit_sha { + return Err(CaptureManifestError::SoftwareRevisionMismatch); + } + + let manifest_record = CaptureManifestRecord { + warc_record_id: record.record_id().to_owned(), + warc_record_digest: sha256_digest(&record.to_warc_bytes()), + prov_json_ld_digest: sha256_digest(bundle.to_json_ld().as_bytes()), + }; + if canonical_records + .insert(record.record_id().to_owned(), manifest_record) + .is_some() + { + return Err(CaptureManifestError::DuplicateRecord); + } + } + + Ok(Self { + version: CAPTURE_MANIFEST_VERSION, + schema_version: schema.version().to_owned(), + schema_digest: extraction_schema_digest(schema), + software_commit_sha: software_commit_sha.to_owned(), + records: canonical_records.into_values().collect(), + }) + } + + /// Return the capture-manifest serialization-contract version. + #[must_use] + pub const fn version(&self) -> u16 { + self.version + } + + /// Return the extraction-schema version identifier bound by this manifest. + #[must_use] + pub fn schema_version(&self) -> &str { + &self.schema_version + } + + /// Return a SHA-256 digest of the complete ordered extraction-schema contract. + #[must_use] + pub fn schema_digest(&self) -> &str { + &self.schema_digest + } + + /// Return the canonical lower-case OriginWeave Git SHA-1 shared by every PROV bundle. + #[must_use] + pub fn software_commit_sha(&self) -> &str { + &self.software_commit_sha + } + + /// Return manifest entries in canonical WARC-record-identifier order. + #[must_use] + pub fn records(&self) -> &[CaptureManifestRecord] { + &self.records + } + + /// Serialize the manifest deterministically without captured payloads or source locations. + #[must_use] + pub fn to_json(&self) -> String { + let records = self + .records + .iter() + .map(|record| { + format!( + "{{\"warcRecordId\":\"{}\",\"warcRecordDigest\":\"{}\",\"provJsonLdDigest\":\"{}\"}}", + record.warc_record_id, record.warc_record_digest, record.prov_json_ld_digest + ) + }) + .collect::>() + .join(","); + format!( + "{{\"manifestVersion\":{},\"schemaVersion\":\"{}\",\"schemaDigest\":\"{}\",\"softwareCommitSha\":\"{}\",\"records\":[{}]}}", + self.version, + self.schema_version, + self.schema_digest, + self.software_commit_sha, + records + ) + } + + /// Return the SHA-256 identity of the deterministic manifest serialization. + #[must_use] + pub fn manifest_digest(&self) -> String { + sha256_digest(self.to_json().as_bytes()) + } + + /// Reconstruct a candidate manifest offline and require exact immutable identity equality. + /// + /// Malformed candidate inputs remain distinguishable from a valid-but-different manifest. + /// Verification performs no network, browser, persistence, model, signing, or authority action. + pub fn verify( + &self, + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + ) -> Result<(), CaptureManifestVerificationError> { + let candidate = Self::new(schema, records) + .map_err(CaptureManifestVerificationError::InvalidCandidate)?; + if candidate == *self { + Ok(()) + } else { + Err(CaptureManifestVerificationError::IdentityMismatch) + } + } +} + +fn extraction_schema_digest(schema: &ExtractionSchema) -> String { + let mut hasher = Sha256::new(); + update_length_prefixed(&mut hasher, b"originweave:capture-schema:v1"); + update_length_prefixed(&mut hasher, schema.version().as_bytes()); + hasher.update((schema.fields().len() as u64).to_be_bytes()); + for field in schema.fields() { + update_length_prefixed(&mut hasher, field.identifier().as_bytes()); + update_length_prefixed(&mut hasher, extraction_value_type_token(field.value_type())); + update_length_prefixed(&mut hasher, extraction_cardinality_token(field.cardinality())); + hasher.update([u8::from(field.required())]); + update_length_prefixed( + &mut hasher, + extraction_normalization_token(field.normalization_rule()), + ); + hasher.update((field.source_channels().len() as u64).to_be_bytes()); + for channel in field.source_channels() { + update_length_prefixed(&mut hasher, extraction_source_channel_token(*channel)); + } + } + encode_sha256(hasher.finalize()) +} + +fn update_length_prefixed(hasher: &mut Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +const fn extraction_value_type_token(value_type: ExtractionValueType) -> &'static [u8] { + match value_type { + ExtractionValueType::Text => b"text", + ExtractionValueType::Integer => b"integer", + ExtractionValueType::Decimal => b"decimal", + ExtractionValueType::Boolean => b"boolean", + ExtractionValueType::Timestamp => b"timestamp", + } +} + +const fn extraction_cardinality_token(cardinality: ExtractionCardinality) -> &'static [u8] { + match cardinality { + ExtractionCardinality::One => b"one", + ExtractionCardinality::ZeroOrOne => b"zero_or_one", + ExtractionCardinality::Many => b"many", + } +} + +const fn extraction_normalization_token( + normalization_rule: ExtractionNormalizationRule, +) -> &'static [u8] { + match normalization_rule { + ExtractionNormalizationRule::Verbatim => b"verbatim", + ExtractionNormalizationRule::TrimTextWhitespace => b"trim_text_whitespace", + ExtractionNormalizationRule::Rfc3339Utc => b"rfc3339_utc", + } +} + +const fn extraction_source_channel_token(channel: ExtractionSourceChannel) -> &'static [u8] { + match channel { + ExtractionSourceChannel::SemanticNode => b"semantic_node", + ExtractionSourceChannel::StructuredData => b"structured_data", + ExtractionSourceChannel::TableCell => b"table_cell", + ExtractionSourceChannel::NetworkResponse => b"network_response", + ExtractionSourceChannel::ModelInterpretation => b"model_interpretation", + } +} + +fn sha256_digest(bytes: &[u8]) -> String { + encode_sha256(Sha256::digest(bytes)) +} + +fn encode_sha256(digest: impl IntoIterator) -> String { + let mut encoded = String::from("sha256:"); + for byte in digest { + encoded.push_str(&format!("{byte:02x}")); + } + encoded +} From 9d712acadeb8ffb22c2bd745fbcd0e0e3bb8c92a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:13:12 -0700 Subject: [PATCH 04/44] feat(evidence): export capture manifest contract --- crates/originweave-evidence/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 313f629d4..d2b5e2bc0 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -7,11 +7,16 @@ #![forbid(unsafe_code)] #![deny(missing_docs)] +mod capture_manifest; mod extraction_schema; mod sensitive_access; mod warc_prov_bundle; mod warc_resource_record; +pub use capture_manifest::{ + CAPTURE_MANIFEST_VERSION, CaptureManifest, CaptureManifestError, CaptureManifestRecord, + CaptureManifestVerificationError, MAX_CAPTURE_MANIFEST_RECORDS, +}; pub use extraction_schema::{ ExtractionCardinality, ExtractionField, ExtractionNormalizationRule, ExtractionSchema, ExtractionSchemaError, ExtractionSourceChannel, ExtractionValueType, From 921ef27534b04a7f6396f8470fc9c982fbe69902 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:15:54 -0700 Subject: [PATCH 05/44] style(evidence): apply canonical capture manifest rustfmt --- .../src/capture_manifest.rs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index a3e3164b4..db2ccfcff 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -37,10 +37,16 @@ pub enum CaptureManifestError { impl fmt::Display for CaptureManifestError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::MissingRecord => formatter.write_str("capture manifest requires at least one record"), + Self::MissingRecord => { + formatter.write_str("capture manifest requires at least one record") + } Self::LimitExceeded => formatter.write_str("capture manifest record limit exceeded"), - Self::DuplicateRecord => formatter.write_str("capture manifest contains a duplicate WARC record"), - Self::BundleMismatch(error) => write!(formatter, "capture manifest WARC/PROV mismatch: {error}"), + Self::DuplicateRecord => { + formatter.write_str("capture manifest contains a duplicate WARC record") + } + Self::BundleMismatch(error) => { + write!(formatter, "capture manifest WARC/PROV mismatch: {error}") + } Self::SoftwareRevisionMismatch => formatter.write_str( "capture manifest records do not share one OriginWeave software revision", ), @@ -72,9 +78,14 @@ pub enum CaptureManifestVerificationError { impl fmt::Display for CaptureManifestVerificationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::IdentityMismatch => formatter.write_str("capture manifest identity does not match"), + Self::IdentityMismatch => { + formatter.write_str("capture manifest identity does not match") + } Self::InvalidCandidate(error) => { - write!(formatter, "invalid capture manifest verification candidate: {error}") + write!( + formatter, + "invalid capture manifest verification candidate: {error}" + ) } } } @@ -265,7 +276,10 @@ fn extraction_schema_digest(schema: &ExtractionSchema) -> String { for field in schema.fields() { update_length_prefixed(&mut hasher, field.identifier().as_bytes()); update_length_prefixed(&mut hasher, extraction_value_type_token(field.value_type())); - update_length_prefixed(&mut hasher, extraction_cardinality_token(field.cardinality())); + update_length_prefixed( + &mut hasher, + extraction_cardinality_token(field.cardinality()), + ); hasher.update([u8::from(field.required())]); update_length_prefixed( &mut hasher, From c78d462b2facb4c525cb6349ac290120390afcab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:20:49 -0700 Subject: [PATCH 06/44] test(evidence): cover complete capture manifest identity contract --- .../tests/capture_manifest.rs | 121 +++++++++++++++++- 1 file changed, 118 insertions(+), 3 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest.rs b/crates/originweave-evidence/tests/capture_manifest.rs index af380582f..d9f3d97bf 100644 --- a/crates/originweave-evidence/tests/capture_manifest.rs +++ b/crates/originweave-evidence/tests/capture_manifest.rs @@ -3,9 +3,9 @@ use originweave_evidence::{ CAPTURE_MANIFEST_VERSION, CaptureManifest, CaptureManifestError, CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, ExtractionField, - ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, MAX_CAPTURE_MANIFEST_RECORDS, - ProvenanceRecord, VerificationResult, WarcProvBundle, WarcProvBundleVerificationError, - WarcResourceRecord, + ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, + MAX_CAPTURE_MANIFEST_RECORDS, ProvenanceRecord, VerificationResult, WarcProvBundle, + WarcProvBundleVerificationError, WarcResourceRecord, }; const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -27,6 +27,59 @@ fn schema(version: &str) -> ExtractionSchema { ExtractionSchema::new(version, vec![field]).expect("schema contract") } +fn schema_covering_all_semantics(version: &str) -> ExtractionSchema { + let text = ExtractionField::new_with_normalization( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + ExtractionNormalizationRule::TrimTextWhitespace, + &[ + ExtractionSourceChannel::SemanticNode, + ExtractionSourceChannel::StructuredData, + ExtractionSourceChannel::TableCell, + ExtractionSourceChannel::NetworkResponse, + ExtractionSourceChannel::ModelInterpretation, + ], + ) + .expect("text field"); + let integer = ExtractionField::new( + "quantity", + ExtractionValueType::Integer, + ExtractionCardinality::ZeroOrOne, + false, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("integer field"); + let decimal = ExtractionField::new( + "price", + ExtractionValueType::Decimal, + ExtractionCardinality::Many, + true, + &[ExtractionSourceChannel::StructuredData], + ) + .expect("decimal field"); + let boolean = ExtractionField::new( + "available", + ExtractionValueType::Boolean, + ExtractionCardinality::Many, + false, + &[ExtractionSourceChannel::TableCell], + ) + .expect("boolean field"); + let timestamp = ExtractionField::new_with_normalization( + "captured_at", + ExtractionValueType::Timestamp, + ExtractionCardinality::One, + true, + ExtractionNormalizationRule::Rfc3339Utc, + &[ExtractionSourceChannel::SemanticNode], + ) + .expect("timestamp field"); + ExtractionSchema::new(version, vec![text, integer, decimal, boolean, timestamp]) + .expect("complete semantic schema") +} + fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { let provenance = ProvenanceRecord::new( "https://example.com/item", @@ -87,6 +140,68 @@ fn capture_manifest_binds_schema_warc_prov_and_software_identity_without_payload assert!(!json.contains(SOURCE_HASH)); } +#[test] +fn capture_manifest_schema_digest_binds_every_typed_schema_dimension() { + let record = resource_record(RECORD_ID_A, b"schema-sensitive"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let complete_schema = schema_covering_all_semantics("catalog-semantic-v1"); + let minimal_schema = schema("catalog-semantic-v1"); + + let complete = CaptureManifest::new(&complete_schema, &[(&record, &bundle)]) + .expect("complete manifest"); + let minimal = + CaptureManifest::new(&minimal_schema, &[(&record, &bundle)]).expect("minimal manifest"); + + assert_ne!(complete.schema_digest(), minimal.schema_digest()); + assert_ne!(complete, minimal); +} + +#[test] +fn capture_manifest_error_contracts_preserve_typed_causes() { + let missing = CaptureManifestError::MissingRecord; + assert_eq!( + missing.to_string(), + "capture manifest requires at least one record" + ); + assert!(std::error::Error::source(&missing).is_none()); + + let limit = CaptureManifestError::LimitExceeded; + assert_eq!(limit.to_string(), "capture manifest record limit exceeded"); + assert!(std::error::Error::source(&limit).is_none()); + + let duplicate = CaptureManifestError::DuplicateRecord; + assert_eq!( + duplicate.to_string(), + "capture manifest contains a duplicate WARC record" + ); + assert!(std::error::Error::source(&duplicate).is_none()); + + let software = CaptureManifestError::SoftwareRevisionMismatch; + assert_eq!( + software.to_string(), + "capture manifest records do not share one OriginWeave software revision" + ); + assert!(std::error::Error::source(&software).is_none()); + + let mismatch = CaptureManifestError::BundleMismatch( + WarcProvBundleVerificationError::RecordIdentityMismatch, + ); + assert!(mismatch.to_string().contains("WARC/PROV mismatch")); + assert!(std::error::Error::source(&mismatch).is_some()); + + let identity = CaptureManifestVerificationError::IdentityMismatch; + assert_eq!(identity.to_string(), "capture manifest identity does not match"); + assert!(std::error::Error::source(&identity).is_none()); + + let invalid = CaptureManifestVerificationError::InvalidCandidate(mismatch); + assert!( + invalid + .to_string() + .starts_with("invalid capture manifest verification candidate:") + ); + assert!(std::error::Error::source(&invalid).is_some()); +} + #[test] fn capture_manifest_is_order_independent_and_rejects_empty_duplicate_or_oversized_sets() { let schema = schema("catalog-v1"); From df70d92b72ea8a4d39e6b9cfea7f3ae8b7b58536 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:05:24 -0700 Subject: [PATCH 07/44] test(evidence): apply canonical rustfmt to capture manifest regression --- crates/originweave-evidence/tests/capture_manifest.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest.rs b/crates/originweave-evidence/tests/capture_manifest.rs index d9f3d97bf..711fa97d6 100644 --- a/crates/originweave-evidence/tests/capture_manifest.rs +++ b/crates/originweave-evidence/tests/capture_manifest.rs @@ -147,8 +147,8 @@ fn capture_manifest_schema_digest_binds_every_typed_schema_dimension() { let complete_schema = schema_covering_all_semantics("catalog-semantic-v1"); let minimal_schema = schema("catalog-semantic-v1"); - let complete = CaptureManifest::new(&complete_schema, &[(&record, &bundle)]) - .expect("complete manifest"); + let complete = + CaptureManifest::new(&complete_schema, &[(&record, &bundle)]).expect("complete manifest"); let minimal = CaptureManifest::new(&minimal_schema, &[(&record, &bundle)]).expect("minimal manifest"); @@ -190,7 +190,10 @@ fn capture_manifest_error_contracts_preserve_typed_causes() { assert!(std::error::Error::source(&mismatch).is_some()); let identity = CaptureManifestVerificationError::IdentityMismatch; - assert_eq!(identity.to_string(), "capture manifest identity does not match"); + assert_eq!( + identity.to_string(), + "capture manifest identity does not match" + ); assert!(std::error::Error::source(&identity).is_none()); let invalid = CaptureManifestVerificationError::InvalidCandidate(mismatch); From 3b16e2245fb7564449eac1cb36ad9adc36410ca7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:09:49 -0700 Subject: [PATCH 08/44] test(evidence): require exact serialized capture manifest identity --- .../tests/capture_manifest_serialized.rs | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 crates/originweave-evidence/tests/capture_manifest_serialized.rs diff --git a/crates/originweave-evidence/tests/capture_manifest_serialized.rs b/crates/originweave-evidence/tests/capture_manifest_serialized.rs new file mode 100644 index 000000000..e483693e4 --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_serialized.rs @@ -0,0 +1,58 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, + ExtractionField, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, + ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, +}; + +const SOURCE_HASH: &str = + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn manifest() -> CaptureManifest { + let field = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("field contract"); + let schema = ExtractionSchema::new("catalog-v1", vec![field]).expect("schema contract"); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + let record = WarcResourceRecord::new( + RECORD_ID, + "2026-08-24T00:00:00Z", + "https://example.com/item", + "text/plain", + b"captured-payload".to_vec(), + provenance, + ) + .expect("WARC resource record"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + CaptureManifest::new(&schema, &[(&record, &bundle)]).expect("capture manifest") +} + +#[test] +fn persisted_capture_manifest_requires_exact_deterministic_serialization() { + let manifest = manifest(); + let exact = manifest.to_json().into_bytes(); + + assert_eq!(manifest.verify_serialized_json(&exact), Ok(())); + + let mut drifted = exact.clone(); + drifted.push(b'\n'); + assert_eq!( + manifest.verify_serialized_json(&drifted), + Err(CaptureManifestVerificationError::IdentityMismatch) + ); +} From 57788f271b9d74362f55d538c953ec58539d0c66 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:13:08 -0700 Subject: [PATCH 09/44] test(evidence): format serialized manifest regression --- .../originweave-evidence/tests/capture_manifest_serialized.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_serialized.rs b/crates/originweave-evidence/tests/capture_manifest_serialized.rs index e483693e4..bb9dbef24 100644 --- a/crates/originweave-evidence/tests/capture_manifest_serialized.rs +++ b/crates/originweave-evidence/tests/capture_manifest_serialized.rs @@ -6,8 +6,7 @@ use originweave_evidence::{ ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, }; -const SOURCE_HASH: &str = - "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; From 140278fddee89c4200dce04b61ee47fb115b74f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:14:31 -0700 Subject: [PATCH 10/44] feat(evidence): verify exact serialized capture manifest identity --- .../src/capture_manifest.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index db2ccfcff..df90505ad 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -249,6 +249,24 @@ impl CaptureManifest { sha256_digest(self.to_json().as_bytes()) } + /// Require candidate bytes to be the exact deterministic serialization of this manifest. + /// + /// This verification deliberately does not parse or normalize JSON: whitespace, member-order, + /// encoding, or other serialization drift is an identity mismatch even when a generic JSON + /// parser could assign equivalent data semantics. It performs no network, browser, persistence, + /// model, signing, or authority action and does not authenticate the producer of either value. + pub fn verify_serialized_json( + &self, + candidate: &[u8], + ) -> Result<(), CaptureManifestVerificationError> { + let expected = self.to_json(); + if candidate == expected.as_bytes() { + Ok(()) + } else { + Err(CaptureManifestVerificationError::IdentityMismatch) + } + } + /// Reconstruct a candidate manifest offline and require exact immutable identity equality. /// /// Malformed candidate inputs remain distinguishable from a valid-but-different manifest. From 1e2ce3d4071a1a75ee891bdcd71c506b3b50d4bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:16:19 -0700 Subject: [PATCH 11/44] docs(changelog): record exact capture-manifest byte verification --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a91615822..f9448e88a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ All notable changes to OriginWeave are documented in this file. The format follo - Universally value-redacted network evidence with explicit path, metadata, and provenance bounds; ambiguous path rejection; validated source URLs; lowercase SHA-256 identifiers; and verification state. - Versioned schema-bound extraction contracts with bounded identifiers and field counts, typed value/cardinality metadata, explicit duplicate-free reviewed source channels, fail-closed schema validation, and deterministic `Display`/`std::error::Error` contracts for public schema failures. - Deterministic W3C PROV-O JSON-LD projection over validated WARC resource records, binding exact source and WARC digests, capture time, and OriginWeave software revision while preserving complete-versus-truncated payload state and exact standard WARC truncation reason without embedding raw payload bytes, with deterministic offline exact-record verification that fails closed on identity, source-evidence, timestamp, payload-digest, completeness, or serialized-record drift. +- Deterministic payload-free capture manifests that bind the complete extraction-schema contract, exact WARC and PROV serialization identities, and one OriginWeave software revision, including exact persisted-byte verification that rejects any JSON serialization drift rather than parsing or normalizing it. - Rust 1.97.1 build contract, strict Clippy and rustdoc gates, and exact production function, line, region, and branch coverage enforcement. - Hourly bounded OpenCode product-development workflow using `NVIDIA_NIM_API_KEY`, an unprivileged disposable workspace, loopback-only model broker, independently verified patches, and publication through a dedicated `OPENCODE_PR_TOKEN` that cannot review or merge. - Architecture, agent, security, contribution, research, database naming, roadmap, quality-gate, and TLS service-identity ADR documentation. From 16fb56db4ee6bf9aed9e8da46e041340cf9de2c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 03:16:06 -0700 Subject: [PATCH 12/44] docs(changelog): record capture manifest contract --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8412f3ddc..64f36d08c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to OriginWeave are documented in this file. The format follo ### Added +- Deterministic payload-free capture manifests bind the complete extraction-schema contract, exact WARC and PROV serialization identities, and one OriginWeave software revision, with exact persisted-byte verification that rejects JSON serialization drift rather than parsing or normalizing it. - Refreshed the product and technical gap baseline with the 2026-08-24 live inventory: 158 open pull requests (44 ready, 114 draft), refreshed exact base/head evidence for the #208–#222 release, enterprise-approval, BAP, and WARC/PROV chains, the governance issue additions #212 and #215, and a required-check provider-failure record for the fail-closed Strix re-dispatches on #208/#218/#220. - Added a dated product and technical gap baseline that separates protected-main implementation truth, active pull-request evidence, live review/check blockers, and the next buyer-visible Phase 1 acceptance work. - Refreshed the product and technical gap baseline with the current open-PR inventory and exact base/head evidence for the newest Chromium, BAP, extraction, WARC, and idempotency slices. From 16a320d0aaea5367b354b070a1ac55d8a9086280 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:08:08 -0700 Subject: [PATCH 13/44] test(evidence): require WARC-backed manifest values --- .../tests/capture_manifest_values.rs | 225 ++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 crates/originweave-evidence/tests/capture_manifest_values.rs diff --git a/crates/originweave-evidence/tests/capture_manifest_values.rs b/crates/originweave-evidence/tests/capture_manifest_values.rs new file mode 100644 index 000000000..4b26d28e3 --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_values.rs @@ -0,0 +1,225 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, EvidenceSourceKind, + ExtractionCardinality, ExtractionField, ExtractionSchema, ExtractionSourceChannel, + ExtractionValueType, MAX_CAPTURE_MANIFEST_VALUES, ProvenanceRecord, VerificationResult, + WarcProvBundle, WarcResourceRecord, +}; + +const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const VALUE_HASH_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const VALUE_HASH_B: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const RECORD_ID_A: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const RECORD_ID_B: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; +const DATE: &str = "2026-08-25T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn schema() -> ExtractionSchema { + let title = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("title field"); + let tags = ExtractionField::new( + "tags", + ExtractionValueType::Text, + ExtractionCardinality::Many, + false, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("tags field"); + ExtractionSchema::new("catalog-v2", vec![title, tags]).expect("schema") +} + +fn semantic_only_schema() -> ExtractionSchema { + let title = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::SemanticNode], + ) + .expect("semantic-only field"); + ExtractionSchema::new("semantic-v1", vec![title]).expect("schema") +} + +fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + SOURCE_HASH, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + record_id, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + ) + .expect("WARC record") +} + +#[test] +fn capture_manifest_binds_required_warc_backed_structured_values() { + let schema = schema(); + let record = resource_record(RECORD_ID_A, b"captured-payload"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let title = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A) + .expect("value binding"); + + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&title), + ) + .expect("manifest with value"); + + assert_eq!(manifest.values(), std::slice::from_ref(&title)); + assert_eq!(manifest.values()[0].field_name(), "title"); + assert_eq!(manifest.values()[0].value_digest(), VALUE_HASH_A); + assert_eq!(manifest.values()[0].source_warc_record_id(), RECORD_ID_A); + assert_eq!( + manifest.verify_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&title), + ), + Ok(()) + ); + + let json = manifest.to_json(); + assert!(json.contains("\"values\":[")); + assert!(json.contains("\"fieldName\":\"title\"")); + assert!(json.contains(VALUE_HASH_A)); + assert!(json.contains(RECORD_ID_A)); + assert!(!json.contains("captured-payload")); +} + +#[test] +fn capture_manifest_value_admission_fails_closed() { + assert_eq!( + CaptureManifestValueBinding::new("Title", VALUE_HASH_A, RECORD_ID_A), + Err(CaptureManifestError::InvalidValueField) + ); + assert_eq!( + CaptureManifestValueBinding::new("title", "sha256:ABC", RECORD_ID_A), + Err(CaptureManifestError::InvalidValueDigest) + ); + + let schema = schema(); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); + let title_a = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A) + .expect("title A"); + let title_b = CaptureManifestValueBinding::new("title", VALUE_HASH_B, RECORD_ID_B) + .expect("title B"); + let unknown = CaptureManifestValueBinding::new("missing", VALUE_HASH_A, RECORD_ID_A) + .expect("syntactically valid unknown field"); + let missing_record = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_B) + .expect("syntactically valid missing record"); + + assert_eq!( + CaptureManifest::new_with_warc_values(&schema, &[(&record_a, &bundle_a)], &[]), + Err(CaptureManifestError::RequiredValueMissing) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a)], + std::slice::from_ref(&unknown), + ), + Err(CaptureManifestError::UnknownValueField) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a)], + std::slice::from_ref(&missing_record), + ), + Err(CaptureManifestError::ValueSourceRecordMissing) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &semantic_only_schema(), + &[(&record_a, &bundle_a)], + std::slice::from_ref(&title_a), + ), + Err(CaptureManifestError::ValueSourceChannelMismatch) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + &[title_a.clone(), title_b], + ), + Err(CaptureManifestError::ValueCardinalityExceeded) + ); + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a)], + &[title_a.clone(), title_a.clone()], + ), + Err(CaptureManifestError::DuplicateValue) + ); + + let oversized = vec![title_a; MAX_CAPTURE_MANIFEST_VALUES + 1]; + assert_eq!( + CaptureManifest::new_with_warc_values(&schema, &[(&record_a, &bundle_a)], &oversized), + Err(CaptureManifestError::ValueLimitExceeded) + ); +} + +#[test] +fn capture_manifest_value_order_is_canonical_and_verification_detects_drift() { + let schema = schema(); + let record_a = resource_record(RECORD_ID_A, b"a"); + let record_b = resource_record(RECORD_ID_B, b"b"); + let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); + let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); + let title = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A) + .expect("title"); + let tag_a = CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_A) + .expect("tag A"); + let tag_b = CaptureManifestValueBinding::new("tags", VALUE_HASH_B, RECORD_ID_B) + .expect("tag B"); + + let first = CaptureManifest::new_with_warc_values( + &schema, + &[(&record_b, &bundle_b), (&record_a, &bundle_a)], + &[tag_b.clone(), title.clone(), tag_a.clone()], + ) + .expect("first manifest"); + let second = CaptureManifest::new_with_warc_values( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + &[tag_a, tag_b.clone(), title], + ) + .expect("second manifest"); + + assert_eq!(first, second); + assert_eq!(first.values()[0].field_name(), "tags"); + assert_eq!(first.values()[1].field_name(), "tags"); + assert_eq!(first.values()[2].field_name(), "title"); + + let drifted_tag = CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_B) + .expect("drifted tag"); + assert_eq!( + first.verify_with_warc_values( + &schema, + &[(&record_a, &bundle_a), (&record_b, &bundle_b)], + &[drifted_tag, tag_b], + ), + Err(originweave_evidence::CaptureManifestVerificationError::IdentityMismatch) + ); +} From e6eaacce05af5b669eeb876a18b8f38cf016d680 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:11:01 -0700 Subject: [PATCH 14/44] test(evidence): format manifest value regressions --- .../tests/capture_manifest_values.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_values.rs b/crates/originweave-evidence/tests/capture_manifest_values.rs index 4b26d28e3..a8dcaabe1 100644 --- a/crates/originweave-evidence/tests/capture_manifest_values.rs +++ b/crates/originweave-evidence/tests/capture_manifest_values.rs @@ -8,8 +8,10 @@ use originweave_evidence::{ }; const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; -const VALUE_HASH_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; -const VALUE_HASH_B: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const VALUE_HASH_A: &str = + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const VALUE_HASH_B: &str = + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; const RECORD_ID_A: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; const RECORD_ID_B: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; const DATE: &str = "2026-08-25T00:00:00Z"; @@ -119,10 +121,10 @@ fn capture_manifest_value_admission_fails_closed() { let record_b = resource_record(RECORD_ID_B, b"b"); let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); - let title_a = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A) - .expect("title A"); - let title_b = CaptureManifestValueBinding::new("title", VALUE_HASH_B, RECORD_ID_B) - .expect("title B"); + let title_a = + CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A).expect("title A"); + let title_b = + CaptureManifestValueBinding::new("title", VALUE_HASH_B, RECORD_ID_B).expect("title B"); let unknown = CaptureManifestValueBinding::new("missing", VALUE_HASH_A, RECORD_ID_A) .expect("syntactically valid unknown field"); let missing_record = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_B) @@ -187,12 +189,10 @@ fn capture_manifest_value_order_is_canonical_and_verification_detects_drift() { let record_b = resource_record(RECORD_ID_B, b"b"); let bundle_a = WarcProvBundle::new(&record_a, SOFTWARE_COMMIT_SHA).expect("PROV A"); let bundle_b = WarcProvBundle::new(&record_b, SOFTWARE_COMMIT_SHA).expect("PROV B"); - let title = CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A) - .expect("title"); - let tag_a = CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_A) - .expect("tag A"); - let tag_b = CaptureManifestValueBinding::new("tags", VALUE_HASH_B, RECORD_ID_B) - .expect("tag B"); + let title = + CaptureManifestValueBinding::new("title", VALUE_HASH_A, RECORD_ID_A).expect("title"); + let tag_a = CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_A).expect("tag A"); + let tag_b = CaptureManifestValueBinding::new("tags", VALUE_HASH_B, RECORD_ID_B).expect("tag B"); let first = CaptureManifest::new_with_warc_values( &schema, @@ -203,7 +203,7 @@ fn capture_manifest_value_order_is_canonical_and_verification_detects_drift() { let second = CaptureManifest::new_with_warc_values( &schema, &[(&record_a, &bundle_a), (&record_b, &bundle_b)], - &[tag_a, tag_b.clone(), title], + &[tag_a, tag_b.clone(), title.clone()], ) .expect("second manifest"); @@ -212,13 +212,13 @@ fn capture_manifest_value_order_is_canonical_and_verification_detects_drift() { assert_eq!(first.values()[1].field_name(), "tags"); assert_eq!(first.values()[2].field_name(), "title"); - let drifted_tag = CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_B) - .expect("drifted tag"); + let drifted_tag = + CaptureManifestValueBinding::new("tags", VALUE_HASH_A, RECORD_ID_B).expect("drifted tag"); assert_eq!( first.verify_with_warc_values( &schema, &[(&record_a, &bundle_a), (&record_b, &bundle_b)], - &[drifted_tag, tag_b], + &[drifted_tag, tag_b, title], ), Err(originweave_evidence::CaptureManifestVerificationError::IdentityMismatch) ); From e550dc68b90bc402deb628bf8f72e65c84899ae9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:13:39 -0700 Subject: [PATCH 15/44] feat(evidence): bind WARC-backed manifest values --- .../src/capture_manifest.rs | 259 +++++++++++++++++- 1 file changed, 250 insertions(+), 9 deletions(-) diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index df90505ad..d7203a0d3 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -5,19 +5,22 @@ //! contains no captured payload, source URL, source locator, credential, browser authority, //! persistence authority, retention decision, signature, or release authority. -use std::{collections::BTreeMap, fmt}; +use std::{collections::BTreeMap, collections::BTreeSet, fmt}; use sha2::{Digest, Sha256}; use crate::{ ExtractionCardinality, ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, - ExtractionValueType, WarcProvBundle, WarcProvBundleVerificationError, WarcResourceRecord, + ExtractionValueType, MAX_EXTRACTION_IDENTIFIER_BYTES, WarcProvBundle, + WarcProvBundleVerificationError, WarcResourceRecord, }; /// Version of the deterministic OriginWeave capture-manifest serialization contract. pub const CAPTURE_MANIFEST_VERSION: u16 = 1; /// Maximum number of WARC/PROV pairs admitted by one capture manifest. pub const MAX_CAPTURE_MANIFEST_RECORDS: usize = 256; +/// Maximum number of structured-value digest bindings admitted by one capture manifest. +pub const MAX_CAPTURE_MANIFEST_VALUES: usize = 1_024; /// A validation failure while constructing one deterministic capture manifest. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -32,6 +35,24 @@ pub enum CaptureManifestError { BundleMismatch(WarcProvBundleVerificationError), /// Supplied PROV bundles referred to different OriginWeave software revisions. SoftwareRevisionMismatch, + /// The number of structured values exceeded the bounded manifest limit. + ValueLimitExceeded, + /// A structured-value field identifier did not use the extraction-schema identifier grammar. + InvalidValueField, + /// A structured-value digest was not a canonical lowercase SHA-256 identifier. + InvalidValueDigest, + /// A structured-value field was not declared by the bound extraction schema. + UnknownValueField, + /// A structured value referenced a WARC record absent from the manifest. + ValueSourceRecordMissing, + /// A structured value used WARC evidence for a field that did not admit network-response evidence. + ValueSourceChannelMismatch, + /// The same field, value digest, and source WARC record were supplied more than once. + DuplicateValue, + /// A structured-value field exceeded its declared extraction cardinality. + ValueCardinalityExceeded, + /// A required extraction field had no structured-value binding. + RequiredValueMissing, } impl fmt::Display for CaptureManifestError { @@ -50,6 +71,30 @@ impl fmt::Display for CaptureManifestError { Self::SoftwareRevisionMismatch => formatter.write_str( "capture manifest records do not share one OriginWeave software revision", ), + Self::ValueLimitExceeded => { + formatter.write_str("capture manifest structured-value limit exceeded") + } + Self::InvalidValueField => { + formatter.write_str("capture manifest structured-value field is invalid") + } + Self::InvalidValueDigest => formatter + .write_str("capture manifest structured-value digest is not canonical SHA-256"), + Self::UnknownValueField => formatter + .write_str("capture manifest structured-value field is absent from the schema"), + Self::ValueSourceRecordMissing => formatter.write_str( + "capture manifest structured value references an absent WARC record", + ), + Self::ValueSourceChannelMismatch => formatter.write_str( + "capture manifest structured value is not admitted by the field source channels", + ), + Self::DuplicateValue => { + formatter.write_str("capture manifest contains a duplicate structured value") + } + Self::ValueCardinalityExceeded => formatter + .write_str("capture manifest structured value exceeds field cardinality"), + Self::RequiredValueMissing => { + formatter.write_str("capture manifest is missing a required structured value") + } } } } @@ -61,7 +106,16 @@ impl std::error::Error for CaptureManifestError { Self::MissingRecord | Self::LimitExceeded | Self::DuplicateRecord - | Self::SoftwareRevisionMismatch => None, + | Self::SoftwareRevisionMismatch + | Self::ValueLimitExceeded + | Self::InvalidValueField + | Self::InvalidValueDigest + | Self::UnknownValueField + | Self::ValueSourceRecordMissing + | Self::ValueSourceChannelMismatch + | Self::DuplicateValue + | Self::ValueCardinalityExceeded + | Self::RequiredValueMissing => None, } } } @@ -128,6 +182,57 @@ impl CaptureManifestRecord { } } +/// Payload-free binding from one schema field digest to its exact source WARC record. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct CaptureManifestValueBinding { + field_name: String, + value_digest: String, + source_warc_record_id: String, +} + +impl CaptureManifestValueBinding { + /// Validate one credential-safe structured-value identity binding. + /// + /// The binding carries only a schema field identifier, canonical value digest, and WARC record + /// identifier. The referenced record and field authority are validated when the binding is + /// admitted into a [`CaptureManifest`]. Raw extracted values are never stored here. + pub fn new( + field_name: &str, + value_digest: &str, + source_warc_record_id: &str, + ) -> Result { + if !valid_value_field_name(field_name) { + return Err(CaptureManifestError::InvalidValueField); + } + if !valid_sha256(value_digest) { + return Err(CaptureManifestError::InvalidValueDigest); + } + Ok(Self { + field_name: field_name.to_owned(), + value_digest: value_digest.to_owned(), + source_warc_record_id: source_warc_record_id.to_owned(), + }) + } + + /// Return the extraction-schema field identifier. + #[must_use] + pub fn field_name(&self) -> &str { + &self.field_name + } + + /// Return the canonical lowercase SHA-256 digest of the extracted value bytes. + #[must_use] + pub fn value_digest(&self) -> &str { + &self.value_digest + } + + /// Return the exact WARC record identifier that supplied this value. + #[must_use] + pub fn source_warc_record_id(&self) -> &str { + &self.source_warc_record_id + } +} + /// Deterministic payload-free identity binding a schema to exact WARC/PROV capture evidence. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CaptureManifest { @@ -136,16 +241,19 @@ pub struct CaptureManifest { schema_digest: String, software_commit_sha: String, records: Vec, + values: Vec, } impl CaptureManifest { - /// Construct one bounded deterministic capture manifest. + /// Construct one bounded deterministic evidence-only capture manifest. /// /// Every supplied PROV bundle must verify the paired WARC record exactly and all pairs must /// name the same canonical OriginWeave software revision. Caller order is not identity: /// records are canonicalized by WARC record identifier before serialization and comparison. - /// This constructor performs no capture, network, browser, persistence, model, signing, or - /// authorization operation. + /// This low-level constructor does not assert that required extraction fields have values; + /// use [`Self::new_with_warc_values`] for a schema-conforming structured-result manifest. + /// It performs no capture, network, browser, persistence, model, signing, or authorization + /// operation. pub fn new( schema: &ExtractionSchema, records: &[(&WarcResourceRecord, &WarcProvBundle)], @@ -186,9 +294,83 @@ impl CaptureManifest { schema_digest: extraction_schema_digest(schema), software_commit_sha: software_commit_sha.to_owned(), records: canonical_records.into_values().collect(), + values: Vec::new(), }) } + /// Construct a schema-conforming manifest with WARC-backed structured-value identities. + /// + /// Every value must name a declared schema field that admits network-response evidence and an + /// exact WARC record present in this manifest. Required fields must be present; `One` and + /// `ZeroOrOne` fields admit at most one value. Duplicate bindings and over-limit collections + /// fail closed. Values are canonicalized independently of caller order. No raw extracted value + /// is retained and no browser, network, persistence, secret, model, or authorization operation + /// is performed. + pub fn new_with_warc_values( + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + values: &[CaptureManifestValueBinding], + ) -> Result { + let mut manifest = Self::new(schema, records)?; + if values.len() > MAX_CAPTURE_MANIFEST_VALUES { + return Err(CaptureManifestError::ValueLimitExceeded); + } + + let mut seen_values = BTreeSet::new(); + let mut field_counts: BTreeMap<&str, usize> = BTreeMap::new(); + for value in values { + let Some(field) = schema.field(value.field_name()) else { + return Err(CaptureManifestError::UnknownValueField); + }; + if !manifest + .records + .iter() + .any(|record| record.warc_record_id() == value.source_warc_record_id()) + { + return Err(CaptureManifestError::ValueSourceRecordMissing); + } + if !field + .source_channels() + .contains(&ExtractionSourceChannel::NetworkResponse) + { + return Err(CaptureManifestError::ValueSourceChannelMismatch); + } + if !seen_values.insert(( + value.field_name(), + value.value_digest(), + value.source_warc_record_id(), + )) { + return Err(CaptureManifestError::DuplicateValue); + } + + let count = field_counts.entry(value.field_name()).or_default(); + *count += 1; + if *count > 1 + && matches!( + field.cardinality(), + ExtractionCardinality::One | ExtractionCardinality::ZeroOrOne + ) + { + return Err(CaptureManifestError::ValueCardinalityExceeded); + } + } + + if schema.fields().iter().any(|field| { + field.required() + && field_counts + .get(field.identifier()) + .copied() + .unwrap_or_default() + == 0 + }) { + return Err(CaptureManifestError::RequiredValueMissing); + } + + manifest.values = values.to_vec(); + manifest.values.sort(); + Ok(manifest) + } + /// Return the capture-manifest serialization-contract version. #[must_use] pub const fn version(&self) -> u16 { @@ -219,6 +401,12 @@ impl CaptureManifest { &self.records } + /// Return structured-value identities in canonical field/digest/source-record order. + #[must_use] + pub fn values(&self) -> &[CaptureManifestValueBinding] { + &self.values + } + /// Serialize the manifest deterministically without captured payloads or source locations. #[must_use] pub fn to_json(&self) -> String { @@ -233,13 +421,25 @@ impl CaptureManifest { }) .collect::>() .join(","); + let values = self + .values + .iter() + .map(|value| { + format!( + "{{\"fieldName\":\"{}\",\"valueDigest\":\"{}\",\"sourceWarcRecordId\":\"{}\"}}", + value.field_name, value.value_digest, value.source_warc_record_id + ) + }) + .collect::>() + .join(","); format!( - "{{\"manifestVersion\":{},\"schemaVersion\":\"{}\",\"schemaDigest\":\"{}\",\"softwareCommitSha\":\"{}\",\"records\":[{}]}}", + "{{\"manifestVersion\":{},\"schemaVersion\":\"{}\",\"schemaDigest\":\"{}\",\"softwareCommitSha\":\"{}\",\"records\":[{}],\"values\":[{}]}}", self.version, self.schema_version, self.schema_digest, self.software_commit_sha, - records + records, + values ) } @@ -267,7 +467,7 @@ impl CaptureManifest { } } - /// Reconstruct a candidate manifest offline and require exact immutable identity equality. + /// Reconstruct a candidate evidence-only manifest and require exact immutable identity equality. /// /// Malformed candidate inputs remain distinguishable from a valid-but-different manifest. /// Verification performs no network, browser, persistence, model, signing, or authority action. @@ -284,6 +484,47 @@ impl CaptureManifest { Err(CaptureManifestVerificationError::IdentityMismatch) } } + + /// Reconstruct a candidate WARC-backed structured-result manifest and require exact identity. + /// + /// Candidate schema, WARC/PROV evidence, field admission, cardinality, requiredness, source + /// identity, and value digests are all revalidated before immutable manifest equality is tested. + pub fn verify_with_warc_values( + &self, + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + values: &[CaptureManifestValueBinding], + ) -> Result<(), CaptureManifestVerificationError> { + let candidate = Self::new_with_warc_values(schema, records, values) + .map_err(CaptureManifestVerificationError::InvalidCandidate)?; + if candidate == *self { + Ok(()) + } else { + Err(CaptureManifestVerificationError::IdentityMismatch) + } + } +} + +fn valid_value_field_name(field_name: &str) -> bool { + if field_name.len() > MAX_EXTRACTION_IDENTIFIER_BYTES { + return false; + } + let mut bytes = field_name.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + first.is_ascii_lowercase() + && bytes.all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'_' | b'-')) +} + +fn valid_sha256(value_digest: &str) -> bool { + let Some(hex_digest) = value_digest.strip_prefix("sha256:") else { + return false; + }; + hex_digest.len() == 64 + && hex_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } fn extraction_schema_digest(schema: &ExtractionSchema) -> String { From a47a11b5996dc03e227e0fb16e9735d639c26c69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:14:42 -0700 Subject: [PATCH 16/44] feat(evidence): export manifest value contract --- crates/originweave-evidence/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 96af9cc4a..0ca683c07 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -15,7 +15,8 @@ mod warc_resource_record; pub use capture_manifest::{ CAPTURE_MANIFEST_VERSION, CaptureManifest, CaptureManifestError, CaptureManifestRecord, - CaptureManifestVerificationError, MAX_CAPTURE_MANIFEST_RECORDS, + CaptureManifestValueBinding, CaptureManifestVerificationError, MAX_CAPTURE_MANIFEST_RECORDS, + MAX_CAPTURE_MANIFEST_VALUES, }; pub use extraction_schema::{ ExtractionCardinality, ExtractionField, ExtractionNormalizationRule, ExtractionSchema, From 2c4d0d3d6f9aa6b75a185f3cfaf03edae271febc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:18:27 -0700 Subject: [PATCH 17/44] style(evidence): apply canonical Rust formatting --- crates/originweave-evidence/src/capture_manifest.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index d7203a0d3..0e593ce67 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -81,17 +81,17 @@ impl fmt::Display for CaptureManifestError { .write_str("capture manifest structured-value digest is not canonical SHA-256"), Self::UnknownValueField => formatter .write_str("capture manifest structured-value field is absent from the schema"), - Self::ValueSourceRecordMissing => formatter.write_str( - "capture manifest structured value references an absent WARC record", - ), + Self::ValueSourceRecordMissing => formatter + .write_str("capture manifest structured value references an absent WARC record"), Self::ValueSourceChannelMismatch => formatter.write_str( "capture manifest structured value is not admitted by the field source channels", ), Self::DuplicateValue => { formatter.write_str("capture manifest contains a duplicate structured value") } - Self::ValueCardinalityExceeded => formatter - .write_str("capture manifest structured value exceeds field cardinality"), + Self::ValueCardinalityExceeded => { + formatter.write_str("capture manifest structured value exceeds field cardinality") + } Self::RequiredValueMissing => { formatter.write_str("capture manifest is missing a required structured value") } From 0b104d7dd7adf128f58a8922ed02fde8bda9b77b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 09:04:01 -0700 Subject: [PATCH 18/44] test(evidence): cover capture manifest value boundaries --- .../tests/capture_manifest_value_edges.rs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 crates/originweave-evidence/tests/capture_manifest_value_edges.rs diff --git a/crates/originweave-evidence/tests/capture_manifest_value_edges.rs b/crates/originweave-evidence/tests/capture_manifest_value_edges.rs new file mode 100644 index 000000000..0a76cdd14 --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_value_edges.rs @@ -0,0 +1,79 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifestError, CaptureManifestValueBinding, MAX_EXTRACTION_IDENTIFIER_BYTES, +}; + +const VALUE_HASH: &str = + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; + +#[test] +fn structured_value_errors_have_deterministic_standard_error_contracts() { + let cases = [ + ( + CaptureManifestError::ValueLimitExceeded, + "capture manifest structured-value limit exceeded", + ), + ( + CaptureManifestError::InvalidValueField, + "capture manifest structured-value field is invalid", + ), + ( + CaptureManifestError::InvalidValueDigest, + "capture manifest structured-value digest is not canonical SHA-256", + ), + ( + CaptureManifestError::UnknownValueField, + "capture manifest structured-value field is absent from the schema", + ), + ( + CaptureManifestError::ValueSourceRecordMissing, + "capture manifest structured value references an absent WARC record", + ), + ( + CaptureManifestError::ValueSourceChannelMismatch, + "capture manifest structured value is not admitted by the field source channels", + ), + ( + CaptureManifestError::DuplicateValue, + "capture manifest contains a duplicate structured value", + ), + ( + CaptureManifestError::ValueCardinalityExceeded, + "capture manifest structured value exceeds field cardinality", + ), + ( + CaptureManifestError::RequiredValueMissing, + "capture manifest is missing a required structured value", + ), + ]; + + for (error, expected_message) in cases { + assert_eq!(error.to_string(), expected_message); + assert!(std::error::Error::source(&error).is_none()); + } +} + +#[test] +fn value_binding_rejects_every_identifier_and_digest_shape_boundary() { + let overlong_field = "a".repeat(MAX_EXTRACTION_IDENTIFIER_BYTES + 1); + for invalid_field in ["", "Title", "title.value", overlong_field.as_str()] { + assert_eq!( + CaptureManifestValueBinding::new(invalid_field, VALUE_HASH, RECORD_ID), + Err(CaptureManifestError::InvalidValueField) + ); + } + + assert_eq!( + CaptureManifestValueBinding::new("title", "not-a-sha256", RECORD_ID), + Err(CaptureManifestError::InvalidValueDigest) + ); + let invalid_hex_digest = format!("sha256:{}", "g".repeat(64)); + assert_eq!( + CaptureManifestValueBinding::new("title", &invalid_hex_digest, RECORD_ID), + Err(CaptureManifestError::InvalidValueDigest) + ); + + assert!(CaptureManifestValueBinding::new("title_1-tag", VALUE_HASH, RECORD_ID).is_ok()); +} From a012cb867ae841bf80a0f4977735a92e3be8ff00 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 09:09:40 -0700 Subject: [PATCH 19/44] test(evidence): close capture value coverage gap --- .../tests/capture_manifest_value_edges.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_value_edges.rs b/crates/originweave-evidence/tests/capture_manifest_value_edges.rs index 0a76cdd14..66cbe07f0 100644 --- a/crates/originweave-evidence/tests/capture_manifest_value_edges.rs +++ b/crates/originweave-evidence/tests/capture_manifest_value_edges.rs @@ -1,11 +1,8 @@ -#![allow(clippy::expect_used)] - use originweave_evidence::{ CaptureManifestError, CaptureManifestValueBinding, MAX_EXTRACTION_IDENTIFIER_BYTES, }; -const VALUE_HASH: &str = - "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const VALUE_HASH: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; #[test] @@ -75,5 +72,7 @@ fn value_binding_rejects_every_identifier_and_digest_shape_boundary() { Err(CaptureManifestError::InvalidValueDigest) ); + let numeric_hex_digest = format!("sha256:{}", "0".repeat(64)); + assert!(CaptureManifestValueBinding::new("title", &numeric_hex_digest, RECORD_ID).is_ok()); assert!(CaptureManifestValueBinding::new("title_1-tag", VALUE_HASH, RECORD_ID).is_ok()); } From 649e30e19c3b9356061ef806d2b8e9b9b6e3ba8e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 09:15:09 -0700 Subject: [PATCH 20/44] test(evidence): cover manifest invalid-candidate propagation --- .../tests/capture_manifest_values.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_values.rs b/crates/originweave-evidence/tests/capture_manifest_values.rs index a8dcaabe1..334466b42 100644 --- a/crates/originweave-evidence/tests/capture_manifest_values.rs +++ b/crates/originweave-evidence/tests/capture_manifest_values.rs @@ -1,10 +1,10 @@ #![allow(clippy::expect_used)] use originweave_evidence::{ - CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, EvidenceSourceKind, - ExtractionCardinality, ExtractionField, ExtractionSchema, ExtractionSourceChannel, - ExtractionValueType, MAX_CAPTURE_MANIFEST_VALUES, ProvenanceRecord, VerificationResult, - WarcProvBundle, WarcResourceRecord, + CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, + CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, ExtractionField, + ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, MAX_CAPTURE_MANIFEST_VALUES, + ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, }; const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -96,6 +96,12 @@ fn capture_manifest_binds_required_warc_backed_structured_values() { ), Ok(()) ); + assert_eq!( + manifest.verify_with_warc_values(&schema, &[], std::slice::from_ref(&title)), + Err(CaptureManifestVerificationError::InvalidCandidate( + CaptureManifestError::MissingRecord, + )) + ); let json = manifest.to_json(); assert!(json.contains("\"values\":[")); @@ -220,6 +226,6 @@ fn capture_manifest_value_order_is_canonical_and_verification_detects_drift() { &[(&record_a, &bundle_a), (&record_b, &bundle_b)], &[drifted_tag, tag_b, title], ), - Err(originweave_evidence::CaptureManifestVerificationError::IdentityMismatch) + Err(CaptureManifestVerificationError::IdentityMismatch) ); } From 7b4ecda39d8075ef3df7074d0ebe29652ff0238e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 23:39:18 -0700 Subject: [PATCH 21/44] test(evidence): bind capture fixtures to retained payload digests --- crates/originweave-evidence/tests/capture_manifest.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest.rs b/crates/originweave-evidence/tests/capture_manifest.rs index 711fa97d6..5ca144f34 100644 --- a/crates/originweave-evidence/tests/capture_manifest.rs +++ b/crates/originweave-evidence/tests/capture_manifest.rs @@ -7,8 +7,8 @@ use originweave_evidence::{ MAX_CAPTURE_MANIFEST_RECORDS, ProvenanceRecord, VerificationResult, WarcProvBundle, WarcProvBundleVerificationError, WarcResourceRecord, }; +use sha2::{Digest, Sha256}; -const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; const RECORD_ID_A: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; const RECORD_ID_B: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174001"; const DATE: &str = "2026-08-24T00:00:00Z"; @@ -81,10 +81,11 @@ fn schema_covering_all_semantics(version: &str) -> ExtractionSchema { } fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); let provenance = ProvenanceRecord::new( "https://example.com/item", "body", - SOURCE_HASH, + &source_hash, EvidenceSourceKind::NetworkResponse, VerificationResult::Verified, ) @@ -137,7 +138,7 @@ fn capture_manifest_binds_schema_warc_prov_and_software_identity_without_payload assert!(json.contains(RECORD_ID_A)); assert!(!json.contains("secret-like-captured-payload")); assert!(!json.contains("https://example.com/item")); - assert!(!json.contains(SOURCE_HASH)); + assert!(!json.contains(record.provenance().source_hash())); } #[test] From a53380d2647f1e0e1cbb4c087ba79675e6056580 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 23:39:48 -0700 Subject: [PATCH 22/44] test(evidence): make serialized manifest fixture provenance-valid --- .../tests/capture_manifest_serialized.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_serialized.rs b/crates/originweave-evidence/tests/capture_manifest_serialized.rs index bb9dbef24..451dcdfaa 100644 --- a/crates/originweave-evidence/tests/capture_manifest_serialized.rs +++ b/crates/originweave-evidence/tests/capture_manifest_serialized.rs @@ -5,8 +5,8 @@ use originweave_evidence::{ ExtractionField, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, }; +use sha2::{Digest, Sha256}; -const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; @@ -20,10 +20,12 @@ fn manifest() -> CaptureManifest { ) .expect("field contract"); let schema = ExtractionSchema::new("catalog-v1", vec![field]).expect("schema contract"); + let payload = b"captured-payload"; + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); let provenance = ProvenanceRecord::new( "https://example.com/item", "body", - SOURCE_HASH, + &source_hash, EvidenceSourceKind::NetworkResponse, VerificationResult::Verified, ) @@ -33,7 +35,7 @@ fn manifest() -> CaptureManifest { "2026-08-24T00:00:00Z", "https://example.com/item", "text/plain", - b"captured-payload".to_vec(), + payload.to_vec(), provenance, ) .expect("WARC resource record"); From d514d5eb1593b12c204061455b12a433906a321f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 23:40:20 -0700 Subject: [PATCH 23/44] test(evidence): bind value fixtures to retained payload digests --- crates/originweave-evidence/tests/capture_manifest_values.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_values.rs b/crates/originweave-evidence/tests/capture_manifest_values.rs index 334466b42..792d9dbc3 100644 --- a/crates/originweave-evidence/tests/capture_manifest_values.rs +++ b/crates/originweave-evidence/tests/capture_manifest_values.rs @@ -6,8 +6,8 @@ use originweave_evidence::{ ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, MAX_CAPTURE_MANIFEST_VALUES, ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, }; +use sha2::{Digest, Sha256}; -const SOURCE_HASH: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; const VALUE_HASH_A: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const VALUE_HASH_B: &str = @@ -50,10 +50,11 @@ fn semantic_only_schema() -> ExtractionSchema { } fn resource_record(record_id: &str, payload: &[u8]) -> WarcResourceRecord { + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); let provenance = ProvenanceRecord::new( "https://example.com/item", "body", - SOURCE_HASH, + &source_hash, EvidenceSourceKind::NetworkResponse, VerificationResult::Verified, ) From 0572dfd2b0767cf9e95e3df2301c4abc21d8d6cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 05:05:38 -0700 Subject: [PATCH 24/44] test(evidence): cover query-bearing provenance URLs --- .../tests/provenance_query_urls.rs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 crates/originweave-evidence/tests/provenance_query_urls.rs diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs new file mode 100644 index 000000000..ad52b5db5 --- /dev/null +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -0,0 +1,73 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceError, EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcResourceRecord, +}; + +const EMPTY_SHA256: &str = + "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-28T00:00:00Z"; + +#[test] +fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclosure() { + let source_url = "https://example.com/search?q=private%20term&access_token=secret"; + let provenance = ProvenanceRecord::new( + source_url, + "body", + EMPTY_SHA256, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("query-bearing provenance URL should be accepted"); + + assert_eq!(provenance.source_url(), source_url); + let provenance_debug = format!("{provenance:?}"); + assert!(!provenance_debug.contains(source_url)); + assert!(!provenance_debug.contains("private%20term")); + assert!(!provenance_debug.contains("access_token")); + assert!(!provenance_debug.contains("secret")); + assert!(!provenance_debug.contains(EMPTY_SHA256)); + assert!(!provenance_debug.contains("body")); + + let record = WarcResourceRecord::new( + RECORD_ID, + DATE, + source_url, + "text/plain", + Vec::new(), + provenance, + ) + .expect("query-bearing WARC target URI should be accepted"); + + assert_eq!(record.target_uri(), source_url); + let warc = String::from_utf8(record.to_warc_bytes()).expect("bounded WARC bytes are UTF-8"); + assert!(warc.contains(&format!("WARC-Target-URI: {source_url}\r\n"))); +} + +#[test] +fn provenance_query_support_does_not_admit_fragments_or_unsafe_uri_octets() { + for source_url in [ + "https://example.com/search?q=value#fragment", + "https://example.com/search#fragment", + "https://example.com/search?q=bad value", + "https://example.com/search?q=bad\\value", + "https://example.com/search?q=raw|pipe", + "https://example.com/search?q=raw-ν•œκΈ€", + "https://example.com/search?q=%", + "https://example.com/search?q=%2", + "https://example.com/search?q=%GG", + ] { + assert_eq!( + ProvenanceRecord::new( + source_url, + "body", + EMPTY_SHA256, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ), + Err(EvidenceError::InvalidSourceUrl), + "source_url={source_url:?}" + ); + } +} From 5474b199a74437bfda8f050e84a7bd2d5cdbe5a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 05:09:23 -0700 Subject: [PATCH 25/44] test(evidence): retire blanket query rejection --- crates/originweave-evidence/tests/evidence.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/originweave-evidence/tests/evidence.rs b/crates/originweave-evidence/tests/evidence.rs index 48d49cbc4..aa415628d 100644 --- a/crates/originweave-evidence/tests/evidence.rs +++ b/crates/originweave-evidence/tests/evidence.rs @@ -126,7 +126,6 @@ fn provenance_rejects_credential_bearing_or_ambiguous_source_urls() { "ftp://example.com/path", "http://example.com/path", "https://user:password@example.com/path", - "https://example.com/path?access_token=secret", "https://example.com/path#fragment", "https://example.com/bad\\path", "https://example.com/\n", From 34936a74c5c5cc022afc296dadf998091bb9b38a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 05:10:11 -0700 Subject: [PATCH 26/44] fix(evidence): preserve safe query-bearing provenance URLs --- crates/originweave-evidence/src/lib.rs | 55 +++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 4ed603600..b8a1aa7c3 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -295,7 +295,7 @@ fn redact_all_values(values: BTreeMap) -> BTreeMap) -> std::fmt::Result { + formatter + .debug_struct("ProvenanceRecord") + .field("source_url_byte_count", &self.source_url.len()) + .field("source_locator_byte_count", &self.source_locator.len()) + .field("source_kind", &self.source_kind) + .field("verification_result", &self.verification_result) + .finish() + } +} + impl ProvenanceRecord { /// Validate and create one provenance record. pub fn new( @@ -372,21 +384,52 @@ fn valid_source_url(source_url: &str) -> bool { || source_url .chars() .any(|character| character.is_control() || character.is_whitespace()) - || source_url.contains(['?', '#', '\\']) + || source_url.contains(['#', '\\']) { return false; } let Some((scheme, remainder)) = source_url.split_once("://") else { return false; }; - let authority_end = remainder.find('/').unwrap_or(remainder.len()); - let authority = &remainder[..authority_end]; + let (hierarchical, query) = remainder + .split_once('?') + .map_or((remainder, None), |(hierarchical, query)| { + (hierarchical, Some(query)) + }); + let authority_end = hierarchical.find('/').unwrap_or(hierarchical.len()); + let authority = &hierarchical[..authority_end]; let origin_text = format!("{scheme}://{authority}"); if Origin::parse(&origin_text).is_err() { return false; } - let path = &remainder[authority_end..]; - path.is_empty() || validate_path(path).is_ok() + let path = &hierarchical[authority_end..]; + if !path.is_empty() && validate_path(path).is_err() { + return false; + } + query.map_or(true, valid_query) +} + +fn valid_query(query: &str) -> bool { + let bytes = query.as_bytes(); + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if byte == b'%' { + if index + 2 >= bytes.len() + || hexadecimal_value(bytes[index + 1]).is_none() + || hexadecimal_value(bytes[index + 2]).is_none() + { + return false; + } + index += 3; + continue; + } + if !is_rfc3986_pchar(byte) && !matches!(byte, b'/' | b'?') { + return false; + } + index += 1; + } + true } fn valid_sha256(source_hash: &str) -> bool { From 5e6671560d13180ff7653271880661d0a64c8c72 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 05:12:46 -0700 Subject: [PATCH 27/44] fix(evidence): satisfy strict query URL lint --- crates/originweave-evidence/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index b8a1aa7c3..569954f82 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -406,7 +406,7 @@ fn valid_source_url(source_url: &str) -> bool { if !path.is_empty() && validate_path(path).is_err() { return false; } - query.map_or(true, valid_query) + query.is_none_or(valid_query) } fn valid_query(query: &str) -> bool { From 2edf09a4a005ad7703da2525c55f45620b296aed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 05:17:13 -0700 Subject: [PATCH 28/44] test(evidence): close query parser branch coverage --- crates/originweave-evidence/tests/provenance_query_urls.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index ad52b5db5..64ba6871e 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -11,7 +11,8 @@ const DATE: &str = "2026-08-28T00:00:00Z"; #[test] fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclosure() { - let source_url = "https://example.com/search?q=private%20term&access_token=secret"; + let source_url = + "https://example.com/search?next=/private?term=private%20term&access_token=secret"; let provenance = ProvenanceRecord::new( source_url, "body", @@ -57,6 +58,7 @@ fn provenance_query_support_does_not_admit_fragments_or_unsafe_uri_octets() { "https://example.com/search?q=%", "https://example.com/search?q=%2", "https://example.com/search?q=%GG", + "https://example.com/search?q=%2G", ] { assert_eq!( ProvenanceRecord::new( From 926529c7b683c05bec6a3f1ba7ba8ae7e8319035 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 05:21:32 -0700 Subject: [PATCH 29/44] test(evidence): reject credential-bearing provenance queries --- .../tests/provenance_query_urls.rs | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index 64ba6871e..ef94e32ef 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -12,7 +12,7 @@ const DATE: &str = "2026-08-28T00:00:00Z"; #[test] fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclosure() { let source_url = - "https://example.com/search?next=/private?term=private%20term&access_token=secret"; + "https://example.com/search?next=/products?category=widgets&q=public%20term"; let provenance = ProvenanceRecord::new( source_url, "body", @@ -25,9 +25,7 @@ fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclo assert_eq!(provenance.source_url(), source_url); let provenance_debug = format!("{provenance:?}"); assert!(!provenance_debug.contains(source_url)); - assert!(!provenance_debug.contains("private%20term")); - assert!(!provenance_debug.contains("access_token")); - assert!(!provenance_debug.contains("secret")); + assert!(!provenance_debug.contains("public%20term")); assert!(!provenance_debug.contains(EMPTY_SHA256)); assert!(!provenance_debug.contains("body")); @@ -46,6 +44,33 @@ fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclo assert!(warc.contains(&format!("WARC-Target-URI: {source_url}\r\n"))); } +#[test] +fn provenance_query_support_rejects_credential_fields() { + for source_url in [ + "https://example.com/callback?access_token=secret", + "https://example.com/callback?ACCESS-TOKEN=secret", + "https://example.com/callback?access%5Ftoken=secret", + "https://example.com/download?api_key=secret", + "https://example.com/download?X-Amz-Signature=secret", + "https://example.com/download?x-goog-credential=secret", + "https://example.com/login?password=secret", + "https://example.com/login?auth=secret", + "https://example.com/login?sig=secret", + ] { + assert_eq!( + ProvenanceRecord::new( + source_url, + "body", + EMPTY_SHA256, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ), + Err(EvidenceError::InvalidSourceUrl), + "credential-bearing source_url={source_url:?}" + ); + } +} + #[test] fn provenance_query_support_does_not_admit_fragments_or_unsafe_uri_octets() { for source_url in [ From 356e063a75336f96f0099d5174b4b1dd3f01001f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:09:11 +0900 Subject: [PATCH 30/44] fix(evidence): reject credential query names --- CHANGELOG.md | 1 + crates/originweave-evidence/src/lib.rs | 51 ++++++++++++++++++- .../tests/provenance_query_urls.rs | 6 ++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 242a69818..3dd099104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ All notable changes to OriginWeave are documented in this file. The format follo ### Security +- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names before query-bearing URLs can be retained or serialized into WARC target metadata. - Explicit proxy server identifiers require ASCII decimal port tokens before numeric range parsing, preventing Rust-specific leading-plus spellings from widening proxy authority. - Raw page content cannot become a trusted instruction. - Raw secrets are rejected and secret-capable actions require an opaque broker handle. diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 569954f82..572906ae0 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -429,7 +429,56 @@ fn valid_query(query: &str) -> bool { } index += 1; } - true + query.split('&').all(|field| { + let name = field.split_once('=').map_or(field, |(name, _value)| name); + !is_credential_query_name(name) + }) +} + +fn is_credential_query_name(name: &str) -> bool { + let bytes = name.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let high = hexadecimal_value(bytes[index + 1]).unwrap_or(0); + let low = hexadecimal_value(bytes[index + 2]).unwrap_or(0); + decoded.push(high * 16 + low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + decoded.make_ascii_lowercase(); + decoded.iter_mut().for_each(|byte| { + if *byte == b'-' { + *byte = b'_'; + } + }); + matches!( + decoded.as_slice(), + b"access_token" + | b"api_key" + | b"auth" + | b"authorization" + | b"client_secret" + | b"credential" + | b"key" + | b"password" + | b"secret" + | b"secret_key" + | b"session" + | b"sig" + | b"signature" + | b"token" + | b"x_api_key" + | b"x_amz_credential" + | b"x_amz_security_token" + | b"x_amz_signature" + | b"x_goog_credential" + | b"x_goog_signature" + ) } fn valid_sha256(source_hash: &str) -> bool { diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index ef94e32ef..9d2cc0335 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -11,8 +11,7 @@ const DATE: &str = "2026-08-28T00:00:00Z"; #[test] fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclosure() { - let source_url = - "https://example.com/search?next=/products?category=widgets&q=public%20term"; + let source_url = "https://example.com/search?next=/products?category=widgets&q=public%20term"; let provenance = ProvenanceRecord::new( source_url, "body", @@ -51,7 +50,10 @@ fn provenance_query_support_rejects_credential_fields() { "https://example.com/callback?ACCESS-TOKEN=secret", "https://example.com/callback?access%5Ftoken=secret", "https://example.com/download?api_key=secret", + "https://example.com/download?client_secret=secret", + "https://example.com/download?X-Amz-Credential=secret", "https://example.com/download?X-Amz-Signature=secret", + "https://example.com/download?X%2Damz%2DSignature=secret", "https://example.com/download?x-goog-credential=secret", "https://example.com/login?password=secret", "https://example.com/login?auth=secret", From 6a0e6ab07233667e95fa49b11653f124dd062749 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:13:29 +0900 Subject: [PATCH 31/44] test(evidence): cover bare query fields --- crates/originweave-evidence/tests/evidence.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/originweave-evidence/tests/evidence.rs b/crates/originweave-evidence/tests/evidence.rs index aa415628d..56532c6c8 100644 --- a/crates/originweave-evidence/tests/evidence.rs +++ b/crates/originweave-evidence/tests/evidence.rs @@ -103,6 +103,7 @@ fn provenance_accepts_safe_root_path_and_loopback_sources() { for source_url in [ "https://example.com", "https://example.com/item/42", + "https://example.com/search?cache", "http://localhost:9222/json/version", "http://[::1]:9222/json/version", ] { From d83748a70bd1b16dbfec46007fe02989ba6ce188 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:26:52 +0900 Subject: [PATCH 32/44] fix(evidence): reject nested credential query values --- CHANGELOG.md | 2 +- crates/originweave-evidence/src/lib.rs | 40 ++++++++++++++++++- .../tests/provenance_query_urls.rs | 2 + docs/doctoring.md | 2 +- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd099104..986c392b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ All notable changes to OriginWeave are documented in this file. The format follo ### Security -- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names before query-bearing URLs can be retained or serialized into WARC target metadata. +- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names, including nested query-like values, before query-bearing URLs can be retained or serialized into WARC target metadata. - Explicit proxy server identifiers require ASCII decimal port tokens before numeric range parsing, preventing Rust-specific leading-plus spellings from widening proxy authority. - Raw page content cannot become a trusted instruction. - Raw secrets are rejected and secret-capable actions require an opaque broker handle. diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 572906ae0..d5d60cc89 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -430,11 +430,42 @@ fn valid_query(query: &str) -> bool { index += 1; } query.split('&').all(|field| { - let name = field.split_once('=').map_or(field, |(name, _value)| name); - !is_credential_query_name(name) + let (name, value) = field + .split_once('=') + .map_or((field, ""), |(name, value)| (name, value)); + !is_credential_query_name(name) && !nested_query_contains_credential(value) }) } +fn nested_query_contains_credential(value: &str) -> bool { + let bytes = value.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let high = hexadecimal_value(bytes[index + 1]).unwrap_or(0); + let low = hexadecimal_value(bytes[index + 2]).unwrap_or(0); + decoded.push(high * 16 + low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + decoded + .split(|byte| *byte == b'?') + .skip(1) + .any(|nested_query| { + nested_query.split(|byte| *byte == b'&').any(|field| { + let name_end = field + .iter() + .position(|byte| *byte == b'=') + .unwrap_or(field.len()); + is_credential_query_name_bytes(&field[..name_end]) + }) + }) +} + fn is_credential_query_name(name: &str) -> bool { let bytes = name.as_bytes(); let mut decoded = Vec::with_capacity(bytes.len()); @@ -450,6 +481,11 @@ fn is_credential_query_name(name: &str) -> bool { index += 1; } } + is_credential_query_name_bytes(&decoded) +} + +fn is_credential_query_name_bytes(decoded: &[u8]) -> bool { + let mut decoded = decoded.to_owned(); decoded.make_ascii_lowercase(); decoded.iter_mut().for_each(|byte| { if *byte == b'-' { diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index 9d2cc0335..c35c863ef 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -58,6 +58,8 @@ fn provenance_query_support_rejects_credential_fields() { "https://example.com/login?password=secret", "https://example.com/login?auth=secret", "https://example.com/login?sig=secret", + "https://example.com/callback?redirect=https://example.com/landing?token=secret", + "https://example.com/callback?redirect=https%3A%2F%2Fexample.com%2Flanding%3Ftoken%3Dsecret", ] { assert_eq!( ProvenanceRecord::new( diff --git a/docs/doctoring.md b/docs/doctoring.md index 92e288fef..9689b0358 100644 --- a/docs/doctoring.md +++ b/docs/doctoring.md @@ -90,7 +90,7 @@ RFC 9309 standardizes robots parsing, matching, error handling, and caching. It W3C PROV-O supplies interoperable Entity, Activity, Agent, derivation, attribution, and responsibility concepts. ISO 28500:2017, confirmed in 2023, defines WARC storage for protocol payloads, control information, metadata, transformations, duplicate detection, integrity, and segmentation. OriginWeave uses source hashes and locators in the safety kernel, then adds WARC and PROV adapters as separately testable modules. -The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. +The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. RFC 3986 permits `?` and `/` within the query component, so OriginWeave keeps safe query-bearing URLs but rejects credential field names after one percent-decoding pass both at the top level and inside nested query-like values; this also follows RFC 9700's prohibition on passing access tokens in URI query parameters. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. The versioned `ExtractionSchema` is an admission and interpretation contract for typed extracted fields: each field is bounded, declares a value type, cardinality, normalization rule, and a canonical duplicate-free set of reviewed source-channel classes. That declaration does not create browser, network, model, secret, storage, retention, disclosure, or governance authority. PROV/WARC interoperability is therefore layered after the schema contract rather than inferred from it. From 83459c49ab803d86b9480774390ce42e95fb0f76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 06:39:52 -0700 Subject: [PATCH 33/44] test(evidence): reject double-encoded nested credential names --- crates/originweave-evidence/tests/provenance_query_urls.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index c35c863ef..2de61b94e 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -60,6 +60,7 @@ fn provenance_query_support_rejects_credential_fields() { "https://example.com/login?sig=secret", "https://example.com/callback?redirect=https://example.com/landing?token=secret", "https://example.com/callback?redirect=https%3A%2F%2Fexample.com%2Flanding%3Ftoken%3Dsecret", + "https://example.com/callback?redirect=https%3A%2F%2Fexample.com%2Flanding%3Faccess%255Ftoken%3Dsecret", ] { assert_eq!( ProvenanceRecord::new( From c8a937678de37b7243291bec5401583eb7b2641f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 06:44:41 -0700 Subject: [PATCH 34/44] fix(evidence): reject recursively encoded nested credential names --- crates/originweave-evidence/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index d5d60cc89..06545c10e 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -461,7 +461,8 @@ fn nested_query_contains_credential(value: &str) -> bool { .iter() .position(|byte| *byte == b'=') .unwrap_or(field.len()); - is_credential_query_name_bytes(&field[..name_end]) + let name = &field[..name_end]; + name.contains(&b'%') || is_credential_query_name_bytes(name) }) }) } From 61243c9f81344084abe9113e0bd1e74344d7001c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 06:50:01 -0700 Subject: [PATCH 35/44] docs(evidence): separate RFC 9700 from credential policy --- docs/doctoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/doctoring.md b/docs/doctoring.md index 9689b0358..7866e2759 100644 --- a/docs/doctoring.md +++ b/docs/doctoring.md @@ -90,7 +90,7 @@ RFC 9309 standardizes robots parsing, matching, error handling, and caching. It W3C PROV-O supplies interoperable Entity, Activity, Agent, derivation, attribution, and responsibility concepts. ISO 28500:2017, confirmed in 2023, defines WARC storage for protocol payloads, control information, metadata, transformations, duplicate detection, integrity, and segmentation. OriginWeave uses source hashes and locators in the safety kernel, then adds WARC and PROV adapters as separately testable modules. -The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. RFC 3986 permits `?` and `/` within the query component, so OriginWeave keeps safe query-bearing URLs but rejects credential field names after one percent-decoding pass both at the top level and inside nested query-like values; this also follows RFC 9700's prohibition on passing access tokens in URI query parameters. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. +The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. RFC 3986 permits `?` and `/` within the query component, so OriginWeave keeps safe query-bearing URLs. RFC 9700 specifically requires clients not to pass access tokens in URI query parameters. Independently, OriginWeave applies a broader evidence-retention policy: decoded credential-like field names are rejected at the top level and inside nested query-like values, and residual nested percent-encoding in a field name fails closed rather than hiding another encoded credential name. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. The versioned `ExtractionSchema` is an admission and interpretation contract for typed extracted fields: each field is bounded, declares a value type, cardinality, normalization rule, and a canonical duplicate-free set of reviewed source-channel classes. That declaration does not create browser, network, model, secret, storage, retention, disclosure, or governance authority. PROV/WARC interoperability is therefore layered after the schema contract rather than inferred from it. From 0341079331f9cea669eb9a5cc21842fd6027431e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:54:49 +0900 Subject: [PATCH 36/44] fix(evidence): reject encoded query controls --- CHANGELOG.md | 2 +- crates/originweave-evidence/src/lib.rs | 17 +++++++++++++---- .../tests/provenance_query_urls.rs | 3 +++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 986c392b5..405cf7659 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ All notable changes to OriginWeave are documented in this file. The format follo ### Security -- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names, including nested query-like values, before query-bearing URLs can be retained or serialized into WARC target metadata. +- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names, including recursively encoded nested query-like values, and percent-encoded controls before query-bearing URLs can be retained or serialized into WARC target metadata. - Explicit proxy server identifiers require ASCII decimal port tokens before numeric range parsing, preventing Rust-specific leading-plus spellings from widening proxy authority. - Raw page content cannot become a trusted instruction. - Raw secrets are rejected and secret-capable actions require an opaque broker handle. diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 06545c10e..5e92bf58f 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -415,10 +415,19 @@ fn valid_query(query: &str) -> bool { while index < bytes.len() { let byte = bytes[index]; if byte == b'%' { - if index + 2 >= bytes.len() - || hexadecimal_value(bytes[index + 1]).is_none() - || hexadecimal_value(bytes[index + 2]).is_none() - { + let Some(high) = bytes + .get(index + 1) + .and_then(|byte| hexadecimal_value(*byte)) + else { + return false; + }; + let Some(low) = bytes + .get(index + 2) + .and_then(|byte| hexadecimal_value(*byte)) + else { + return false; + }; + if (high * 16 + low).is_ascii_control() { return false; } index += 3; diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index 2de61b94e..fe0276acc 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -89,6 +89,9 @@ fn provenance_query_support_does_not_admit_fragments_or_unsafe_uri_octets() { "https://example.com/search?q=%2", "https://example.com/search?q=%GG", "https://example.com/search?q=%2G", + "https://example.com/search?q=%0A", + "https://example.com/search?q=%09", + "https://example.com/search?q=%7F", ] { assert_eq!( ProvenanceRecord::new( From 6ecb3b0a4f7c57af50013f48d0ad09bdb334bb70 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 08:04:35 -0700 Subject: [PATCH 37/44] test(evidence): reject recursively encoded credential names --- crates/originweave-evidence/tests/provenance_query_urls.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs index fe0276acc..a01762e11 100644 --- a/crates/originweave-evidence/tests/provenance_query_urls.rs +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -49,6 +49,7 @@ fn provenance_query_support_rejects_credential_fields() { "https://example.com/callback?access_token=secret", "https://example.com/callback?ACCESS-TOKEN=secret", "https://example.com/callback?access%5Ftoken=secret", + "https://example.com/callback?access%255Ftoken=secret", "https://example.com/download?api_key=secret", "https://example.com/download?client_secret=secret", "https://example.com/download?X-Amz-Credential=secret", From 9f0117cf44e2ee947817924d68e86f266890d6b9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 08:09:36 -0700 Subject: [PATCH 38/44] fix(evidence): reject recursively encoded credential names --- crates/originweave-evidence/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 5e92bf58f..7962ee442 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -491,7 +491,7 @@ fn is_credential_query_name(name: &str) -> bool { index += 1; } } - is_credential_query_name_bytes(&decoded) + decoded.contains(&b'%') || is_credential_query_name_bytes(&decoded) } fn is_credential_query_name_bytes(decoded: &[u8]) -> bool { @@ -535,4 +535,4 @@ fn valid_sha256(source_hash: &str) -> bool { && hex_digest .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) -} +} \ No newline at end of file From bea65643109449d63d367a35b8d9bf327ee7cb2c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 08:11:35 -0700 Subject: [PATCH 39/44] style(evidence): preserve rustfmt newline --- crates/originweave-evidence/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index 7962ee442..b3cca4154 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -535,4 +535,4 @@ fn valid_sha256(source_hash: &str) -> bool { && hex_digest .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) -} \ No newline at end of file +} From 9d8e28f1febdad7353dccdceeb8171da67644f23 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 16:07:43 -0700 Subject: [PATCH 40/44] test(evidence): reject truncated structured-value source --- .../tests/capture_manifest_partial_source.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 crates/originweave-evidence/tests/capture_manifest_partial_source.rs diff --git a/crates/originweave-evidence/tests/capture_manifest_partial_source.rs b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs new file mode 100644 index 000000000..06284ac33 --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs @@ -0,0 +1,68 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestValueBinding, EvidenceSourceKind, ExtractionCardinality, + ExtractionField, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, + ProvenanceRecord, VerificationResult, WarcPayloadCompleteness, WarcProvBundle, + WarcResourceRecord, WarcTruncationReason, +}; +use sha2::{Digest, Sha256}; + +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-29T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; +const VALUE_HASH: &str = + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +#[test] +fn structured_value_rejects_truncated_warc_source() { + let schema = ExtractionSchema::new( + "catalog-v3", + vec![ + ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("title field"), + ], + ) + .expect("schema"); + + let payload = b"partial-response"; + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + &source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + let record = WarcResourceRecord::new_with_completeness( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + WarcPayloadCompleteness::Truncated(WarcTruncationReason::Disconnect), + ) + .expect("truncated WARC evidence remains representable"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID) + .expect("structured value binding"); + + let result = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ); + + assert!( + result.is_err(), + "a structured value must not be promoted from a truncated network source" + ); +} From 74c971b76d1bcd58263975d9f09ca881fd58e1ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 16:08:50 -0700 Subject: [PATCH 41/44] test(evidence): format partial-source regression --- .../tests/capture_manifest_partial_source.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_partial_source.rs b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs index 06284ac33..6a6f160f2 100644 --- a/crates/originweave-evidence/tests/capture_manifest_partial_source.rs +++ b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs @@ -11,8 +11,7 @@ use sha2::{Digest, Sha256}; const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; const DATE: &str = "2026-08-29T00:00:00Z"; const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; -const VALUE_HASH: &str = - "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const VALUE_HASH: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; #[test] fn structured_value_rejects_truncated_warc_source() { From b82940044a1d8934358a86586754ca1a09c3a06b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 16:12:18 -0700 Subject: [PATCH 42/44] fix(evidence): reject truncated structured-value sources --- .../src/capture_manifest.rs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index 0e593ce67..0805c8143 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -11,7 +11,7 @@ use sha2::{Digest, Sha256}; use crate::{ ExtractionCardinality, ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, - ExtractionValueType, MAX_EXTRACTION_IDENTIFIER_BYTES, WarcProvBundle, + ExtractionValueType, MAX_EXTRACTION_IDENTIFIER_BYTES, WarcPayloadCompleteness, WarcProvBundle, WarcProvBundleVerificationError, WarcResourceRecord, }; @@ -45,6 +45,8 @@ pub enum CaptureManifestError { UnknownValueField, /// A structured value referenced a WARC record absent from the manifest. ValueSourceRecordMissing, + /// A structured value referenced a WARC record whose retained payload was truncated. + ValueSourceRecordTruncated, /// A structured value used WARC evidence for a field that did not admit network-response evidence. ValueSourceChannelMismatch, /// The same field, value digest, and source WARC record were supplied more than once. @@ -83,6 +85,9 @@ impl fmt::Display for CaptureManifestError { .write_str("capture manifest structured-value field is absent from the schema"), Self::ValueSourceRecordMissing => formatter .write_str("capture manifest structured value references an absent WARC record"), + Self::ValueSourceRecordTruncated => formatter.write_str( + "capture manifest structured value references a truncated WARC record", + ), Self::ValueSourceChannelMismatch => formatter.write_str( "capture manifest structured value is not admitted by the field source channels", ), @@ -112,6 +117,7 @@ impl std::error::Error for CaptureManifestError { | Self::InvalidValueDigest | Self::UnknownValueField | Self::ValueSourceRecordMissing + | Self::ValueSourceRecordTruncated | Self::ValueSourceChannelMismatch | Self::DuplicateValue | Self::ValueCardinalityExceeded @@ -301,11 +307,11 @@ impl CaptureManifest { /// Construct a schema-conforming manifest with WARC-backed structured-value identities. /// /// Every value must name a declared schema field that admits network-response evidence and an - /// exact WARC record present in this manifest. Required fields must be present; `One` and - /// `ZeroOrOne` fields admit at most one value. Duplicate bindings and over-limit collections - /// fail closed. Values are canonicalized independently of caller order. No raw extracted value - /// is retained and no browser, network, persistence, secret, model, or authorization operation - /// is performed. + /// exact complete WARC record present in this manifest. Required fields must be present; `One` + /// and `ZeroOrOne` fields admit at most one value. Duplicate bindings, truncated source records, + /// and over-limit collections fail closed. Values are canonicalized independently of caller + /// order. No raw extracted value is retained and no browser, network, persistence, secret, + /// model, or authorization operation is performed. pub fn new_with_warc_values( schema: &ExtractionSchema, records: &[(&WarcResourceRecord, &WarcProvBundle)], @@ -322,12 +328,14 @@ impl CaptureManifest { let Some(field) = schema.field(value.field_name()) else { return Err(CaptureManifestError::UnknownValueField); }; - if !manifest - .records + let Some((source_record, _)) = records .iter() - .any(|record| record.warc_record_id() == value.source_warc_record_id()) - { + .find(|(record, _)| record.record_id() == value.source_warc_record_id()) + else { return Err(CaptureManifestError::ValueSourceRecordMissing); + }; + if source_record.completeness() != WarcPayloadCompleteness::Complete { + return Err(CaptureManifestError::ValueSourceRecordTruncated); } if !field .source_channels() From 303da3ba88ad98f0c8a2a7e877172e2ac8e4b251 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 16:12:40 -0700 Subject: [PATCH 43/44] test(evidence): pin truncated-source error contract --- .../tests/capture_manifest_partial_source.rs | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/originweave-evidence/tests/capture_manifest_partial_source.rs b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs index 6a6f160f2..3a57db51d 100644 --- a/crates/originweave-evidence/tests/capture_manifest_partial_source.rs +++ b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs @@ -1,10 +1,10 @@ #![allow(clippy::expect_used)] use originweave_evidence::{ - CaptureManifest, CaptureManifestValueBinding, EvidenceSourceKind, ExtractionCardinality, - ExtractionField, ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, - ProvenanceRecord, VerificationResult, WarcPayloadCompleteness, WarcProvBundle, - WarcResourceRecord, WarcTruncationReason, + CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, EvidenceSourceKind, + ExtractionCardinality, ExtractionField, ExtractionSchema, ExtractionSourceChannel, + ExtractionValueType, ProvenanceRecord, VerificationResult, WarcPayloadCompleteness, + WarcProvBundle, WarcResourceRecord, WarcTruncationReason, }; use sha2::{Digest, Sha256}; @@ -54,14 +54,16 @@ fn structured_value_rejects_truncated_warc_source() { let value = CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID) .expect("structured value binding"); - let result = CaptureManifest::new_with_warc_values( - &schema, - &[(&record, &bundle)], - std::slice::from_ref(&value), + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ), + Err(CaptureManifestError::ValueSourceRecordTruncated) ); - - assert!( - result.is_err(), - "a structured value must not be promoted from a truncated network source" + assert_eq!( + CaptureManifestError::ValueSourceRecordTruncated.to_string(), + "capture manifest structured value references a truncated WARC record" ); } From 299a2bb946a5c0cc1bdc158d473e3138a7ca971c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 16:19:06 -0700 Subject: [PATCH 44/44] style(evidence): apply canonical rustfmt --- crates/originweave-evidence/src/capture_manifest.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index 0805c8143..dfaaa5ee8 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -85,9 +85,8 @@ impl fmt::Display for CaptureManifestError { .write_str("capture manifest structured-value field is absent from the schema"), Self::ValueSourceRecordMissing => formatter .write_str("capture manifest structured value references an absent WARC record"), - Self::ValueSourceRecordTruncated => formatter.write_str( - "capture manifest structured value references a truncated WARC record", - ), + Self::ValueSourceRecordTruncated => formatter + .write_str("capture manifest structured value references a truncated WARC record"), Self::ValueSourceChannelMismatch => formatter.write_str( "capture manifest structured value is not admitted by the field source channels", ),