From 8c6a8a98fe9261404ef5d0a0c27d20a8bdaf9af2 Mon Sep 17 00:00:00 2001 From: Tom Tang <4220945+shiba4life@users.noreply.github.com> Date: Sun, 19 Apr 2026 21:02:27 -0700 Subject: [PATCH 1/2] fix(query): skip unresolvable pre-tag atom refs with warn (alpha BLOCKER 4b171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Alice tags an existing schema with `org_hash` and then writes more molecules, the set-org-hash design intentionally does not migrate pre-tag atom data. Post-tag molecules ship through the org log fine, but a molecule whose ref contains a pre-tag atom uuid ships the ref without the backing atom — leaving a receiver (Bob) with an orphan ref that fails every unfiltered query on the shared schema with `Atom not found for key`. Fix: in `fetch_atoms_with_key_metadata_async_with_org`, when both the org-prefixed and unprefixed atom lookups miss for an org-scoped read, log a warning and skip the ref instead of erroring the whole query. Personal reads keep the strict error so genuine data integrity bugs still surface. Adds two regression tests in `tests/org_key_prefixing_test.rs`: - `test_org_query_skips_unresolvable_pretag_atom_refs_with_warn` — org schema with a simulated sync gap (delete one atom at the Sled level), unfiltered query must succeed and return the resolvable molecules. - `test_personal_query_still_errors_on_missing_atom` — personal schema with a missing atom must still raise `InvalidField`. See gbrain `projects/alpha-e2e-dogfood-run-5` for repro context. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/schema/types/field/filter_utils.rs | 18 +++ tests/org_key_prefixing_test.rs | 152 +++++++++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/src/schema/types/field/filter_utils.rs b/src/schema/types/field/filter_utils.rs index b80cf9c07..591207f32 100644 --- a/src/schema/types/field/filter_utils.rs +++ b/src/schema/types/field/filter_utils.rs @@ -74,6 +74,14 @@ pub async fn fetch_atoms_with_key_metadata_async( /// falls back to the unprefixed (personal) key so atoms written before the /// schema was tagged remain readable. See `docs/designs/org_shared_sync.md` — /// `set-org-hash` does not rewrite pre-existing keys. +/// +/// For org-scoped reads, if BOTH lookups miss we log a warning and SKIP the +/// ref rather than returning an error (alpha BLOCKER 4b171). Rationale: when +/// Alice tags a schema post-ingest, only post-tag atoms ship through the org +/// log; a receiver (Bob) may hold a molecule ref whose pre-tag atom never +/// replayed. Failing the whole query on one orphan ref would make every +/// unfiltered query on a shared schema unusable. Personal reads keep the +/// strict error so genuine data integrity bugs still surface. pub async fn fetch_atoms_with_key_metadata_async_with_org( db_ops: &Arc, matches: impl IntoIterator)>, @@ -117,6 +125,16 @@ pub async fn fetch_atoms_with_key_metadata_async_with_org( } Ok(None) => { let key_str = key.to_string(); + if let Some(org) = org_hash { + log::warn!( + "Skipping unresolvable atom ref '{}' (org='{}', key='{}') — \ + pre-tag molecule ref leaked without atom data (alpha BLOCKER 4b171)", + atom_uuid, + org, + key_str + ); + continue; + } if key_str.is_empty() { return Err(SchemaError::InvalidField(format!( "Atom '{}' not found", diff --git a/tests/org_key_prefixing_test.rs b/tests/org_key_prefixing_test.rs index 16526948f..20c503696 100644 --- a/tests/org_key_prefixing_test.rs +++ b/tests/org_key_prefixing_test.rs @@ -587,3 +587,155 @@ async fn test_pre_tag_molecules_remain_queryable_after_set_org_hash() { all_keys ); } + +// === Unresolvable pre-tag ref leak: graceful skip on unfiltered query === + +/// Alpha BLOCKER 4b171 regression: when a pre-tag molecule's ref replays to a +/// peer via the org log but its backing atom never ships (because +/// `set-org-hash` doesn't migrate pre-tag data), the receiver holds an orphan +/// ref. A previous implementation returned +/// `InvalidField("Atom … not found for key …")` on the entire unfiltered +/// query, rendering every shared-schema query unusable for the receiver. +/// +/// The fix: for org-scoped reads, if an atom is missing at BOTH the +/// org-prefixed and unprefixed keys, log a warning and skip the ref — the +/// query still returns the resolvable molecules. Simulated here by deleting an +/// atom at the Sled level after ingest, which mirrors the on-receiver state +/// without the two-node sync harness. +#[tokio::test] +async fn test_org_query_skips_unresolvable_pretag_atom_refs_with_warn() { + let tmp = tempfile::tempdir().unwrap(); + let db = make_folddb(&tmp).await; + + register_test_org(&db, ORG_HASH); + register_schema(&db, "leaky_notes", Some(ORG_HASH)).await; + + // Write two molecules — both end up fully org-prefixed. + write_mutation(&db, "leaky_notes", "m1", "2026-04-01", "present body").await; + write_mutation(&db, "leaky_notes", "m2", "2026-04-02", "orphan body").await; + + // Simulate the sync gap: remove exactly one org-prefixed atom key while + // leaving its ref intact. This mirrors a receiver that replayed a pre-tag + // ref without the matching atom data. + let pool = db.sled_pool().expect("Expected sled backend"); + let guard = pool.acquire_arc().unwrap(); + let main_tree = guard.db().open_tree("main").unwrap(); + + let atom_prefix = format!("{ORG_HASH}:atom:"); + let atom_keys: Vec> = main_tree + .iter() + .filter_map(|r| r.ok()) + .filter(|(k, _)| k.starts_with(atom_prefix.as_bytes())) + .map(|(k, _)| k.to_vec()) + .collect(); + assert!( + atom_keys.len() >= 2, + "expected at least 2 org-prefixed atom keys before gap simulation, got {}", + atom_keys.len() + ); + // Delete ALL atoms at the org prefix for ONE of the molecules. Pick the + // first atom, figure out its uuid, and delete that uuid's atom at every + // key variant (org-prefixed + unprefixed) so both lookups miss — this is + // the exact on-wire state for a pre-tag-ref leak. + let victim_bytes = atom_keys.first().expect("at least one atom").clone(); + let victim_str = String::from_utf8_lossy(&victim_bytes).to_string(); + let victim_uuid = victim_str + .strip_prefix(&atom_prefix) + .expect("victim key must start with org atom prefix") + .to_string(); + let unprefixed = format!("atom:{}", victim_uuid); + + main_tree + .remove(&victim_bytes) + .expect("remove org-prefixed victim"); + main_tree + .remove(unprefixed.as_bytes()) + .expect("remove unprefixed victim (idempotent if absent)"); + main_tree.flush().expect("flush after remove"); + drop(guard); + + // Unfiltered query must NOT fail — it should skip the unresolvable ref. + let access = AccessContext::owner("test-owner"); + let result = db + .query_executor() + .query_with_access( + Query::new("leaky_notes".to_string(), vec!["body".to_string()]), + &access, + None, + ) + .await + .expect("unfiltered query must gracefully skip orphan atom refs (4b171)"); + + let bodies: Vec = result + .get("body") + .expect("missing body field") + .values() + .map(|fv| fv.value.clone()) + .collect(); + + // At least one body survives — before the fix, the whole query errored + // with `Atom … not found`. After the fix, unresolvable refs are skipped + // with a warn log and the query still returns the resolvable molecules. + assert!( + !bodies.is_empty(), + "expected at least one resolved molecule after gap simulation, got empty" + ); + for b in &bodies { + assert!( + b == &json!("present body") || b == &json!("orphan body"), + "resolved body must be one of the two ingested values, got {:?}", + b + ); + } +} + +/// Personal-mode reads keep the strict error behavior so genuine data +/// integrity bugs still surface. The graceful skip is intentionally scoped to +/// org reads (`org_hash.is_some()`) because the sync gap only exists there. +#[tokio::test] +async fn test_personal_query_still_errors_on_missing_atom() { + let tmp = tempfile::tempdir().unwrap(); + let db = make_folddb(&tmp).await; + + register_schema(&db, "strict_notes", None).await; + write_mutation(&db, "strict_notes", "k1", "2026-04-01", "body").await; + + // Remove the personal atom key at the Sled level. + let pool = db.sled_pool().expect("Expected sled backend"); + let guard = pool.acquire_arc().unwrap(); + let main_tree = guard.db().open_tree("main").unwrap(); + let atom_key: Vec = main_tree + .iter() + .filter_map(|r| r.ok()) + .find(|(k, _)| { + let s = String::from_utf8_lossy(k); + s.starts_with("atom:") + }) + .map(|(k, _)| k.to_vec()) + .expect("expected at least one personal atom key"); + main_tree.remove(&atom_key).unwrap(); + main_tree.flush().expect("flush after remove"); + drop(guard); + + let access = AccessContext::owner("test-owner"); + let err = db + .query_executor() + .query_with_access( + Query::new("strict_notes".to_string(), vec!["body".to_string()]), + &access, + None, + ) + .await + .expect_err("personal query with missing atom must return an error"); + + match err { + fold_db::schema::SchemaError::InvalidField(msg) => { + assert!( + msg.contains("Atom") && msg.contains("not found"), + "expected 'Atom … not found' error, got: {}", + msg + ); + } + other => panic!("expected SchemaError::InvalidField, got {:?}", other), + } +} From c668f1ccc5e695a40543aa4a3e9f1b9fb3787b47 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Sun, 19 Apr 2026 21:11:46 -0700 Subject: [PATCH 2/2] chore(log): strip atom_uuid/org/key from 4b171 warn log (CodeQL cleartext-logging) --- src/schema/types/field/filter_utils.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/schema/types/field/filter_utils.rs b/src/schema/types/field/filter_utils.rs index 591207f32..5a5d957de 100644 --- a/src/schema/types/field/filter_utils.rs +++ b/src/schema/types/field/filter_utils.rs @@ -125,13 +125,10 @@ pub async fn fetch_atoms_with_key_metadata_async_with_org( } Ok(None) => { let key_str = key.to_string(); - if let Some(org) = org_hash { + if org_hash.is_some() { log::warn!( - "Skipping unresolvable atom ref '{}' (org='{}', key='{}') — \ - pre-tag molecule ref leaked without atom data (alpha BLOCKER 4b171)", - atom_uuid, - org, - key_str + "Skipping unresolvable atom ref — pre-tag molecule ref leaked \ + without atom data (alpha BLOCKER 4b171)" ); continue; }