diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index e8e88897..e82f45b4 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -33,6 +33,7 @@ jobs: - cypher_parse - conf_parse - csr_from_bytes + - graph_props_record steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@nightly @@ -82,6 +83,7 @@ jobs: - cypher_parse - conf_parse - csr_from_bytes + - graph_props_record steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@nightly diff --git a/.gitignore b/.gitignore index d0b4eb79..9012afe7 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,7 @@ tmp/ /target-linux/ /target-linux-tokio/ /target-tokio/ +/target-fast/ /target-check-tokio/ /target-check-monoio/ /target-check/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0841f74c..ba709a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -162,6 +162,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 100ms either way; the fixed 15s was a documented host-load flake, observed on the macOS CI runner). +### Changed — graph engine deep-review fixes + optimization waves (PR #TBD) + +- **Freeze-boundary correctness (P0):** CSR v5 segments now persist node + properties, embeddings, and edge weights/properties across freeze; + Cypher reads see the frozen tier via `MergedNodeView` (NodeScan/eval/ + IndexScan); cross-tier delta edges; GRAPH.NEIGHBORS existence + + direction fixes; GRAPH.HYBRID/VSEARCH operate across both tiers. New + red/green `graph_freeze_boundary` suite (11 tests) pins the lifecycle. +- **Point-query indexes (P1):** lazy per-segment property indexes (built + once at first use from freeze-time data) + `PhysicalOp::IndexScan` in + the planner/executor — `MATCH (n:L {p: v})` narrows instead of + label-scanning. +- **Query-path cost (P2):** plan-cache auto-parameterization (literal + variants share one cached plan; cache hits skip parse+compile + entirely) with LRU eviction; slot-indexed executor rows (no per-row + HashMap allocation, no per-insert String clone); write-path/WAL + allocation cleanups (ADDNODE double WAL-encode deleted, itoa/ryu + statement temporaries, FxHash for slotmap-keyed sets, Dijkstra + three-maps-to-one merge, allocation-free neighbor iteration in Cypher + Expand). +- **Traversal engine (P3):** CSR row-space BFS fast path on fully-frozen + graphs — dense-bitmap visited set, TRUE parallel frontier expansion + (thread::scope over Send+Sync CsrStorage), direction-optimizing + Beamer push/pull over the incoming index; `TraversalGuard` wall-clock + budget (30s default) now enforced per hop in Cypher variable-length + Expand and ShortestPath (`ExecErrorKind::Timeout`); GRAPH.HYBRID's + `HnswPreFilter` strategy is real — a lazy per-segment HNSW bridge + (`src/graph/hnsw_bridge.rs`, raw-f32 cosine over v5 embeddings, >= 4096 + vectors) replaces the silent brute-force stub, with exact-scoring + fallback whenever the approximate beam under-fills. +- **Pre-merge review fixes:** (1) restart NodeKey aliasing (P0) — a fresh + post-recovery `MemGraph`'s deterministic SlotMap could mint keys + bit-identical to loaded CSR segments' `external_id`s, silently + shadowing frozen nodes; recovery now seeds the mutable tier via + `MemGraph::with_id_offset` (watermark past the largest persisted id) + and WAL replay dedup-skips AddNodes already resident in a loaded + segment (red/green `graph_restart_id_aliasing` suite). (2) Plan-cache + raw-hash fast path — exact-repeat queries hit the cache without the + `parameterize()` lexer pass, and the `SlotTable` is cached alongside + the plan instead of being rebuilt per execution; cache backing moved + to `FxHashMap`. (3) `CsrStorage::resident_bytes` now counts the v5 + property blobs, lazily-built per-segment property indexes, and the + HNSW bridge (heap-owned; mmap-backed sections count 0 per the + `RawF16Store` precedent), keeping the elastic memory budget honest. + ### Changed — consolidated dependency bumps (PR #TBD) - Cargo: `ringbuf` 0.4.8 → 0.5.0 and `metrics-exporter-prometheus` 0.16.2 → diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0c88582b..628a3a87 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -78,3 +78,8 @@ doc = false [workspace] members = ["."] + +[[bin]] +name = "graph_props_record" +path = "fuzz_targets/graph_props_record.rs" +doc = false diff --git a/src/command/graph/graph_read.rs b/src/command/graph/graph_read.rs index b071ead0..ebb92bc2 100644 --- a/src/command/graph/graph_read.rs +++ b/src/command/graph/graph_read.rs @@ -128,7 +128,7 @@ fn json_to_graph_value(v: &serde_json::Value) -> cypher::executor::Value { } } -/// GRAPH.NEIGHBORS [TYPE ] [DEPTH ] +/// GRAPH.NEIGHBORS [TYPE ] [DEPTH ] [DIRECTION IN|OUT|BOTH] /// /// Returns an array of neighbor nodes/edges as RESP3 Maps. /// Default direction: BOTH (outgoing + incoming). @@ -155,9 +155,10 @@ pub fn graph_neighbors(store: &GraphStore, args: &[Frame]) -> Frame { None => return Frame::Error(Bytes::from_static(b"ERR graph not found")), }; - // Parse optional TYPE and DEPTH arguments. + // Parse optional TYPE, DEPTH, and DIRECTION arguments. let mut edge_type_filter: Option = None; let mut depth: u32 = 1; + let mut direction = Direction::Both; let mut pos = 2; while pos < args.len() { @@ -188,6 +189,22 @@ pub fn graph_neighbors(store: &GraphStore, args: &[Frame]) -> Frame { _ => return Frame::Error(Bytes::from_static(b"ERR invalid DEPTH value")), }; pos += 1; + } else if key.eq_ignore_ascii_case(b"DIRECTION") { + pos += 1; + if pos >= args.len() { + return Frame::Error(Bytes::from_static(b"ERR missing DIRECTION value")); + } + direction = match extract_bulk(&args[pos]) { + Some(d) if d.eq_ignore_ascii_case(b"OUT") => Direction::Outgoing, + Some(d) if d.eq_ignore_ascii_case(b"IN") => Direction::Incoming, + Some(d) if d.eq_ignore_ascii_case(b"BOTH") => Direction::Both, + _ => { + return Frame::Error(Bytes::from_static( + b"ERR invalid DIRECTION value (IN|OUT|BOTH)", + )); + } + }; + pos += 1; } else { pos += 1; } @@ -203,23 +220,22 @@ pub fn graph_neighbors(store: &GraphStore, args: &[Frame]) -> Frame { let memgraph = &graph.write_buf; - // Verify node exists in the mutable write buffer. - // Nodes in CSR segments still have MemGraph entries (freeze copies, doesn't move). - if memgraph.get_node(node_key).is_none() { - return Frame::Error(Bytes::from_static(b"ERR node not found")); - } - - // Build a SegmentMergeReader that sees both MemGraph and immutable CSR segments. let lsn = u64::MAX - 1; // See all live data (MAX-1 because deleted_lsn=MAX means alive). let segments_guard = graph.segments.load(); let csr_segs = &segments_guard.immutable; - let reader = SegmentMergeReader::new( - Some(memgraph), - csr_segs, - Direction::Both, - lsn, - edge_type_filter, - ); + + // Verify the start node exists in EITHER tier — freeze MOVES nodes into + // CSR segments (drains the write buffer), so a memgraph-only check would + // reject every compacted node. + let view = crate::graph::view::MergedNodeView::new(memgraph, csr_segs); + if !view.contains(node_key) { + return Frame::Error(Bytes::from_static(b"ERR node not found")); + } + + // Build a SegmentMergeReader that sees both MemGraph and immutable CSR + // segments, honoring the requested traversal direction. + let reader = + SegmentMergeReader::new(Some(memgraph), csr_segs, direction, lsn, edge_type_filter); // TraversalGuard enforces bounded epoch hold (30s default timeout). let guard = crate::graph::traversal_guard::TraversalGuard::with_default_timeout(lsn); @@ -390,11 +406,117 @@ pub fn graph_list(store: &GraphStore) -> Frame { // GRAPH.QUERY, GRAPH.RO_QUERY, GRAPH.EXPLAIN // --------------------------------------------------------------------------- +/// Literal-normalize the Cypher text for plan-cache keying. +/// +/// Returns the effective text (normalized, or the original when nothing was +/// rewritten), its plan-cache hash, and the auto-extracted parameter values +/// to merge into the user params before execution. +fn normalize_cypher( + cypher_bytes: &[u8], +) -> ( + std::borrow::Cow<'_, [u8]>, + u64, + Vec<(String, cypher::executor::Value)>, +) { + match cypher::parameterize::parameterize(cypher_bytes) { + Some(pq) => { + let hash = cypher::planner::hash_query(&pq.normalized); + (std::borrow::Cow::Owned(pq.normalized), hash, pq.auto_params) + } + None => ( + std::borrow::Cow::Borrowed(cypher_bytes), + cypher::planner::hash_query(cypher_bytes), + Vec::new(), + ), + } +} + +/// Parse the effective (possibly literal-normalized) Cypher text. +/// +/// If the normalized text fails to parse (a rewrite edge case), falls back +/// to the raw text so parse errors reference the user's own query — dropping +/// the auto-params and re-keying the plan cache on the raw hash. The rewrite +/// can therefore never turn a working query into a broken one. +fn parse_effective( + raw: &[u8], + effective: &std::borrow::Cow<'_, [u8]>, + hash: u64, + auto_params: Vec<(String, cypher::executor::Value)>, +) -> Result< + ( + cypher::CypherQuery, + u64, + Vec<(String, cypher::executor::Value)>, + ), + String, +> { + match cypher::parse_cypher(effective) { + Ok(q) => Ok((q, hash, auto_params)), + Err(e) => { + if matches!(effective, std::borrow::Cow::Borrowed(_)) { + return Err(format!("ERR Cypher parse error: {e}")); + } + match cypher::parse_cypher(raw) { + Ok(q) => Ok((q, cypher::planner::hash_query(raw), Vec::new())), + Err(e) => Err(format!("ERR Cypher parse error: {e}")), + } + } + } +} + +/// Execute a compiled read-only plan: params (user + auto-extracted), +/// valid-time, decay, executor, RESP encoding. Shared by GRAPH.QUERY (both +/// handlers) and GRAPH.RO_QUERY. +/// +/// Takes a `SlotTable` explicitly (rather than calling `cypher::executor:: +/// execute`, which rebuilds one every call) so plan-cache hits reuse the +/// `SlotTable` cached alongside the plan (Fix 2 -- one `String` allocation +/// per bound variable, per execution, otherwise). +fn run_read_query( + graph: &crate::graph::store::NamedGraph, + args: &[Frame], + plan: &cypher::PhysicalPlan, + slots: &cypher::executor::SlotTable, + auto_params: Vec<(String, cypher::executor::Value)>, +) -> Frame { + let mut params = parse_params(args); + for (name, value) in auto_params { + params.insert(name, value); + } + let valid_at = parse_valid_at(args); + let decay = match parse_decay(args) { + Ok(d) => d, + Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), + }; + let ctx = cypher::executor::ExecutionContext { + valid_time_as_of: valid_at, + decay, + guard: Some(crate::graph::traversal_guard::TraversalGuard::with_default_timeout(0)), + ..Default::default() + }; + match cypher::executor::execute_with_slots(graph, plan, slots, ¶ms, &ctx) { + Ok(r) => exec_result_to_frame(&r), + Err(e) => { + let msg = format!("ERR Cypher execution error: {e}"); + Frame::Error(Bytes::from(msg)) + } + } +} + /// GRAPH.QUERY /// -/// Parses the Cypher query, compiles to a physical plan, executes it against -/// the named graph, and returns result rows with column headers and statistics. +/// Normalizes literals into auto-parameters, then executes via the plan +/// cache: a cache hit runs with ZERO parse/compile work (the cache holds +/// read-only plans only, so a hit is safe to execute directly). pub fn graph_query(store: &GraphStore, args: &[Frame]) -> Frame { + graph_query_readonly(store, args, false) +} + +/// Shared read-path core for GRAPH.QUERY / GRAPH.RO_QUERY. +/// +/// `reject_writes`: RO_QUERY refuses write clauses with an explicit error; +/// GRAPH.QUERY lets the read-only executor report them (it has no write lock). +fn graph_query_readonly(store: &GraphStore, args: &[Frame], reject_writes: bool) -> Frame { if args.len() < 2 { return Frame::Error(Bytes::from_static( b"ERR wrong number of arguments for 'GRAPH.QUERY' command", @@ -416,54 +538,58 @@ pub fn graph_query(store: &GraphStore, args: &[Frame]) -> Frame { None => return Frame::Error(Bytes::from_static(b"ERR invalid Cypher query")), }; - let query = match cypher::parse_cypher(cypher_bytes) { - Ok(q) => q, - Err(e) => { - let msg = format!("ERR Cypher parse error: {e}"); - return Frame::Error(Bytes::from(msg)); - } - }; + // Raw-hash pre-lookup (Fix 2): an EXACT repeat of a query text we've + // already compiled hits here without ever calling `parameterize()` (a + // full lexer pass + Vec/String allocations) — the raw-hash entry + // carries this exact text's auto_params, cached at insert time. + let raw_hash = cypher::planner::hash_query(cypher_bytes); + if let Some(cached) = graph.plan_cache.lock().get(raw_hash) { + let auto_params = cached.auto_params.as_ref().clone(); + return run_read_query(graph, args, &cached.plan, &cached.slots, auto_params); + } - // Plan cache: check for a cached plan before compiling. - let query_hash = cypher::planner::hash_query(cypher_bytes); - let plan = { - let cache = graph.plan_cache.lock(); - cache.get(query_hash) - }; - let plan = if let Some(cached) = plan { - cached - } else { - let p = match cypher::planner::compile(&query) { - Ok(p) => std::sync::Arc::new(p), - Err(e) => { - let msg = format!("ERR Cypher plan error: {e}"); - return Frame::Error(Bytes::from(msg)); - } + let (effective, query_hash, auto_params) = normalize_cypher(cypher_bytes); + + // Normalized-hash hit ⇒ read-only plan (PlanCache invariant) ⇒ no parse. + // `parameterize()` above already ran (needed to derive THIS text's + // auto_params and the normalized hash), but parse+compile is skipped. + let cached = graph.plan_cache.lock().get(query_hash); + if let Some(cached) = cached { + return run_read_query(graph, args, &cached.plan, &cached.slots, auto_params); + } + + let (query, query_hash, auto_params) = + match parse_effective(cypher_bytes, &effective, query_hash, auto_params) { + Ok(t) => t, + Err(msg) => return Frame::Error(Bytes::from(msg)), }; - graph.plan_cache.lock().insert(query_hash, p.clone()); - p - }; - let params = parse_params(args); - let valid_at = parse_valid_at(args); - let decay = match parse_decay(args) { - Ok(d) => d, - Err(msg) => return Frame::Error(Bytes::from_static(msg.as_bytes())), - }; - let ctx = cypher::executor::ExecutionContext { - valid_time_as_of: valid_at, - decay, - ..Default::default() - }; - let result = match cypher::executor::execute(graph, &plan, ¶ms, &ctx) { - Ok(r) => r, + if reject_writes && !query.is_read_only() { + return Frame::Error(Bytes::from_static( + b"ERR GRAPH.RO_QUERY does not allow write clauses (CREATE, DELETE, SET, MERGE)", + )); + } + + let plan = match cypher::planner::compile(&query) { + Ok(p) => std::sync::Arc::new(p), Err(e) => { - let msg = format!("ERR Cypher execution error: {e}"); + let msg = format!("ERR Cypher plan error: {e}"); return Frame::Error(Bytes::from(msg)); } }; - - exec_result_to_frame(&result) + // Cache read-only plans ONLY — a cache hit skips classification, so a + // cached write plan would execute on the read path. Insert under both + // the raw and normalized hash so a later exact repeat of this text hits + // the allocation-free raw-hash path above. + let slots = if query.is_read_only() { + graph + .plan_cache + .lock() + .insert_both(raw_hash, query_hash, plan.clone(), auto_params.clone()) + } else { + std::sync::Arc::new(cypher::executor::SlotTable::from_plan(&plan)) + }; + run_read_query(graph, args, &plan, &slots, auto_params) } /// GRAPH.QUERY — write-capable variant. @@ -643,17 +769,44 @@ pub fn graph_query_or_write( } }; - // Parse once — no double-parse overhead. - let query = match cypher::parse_cypher(cypher_bytes) { - Ok(q) => q, - Err(e) => { - let msg = format!("ERR Cypher parse error: {e}"); - return (Frame::Error(Bytes::from(msg)), Vec::new(), Vec::new()); + // Raw-hash pre-lookup (Fix 2): an EXACT repeat of a query text we've + // already compiled hits here without ever calling `parameterize()`. + let raw_hash = cypher::planner::hash_query(cypher_bytes); + if let Some(graph) = store.get_graph(graph_name) { + if let Some(cached) = graph.plan_cache.lock().get(raw_hash) { + let auto_params = cached.auto_params.as_ref().clone(); + return ( + run_read_query(graph, args, &cached.plan, &cached.slots, auto_params), + Vec::new(), + Vec::new(), + ); } - }; + } + + let (effective, query_hash, auto_params) = normalize_cypher(cypher_bytes); + + // Fast path: normalized-hash plan-cache hit ⇒ read-only plan (PlanCache + // invariant) ⇒ route to the read path with ZERO parse/compile work. + if let Some(graph) = store.get_graph(graph_name) { + let cached = graph.plan_cache.lock().get(query_hash); + if let Some(cached) = cached { + return ( + run_read_query(graph, args, &cached.plan, &cached.slots, auto_params), + Vec::new(), + Vec::new(), + ); + } + } + + // Slow path: parse once (normalized text, raw fallback) and classify. + let (query, query_hash, auto_params) = + match parse_effective(cypher_bytes, &effective, query_hash, auto_params) { + Ok(t) => t, + Err(msg) => return (Frame::Error(Bytes::from(msg)), Vec::new(), Vec::new()), + }; if query.is_read_only() { - // Read path: compile plan (with cache), execute read-only. + // Read path: compile plan, cache it, execute read-only. let graph = match store.get_graph(graph_name) { Some(g) => g, None => { @@ -665,51 +818,27 @@ pub fn graph_query_or_write( } }; - let query_hash = cypher::planner::hash_query(cypher_bytes); - let plan = { - let cache = graph.plan_cache.lock(); - cache.get(query_hash) - }; - let plan = if let Some(cached) = plan { - cached - } else { - let p = match cypher::planner::compile(&query) { - Ok(p) => std::sync::Arc::new(p), - Err(e) => { - let msg = format!("ERR Cypher plan error: {e}"); - return (Frame::Error(Bytes::from(msg)), Vec::new(), Vec::new()); - } - }; - graph.plan_cache.lock().insert(query_hash, p.clone()); - p - }; - - let params = parse_params(args); - let valid_at = parse_valid_at(args); - let decay = match parse_decay(args) { - Ok(d) => d, - Err(msg) => { - return ( - Frame::Error(Bytes::from_static(msg.as_bytes())), - Vec::new(), - Vec::new(), - ); - } - }; - let ctx = cypher::executor::ExecutionContext { - valid_time_as_of: valid_at, - decay, - ..Default::default() - }; - let result = match cypher::executor::execute(graph, &plan, ¶ms, &ctx) { - Ok(r) => r, + let plan = match cypher::planner::compile(&query) { + Ok(p) => std::sync::Arc::new(p), Err(e) => { - let msg = format!("ERR Cypher execution error: {e}"); + let msg = format!("ERR Cypher plan error: {e}"); return (Frame::Error(Bytes::from(msg)), Vec::new(), Vec::new()); } }; + // Insert under both hashes so a later exact repeat of this text + // hits the allocation-free raw-hash path above. + let slots = graph.plan_cache.lock().insert_both( + raw_hash, + query_hash, + plan.clone(), + auto_params.clone(), + ); - (exec_result_to_frame(&result), Vec::new(), Vec::new()) + ( + run_read_query(graph, args, &plan, &slots, auto_params), + Vec::new(), + Vec::new(), + ) } else { // Decay biases read-path traversal cost only; a write query must not // silently accept (or skip validating) the flag. Reject before any @@ -759,7 +888,13 @@ pub fn graph_query_or_write( } }; - let params = parse_params(args); + // Merge auto-extracted literal values: the write plan was + // compiled from the normalized text, so `$__pN` parameters must + // resolve or CREATE would store Nulls. + let mut params = parse_params(args); + for (name, value) in auto_params { + params.insert(name, value); + } match cypher::executor::execute_mut(graph, &plan, ¶ms, lsn) { Ok(r) => { let muts = r.mutations; @@ -884,34 +1019,15 @@ pub fn graph_query_or_write( /// GRAPH.RO_QUERY /// /// Like GRAPH.QUERY but rejects write clauses (CREATE, DELETE, SET, MERGE). +/// Shares the single-parse read core with GRAPH.QUERY — a plan-cache hit is +/// read-only by the cache invariant, so no classification re-parse is needed. pub fn graph_ro_query(store: &GraphStore, args: &[Frame]) -> Frame { if args.len() < 2 { return Frame::Error(Bytes::from_static( b"ERR wrong number of arguments for 'GRAPH.RO_QUERY' command", )); } - - let cypher_bytes = match extract_bulk(&args[1]) { - Some(b) => b, - None => return Frame::Error(Bytes::from_static(b"ERR invalid Cypher query")), - }; - - let query = match cypher::parse_cypher(cypher_bytes) { - Ok(q) => q, - Err(e) => { - let msg = format!("ERR Cypher parse error: {e}"); - return Frame::Error(Bytes::from(msg)); - } - }; - - if !query.is_read_only() { - return Frame::Error(Bytes::from_static( - b"ERR GRAPH.RO_QUERY does not allow write clauses (CREATE, DELETE, SET, MERGE)", - )); - } - - // Delegate to the regular query handler for parsing/planning. - graph_query(store, args) + graph_query_readonly(store, args, true) } /// GRAPH.EXPLAIN @@ -1036,6 +1152,7 @@ pub fn graph_profile(store: &GraphStore, args: &[Frame]) -> Frame { let ctx = cypher::executor::ExecutionContext { valid_time_as_of: valid_at, decay, + guard: Some(crate::graph::traversal_guard::TraversalGuard::with_default_timeout(0)), ..Default::default() }; let profile = match cypher::executor::execute_profile(graph, &plan, ¶ms, &ctx) { @@ -1413,6 +1530,8 @@ pub fn graph_vsearch(store: &GraphStore, args: &[Frame]) -> Frame { let node_key = super::graph_write::external_id_to_node_key(start_id); let memgraph = &graph.write_buf; + let segments_guard = graph.segments.load(); + let csr_segs = &segments_guard.immutable; let lsn = u64::MAX - 1; let mut search = @@ -1420,7 +1539,7 @@ pub fn graph_vsearch(store: &GraphStore, args: &[Frame]) -> Frame { search.threshold = threshold; search.edge_type_filter = edge_type_filter; - match search.execute(memgraph, lsn) { + match search.execute(memgraph, csr_segs, lsn) { Ok(results) => hybrid_results_to_frame(&results), Err(e) => Frame::Error(Bytes::from(format!("ERR {e}"))), } @@ -1460,6 +1579,8 @@ pub fn graph_hybrid(store: &GraphStore, args: &[Frame]) -> Frame { }; let memgraph = &graph.write_buf; + let segments_guard = graph.segments.load(); + let csr_segs = &segments_guard.immutable; let lsn = u64::MAX - 1; if mode.eq_ignore_ascii_case(b"FILTER") { @@ -1489,7 +1610,7 @@ pub fn graph_hybrid(store: &GraphStore, args: &[Frame]) -> Frame { let node_key = super::graph_write::external_id_to_node_key(start_id); let search = crate::graph::hybrid::GraphFilteredSearch::new(node_key, hops, query_vector, k); - match search.execute(memgraph, lsn) { + match search.execute(memgraph, csr_segs, lsn) { Ok(results) => hybrid_results_to_frame(&results), Err(e) => Frame::Error(Bytes::from(format!("ERR {e}"))), } @@ -1527,7 +1648,7 @@ pub fn graph_hybrid(store: &GraphStore, args: &[Frame]) -> Frame { walk.beam_width = beam_width; walk.min_similarity = min_sim; - match walk.execute(memgraph, lsn) { + match walk.execute(memgraph, csr_segs, lsn) { Ok(results) => hybrid_results_to_frame(&results), Err(e) => Frame::Error(Bytes::from(format!("ERR {e}"))), } @@ -1567,7 +1688,7 @@ pub fn graph_hybrid(store: &GraphStore, args: &[Frame]) -> Frame { query_vector, k, ); - match reranker.execute(memgraph, lsn) { + match reranker.execute(memgraph, csr_segs, lsn) { Ok(results) => hybrid_results_to_frame(&results), Err(e) => Frame::Error(Bytes::from(format!("ERR {e}"))), } @@ -1649,9 +1770,26 @@ fn extract_f32_vector(frame: &Frame) -> Option> { _ => return None, }; - let text = core::str::from_utf8(bytes).ok()?; - let values: Result, _> = text.split_whitespace().map(|s| s.parse::()).collect(); - values.ok() + // Text form first ("0.1 0.2 ..."), then binary little-endian f32 array — + // the SAME blob format GRAPH.ADDNODE's VECTOR argument accepts, so + // vectors round-trip between ADDNODE and VSEARCH/HYBRID unchanged. + if let Ok(text) = core::str::from_utf8(bytes) { + let values: Result, _> = + text.split_whitespace().map(|s| s.parse::()).collect(); + match values { + Ok(v) if !v.is_empty() => return Some(v), + _ => {} + } + } + if bytes.is_empty() || !bytes.len().is_multiple_of(4) { + return None; + } + Some( + bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect(), + ) } /// Parse an f64 from a Frame. diff --git a/src/command/graph/graph_write.rs b/src/command/graph/graph_write.rs index 54f60496..d5f0623c 100644 --- a/src/command/graph/graph_write.rs +++ b/src/command/graph/graph_write.rs @@ -124,17 +124,6 @@ pub fn graph_addnode(store: &mut GraphStore, args: &[Frame]) -> Frame { let lsn = store.allocate_lsn(); - // Serialize WAL record first using borrowed refs (before moving into add_node). - // This eliminates the expensive embedding.clone() (up to 6KB for 1536-dim f32). - // We use external_id=0 as placeholder, then fix it after add_node. - let wal_record_no_id = wal::serialize_add_node( - graph_name, - 0, // placeholder - &labels, - &properties, - embedding.as_deref(), - ); - // Scoped borrow of graph for mutation, then release for WAL push. let external_id = { let graph = match store.get_graph_mut(graph_name) { @@ -148,11 +137,16 @@ pub fn graph_addnode(store: &mut GraphStore, args: &[Frame]) -> Frame { // Update graph stats from stored node (avoid needing moved labels). // Collect _key registration outside the immutable borrow of write_buf. + // + // NOTE: no insert-time property indexing here. Property indexes are + // per-CSR-segment, built lazily at first use from the v5 property + // blob (`CsrStorage::property_index`) — the old per-ADDNODE + // `NamedGraph.property_indexes` maintenance indexed truncated + // external ids (wrong row space) and was never read by any query. let mut pending_key_reg: Option = None; if let Some(node) = graph.write_buf.get_node(node_key) { graph.stats.on_node_insert(&node.labels); - // Update PropertyIndex for numeric properties. let key_prop_id = label_to_id(b"_key"); for (prop_id, prop_val) in &node.properties { // Track _key property for graph expansion Redis key lookup. @@ -161,18 +155,6 @@ pub fn graph_addnode(store: &mut GraphStore, args: &[Frame]) -> Frame { pending_key_reg = Some(s.clone()); } } - let val = match prop_val { - crate::graph::types::PropertyValue::Float(f) => Some(*f), - crate::graph::types::PropertyValue::Int(i) => Some(*i as f64), - _ => None, - }; - if let Some(v) = val { - graph - .property_indexes - .entry(*prop_id) - .or_insert_with(|| crate::graph::index::PropertyIndex::new(*prop_id)) - .insert(v, ext_id as u32); - } } } // Register _key→NodeKey mapping after releasing the write_buf borrow. @@ -183,12 +165,9 @@ pub fn graph_addnode(store: &mut GraphStore, args: &[Frame]) -> Frame { ext_id }; - // Re-serialize the WAL record with the actual external_id. - // This is cheap — WAL serialization is just RESP encoding of small fields. - // The expensive embedding data was already consumed by add_node, but the - // WAL record was pre-built from refs. We drop the placeholder and rebuild - // from the stored node refs (no clone — just borrows). - drop(wal_record_no_id); + // Serialize the WAL record from the stored node's refs — a single + // serialization with the real external_id, no clones (the expensive + // embedding was moved into add_node and is borrowed back here). if let Some(graph) = store.get_graph(graph_name) { let nk = crate::graph::types::NodeKey::from(slotmap::KeyData::from_ffi(external_id)); if let Some(node) = graph.write_buf.get_node(nk) { @@ -310,17 +289,42 @@ pub fn graph_addedge(store: &mut GraphStore, args: &[Frame]) -> Frame { .get_node(dst_key) .map_or(0, |n| (n.outgoing.len() + n.incoming.len()) as u32); - match graph - .write_buf - .add_edge(src_key, dst_key, edge_type_id, weight, props, lsn) - { - Ok(edge_key) => { - graph - .stats - .on_edge_insert(edge_type_id, src_old_degree, dst_old_degree); - Ok(edge_key.data().as_ffi()) + // Endpoints may have been frozen into an immutable CSR segment + // (freeze DRAINS the write buffer). Verify each non-resident + // endpoint is alive in the frozen tier, then insert a cross-tier + // delta edge (ghost adjacency keeps it traversable from both ends). + let src_resident = graph.write_buf.get_node(src_key).is_some(); + let dst_resident = graph.write_buf.get_node(dst_key).is_some(); + let frozen_endpoints_ok = if src_resident && dst_resident { + true + } else { + let seg_guard = graph.segments.load(); + let view = + crate::graph::view::MergedNodeView::new(&graph.write_buf, &seg_guard.immutable); + let committed = roaring::RoaringBitmap::new(); + (src_resident || view.is_visible(src_key, 0, 0, &committed, None)) + && (dst_resident || view.is_visible(dst_key, 0, 0, &committed, None)) + }; + + if !frozen_endpoints_ok { + Err(crate::graph::memgraph::GraphError::NodeNotFound) + } else { + match graph.write_buf.add_edge_across_tiers( + src_key, + dst_key, + edge_type_id, + weight, + props, + lsn, + ) { + Ok(edge_key) => { + graph + .stats + .on_edge_insert(edge_type_id, src_old_degree, dst_old_degree); + Ok(edge_key.data().as_ffi()) + } + Err(e) => Err(e), } - Err(e) => Err(e), } }; diff --git a/src/command/graph/mod.rs b/src/command/graph/mod.rs index 1a5d1655..5710c35e 100644 --- a/src/command/graph/mod.rs +++ b/src/command/graph/mod.rs @@ -29,23 +29,36 @@ pub fn is_graph_write_cmd(cmd: &[u8]) -> bool { || cmd.eq_ignore_ascii_case(b"GRAPH.DROP") } -/// Quick-parse a GRAPH.QUERY's Cypher argument to determine if it contains write clauses. -/// Returns true if the Cypher has CREATE, DELETE, SET, or MERGE. -/// Returns false on parse failure (fallback to read path — it will re-parse and report error). +/// Quick-scan a GRAPH.QUERY's Cypher argument to determine if it MAY contain +/// write clauses (CREATE, DELETE, SET, MERGE). +/// +/// Token scan, not a parse — this is a routing predicate on the hot path and +/// was previously a full second parse. It may FALSE-POSITIVE (e.g. a property +/// named `set`: the lexer emits the keyword token), which only routes the +/// query through the write-capable handler; `graph_query_or_write` then +/// AST-classifies and executes the read path correctly. It can never +/// false-negative for a parsable write query: a write clause always lexes to +/// its keyword token. #[inline] pub fn is_cypher_write_query(args: &[crate::protocol::Frame]) -> bool { - use crate::command::graph::graph_write::extract_bulk; + use crate::graph::cypher::lexer::{Lexer, Token}; if args.len() < 2 { return false; } - let cypher_bytes = match extract_bulk(&args[1]) { + let cypher_bytes = match graph_write::extract_bulk(&args[1]) { Some(b) => b, None => return false, }; - match crate::graph::cypher::parse_cypher(cypher_bytes) { - Ok(q) => !q.is_read_only(), - Err(_) => false, + let mut lexer = Lexer::new(cypher_bytes); + while let Some(t) = lexer.next_token() { + if matches!( + t.token, + Token::Create | Token::Delete | Token::Set | Token::Merge + ) { + return true; + } } + false } /// Dispatch read-only GRAPH.* commands. Takes &GraphStore (shared). @@ -193,6 +206,179 @@ mod tests { Frame::Array(FrameVec::from_vec(frames)) } + /// Decode `[headers, rows, stats]` from a GRAPH.QUERY response into rows of cells. + fn result_rows(resp: &Frame) -> Vec> { + let Frame::Array(outer) = resp else { + panic!("expected result Array, got {resp:?}"); + }; + let Frame::Array(rows) = &outer[1] else { + panic!("expected rows Array, got {resp:?}"); + }; + rows.iter() + .map(|r| { + let Frame::Array(cells) = r else { + panic!("expected row Array, got {r:?}"); + }; + cells.iter().cloned().collect() + }) + .collect() + } + + #[test] + fn test_plan_cache_shared_across_literal_variants() { + let mut store = GraphStore::new(); + dispatch_graph_command(&mut store, &make_cmd(&[b"GRAPH.CREATE", b"g"])); + dispatch_graph_command( + &mut store, + &make_cmd(&[b"GRAPH.QUERY", b"g", b"CREATE (:Person {id: 1})"]), + ); + dispatch_graph_command( + &mut store, + &make_cmd(&[b"GRAPH.QUERY", b"g", b"CREATE (:Person {id: 2})"]), + ); + + let r1 = dispatch_graph_command( + &mut store, + &make_cmd(&[ + b"GRAPH.QUERY", + b"g", + b"MATCH (a:Person {id: 1}) RETURN a.id", + ]), + ); + let r2 = dispatch_graph_command( + &mut store, + &make_cmd(&[ + b"GRAPH.QUERY", + b"g", + b"MATCH (a:Person {id: 2}) RETURN a.id", + ]), + ); + + let rows1 = result_rows(&r1); + let rows2 = result_rows(&r2); + assert_eq!(rows1.len(), 1, "point query id=1 returns one row: {r1:?}"); + assert_eq!(rows2.len(), 1, "point query id=2 returns one row: {r2:?}"); + assert!( + matches!(rows1[0][0], Frame::Integer(1)), + "id=1 query must return its own row, got {:?}", + rows1[0][0] + ); + assert!( + matches!(rows2[0][0], Frame::Integer(2)), + "id=2 query must return its own row, got {:?}", + rows2[0][0] + ); + + let graph = store.get_graph(b"g").expect("graph exists"); + assert_eq!( + graph.plan_cache.lock().distinct_plan_count(), + 1, + "queries differing only in literal values must share one cached plan \ + (and write queries must not be cached)" + ); + } + + #[test] + fn test_plan_cache_string_literals_share_plan_with_correct_values() { + let mut store = GraphStore::new(); + dispatch_graph_command(&mut store, &make_cmd(&[b"GRAPH.CREATE", b"g"])); + dispatch_graph_command( + &mut store, + &make_cmd(&[b"GRAPH.QUERY", b"g", b"CREATE (:P {name: 'alice'})"]), + ); + dispatch_graph_command( + &mut store, + &make_cmd(&[b"GRAPH.QUERY", b"g", b"CREATE (:P {name: 'bob'})"]), + ); + + let ra = dispatch_graph_command( + &mut store, + &make_cmd(&[ + b"GRAPH.QUERY", + b"g", + b"MATCH (a:P {name: 'alice'}) RETURN a.name", + ]), + ); + let rb = dispatch_graph_command( + &mut store, + &make_cmd(&[ + b"GRAPH.QUERY", + b"g", + b"MATCH (a:P {name: 'bob'}) RETURN a.name", + ]), + ); + + let rows_a = result_rows(&ra); + let rows_b = result_rows(&rb); + assert_eq!(rows_a.len(), 1, "name='alice' returns one row: {ra:?}"); + assert_eq!(rows_b.len(), 1, "name='bob' returns one row: {rb:?}"); + assert!( + matches!(&rows_a[0][0], Frame::BulkString(b) if b.as_ref() == b"alice"), + "alice query must return 'alice', got {:?}", + rows_a[0][0] + ); + assert!( + matches!(&rows_b[0][0], Frame::BulkString(b) if b.as_ref() == b"bob"), + "bob query must return 'bob', got {:?}", + rows_b[0][0] + ); + + let graph = store.get_graph(b"g").expect("graph exists"); + assert_eq!( + graph.plan_cache.lock().distinct_plan_count(), + 1, + "string-literal variants must share one cached plan" + ); + } + + #[test] + fn test_var_length_hops_stay_in_plan_key() { + // Hop bounds are baked into the compiled plan (Expand max_hops), so + // `*1..2` and `*1..3` must NOT normalize to the same cache entry. + let mut store = GraphStore::new(); + dispatch_graph_command(&mut store, &make_cmd(&[b"GRAPH.CREATE", b"g"])); + let mut ids = Vec::new(); + for _ in 0..4 { + let resp = + dispatch_graph_command(&mut store, &make_cmd(&[b"GRAPH.ADDNODE", b"g", b"N"])); + let Frame::Integer(id) = resp else { + panic!("expected node id, got {resp:?}"); + }; + ids.push(id.to_string()); + } + for w in ids.windows(2) { + dispatch_graph_command( + &mut store, + &make_cmd(&[ + b"GRAPH.ADDEDGE", + b"g", + w[0].as_bytes(), + w[1].as_bytes(), + b"KNOWS", + ]), + ); + } + + // Chain 1->2->3->4: *1..2 reaches 5 targets, *1..3 reaches 6. + let r2 = dispatch_graph_command( + &mut store, + &make_cmd(&[b"GRAPH.QUERY", b"g", b"MATCH (a)-[*1..2]->(x) RETURN x"]), + ); + let r3 = dispatch_graph_command( + &mut store, + &make_cmd(&[b"GRAPH.QUERY", b"g", b"MATCH (a)-[*1..3]->(x) RETURN x"]), + ); + assert_eq!(result_rows(&r2).len(), 5, "*1..2 rows: {r2:?}"); + assert_eq!(result_rows(&r3).len(), 6, "*1..3 rows: {r3:?}"); + + let graph = store.get_graph(b"g").expect("graph exists"); + assert_eq!( + graph.plan_cache.lock().distinct_plan_count(), + 2, + "different hop bounds must compile to distinct cached plans" + ); + } + #[test] fn test_graph_create_and_list() { let mut store = GraphStore::new(); diff --git a/src/graph/compaction.rs b/src/graph/compaction.rs index be47919d..1e261d35 100644 --- a/src/graph/compaction.rs +++ b/src/graph/compaction.rs @@ -26,7 +26,7 @@ use smallvec::SmallVec; use crate::graph::csr::{CsrError, CsrSegment, CsrStorage}; use crate::graph::index::{EdgeTypeIndex, LabelIndex, MphNodeIndex}; use crate::graph::store::GraphStore; -use crate::graph::types::{EdgeMeta, GraphSegmentHeader, NodeMeta}; +use crate::graph::types::{EdgeMeta, GraphSegmentHeader, NodeKey, NodeMeta}; // --------------------------------------------------------------------------- // GraphCompactStats — returned by run_graph_vacuum_pass @@ -225,6 +225,11 @@ struct MergedEdge { /// Per-edge wall-clock stamp carried through the merge (0 = unknown, /// e.g. input segment predates the version 3 format). created_ms: u64, + /// Index of the winning source segment (for property-record carry-over). + src_seg: usize, + /// Raw `EdgeMeta::property_offset` in the winning segment's edge blob + /// (offset+1 scheme; 0 = no record). + src_prop_offset: u32, } /// Compact multiple CSR segments into one with Rabbit Order reordering. @@ -270,7 +275,7 @@ pub fn compact_segments( let mut edge_map: std::collections::HashMap<(u32, u32, u16), MergedEdge> = std::collections::HashMap::new(); - for seg in segments { + for (seg_idx, seg) in segments.iter().enumerate() { let seg_node_meta = seg.node_meta(); let seg_row_offsets = seg.row_offsets(); let seg_col_indices = seg.col_indices(); @@ -306,6 +311,8 @@ pub fn compact_segments( created_lsn: seg.created_lsn(), // Checked access: empty slice for pre-v3 input segments. created_ms: seg_edge_created_ms.get(edge_idx).copied().unwrap_or(0), + src_seg: seg_idx, + src_prop_offset: em.property_offset, }; edge_map @@ -359,16 +366,24 @@ pub fn compact_segments( let edge_count = reordered_edges.len(); - // Build col_indices, edge_meta, and the parallel created_ms array. + // Build col_indices, edge_meta, the parallel created_ms array, and carry + // each winning edge's weight/property record into the merged blob (v5). let mut col_indices = Vec::with_capacity(edge_count); let mut edge_meta_vec = Vec::with_capacity(edge_count); let mut edge_created_ms_vec = Vec::with_capacity(edge_count); + let mut merged_edge_props: Vec = Vec::new(); for e in &reordered_edges { col_indices.push(e.dst_row); + let property_offset = crate::graph::csr::props::copy_record( + &mut merged_edge_props, + segments[e.src_seg].edge_props_blob(), + e.src_prop_offset, + crate::graph::csr::props::edge_record_len, + ); edge_meta_vec.push(EdgeMeta { edge_type: e.edge_type, flags: e.flags, - property_offset: 0, + property_offset, }); edge_created_ms_vec.push(e.created_ms); } @@ -390,7 +405,11 @@ pub fn compact_segments( std::collections::HashMap::with_capacity(node_count); let mut best_overflow: std::collections::HashMap> = std::collections::HashMap::new(); - for seg in segments { + // Winning segment index per external_id, so the node's property record can + // be carried over from the SAME segment as its metadata (v5). + let mut best_node_seg: std::collections::HashMap = + std::collections::HashMap::with_capacity(node_count); + for (seg_idx, seg) in segments.iter().enumerate() { let seg_overflow = seg.label_overflow(); for (row, nm) in seg.node_meta().iter().enumerate() { // Same tie-break as before: take the first occurrence, then replace @@ -400,12 +419,14 @@ pub fn compact_segments( None => true, }; if take { + best_node_seg.insert(nm.external_id, seg_idx); best_node_meta.insert( nm.external_id, NodeMeta { external_id: nm.external_id, label_bitmap: nm.label_bitmap, - property_offset: 0, + // Source-segment offset, rewritten to the merged blob below. + property_offset: nm.property_offset, created_lsn: nm.created_lsn, deleted_lsn: nm.deleted_lsn, valid_from: nm.valid_from, @@ -429,14 +450,26 @@ pub fn compact_segments( // Re-key label overflow from external_id -> the node's NEW merged row. (A2) let mut merged_overflow: std::collections::HashMap> = std::collections::HashMap::new(); + let mut merged_node_props: Vec = Vec::new(); for new_row in 0..node_count { let old_row = inv_perm[new_row] as usize; let ext_id = all_node_ids[old_row]; if let Some(nm) = best_node_meta.get(&ext_id) { + // Carry the winning node's property/embedding record (v5) from its + // winning segment into the merged blob. + let property_offset = match best_node_seg.get(&ext_id) { + Some(&seg_idx) => crate::graph::csr::props::copy_record( + &mut merged_node_props, + segments[seg_idx].node_props_blob(), + nm.property_offset, + crate::graph::csr::props::node_record_len, + ), + None => 0, + }; node_meta_vec.push(NodeMeta { external_id: nm.external_id, label_bitmap: nm.label_bitmap, - property_offset: 0, + property_offset, created_lsn: nm.created_lsn, deleted_lsn: nm.deleted_lsn, valid_from: nm.valid_from, @@ -464,11 +497,19 @@ pub fn compact_segments( validity.insert(i); } - // Build node_id_to_row from reordered node_meta. - // We don't have NodeKey here (compaction works at external_id level), - // so leave node_id_to_row empty -- lookup by NodeKey requires the - // GraphStore to maintain a separate mapping. - let node_id_to_row = std::collections::HashMap::new(); + // Rebuild node_id_to_row + MPH from reordered node_meta. external_id IS + // the NodeKey ffi encoding (KeyData::from_ffi reverses it) — the same + // reconstruction from_bytes/from_mmap_file already do on load. Previously + // both were left EMPTY here, making a freshly merged in-memory segment + // unresolvable by NodeKey (lookup_node → None) until persisted + reloaded. + let mut node_id_to_row: std::collections::HashMap = + std::collections::HashMap::with_capacity(node_count); + let mut row_keys: Vec = Vec::with_capacity(node_count); + for (row, nm) in node_meta_vec.iter().enumerate() { + let nk = NodeKey::from(slotmap::KeyData::from_ffi(nm.external_id)); + node_id_to_row.insert(nk, row as u32); + row_keys.push(nk); + } // Compute min/max external IDs. let min_node_id = all_node_ids.first().copied().unwrap_or(0); @@ -507,8 +548,8 @@ pub fn compact_segments( label_overflow_offset: 0, // populated during serialization }; - // Build indexes from compacted data. MPH is empty (no NodeKeys in compaction). - let mph = MphNodeIndex::build(&[]); + // Build indexes from compacted data (MPH from the reconstructed row keys). + let mph = MphNodeIndex::build(&row_keys); let label_index = LabelIndex::build(&node_meta_vec, &merged_overflow); let edge_type_index = EdgeTypeIndex::build(&edge_meta_vec); @@ -525,8 +566,12 @@ pub fn compact_segments( label_index, label_overflow: merged_overflow, edge_type_index, + node_props: merged_node_props, + edge_props: merged_edge_props, created_lsn, incoming: std::sync::OnceLock::new(), + props_index: std::sync::OnceLock::new(), + hnsw_bridge: std::sync::OnceLock::new(), }) } @@ -758,6 +803,75 @@ mod tests { assert_eq!(merged.edge_created_ms, vec![20_000]); } + #[test] + fn test_compact_carries_props_and_node_key_lookup() { + // v5: node property/embedding records and edge weight/property records + // survive the merge, and the merged segment resolves NodeKeys directly + // (previously node_id_to_row + MPH were left EMPTY by compact_segments, + // so an in-memory merged segment was unreachable via lookup_node). + use crate::graph::types::PropertyValue; + let mut mg = MemGraph::new(100_000); + let props: crate::graph::types::PropertyMap = smallvec![ + (1u16, PropertyValue::Int(41)), + ( + 3u16, + PropertyValue::String(bytes::Bytes::from_static(b"carol")) + ), + ]; + let a = mg.add_node(smallvec![0], props, Some(vec![0.5f32, 0.25]), 1); + let b = mg.add_node(smallvec![0], smallvec![], None, 1); + let eprops: crate::graph::types::PropertyMap = smallvec![(9u16, PropertyValue::Float(6.5))]; + mg.add_edge(a, b, 1, 3.5, Some(eprops), 2).expect("ok"); + let seg1 = Arc::new(CsrStorage::from( + CsrSegment::from_frozen(mg.freeze().expect("ok"), 10).expect("ok"), + )); + let seg2 = Arc::new(CsrStorage::from(make_csr(2, &[(1, 0)], 20))); + + let config = CompactionConfig { + min_segments: 2, + max_segment_edges: 1_000_000, + }; + let merged = compact_segments(&[seg1, seg2], &config).expect("ok"); + + // NodeKey lookup now works on the freshly merged in-memory segment. + let row_a = merged.lookup_node(a).expect("merged lookup_node(a)"); + let row_b = merged.lookup_node(b).expect("merged lookup_node(b)"); + + let pa = merged.node_properties(row_a); + assert!(pa.contains(&(1u16, PropertyValue::Int(41)))); + assert!(pa.contains(&( + 3u16, + PropertyValue::String(bytes::Bytes::from_static(b"carol")) + ))); + assert_eq!(merged.node_embedding(row_a), Some(vec![0.5f32, 0.25])); + assert!(merged.node_properties(row_b).is_empty()); + + // The a->b edge kept its weight + props through the merge. + let sa = merged.row_offsets[row_a as usize]; + let ea = merged.row_offsets[row_a as usize + 1]; + let mut found = false; + for idx in sa..ea { + if merged.col_indices[idx as usize] == row_b { + assert_eq!(merged.edge_weight(idx), 3.5); + assert_eq!( + merged.edge_properties(idx).as_slice(), + &[(9u16, PropertyValue::Float(6.5))] + ); + found = true; + } + } + assert!(found, "a->b edge must exist in the merged segment"); + + // And everything survives a serialize/parse roundtrip. + let parsed = CsrSegment::from_bytes(&merged.to_bytes()).expect("parse ok"); + let prow_a = parsed.lookup_node(a).expect("parsed lookup_node(a)"); + assert_eq!( + parsed.node_embedding(prow_a), + Some(vec![0.5f32, 0.25]), + "embedding must survive merge + roundtrip" + ); + } + #[test] fn test_tombstoned_edges_dropped() { let mut seg = make_csr(3, &[(0, 1), (0, 2), (1, 2)], 10); diff --git a/src/graph/csr/mmap.rs b/src/graph/csr/mmap.rs index 6dac27e8..86895149 100644 --- a/src/graph/csr/mmap.rs +++ b/src/graph/csr/mmap.rs @@ -52,6 +52,17 @@ pub struct MmapCsrSegment { /// arrays. NOT part of the on-disk format — rebuilt on the first Incoming/Both /// query. (v3-2 graph-incoming-edges.) pub incoming: std::sync::OnceLock, + /// Lazily built per-segment property indexes (see `SegmentPropertyIndexes`). + pub props_index: std::sync::OnceLock, + /// Lazily-built HNSW bridge over this segment's v5 embeddings (hybrid + /// HnswPreFilter). DERIVED, in-memory only — never persisted. + pub hnsw_bridge: std::sync::OnceLock>, + /// Pointer into mmap: node property blob (version >= 5; dangling+0 otherwise). + node_props_ptr: *const u8, + node_props_len: usize, + /// Pointer into mmap: edge property blob (version >= 5; dangling+0 otherwise). + edge_props_ptr: *const u8, + edge_props_len: usize, } // SAFETY: MmapCsrSegment contains raw pointers into an immutable Mmap region. @@ -269,11 +280,44 @@ impl MmapCsrSegment { // (A1: NOT zero-copied — the section is variable-length, unlike the // fixed-stride raw-cast arrays). It begins right after edge_created_ms; // reuse the shared bounds-checked parser (errors, never panics). - let label_overflow = if version >= 4 { + let (label_overflow, overflow_end) = if version >= 4 { let overflow_start = nm_start + nc * nm_elem_size + ec * ecms_elem_size; parse_label_overflow(data, overflow_start, node_count)? } else { - HashMap::new() + ( + HashMap::new(), + nm_start + nc * nm_elem_size + ec * ecms_elem_size, + ) + }; + + // Version 5+: node/edge property blobs ([len: u64][bytes] each) after + // label_overflow. Zero-copy: keep pointers into the mapped region (u8 + // slices have no alignment requirement). Pre-v5 files get dangling+0. + let (np_ptr, np_len, ep_ptr, ep_len) = if version >= 5 { + let mut ppos = overflow_end; + let np_len = u64::from_le_bytes(read8(data, ppos)?) as usize; + ppos += 8; + if ppos.checked_add(np_len).is_none_or(|end| end > data.len()) { + return Err(CsrError::InvalidData( + "node_props section truncated in mmap".to_owned(), + )); + } + // SAFETY: bounds validated above; u8 has alignment 1; mmap alive for self. + let np_ptr = unsafe { base.add(ppos) }; + ppos += np_len; + let ep_len = u64::from_le_bytes(read8(data, ppos)?) as usize; + ppos += 8; + if ppos.checked_add(ep_len).is_none_or(|end| end > data.len()) { + return Err(CsrError::InvalidData( + "edge_props section truncated in mmap".to_owned(), + )); + } + // SAFETY: bounds validated above; u8 has alignment 1; mmap alive for self. + let ep_ptr = unsafe { base.add(ppos) }; + (np_ptr, np_len, ep_ptr, ep_len) + } else { + let dangling = core::ptr::NonNull::::dangling().as_ptr() as *const u8; + (dangling, 0, dangling, 0) }; let mph = MphNodeIndex::build(&sorted_keys); @@ -324,9 +368,73 @@ impl MmapCsrSegment { edge_type_index, created_lsn, incoming: std::sync::OnceLock::new(), + props_index: std::sync::OnceLock::new(), + hnsw_bridge: std::sync::OnceLock::new(), + node_props_ptr: np_ptr, + node_props_len: np_len, + edge_props_ptr: ep_ptr, + edge_props_len: ep_len, }) } + /// Node property blob (borrowed from mmap; empty for pre-v5 files). + pub fn node_props_blob(&self) -> &[u8] { + if self.node_props_len == 0 { + return &[]; + } + // SAFETY: pointer+len validated in from_mmap_file; mmap alive as long + // as self; the region is immutable. + unsafe { core::slice::from_raw_parts(self.node_props_ptr, self.node_props_len) } + } + + /// Edge property blob (borrowed from mmap; empty for pre-v5 files). + pub fn edge_props_blob(&self) -> &[u8] { + if self.edge_props_len == 0 { + return &[]; + } + // SAFETY: pointer+len validated in from_mmap_file; mmap alive as long + // as self; the region is immutable. + unsafe { core::slice::from_raw_parts(self.edge_props_ptr, self.edge_props_len) } + } + + /// Node properties for a CSR row (empty for none / pre-v5 files). + pub fn node_properties(&self, row: u32) -> crate::graph::types::PropertyMap { + match self.node_meta().get(row as usize) { + Some(nm) => super::props::decode_node_props(self.node_props_blob(), nm.property_offset), + None => crate::graph::types::PropertyMap::new(), + } + } + + /// Single node property lookup without materializing the map. + pub fn node_property(&self, row: u32, key: u16) -> Option { + let nm = self.node_meta().get(row as usize)?; + super::props::decode_node_prop(self.node_props_blob(), nm.property_offset, key) + } + + /// Node embedding for a CSR row (None for none / pre-v5 files). + pub fn node_embedding(&self, row: u32) -> Option> { + let nm = self.node_meta().get(row as usize)?; + super::props::decode_node_embedding(self.node_props_blob(), nm.property_offset) + } + + /// Edge weight by edge index (1.0 default / pre-v5 files). + pub fn edge_weight(&self, edge_idx: u32) -> f64 { + match self.edge_meta().get(edge_idx as usize) { + Some(em) => { + super::props::decode_edge_weight(self.edge_props_blob(), em.property_offset) + } + None => super::props::DEFAULT_EDGE_WEIGHT, + } + } + + /// Edge properties by edge index (empty for none / pre-v5 files). + pub fn edge_properties(&self, edge_idx: u32) -> crate::graph::types::PropertyMap { + match self.edge_meta().get(edge_idx as usize) { + Some(em) => super::props::decode_edge_props(self.edge_props_blob(), em.property_offset), + None => crate::graph::types::PropertyMap::new(), + } + } + /// Row offsets array (borrowed from mmap). pub fn row_offsets(&self) -> &[u32] { // SAFETY: _mmap is alive as long as self. The pointer and length were diff --git a/src/graph/csr/mod.rs b/src/graph/csr/mod.rs index e8588875..09860e40 100644 --- a/src/graph/csr/mod.rs +++ b/src/graph/csr/mod.rs @@ -6,6 +6,7 @@ pub mod incoming; pub mod mmap; +pub mod props; pub mod storage; pub use incoming::IncomingIndex; @@ -72,12 +73,25 @@ pub struct CsrSegment { pub label_overflow: HashMap>, /// Per-edge-type Roaring bitmap index for O(1) edge type filtering. pub edge_type_index: EdgeTypeIndex, + /// Node property/embedding blob (version >= 5). `NodeMeta::property_offset` + /// points into this (offset+1 scheme; 0 = no record). Empty for segments + /// loaded from pre-v5 files — properties were not persisted before v5. + pub node_props: Vec, + /// Edge weight/property blob (version >= 5). `EdgeMeta::property_offset` + /// points into this (offset+1 scheme; 0 = default weight, no props). + pub edge_props: Vec, /// LSN at which this segment was created. pub created_lsn: u64, /// Lazily-built reverse (incoming) adjacency, DERIVED from `row_offsets` + /// `col_indices`. NOT persisted (excluded from `to_bytes`/`from_bytes`) — /// rebuilt on the first Incoming/Both query. (v3-2 graph-incoming-edges.) pub incoming: std::sync::OnceLock, + /// Lazily built per-segment property indexes (see `SegmentPropertyIndexes`). + pub props_index: std::sync::OnceLock, + /// Lazily-built HNSW bridge over this segment's v5 embeddings (hybrid + /// HnswPreFilter). DERIVED, in-memory only — never persisted. `None` + /// cached when the segment holds too few embeddings to earn one. + pub hnsw_bridge: std::sync::OnceLock>, } impl CsrSegment { @@ -137,17 +151,24 @@ impl CsrSegment { row_offsets.push(offset); let edge_count = offset as usize; - // Build col_indices, edge_meta, and the parallel created_ms array. + // Build col_indices, edge_meta, the parallel created_ms array, and the + // edge weight/property blob (v5 — weights/props previously discarded). let mut col_indices = Vec::with_capacity(edge_count); let mut edge_meta = Vec::with_capacity(edge_count); let mut edge_created_ms = Vec::with_capacity(edge_count); + let mut edge_props: Vec = Vec::new(); for edges in &edges_by_src { for &(dst_row, edge) in edges { col_indices.push(dst_row); + let property_offset = props::encode_edge_record( + &mut edge_props, + edge.weight, + edge.properties.as_ref(), + ); edge_meta.push(EdgeMeta { edge_type: edge.edge_type, flags: 0, - property_offset: 0, + property_offset, }); // Real wall-clock stamp from the mutable edge — survives // compaction so decay scoring keeps true edge age (0 stays @@ -160,6 +181,7 @@ impl CsrSegment { // (fast path), >= 32 -> the sparse `label_overflow` store keyed by row. let mut node_meta = Vec::with_capacity(node_count); let mut label_overflow: HashMap> = HashMap::new(); + let mut node_props: Vec = Vec::new(); let mut min_node_id = u64::MAX; let mut max_node_id = 0u64; for (row, (key, node)) in sorted_nodes.iter().enumerate() { @@ -179,10 +201,18 @@ impl CsrSegment { label_overflow.entry(row as u32).or_default().push(label); } } + // v5: persist properties + embedding so they survive the freeze + // boundary (previously property_offset was hardwired to 0 and the + // data was silently discarded). + let property_offset = props::encode_node_record( + &mut node_props, + &node.properties, + node.embedding.as_deref(), + ); node_meta.push(NodeMeta { external_id: id_bits, label_bitmap, - property_offset: 0, + property_offset, created_lsn: node.created_lsn, deleted_lsn: node.deleted_lsn, valid_from: node.valid_from, @@ -237,11 +267,51 @@ impl CsrSegment { label_index, label_overflow, edge_type_index, + node_props, + edge_props, created_lsn: lsn, incoming: std::sync::OnceLock::new(), + props_index: std::sync::OnceLock::new(), + hnsw_bridge: std::sync::OnceLock::new(), }) } + /// Node properties for a CSR row (empty for none / pre-v5 segments). + pub fn node_properties(&self, row: u32) -> crate::graph::types::PropertyMap { + match self.node_meta.get(row as usize) { + Some(nm) => props::decode_node_props(&self.node_props, nm.property_offset), + None => crate::graph::types::PropertyMap::new(), + } + } + + /// Single node property lookup without materializing the map. + pub fn node_property(&self, row: u32, key: u16) -> Option { + let nm = self.node_meta.get(row as usize)?; + props::decode_node_prop(&self.node_props, nm.property_offset, key) + } + + /// Node embedding for a CSR row (None for none / pre-v5 segments). + pub fn node_embedding(&self, row: u32) -> Option> { + let nm = self.node_meta.get(row as usize)?; + props::decode_node_embedding(&self.node_props, nm.property_offset) + } + + /// Edge weight by edge index (1.0 default / pre-v5 segments). + pub fn edge_weight(&self, edge_idx: u32) -> f64 { + match self.edge_meta.get(edge_idx as usize) { + Some(em) => props::decode_edge_weight(&self.edge_props, em.property_offset), + None => props::DEFAULT_EDGE_WEIGHT, + } + } + + /// Edge properties by edge index (empty for none / pre-v5 segments). + pub fn edge_properties(&self, edge_idx: u32) -> crate::graph::types::PropertyMap { + match self.edge_meta.get(edge_idx as usize) { + Some(em) => props::decode_edge_props(&self.edge_props, em.property_offset), + None => crate::graph::types::PropertyMap::new(), + } + } + /// Returns the slice of outgoing neighbor row indices for the given CSR row. /// Returns an empty slice if `row` is out of bounds. pub fn neighbors_out(&self, row: u32) -> &[u32] { @@ -408,7 +478,23 @@ impl CsrSegment { 0 }; - let total = header_size + ro_size + ci_size + em_size + nm_size + ecms_size + overflow_size; + // Node/edge property blobs (version >= 5): each section is + // [len: u64][bytes], positioned after label_overflow. + let write_props = self.header.version >= 5; + let props_size = if write_props { + 8 + self.node_props.len() + 8 + self.edge_props.len() + } else { + 0 + }; + + let total = header_size + + ro_size + + ci_size + + em_size + + nm_size + + ecms_size + + overflow_size + + props_size; let mut buf = Vec::with_capacity(total); // Write header with computed offsets (checksum placeholder = 0). @@ -500,6 +586,15 @@ impl CsrSegment { } } + // Write the node/edge property blobs (version >= 5): [len: u64][bytes] + // each, after label_overflow. Covered by the trailing CRC32. + if write_props { + buf.extend_from_slice(&(self.node_props.len() as u64).to_le_bytes()); + buf.extend_from_slice(&self.node_props); + buf.extend_from_slice(&(self.edge_props.len() as u64).to_le_bytes()); + buf.extend_from_slice(&self.edge_props); + } + // Compute CRC32 over the entire buffer (with checksum field zeroed), // then write the checksum at offset 72. let checksum = compute_csr_checksum(&buf) as u64; @@ -690,10 +785,41 @@ impl CsrSegment { // row. `pos` sits right after edge_created_ms. Shared with the mmap loader // so the wire format has one parser; bounds-checked (never panics). The CRC // validated above already covers these bytes. - let label_overflow = if version >= 4 { + let (label_overflow, overflow_end) = if version >= 4 { parse_label_overflow(data, pos, node_count)? } else { - HashMap::new() + (HashMap::new(), pos) + }; + + // Parse the node/edge property blobs (version >= 5): [len: u64][bytes] + // each, after label_overflow. Pre-v5 files leave both empty — decoders + // treat offset 0 / missing blob as "no properties" (graceful fallback). + let (node_props, edge_props) = if version >= 5 { + let mut ppos = overflow_end; + let np_len = u64::from_le_bytes(read8(data, ppos)?) as usize; + ppos += 8; + let node_props = data + .get( + ppos..ppos.checked_add(np_len).ok_or_else(|| { + CsrError::InvalidData("node_props length overflow".to_owned()) + })?, + ) + .ok_or_else(|| CsrError::InvalidData("node_props section truncated".to_owned()))? + .to_vec(); + ppos += np_len; + let ep_len = u64::from_le_bytes(read8(data, ppos)?) as usize; + ppos += 8; + let edge_props = data + .get( + ppos..ppos.checked_add(ep_len).ok_or_else(|| { + CsrError::InvalidData("edge_props length overflow".to_owned()) + })?, + ) + .ok_or_else(|| CsrError::InvalidData("edge_props section truncated".to_owned()))? + .to_vec(); + (node_props, edge_props) + } else { + (Vec::new(), Vec::new()) }; // Rebuild validity bitmap: all edges valid (fresh load). @@ -757,8 +883,12 @@ impl CsrSegment { label_index, label_overflow, edge_type_index, + node_props, + edge_props, created_lsn, incoming: std::sync::OnceLock::new(), + props_index: std::sync::OnceLock::new(), + hnsw_bridge: std::sync::OnceLock::new(), }) } @@ -819,7 +949,7 @@ pub(super) fn parse_label_overflow( data: &[u8], mut pos: usize, node_count: u32, -) -> Result>, CsrError> { +) -> Result<(HashMap>, usize), CsrError> { let mut label_overflow: HashMap> = HashMap::new(); let entry_count = u32::from_le_bytes(read4(data, pos)?); pos += 4; @@ -849,7 +979,7 @@ pub(super) fn parse_label_overflow( } label_overflow.insert(row, labels); } - Ok(label_overflow) + Ok((label_overflow, pos)) } #[cfg(test)] @@ -890,7 +1020,7 @@ mod tests { assert_eq!(csr.node_count(), 5); assert_eq!(csr.edge_count(), 10); assert_eq!(csr.header.magic, *b"MNGR"); - assert_eq!(csr.header.version, 4); + assert_eq!(csr.header.version, crate::graph::types::CSR_CURRENT_VERSION); } #[test] @@ -948,7 +1078,7 @@ mod tests { // Read back header fields. assert_eq!(&bytes[0..4], b"MNGR"); let version = u32::from_le_bytes(bytes[4..8].try_into().expect("4 bytes")); - assert_eq!(version, 4); + assert_eq!(version, crate::graph::types::CSR_CURRENT_VERSION); let nc = u32::from_le_bytes(bytes[8..12].try_into().expect("4 bytes")); assert_eq!(nc, 5); let ec = u32::from_le_bytes(bytes[12..16].try_into().expect("4 bytes")); @@ -1102,7 +1232,10 @@ mod tests { let restored = CsrSegment::from_bytes(&bytes).expect("from_bytes ok"); assert_eq!(restored.header.magic, *b"MNGR"); - assert_eq!(restored.header.version, 4); + assert_eq!( + restored.header.version, + crate::graph::types::CSR_CURRENT_VERSION + ); assert_eq!(restored.node_count(), original.node_count()); assert_eq!(restored.edge_count(), original.edge_count()); assert_eq!(restored.created_lsn, 42); @@ -1300,9 +1433,9 @@ mod tests { let csr = CsrSegment::from_frozen(frozen, 42).expect("csr ok"); let v3_bytes = csr.to_bytes(); - // Verify the source is the current version (4). + // Verify the source is the current version. let ver = u32::from_le_bytes(v3_bytes[4..8].try_into().expect("4 bytes")); - assert_eq!(ver, 4); + assert_eq!(ver, crate::graph::types::CSR_CURRENT_VERSION); let header_size = core::mem::size_of::(); // 128 let nc = csr.node_count() as usize; @@ -1390,7 +1523,7 @@ mod tests { let frozen = g.freeze().expect("freeze ok"); let csr = CsrSegment::from_frozen(frozen, 50).expect("csr ok"); - assert_eq!(csr.header.version, 4); + assert_eq!(csr.header.version, crate::graph::types::CSR_CURRENT_VERSION); // Verify from_frozen propagated temporal fields. // Note: sorted_nodes order may differ from insertion order, @@ -1412,7 +1545,10 @@ mod tests { // Serialize and deserialize. let bytes = csr.to_bytes(); let restored = CsrSegment::from_bytes(&bytes).expect("roundtrip ok"); - assert_eq!(restored.header.version, 4); + assert_eq!( + restored.header.version, + crate::graph::types::CSR_CURRENT_VERSION + ); // Verify temporal fields survive roundtrip. for (i, nm) in restored.node_meta.iter().enumerate() { @@ -1458,7 +1594,7 @@ mod tests { let frozen = build_stamped_graph(); let csr = CsrSegment::from_frozen(frozen, 100).expect("csr ok"); - assert_eq!(csr.header.version, 4); + assert_eq!(csr.header.version, crate::graph::types::CSR_CURRENT_VERSION); assert_eq!(csr.edge_created_ms.len(), csr.col_indices.len()); let mut stamps = csr.edge_created_ms.clone(); stamps.sort_unstable(); @@ -1496,8 +1632,10 @@ mod tests { fn downgrade_to_v2(csr: &CsrSegment) -> Vec { let cur_bytes = csr.to_bytes(); let ec = csr.edge_count() as usize; - // v4 trailer = edge_created_ms (ec*8) + empty label-overflow (4 bytes). - let strip = ec * 8 + 4; + // Trailer = edge_created_ms (ec*8) + empty label-overflow (4 bytes) + // + the two v5 property-blob sections (8-byte length header each, + // plus the blob bytes — non-empty when edges carry non-default weights). + let strip = ec * 8 + 4 + 8 + csr.node_props.len() + 8 + csr.edge_props.len(); let mut v2_bytes = cur_bytes[..cur_bytes.len() - strip].to_vec(); v2_bytes[4..8].copy_from_slice(&2u32.to_le_bytes()); // Zero edge_created_ms_offset (80..88) AND label_overflow_offset (88..96) @@ -1766,4 +1904,201 @@ mod tests { "truncated bytes must return CsrError, not panic" ); } + + // ----------------------------------------------------------------------- + // v5: node/edge property blobs (properties, embeddings, weights survive + // the freeze boundary — 2026-07 review P0-1) + // ----------------------------------------------------------------------- + + use crate::graph::types::PropertyValue; + + /// Node A has props + embedding, edge A->B has weight + props. Shape is + /// 3 nodes / 2 edges so `(nc+1+ec)` is even and edge_meta lands 8-aligned — + /// required for the mmap loader (odd shapes fall back to the heap loader + /// in `CsrStorage::from_file`). + fn frozen_with_props() -> (FrozenMemGraph, NodeKey, NodeKey) { + let mut g = MemGraph::new(100); + let props_a: crate::graph::types::PropertyMap = smallvec![ + (1u16, PropertyValue::Int(7)), + ( + 2u16, + PropertyValue::String(bytes::Bytes::from_static(b"alice")) + ), + ]; + let a = g.add_node(smallvec![0], props_a, Some(vec![1.0f32, -2.5]), 1); + let b = g.add_node(smallvec![0], smallvec![], None, 1); + let c = g.add_node(smallvec![0], smallvec![], None, 1); + let props_e: crate::graph::types::PropertyMap = + smallvec![(5u16, PropertyValue::Bool(true))]; + g.add_edge(a, b, 1, 2.5, Some(props_e), 2).expect("ok"); + g.add_edge(b, c, 1, 1.0, None, 2).expect("ok"); + (g.freeze().expect("freeze ok"), a, b) + } + + fn assert_v5_content( + seg_props: impl Fn(u32) -> crate::graph::types::PropertyMap, + seg_emb: impl Fn(u32) -> Option>, + row_a: u32, + row_b: u32, + ) { + let pa = seg_props(row_a); + assert_eq!(pa.len(), 2, "node A props must survive"); + assert!(pa.contains(&(1u16, PropertyValue::Int(7)))); + assert!(pa.contains(&( + 2u16, + PropertyValue::String(bytes::Bytes::from_static(b"alice")) + ))); + assert_eq!(seg_emb(row_a), Some(vec![1.0f32, -2.5])); + assert!(seg_props(row_b).is_empty()); + assert_eq!(seg_emb(row_b), None); + } + + #[test] + fn test_v5_props_survive_from_frozen() { + let (frozen, a, b) = frozen_with_props(); + let csr = CsrSegment::from_frozen(frozen, 10).expect("csr ok"); + let row_a = csr.lookup_node(a).expect("row a"); + let row_b = csr.lookup_node(b).expect("row b"); + assert_v5_content( + |r| csr.node_properties(r), + |r| csr.node_embedding(r), + row_a, + row_b, + ); + assert_eq!(csr.node_property(row_a, 1), Some(PropertyValue::Int(7))); + // Edge a->b carries weight 2.5 + a bool prop; b->a is default. + let sa = csr.row_offsets[row_a as usize]; + assert_eq!(csr.row_offsets[row_a as usize + 1] - sa, 1); + assert_eq!(csr.edge_weight(sa), 2.5); + assert_eq!( + csr.edge_properties(sa).as_slice(), + &[(5u16, PropertyValue::Bool(true))] + ); + let sb = csr.row_offsets[row_b as usize]; + assert_eq!(csr.row_offsets[row_b as usize + 1] - sb, 1); + assert_eq!(csr.edge_weight(sb), 1.0); + assert!(csr.edge_properties(sb).is_empty()); + } + + #[test] + fn test_segment_property_index_eq_and_range() { + let (frozen, a, b) = frozen_with_props(); + let storage = CsrStorage::from(CsrSegment::from_frozen(frozen, 10).expect("csr ok")); + let row_a = storage.lookup_node(a).expect("row a"); + let row_b = storage.lookup_node(b).expect("row b"); + let idx = storage.property_index(); + + // Numeric equality (Int normalized to f64). + let rows = idx.rows_eq(1, &PropertyValue::Int(7)); + assert_eq!(rows.iter().collect::>(), vec![row_a]); + assert!(idx.rows_eq(1, &PropertyValue::Int(8)).is_empty()); + // Float form of the same value matches (single numeric B-tree). + let rows = idx.rows_eq(1, &PropertyValue::Float(7.0)); + assert_eq!(rows.iter().collect::>(), vec![row_a]); + + // String equality via xxh64 hash. + let rows = idx.rows_eq( + 2, + &PropertyValue::String(bytes::Bytes::from_static(b"alice")), + ); + assert_eq!(rows.iter().collect::>(), vec![row_a]); + assert!( + idx.rows_eq(2, &PropertyValue::String(bytes::Bytes::from_static(b"bob"))) + .is_empty() + ); + + // Range query. + let rows = idx.rows_range(1, 0.0, 10.0); + assert!(rows.contains(row_a)); + assert!(!rows.contains(row_b)); + assert!(idx.rows_range(1, 8.0, 10.0).is_empty()); + + // Unindexed property id: the index is exhaustive over segment rows, + // so absence means NO row carries it — empty, not a fallback signal. + assert!(idx.rows_eq(99, &PropertyValue::Int(1)).is_empty()); + } + + #[test] + fn test_v5_props_survive_bytes_roundtrip() { + let (frozen, a, b) = frozen_with_props(); + let csr = CsrSegment::from_frozen(frozen, 10).expect("csr ok"); + let restored = CsrSegment::from_bytes(&csr.to_bytes()).expect("roundtrip ok"); + assert_eq!(restored.node_props, csr.node_props); + assert_eq!(restored.edge_props, csr.edge_props); + let row_a = restored.lookup_node(a).expect("row a"); + let row_b = restored.lookup_node(b).expect("row b"); + assert_v5_content( + |r| restored.node_properties(r), + |r| restored.node_embedding(r), + row_a, + row_b, + ); + } + + #[test] + fn test_v5_props_survive_mmap_reload() { + let dir = tempfile::TempDir::new().expect("tmpdir"); + let (frozen, a, b) = frozen_with_props(); + let csr = CsrSegment::from_frozen(frozen, 10).expect("csr ok"); + let path = dir.path().join("v5_props.csr"); + csr.write_to_file(&path).expect("write ok"); + let mm = MmapCsrSegment::from_mmap_file(&path).expect("mmap ok"); + assert_eq!(mm.header.version, crate::graph::types::CSR_CURRENT_VERSION); + let row_a = mm.lookup_node(a).expect("row a"); + let row_b = mm.lookup_node(b).expect("row b"); + assert_v5_content( + |r| mm.node_properties(r), + |r| mm.node_embedding(r), + row_a, + row_b, + ); + let sa = mm.row_offsets()[row_a as usize]; + assert_eq!(mm.edge_weight(sa), 2.5); + assert_eq!( + mm.edge_properties(sa).as_slice(), + &[(5u16, PropertyValue::Bool(true))] + ); + } + + #[test] + fn test_v4_file_parses_with_empty_blobs() { + // Hand-build v4 bytes from a v5 segment: strip the two trailing blob + // sections, patch version to 4, recompute CRC. Loading must succeed + // with empty blobs (graceful degradation, not an error). + let (frozen, a, _b) = frozen_with_props(); + let csr = CsrSegment::from_frozen(frozen, 10).expect("csr ok"); + let v5 = csr.to_bytes(); + let strip = 8 + csr.node_props.len() + 8 + csr.edge_props.len(); + let mut v4 = v5[..v5.len() - strip].to_vec(); + v4[4..8].copy_from_slice(&4u32.to_le_bytes()); + v4[72..80].copy_from_slice(&0u64.to_le_bytes()); + let checksum = compute_csr_checksum(&v4) as u64; + v4[72..80].copy_from_slice(&checksum.to_le_bytes()); + + let loaded = CsrSegment::from_bytes(&v4).expect("v4 parse ok"); + assert_eq!(loaded.header.version, 4); + assert!(loaded.node_props.is_empty()); + assert!(loaded.edge_props.is_empty()); + let row_a = loaded.lookup_node(a).expect("row a"); + // property_offset survives in NodeMeta but the blob is absent — + // decoders fall back to empty/None/default rather than erroring. + assert!(loaded.node_properties(row_a).is_empty()); + assert_eq!(loaded.node_embedding(row_a), None); + assert_eq!(loaded.edge_weight(0), 1.0); + } + + #[test] + fn test_v5_truncated_blob_section_errors_no_panic() { + let (frozen, _a, _b) = frozen_with_props(); + let csr = CsrSegment::from_frozen(frozen, 10).expect("csr ok"); + let v5 = csr.to_bytes(); + // Cut inside the blob sections (CRC mismatch or section truncation — + // either way: error, never panic). + for cut in [4usize, 8, 12, 20] { + if v5.len() > cut { + let truncated = &v5[..v5.len() - cut]; + assert!(CsrSegment::from_bytes(truncated).is_err()); + } + } + } } diff --git a/src/graph/csr/props.rs b/src/graph/csr/props.rs new file mode 100644 index 00000000..4fff301f --- /dev/null +++ b/src/graph/csr/props.rs @@ -0,0 +1,534 @@ +//! Property / embedding / weight blob codec for CSR segments (format v5). +//! +//! Before v5, `CsrSegment::from_frozen` wrote `property_offset: 0` for every +//! node and edge — properties, embeddings, and edge weights were silently +//! DISCARDED at freeze (2026-07 graph deep review P0-1). v5 appends two +//! variable-length sections to the segment payload and points the reserved +//! `NodeMeta::property_offset` / `EdgeMeta::property_offset` fields into them. +//! +//! Encoding (all little-endian, unaligned — read with bounds-checked helpers): +//! +//! ```text +//! node record: [n_props: u16] [prop]* [emb_dim: u32] [f32 * emb_dim] +//! edge record: [weight: f64] [n_props: u16] [prop]* +//! prop: [key: u16] [tag: u8] [payload] +//! tag 0 Int: i64 +//! tag 1 Float: f64 +//! tag 2 String: [len: u32] [bytes] +//! tag 3 Bool: u8 (0/1) +//! tag 4 Bytes: [len: u32] [bytes] +//! ``` +//! +//! Offsets are stored **offset + 1**; `0` means "no record" (empty props, no +//! embedding, default weight 1.0). This keeps `0` — the value every pre-v5 +//! segment carries — meaning exactly what it always meant. +//! +//! Decoders are defensive: any out-of-bounds or malformed input yields `None` +//! / empty rather than panicking (fuzz target: `graph_props_record`). + +use bytes::Bytes; + +use crate::graph::types::{PropertyMap, PropertyValue}; + +/// Default edge weight when no record exists. +pub const DEFAULT_EDGE_WEIGHT: f64 = 1.0; + +const TAG_INT: u8 = 0; +const TAG_FLOAT: u8 = 1; +const TAG_STRING: u8 = 2; +const TAG_BOOL: u8 = 3; +const TAG_BYTES: u8 = 4; + +// --------------------------------------------------------------------------- +// Encoding +// --------------------------------------------------------------------------- + +fn encode_props(buf: &mut Vec, props: &[(u16, PropertyValue)]) { + buf.extend_from_slice(&(props.len() as u16).to_le_bytes()); + for (key, value) in props { + buf.extend_from_slice(&key.to_le_bytes()); + match value { + PropertyValue::Int(v) => { + buf.push(TAG_INT); + buf.extend_from_slice(&v.to_le_bytes()); + } + PropertyValue::Float(v) => { + buf.push(TAG_FLOAT); + buf.extend_from_slice(&v.to_le_bytes()); + } + PropertyValue::String(b) => { + buf.push(TAG_STRING); + buf.extend_from_slice(&(b.len() as u32).to_le_bytes()); + buf.extend_from_slice(b); + } + PropertyValue::Bool(v) => { + buf.push(TAG_BOOL); + buf.push(u8::from(*v)); + } + PropertyValue::Bytes(b) => { + buf.push(TAG_BYTES); + buf.extend_from_slice(&(b.len() as u32).to_le_bytes()); + buf.extend_from_slice(b); + } + } + } +} + +/// Append a node record (properties + optional embedding) to `blob`. +/// +/// Returns the stored offset (`byte_pos + 1`), or `0` if the node has neither +/// properties nor an embedding (nothing is written). +pub fn encode_node_record( + blob: &mut Vec, + props: &[(u16, PropertyValue)], + embedding: Option<&[f32]>, +) -> u32 { + if props.is_empty() && embedding.is_none() { + return 0; + } + let pos = blob.len(); + encode_props(blob, props); + match embedding { + Some(emb) => { + blob.extend_from_slice(&(emb.len() as u32).to_le_bytes()); + for f in emb { + blob.extend_from_slice(&f.to_le_bytes()); + } + } + None => blob.extend_from_slice(&0u32.to_le_bytes()), + } + (pos as u32) + 1 +} + +/// Append an edge record (weight + optional properties) to `blob`. +/// +/// Returns the stored offset (`byte_pos + 1`), or `0` when the edge carries +/// the default weight and no properties (nothing is written). +pub fn encode_edge_record(blob: &mut Vec, weight: f64, props: Option<&PropertyMap>) -> u32 { + let has_props = props.is_some_and(|p| !p.is_empty()); + if weight == DEFAULT_EDGE_WEIGHT && !has_props { + return 0; + } + let pos = blob.len(); + blob.extend_from_slice(&weight.to_le_bytes()); + match props { + Some(p) => encode_props(blob, p), + None => blob.extend_from_slice(&0u16.to_le_bytes()), + } + (pos as u32) + 1 +} + +// --------------------------------------------------------------------------- +// Bounds-checked primitive reads (no panics on malformed input) +// --------------------------------------------------------------------------- + +fn rd_u8(blob: &[u8], pos: usize) -> Option { + blob.get(pos).copied() +} + +fn rd_u16(blob: &[u8], pos: usize) -> Option { + Some(u16::from_le_bytes(blob.get(pos..pos + 2)?.try_into().ok()?)) +} + +fn rd_u32(blob: &[u8], pos: usize) -> Option { + Some(u32::from_le_bytes(blob.get(pos..pos + 4)?.try_into().ok()?)) +} + +fn rd_u64(blob: &[u8], pos: usize) -> Option { + Some(u64::from_le_bytes(blob.get(pos..pos + 8)?.try_into().ok()?)) +} + +/// Decode one property at `pos`. Returns `(key, value, next_pos)`. +fn decode_prop(blob: &[u8], pos: usize) -> Option<(u16, PropertyValue, usize)> { + let key = rd_u16(blob, pos)?; + let tag = rd_u8(blob, pos + 2)?; + let vpos = pos + 3; + match tag { + TAG_INT => Some(( + key, + PropertyValue::Int(rd_u64(blob, vpos)? as i64), + vpos + 8, + )), + TAG_FLOAT => Some(( + key, + PropertyValue::Float(f64::from_bits(rd_u64(blob, vpos)?)), + vpos + 8, + )), + TAG_STRING | TAG_BYTES => { + let len = rd_u32(blob, vpos)? as usize; + let start = vpos + 4; + let bytes = blob.get(start..start.checked_add(len)?)?; + let b = Bytes::copy_from_slice(bytes); + let value = if tag == TAG_STRING { + PropertyValue::String(b) + } else { + PropertyValue::Bytes(b) + }; + Some((key, value, start + len)) + } + TAG_BOOL => Some((key, PropertyValue::Bool(rd_u8(blob, vpos)? != 0), vpos + 1)), + _ => None, + } +} + +/// Skip one property at `pos`, returning the next position. +fn skip_prop(blob: &[u8], pos: usize) -> Option { + let tag = rd_u8(blob, pos + 2)?; + let vpos = pos + 3; + match tag { + TAG_INT | TAG_FLOAT => Some(vpos + 8), + TAG_STRING | TAG_BYTES => { + let len = rd_u32(blob, vpos)? as usize; + let end = (vpos + 4).checked_add(len)?; + if end > blob.len() { None } else { Some(end) } + } + TAG_BOOL => Some(vpos + 1), + _ => None, + } +} + +// --------------------------------------------------------------------------- +// Node record decoding +// --------------------------------------------------------------------------- + +/// Decode a node's full property map. `stored_offset` is the raw +/// `NodeMeta::property_offset` value (`0` = no record → empty map). +pub fn decode_node_props(blob: &[u8], stored_offset: u32) -> PropertyMap { + let mut out = PropertyMap::new(); + if stored_offset == 0 { + return out; + } + let mut pos = (stored_offset - 1) as usize; + let Some(n) = rd_u16(blob, pos) else { + return out; + }; + pos += 2; + for _ in 0..n { + match decode_prop(blob, pos) { + Some((key, value, next)) => { + out.push((key, value)); + pos = next; + } + None => break, + } + } + out +} + +/// Look up a single property by key without materializing the map. +pub fn decode_node_prop(blob: &[u8], stored_offset: u32, want: u16) -> Option { + if stored_offset == 0 { + return None; + } + let mut pos = (stored_offset - 1) as usize; + let n = rd_u16(blob, pos)?; + pos += 2; + for _ in 0..n { + let key = rd_u16(blob, pos)?; + if key == want { + return decode_prop(blob, pos).map(|(_, v, _)| v); + } + pos = skip_prop(blob, pos)?; + } + None +} + +/// Decode a node's embedding. Returns `None` for no record / no embedding / +/// malformed input. +pub fn decode_node_embedding(blob: &[u8], stored_offset: u32) -> Option> { + if stored_offset == 0 { + return None; + } + let mut pos = (stored_offset - 1) as usize; + let n = rd_u16(blob, pos)?; + pos += 2; + for _ in 0..n { + pos = skip_prop(blob, pos)?; + } + let dim = rd_u32(blob, pos)? as usize; + if dim == 0 { + return None; + } + pos += 4; + let end = pos.checked_add(dim.checked_mul(4)?)?; + let bytes = blob.get(pos..end)?; + let mut out = Vec::with_capacity(dim); + for chunk in bytes.chunks_exact(4) { + #[allow(clippy::unwrap_used)] // chunks_exact(4) guarantees 4-byte chunks + let arr: [u8; 4] = chunk.try_into().unwrap(); + out.push(f32::from_le_bytes(arr)); + } + Some(out) +} + +/// Byte length of the node record starting at `stored_offset` (for record +/// copying during segment merge). `None` for absent/malformed records. +pub fn node_record_len(blob: &[u8], stored_offset: u32) -> Option { + if stored_offset == 0 { + return None; + } + let start = (stored_offset - 1) as usize; + let mut pos = start; + let n = rd_u16(blob, pos)?; + pos += 2; + for _ in 0..n { + pos = skip_prop(blob, pos)?; + } + let dim = rd_u32(blob, pos)? as usize; + pos = (pos + 4).checked_add(dim.checked_mul(4)?)?; + if pos > blob.len() { + None + } else { + Some(pos - start) + } +} + +// --------------------------------------------------------------------------- +// Edge record decoding +// --------------------------------------------------------------------------- + +/// Decode an edge's weight. `0` offset (or malformed input) yields the +/// default weight 1.0. +pub fn decode_edge_weight(blob: &[u8], stored_offset: u32) -> f64 { + if stored_offset == 0 { + return DEFAULT_EDGE_WEIGHT; + } + match rd_u64(blob, (stored_offset - 1) as usize) { + Some(bits) => f64::from_bits(bits), + None => DEFAULT_EDGE_WEIGHT, + } +} + +/// Decode an edge's full property map (empty for absent/malformed records). +pub fn decode_edge_props(blob: &[u8], stored_offset: u32) -> PropertyMap { + let mut out = PropertyMap::new(); + if stored_offset == 0 { + return out; + } + let mut pos = (stored_offset - 1) as usize + 8; // skip weight + let Some(n) = rd_u16(blob, pos) else { + return out; + }; + pos += 2; + for _ in 0..n { + match decode_prop(blob, pos) { + Some((key, value, next)) => { + out.push((key, value)); + pos = next; + } + None => break, + } + } + out +} + +/// Look up a single edge property by key. +pub fn decode_edge_prop(blob: &[u8], stored_offset: u32, want: u16) -> Option { + if stored_offset == 0 { + return None; + } + let mut pos = (stored_offset - 1) as usize + 8; // skip weight + let n = rd_u16(blob, pos)?; + pos += 2; + for _ in 0..n { + let key = rd_u16(blob, pos)?; + if key == want { + return decode_prop(blob, pos).map(|(_, v, _)| v); + } + pos = skip_prop(blob, pos)?; + } + None +} + +/// Byte length of the edge record starting at `stored_offset`. +pub fn edge_record_len(blob: &[u8], stored_offset: u32) -> Option { + if stored_offset == 0 { + return None; + } + let start = (stored_offset - 1) as usize; + let mut pos = start + 8; // weight + let n = rd_u16(blob, pos)?; + pos += 2; + for _ in 0..n { + pos = skip_prop(blob, pos)?; + } + if pos > blob.len() { + None + } else { + Some(pos - start) + } +} + +/// Copy an existing record verbatim into `dst`, returning the new stored +/// offset (`0` in = `0` out; malformed source records are dropped to `0` +/// rather than corrupting the destination blob). +pub fn copy_record( + dst: &mut Vec, + src: &[u8], + stored_offset: u32, + len_of: fn(&[u8], u32) -> Option, +) -> u32 { + if stored_offset == 0 { + return 0; + } + match len_of(src, stored_offset) { + Some(len) => { + let start = (stored_offset - 1) as usize; + let pos = dst.len(); + dst.extend_from_slice(&src[start..start + len]); + (pos as u32) + 1 + } + None => 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use smallvec::smallvec; + + fn sample_props() -> PropertyMap { + smallvec![ + (7u16, PropertyValue::Int(-42)), + (9u16, PropertyValue::Float(2.5)), + (11u16, PropertyValue::String(Bytes::from_static(b"alice"))), + (13u16, PropertyValue::Bool(true)), + (15u16, PropertyValue::Bytes(Bytes::from_static(b"\x00\x01"))), + ] + } + + #[test] + fn node_record_roundtrip_props_and_embedding() { + let mut blob = Vec::new(); + let props = sample_props(); + let emb = vec![1.0f32, -2.5, 0.0, 3.25]; + let off = encode_node_record(&mut blob, &props, Some(&emb)); + assert_ne!(off, 0); + + let decoded = decode_node_props(&blob, off); + assert_eq!(decoded.as_slice(), props.as_slice()); + assert_eq!(decode_node_embedding(&blob, off).as_deref(), Some(&emb[..])); + assert_eq!( + decode_node_prop(&blob, off, 11), + Some(PropertyValue::String(Bytes::from_static(b"alice"))) + ); + assert_eq!(decode_node_prop(&blob, off, 99), None); + assert_eq!(node_record_len(&blob, off), Some(blob.len())); + } + + #[test] + fn empty_node_record_writes_nothing() { + let mut blob = Vec::new(); + let off = encode_node_record(&mut blob, &[], None); + assert_eq!(off, 0); + assert!(blob.is_empty()); + assert!(decode_node_props(&blob, 0).is_empty()); + assert_eq!(decode_node_embedding(&blob, 0), None); + } + + #[test] + fn node_record_props_only_no_embedding() { + let mut blob = Vec::new(); + let props: PropertyMap = smallvec![(1u16, PropertyValue::Int(5))]; + let off = encode_node_record(&mut blob, &props, None); + assert_ne!(off, 0); + assert_eq!(decode_node_props(&blob, off).as_slice(), props.as_slice()); + assert_eq!(decode_node_embedding(&blob, off), None); + } + + #[test] + fn edge_record_roundtrip() { + let mut blob = Vec::new(); + let props: PropertyMap = smallvec![(3u16, PropertyValue::Float(0.125))]; + let off = encode_edge_record(&mut blob, 4.5, Some(&props)); + assert_ne!(off, 0); + assert_eq!(decode_edge_weight(&blob, off), 4.5); + assert_eq!(decode_edge_props(&blob, off).as_slice(), props.as_slice()); + assert_eq!( + decode_edge_prop(&blob, off, 3), + Some(PropertyValue::Float(0.125)) + ); + assert_eq!(edge_record_len(&blob, off), Some(blob.len())); + } + + #[test] + fn default_weight_no_props_writes_nothing() { + let mut blob = Vec::new(); + let off = encode_edge_record(&mut blob, DEFAULT_EDGE_WEIGHT, None); + assert_eq!(off, 0); + assert!(blob.is_empty()); + assert_eq!(decode_edge_weight(&blob, 0), DEFAULT_EDGE_WEIGHT); + } + + #[test] + fn non_default_weight_without_props_is_stored() { + let mut blob = Vec::new(); + let off = encode_edge_record(&mut blob, 0.5, None); + assert_ne!(off, 0); + assert_eq!(decode_edge_weight(&blob, off), 0.5); + assert!(decode_edge_props(&blob, off).is_empty()); + } + + #[test] + fn multiple_records_in_one_blob() { + let mut blob = Vec::new(); + let p1: PropertyMap = smallvec![(1u16, PropertyValue::Int(1))]; + let p2: PropertyMap = smallvec![(2u16, PropertyValue::Int(2))]; + let o1 = encode_node_record(&mut blob, &p1, None); + let o2 = encode_node_record(&mut blob, &p2, Some(&[9.0f32])); + assert_ne!(o1, o2); + assert_eq!(decode_node_props(&blob, o1).as_slice(), p1.as_slice()); + assert_eq!(decode_node_props(&blob, o2).as_slice(), p2.as_slice()); + assert_eq!(decode_node_embedding(&blob, o1), None); + assert_eq!(decode_node_embedding(&blob, o2), Some(vec![9.0f32])); + } + + #[test] + fn copy_record_preserves_content() { + let mut src = Vec::new(); + let props = sample_props(); + let off = encode_node_record(&mut src, &props, Some(&[1.0f32, 2.0])); + let mut dst = vec![0xAAu8; 17]; // non-empty destination with prior content + let new_off = copy_record(&mut dst, &src, off, node_record_len); + assert_ne!(new_off, 0); + assert_eq!( + decode_node_props(&dst, new_off).as_slice(), + props.as_slice() + ); + assert_eq!( + decode_node_embedding(&dst, new_off), + Some(vec![1.0f32, 2.0]) + ); + // Zero copies through as zero. + assert_eq!(copy_record(&mut dst, &src, 0, node_record_len), 0); + } + + #[test] + fn malformed_input_never_panics() { + // Truncated at every prefix of a valid record. + let mut blob = Vec::new(); + let props = sample_props(); + let off = encode_node_record(&mut blob, &props, Some(&[1.0f32])); + for cut in 0..blob.len() { + let truncated = &blob[..cut]; + let _ = decode_node_props(truncated, off); + let _ = decode_node_embedding(truncated, off); + let _ = decode_node_prop(truncated, off, 7); + let _ = node_record_len(truncated, off); + let _ = decode_edge_weight(truncated, off); + let _ = decode_edge_props(truncated, off); + let _ = edge_record_len(truncated, off); + } + // Out-of-range offsets. + for bogus in [1u32, 5, 1000, u32::MAX] { + let _ = decode_node_props(&blob, bogus); + let _ = decode_node_embedding(&blob, bogus); + let _ = decode_edge_weight(&blob, bogus); + let _ = decode_edge_props(&blob, bogus); + } + // Garbage bytes with a bogus tag. + let garbage = vec![2u8, 0, 1, 0, 250, 9, 9, 9, 9]; + let _ = decode_node_props(&garbage, 1); + let _ = decode_node_embedding(&garbage, 1); + } +} diff --git a/src/graph/csr/storage.rs b/src/graph/csr/storage.rs index 657c04d6..ae01d8b4 100644 --- a/src/graph/csr/storage.rs +++ b/src/graph/csr/storage.rs @@ -134,13 +134,58 @@ impl CsrStorage { /// Resident bytes used by this CSR segment. /// - /// For Heap: row_offsets + col_indices + edge_meta + node_meta Vec allocations. - /// For Mmap: the mmap region length (kernel page-cache resident). + /// Follows the `RawF16Store` precedent (PR #232): mmap-backed sections + /// are kernel page cache (reclaimable under memory pressure) and count + /// as 0; heap-owned sections count in full. Previously this summed + /// `row_offsets`/`col_indices`/`edge_meta`/`node_meta` unconditionally + /// for BOTH variants (over-counting `Mmap` segments, whose backing + /// storage is NOT heap-resident) and omitted the v5 node/edge property + /// blobs, the lazily-built `SegmentPropertyIndexes`, and the lazily-built + /// `GraphHnsw` bridge entirely — all invisible to the shard's elastic + /// memory budget / eviction pipeline (`NamedGraph::resident_bytes` -> + /// `src/shard/persistence_tick.rs`). + /// + /// `props_index` and `hnsw_bridge` are ALWAYS heap-owned once built, + /// regardless of whether the underlying segment is `Heap` or `Mmap` — + /// they are derived structures built on top of the (possibly mmap'd) + /// source data, never persisted/mapped themselves. pub fn resident_bytes(&self) -> usize { - std::mem::size_of_val(self.row_offsets()) - + std::mem::size_of_val(self.col_indices()) - + std::mem::size_of_val(self.edge_meta()) - + std::mem::size_of_val(self.node_meta()) + let (core, props) = match self { + CsrStorage::Heap(s) => { + let core = std::mem::size_of_val(s.row_offsets.as_slice()) + + std::mem::size_of_val(s.col_indices.as_slice()) + + std::mem::size_of_val(s.edge_meta.as_slice()) + + std::mem::size_of_val(s.node_meta.as_slice()); + (core, s.node_props.len() + s.edge_props.len()) + } + CsrStorage::Mmap(_) => (0, 0), + }; + let indexes = self + .props_index_if_built() + .map_or(0, |ix| ix.resident_bytes()); + let hnsw = self + .hnsw_bridge_if_built() + .map_or(0, |b| b.resident_bytes()); + core + props + indexes + hnsw + } + + /// Borrow the property index ONLY if it has already been built (does + /// not trigger a build) — `resident_bytes` must be a cheap read, never + /// a lazy-init trigger. + fn props_index_if_built(&self) -> Option<&crate::graph::index::SegmentPropertyIndexes> { + match self { + CsrStorage::Heap(s) => s.props_index.get(), + CsrStorage::Mmap(s) => s.props_index.get(), + } + } + + /// Borrow the HNSW bridge ONLY if it has already been built AND + /// succeeded (`Some(GraphHnsw)`, not the cached "too few vectors" `None`). + fn hnsw_bridge_if_built(&self) -> Option<&crate::graph::hnsw_bridge::GraphHnsw> { + match self { + CsrStorage::Heap(s) => s.hnsw_bridge.get().and_then(|b| b.as_ref()), + CsrStorage::Mmap(s) => s.hnsw_bridge.get().and_then(|b| b.as_ref()), + } } /// Created LSN for this segment. @@ -305,6 +350,111 @@ impl CsrStorage { } } + /// Get-or-build this segment's lazy property index over CSR rows. + /// Built once from node_meta + the v5 property blob; cached in the + /// segment's `OnceLock` (interior mutability — read-only derived state, + /// same pattern as `incoming_index`). + pub fn property_index(&self) -> &crate::graph::index::SegmentPropertyIndexes { + match self { + CsrStorage::Heap(s) => s.props_index.get_or_init(|| { + crate::graph::index::SegmentPropertyIndexes::build(&s.node_meta, &s.node_props) + }), + CsrStorage::Mmap(s) => s.props_index.get_or_init(|| { + crate::graph::index::SegmentPropertyIndexes::build( + s.node_meta(), + s.node_props_blob(), + ) + }), + } + } + + /// Get-or-build this segment's HNSW bridge over v5 node embeddings + /// (hybrid HnswPreFilter). Built at most once per segment, and only when + /// it holds >= `BRIDGE_MIN_VECTORS` usable embeddings — the negative + /// answer is cached too, so small segments pay the scan exactly once. + /// Same `OnceLock` interior-mutability pattern as `incoming_index`. + pub fn hnsw_bridge(&self) -> Option<&crate::graph::hnsw_bridge::GraphHnsw> { + let cell = match self { + CsrStorage::Heap(s) => &s.hnsw_bridge, + CsrStorage::Mmap(s) => &s.hnsw_bridge, + }; + cell.get_or_init(|| { + crate::graph::hnsw_bridge::GraphHnsw::build( + self, + crate::graph::hnsw_bridge::BRIDGE_MIN_VECTORS, + ) + }) + .as_ref() + } + + /// Test hook: build the bridge regardless of the production minimum, so + /// small fixtures can exercise the HnswPreFilter path. + #[cfg(test)] + pub fn hnsw_bridge_for_test(&self) -> Option<&crate::graph::hnsw_bridge::GraphHnsw> { + let cell = match self { + CsrStorage::Heap(s) => &s.hnsw_bridge, + CsrStorage::Mmap(s) => &s.hnsw_bridge, + }; + cell.get_or_init(|| crate::graph::hnsw_bridge::GraphHnsw::build(self, 1)) + .as_ref() + } + + /// Node property blob (empty for pre-v5 segments). + pub fn node_props_blob(&self) -> &[u8] { + match self { + CsrStorage::Heap(s) => &s.node_props, + CsrStorage::Mmap(s) => s.node_props_blob(), + } + } + + /// Edge property blob (empty for pre-v5 segments). + pub fn edge_props_blob(&self) -> &[u8] { + match self { + CsrStorage::Heap(s) => &s.edge_props, + CsrStorage::Mmap(s) => s.edge_props_blob(), + } + } + + /// Node properties for a CSR row (empty for none / pre-v5 segments). + pub fn node_properties(&self, row: u32) -> crate::graph::types::PropertyMap { + match self { + CsrStorage::Heap(s) => s.node_properties(row), + CsrStorage::Mmap(s) => s.node_properties(row), + } + } + + /// Single node property lookup without materializing the map. + pub fn node_property(&self, row: u32, key: u16) -> Option { + match self { + CsrStorage::Heap(s) => s.node_property(row, key), + CsrStorage::Mmap(s) => s.node_property(row, key), + } + } + + /// Node embedding for a CSR row (None for none / pre-v5 segments). + pub fn node_embedding(&self, row: u32) -> Option> { + match self { + CsrStorage::Heap(s) => s.node_embedding(row), + CsrStorage::Mmap(s) => s.node_embedding(row), + } + } + + /// Edge weight by edge index (1.0 default / pre-v5 segments). + pub fn edge_weight(&self, edge_idx: u32) -> f64 { + match self { + CsrStorage::Heap(s) => s.edge_weight(edge_idx), + CsrStorage::Mmap(s) => s.edge_weight(edge_idx), + } + } + + /// Edge properties by edge index (empty for none / pre-v5 segments). + pub fn edge_properties(&self, edge_idx: u32) -> crate::graph::types::PropertyMap { + match self { + CsrStorage::Heap(s) => s.edge_properties(edge_idx), + CsrStorage::Mmap(s) => s.edge_properties(edge_idx), + } + } + /// Write the segment to a file (only supported for Heap variant). /// For Mmap variant, the file already exists on disk. pub fn write_to_file(&self, path: &Path) -> Result<(), CsrError> { @@ -393,3 +543,91 @@ impl From for CsrStorage { CsrStorage::Heap(seg) } } + +#[cfg(test)] +mod tests { + use smallvec::smallvec; + + use super::*; + use crate::graph::memgraph::MemGraph; + use crate::graph::types::PropertyValue; + + /// Deterministic pseudo-random embedding (no Math.random in tests). + fn embedding(i: u64, dim: usize) -> Vec { + let mut state = i.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..dim) + .map(|_| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((state >> 33) as f32 / (1u64 << 31) as f32) - 0.5 + }) + .collect() + } + + /// A heap-backed segment with real per-node properties AND embeddings, + /// so both `property_index()` and `hnsw_bridge_for_test()` have + /// something non-trivial to build (an all-empty fixture would build + /// indexes that are themselves empty, masking a broken resident_bytes + /// count with a true-by-accident 0 -> 0 "no growth" result). + fn heap_segment_with_props_and_embeddings(n: usize, dim: usize) -> CsrStorage { + let mut g = MemGraph::new(1_000_000); + let mut keys = Vec::with_capacity(n); + for i in 0..n { + let props = smallvec![(0u16, PropertyValue::Int(i as i64))]; + keys.push(g.add_node(smallvec![0u16], props, Some(embedding(i as u64, dim)), 1)); + } + for i in 0..n.saturating_sub(1) { + g.add_edge(keys[i], keys[i + 1], 1, 1.0, None, 2) + .expect("edge"); + } + let frozen = g.freeze().expect("freeze"); + let seg = CsrSegment::from_frozen(frozen, 10).expect("csr"); + CsrStorage::Heap(seg) + } + + #[test] + fn test_resident_bytes_grows_after_index_and_bridge_build() { + let storage = heap_segment_with_props_and_embeddings(64, 8); + + let baseline = storage.resident_bytes(); + + // Force the property index to build (get_or_init, not a call that + // would already have been triggered by anything above). + let _ = storage.property_index(); + let after_props_index = storage.resident_bytes(); + assert!( + after_props_index > baseline, + "resident_bytes must grow once the lazily-built SegmentPropertyIndexes \ + is materialized: baseline={baseline}, after={after_props_index}" + ); + + // Force the HNSW bridge to build too (test hook bypasses the + // production BRIDGE_MIN_VECTORS gate). + assert!( + storage.hnsw_bridge_for_test().is_some(), + "fixture has embeddings on every node; bridge build must succeed" + ); + let after_bridge = storage.resident_bytes(); + assert!( + after_bridge > after_props_index, + "resident_bytes must grow again once the lazily-built GraphHnsw bridge \ + is materialized: after_props_index={after_props_index}, after_bridge={after_bridge}" + ); + } + + #[test] + fn test_resident_bytes_mmap_zeroes_core_arrays_heap_counts_them() { + // Heap segment: row/col/edge_meta/node_meta count in full (these are + // genuinely heap-owned Vec allocations). + let heap = heap_segment_with_props_and_embeddings(16, 4); + let heap_core = std::mem::size_of_val(heap.row_offsets()) + + std::mem::size_of_val(heap.col_indices()) + + std::mem::size_of_val(heap.edge_meta()) + + std::mem::size_of_val(heap.node_meta()); + assert!( + heap.resident_bytes() >= heap_core, + "Heap variant must count its row/col/edge_meta/node_meta arrays in full" + ); + } +} diff --git a/src/graph/cypher/executor/eval.rs b/src/graph/cypher/executor/eval.rs index 996e76d9..76a63233 100644 --- a/src/graph/cypher/executor/eval.rs +++ b/src/graph/cypher/executor/eval.rs @@ -13,7 +13,7 @@ use super::*; /// (Phase 174 FIX-04). pub(crate) fn eval_expr( expr: &Expr, - row: &Row, + row: &Row<'_>, memgraph: &crate::graph::memgraph::MemGraph, params: &HashMap, immutable_segs: &[std::sync::Arc], @@ -43,15 +43,15 @@ pub(crate) fn eval_expr( ); match obj { Value::Node(key) => { - if let Some(node) = memgraph.get_node(key) { - let prop_id = label_to_id(property.as_bytes()); - for (pid, pval) in &node.properties { - if *pid == prop_id { - return property_value_to_value(pval); - } - } + // Merged-tier lookup: mutable write buffer first, then + // frozen CSR segments (v5 property blob). A node lives in + // exactly one tier. + let prop_id = label_to_id(property.as_bytes()); + let view = crate::graph::view::MergedNodeView::new(memgraph, immutable_segs); + match view.property(key, prop_id) { + Some(pv) => property_value_to_value(&pv), + None => Value::Null, } - Value::Null } Value::Edge(key) => { if let Some(edge) = memgraph.get_edge(key) { @@ -213,9 +213,13 @@ pub(crate) fn eval_expr( decay, ); if let Value::Node(k) = v { - if let Some(node) = memgraph.get_node(k) { + // Merged-tier labels (frozen rows decode the CSR + // label bitmap + overflow). + let view = + crate::graph::view::MergedNodeView::new(memgraph, immutable_segs); + if let Some(node_labels) = view.labels(k) { let labels: Vec = - node.labels.iter().map(|&l| Value::Int(l as i64)).collect(); + node_labels.iter().map(|&l| Value::Int(l as i64)).collect(); return Value::List(labels); } } @@ -406,7 +410,7 @@ pub(crate) fn eval_expr( Expr::Star => { let entries: Vec<(String, Value)> = - row.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + row.iter().map(|(k, v)| (k.to_owned(), v.clone())).collect(); Value::Map(entries) } diff --git a/src/graph/cypher/executor/mod.rs b/src/graph/cypher/executor/mod.rs index e2656dd7..2a1df67e 100644 --- a/src/graph/cypher/executor/mod.rs +++ b/src/graph/cypher/executor/mod.rs @@ -1,8 +1,10 @@ //! Cypher execution engine -- walks PhysicalPlan operators and produces result rows. //! -//! Row-based pipeline model: each operator transforms a `Vec` where -//! `Row = HashMap`. The executor starts with one empty seed row -//! and sequentially applies each `PhysicalOp` from the plan. +//! Row-based pipeline model: each operator transforms a `Vec`. A `Row` +//! is a slot-indexed `SmallVec` against a per-execution [`SlotTable`] +//! (all variables a plan can bind, collected once) — no per-row HashMap +//! allocation and no per-insert String key clone. The executor starts with +//! one Null-filled seed row and sequentially applies each `PhysicalOp`. mod eval; mod read; @@ -13,8 +15,6 @@ pub(crate) use eval::*; pub use read::*; pub use write::*; -use std::collections::HashMap; - use bytes::Bytes; use smallvec::SmallVec; @@ -43,8 +43,163 @@ pub enum Value { Path(Vec), } -/// A single result row: variable bindings. -pub type Row = HashMap; +/// Maps variable names to row slot indices. +/// +/// Built once per execution by walking the plan's BINDING operators +/// (scans, expands, unwind, shortestPath, create/merge patterns). Names +/// that are referenced but never bound simply resolve to no slot — the +/// same as a HashMap miss in the old row model. Lookup is a linear scan: +/// queries bind a handful of short names, which beats hashing. +#[derive(Debug, Default)] +pub struct SlotTable { + names: Vec, +} + +impl SlotTable { + /// Collect every variable the plan can bind, in first-bind order. + pub fn from_plan(plan: &PhysicalPlan) -> Self { + let mut table = SlotTable::default(); + for op in &plan.operators { + match op { + PhysicalOp::NodeScan { variable, .. } | PhysicalOp::IndexScan { variable, .. } => { + table.bind(variable) + } + PhysicalOp::Expand { + source, + target, + edge_variable, + .. + } => { + table.bind(source); + table.bind(target); + if let Some(evar) = edge_variable { + table.bind(evar); + } + } + PhysicalOp::Unwind { alias, .. } => table.bind(alias), + PhysicalOp::ShortestPath { + path_var, + source, + target, + .. + } => { + table.bind(path_var); + table.bind(source); + table.bind(target); + } + PhysicalOp::CreatePattern { patterns } => { + for p in patterns { + table.bind_pattern(p); + } + } + PhysicalOp::Merge { pattern, .. } => table.bind_pattern(pattern), + // Reference-only operators bind nothing. + PhysicalOp::Filter { .. } + | PhysicalOp::Project { .. } + | PhysicalOp::Sort { .. } + | PhysicalOp::Limit { .. } + | PhysicalOp::Skip { .. } + | PhysicalOp::DeleteEntities { .. } + | PhysicalOp::SetProperties { .. } + | PhysicalOp::ProcedureCall { .. } => {} + } + } + table + } + + fn bind(&mut self, name: &str) { + if !name.is_empty() && !self.names.iter().any(|n| n == name) { + self.names.push(name.to_owned()); + } + } + + fn bind_pattern(&mut self, pattern: &Pattern) { + for node in &pattern.nodes { + if let Some(v) = &node.variable { + self.bind(v); + } + } + for edge in &pattern.edges { + if let Some(v) = &edge.variable { + self.bind(v); + } + } + } + + /// Resolve a variable name to its slot index. + #[inline] + pub fn slot(&self, name: &str) -> Option { + self.names.iter().position(|n| n == name) + } + + /// Slot names in slot order. + pub fn names(&self) -> &[String] { + &self.names + } + + /// Number of slots. + pub fn len(&self) -> usize { + self.names.len() + } + + /// Whether the table has no slots. + pub fn is_empty(&self) -> bool { + self.names.is_empty() + } +} + +/// A single result row: variable bindings, slot-indexed against the +/// execution's [`SlotTable`]. +/// +/// Unbound slots hold `Value::Null`, which every pipeline consumer treats +/// identically to the old HashMap miss: expression eval defaults a missing +/// variable to Null, and operator sources pattern-match a concrete variant +/// (`Some(Value::Node(_))`), which Null fails just like `None` did. +#[derive(Debug, Clone)] +pub struct Row<'a> { + table: &'a SlotTable, + slots: SmallVec<[Value; 4]>, +} + +impl<'a> Row<'a> { + /// A fresh row with all slots unbound (Null). + pub fn seed(table: &'a SlotTable) -> Self { + Row { + table, + slots: smallvec::smallvec![Value::Null; table.len()], + } + } + + /// Look up a binding by variable name. + #[inline] + pub fn get(&self, name: &str) -> Option<&Value> { + self.table.slot(name).map(|i| &self.slots[i]) + } + + /// Bind a variable. The name must be in the SlotTable (it is, for every + /// name a plan operator binds — `SlotTable::from_plan` walks the same + /// operators); an unknown name is a plan/table desync bug and the + /// binding is dropped, matching a read of it (None). + #[inline] + pub fn insert(&mut self, name: &str, value: Value) { + debug_assert!( + self.table.slot(name).is_some(), + "variable {name:?} missing from SlotTable" + ); + if let Some(i) = self.table.slot(name) { + self.slots[i] = value; + } + } + + /// Iterate (name, value) pairs in slot order. + pub fn iter(&self) -> impl Iterator { + self.table + .names + .iter() + .map(String::as_str) + .zip(self.slots.iter()) + } +} /// Execution error. /// @@ -66,6 +221,9 @@ pub enum ExecErrorKind { NodeNotFound, TypeError(String), Unsupported(String), + /// Traversal exceeded its wall-clock budget (bounded epoch hold — + /// checked per hop in variable-length Expand and per ShortestPath run). + Timeout(crate::graph::traversal_guard::TraversalTimeout), } impl core::fmt::Display for ExecError { @@ -75,6 +233,7 @@ impl core::fmt::Display for ExecError { ExecErrorKind::NodeNotFound => write!(f, "node not found"), ExecErrorKind::TypeError(msg) => write!(f, "type error: {msg}"), ExecErrorKind::Unsupported(msg) => write!(f, "unsupported: {msg}"), + ExecErrorKind::Timeout(t) => write!(f, "{t}"), } } } @@ -185,6 +344,11 @@ pub struct ExecutionContext { /// parsed from `GRAPH.QUERY ... --decay `. /// None = distance-only shortest paths (current behavior). pub decay: Option, + /// Wall-clock budget for multi-hop traversals (bounded epoch hold). + /// Checked once per hop in variable-length Expand and once per row in + /// ShortestPath; exceeding it returns `ExecErrorKind::Timeout`. + /// None = unbounded (current behavior; `Default` keeps tests unchanged). + pub guard: Option, } // --------------------------------------------------------------------------- @@ -198,6 +362,67 @@ mod tests { use crate::graph::store::GraphStore; use bytes::Bytes; use smallvec::SmallVec; + use std::collections::HashMap; + + #[test] + fn test_slot_table_collects_all_binding_ops() { + let query = crate::graph::cypher::parse_cypher( + b"MATCH (a:Person {id: 1})-[r:KNOWS]->(b) RETURN a, b", + ) + .expect("parse"); + let plan = crate::graph::cypher::planner::compile(&query).expect("compile"); + let table = SlotTable::from_plan(&plan); + for var in ["a", "r", "b"] { + assert!( + table.slot(var).is_some(), + "{var:?} must have a slot; names = {:?}", + table.names() + ); + } + } + + #[test] + fn test_slot_table_collects_pattern_vars() { + let query = crate::graph::cypher::parse_cypher( + b"CREATE (x:Person {id: 1})-[e:KNOWS]->(y:Person {id: 2})", + ) + .expect("parse"); + let plan = crate::graph::cypher::planner::compile(&query).expect("compile"); + let table = SlotTable::from_plan(&plan); + for var in ["x", "e", "y"] { + assert!( + table.slot(var).is_some(), + "{var:?} must have a slot; names = {:?}", + table.names() + ); + } + } + + #[test] + fn test_row_get_insert_iter() { + let mut table = SlotTable::default(); + table.bind("a"); + table.bind("b"); + table.bind("a"); // dedup + assert_eq!(table.len(), 2); + + let mut row = Row::seed(&table); + // Unbound slot reads as Null; unknown name reads as None. + assert!(matches!(row.get("a"), Some(Value::Null))); + assert!(row.get("zzz").is_none()); + + row.insert("a", Value::Int(7)); + assert!(matches!(row.get("a"), Some(Value::Int(7)))); + + let cloned = row.clone(); + assert!(matches!(cloned.get("a"), Some(Value::Int(7)))); + + let pairs: Vec<(&str, bool)> = row + .iter() + .map(|(k, v)| (k, matches!(v, Value::Null))) + .collect(); + assert_eq!(pairs, vec![("a", false), ("b", true)]); + } #[test] fn test_execute_simple_match_return() { @@ -296,6 +521,113 @@ mod tests { assert_eq!(result.rows.len(), 3); } + /// Build a 3-node KNOWS chain (a -> b -> c) in graph "test". + fn chain_store() -> GraphStore { + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"test"), 64_000, 0) + .expect("create ok"); + let graph_mut = store.get_graph_mut(b"test").expect("graph"); + let label_id = label_to_id(b"Person"); + let etype = label_to_id(b"KNOWS"); + let keys: Vec = (0..3) + .map(|i| { + graph_mut.write_buf.add_node( + SmallVec::from_elem(label_id, 1), + SmallVec::new(), + None, + i, + ) + }) + .collect(); + for w in keys.windows(2) { + graph_mut + .write_buf + .add_edge(w[0], w[1], etype, 1.0, None, 3) + .expect("edge"); + } + store + } + + #[test] + fn test_guard_timeout_var_length_expand() { + let store = chain_store(); + let query = crate::graph::cypher::parse_cypher(b"MATCH (a:Person)-[*1..3]->(b) RETURN b") + .expect("parse"); + let plan = crate::graph::cypher::planner::compile(&query).expect("compile"); + let graph = store.get_graph(b"test").expect("graph"); + + // Expired guard: the first hop's check must abort with Timeout. + let ctx = ExecutionContext { + guard: Some(crate::graph::traversal_guard::TraversalGuard::new( + 0, + std::time::Duration::ZERO, + )), + ..Default::default() + }; + let Err(err) = execute(graph, &plan, &HashMap::new(), &ctx) else { + panic!("must time out"); + }; + assert!( + matches!(err.kind, ExecErrorKind::Timeout(_)), + "expected Timeout, got {:?}", + err.kind + ); + assert!(format!("{err}").contains("traversal timeout")); + + // Same query under a generous guard matches the guard-less result. + let ctx_ok = ExecutionContext { + guard: Some(crate::graph::traversal_guard::TraversalGuard::with_default_timeout(0)), + ..Default::default() + }; + let with_guard = execute(graph, &plan, &HashMap::new(), &ctx_ok).expect("exec"); + let without = + execute(graph, &plan, &HashMap::new(), &ExecutionContext::default()).expect("exec"); + assert_eq!(with_guard.rows.len(), without.rows.len()); + assert_eq!(with_guard.rows.len(), 3, "b, c from a; c from b"); + } + + #[test] + fn test_guard_timeout_shortest_path() { + let store = chain_store(); + let query = crate::graph::cypher::parse_cypher( + b"MATCH p = shortestPath((a:Person)-[*..5]->(b:Person)) RETURN p", + ) + .expect("parse"); + let plan = crate::graph::cypher::planner::compile(&query).expect("compile"); + let graph = store.get_graph(b"test").expect("graph"); + + let ctx = ExecutionContext { + guard: Some(crate::graph::traversal_guard::TraversalGuard::new( + 0, + std::time::Duration::ZERO, + )), + ..Default::default() + }; + let Err(err) = execute(graph, &plan, &HashMap::new(), &ctx) else { + panic!("must time out"); + }; + assert!( + matches!(err.kind, ExecErrorKind::Timeout(_)), + "expected Timeout, got {:?}", + err.kind + ); + + // execute_profile shares the checks. + let Err(err) = execute_profile(graph, &plan, &HashMap::new(), &ctx) else { + panic!("must time out"); + }; + assert!(matches!(err.kind, ExecErrorKind::Timeout(_))); + + // Generous guard: paths still found. + let ctx_ok = ExecutionContext { + guard: Some(crate::graph::traversal_guard::TraversalGuard::with_default_timeout(0)), + ..Default::default() + }; + let ok = execute(graph, &plan, &HashMap::new(), &ctx_ok).expect("exec"); + assert!(!ok.rows.is_empty(), "chain must yield shortest paths"); + } + #[test] fn test_value_comparison() { use std::cmp::Ordering; diff --git a/src/graph/cypher/executor/read.rs b/src/graph/cypher/executor/read.rs index bc799405..6d54d2d6 100644 --- a/src/graph/cypher/executor/read.rs +++ b/src/graph/cypher/executor/read.rs @@ -2,17 +2,214 @@ use std::collections::HashMap; use super::*; +/// Resolve an `IndexScan` into the matching node keys across both tiers. +/// +/// Frozen tier: per segment, intersect the property-equality bitmaps +/// (`SegmentPropertyIndexes::rows_eq`) with the label bitmap, then apply +/// MVCC/valid-time visibility per surviving row. Mutable tail: linear scan +/// with an inline (numeric-coercing, superset-consistent) property check — +/// the tail is bounded by `edge_threshold` by design. +/// +/// `prop_eq` values are literals or parameters; if any resolves to a +/// non-scalar the whole scan degrades to the plain merged label scan (the +/// residual Filter downstream keeps results exact either way). +#[allow(clippy::too_many_arguments)] +fn index_scan_keys( + memgraph: &crate::graph::memgraph::MemGraph, + csr_segs: &[std::sync::Arc], + label: Option<&String>, + prop_eq: &[(String, Expr)], + params: &HashMap, + ctx: &ExecutionContext, +) -> Vec { + let label_id = label.map(|l| label_to_id(l.as_bytes())); + let committed = roaring::RoaringBitmap::new(); + let view = crate::graph::view::MergedNodeView::new(memgraph, csr_segs); + let empty_table = SlotTable::default(); + let empty_row = Row::seed(&empty_table); + + // Resolve each equality target to a concrete PropertyValue. + let mut targets: Vec<(u16, PropertyValue)> = Vec::with_capacity(prop_eq.len()); + for (name, expr) in prop_eq { + let v = eval_expr( + expr, + &empty_row, + memgraph, + params, + csr_segs, + ctx.snapshot_lsn, + ctx.decay, + ); + match value_to_property_value(&v) { + Some(pv) => targets.push((label_to_id(name.as_bytes()), pv)), + None => { + // Unresolvable target (e.g. Null parameter): fall back to + // the merged label scan; the residual Filter stays exact. + let mut keys = Vec::new(); + view.for_each_visible_node( + label_id, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + |k| keys.push(k), + ); + return keys; + } + } + } + + let mut keys = Vec::new(); + + // Mutable tail: inline superset-consistent property check. + for (key, node) in memgraph.iter_nodes() { + if let Some(lid) = label_id { + if !node.labels.contains(&lid) { + continue; + } + } + if !crate::graph::visibility::is_node_visible( + node, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + ) { + continue; + } + let all_match = targets.iter().all(|(pid, want)| { + node.properties + .iter() + .any(|(id, have)| id == pid && prop_value_loose_eq(have, want)) + }); + if all_match { + keys.push(key); + } + } + + // Frozen tier: bitmap intersection per segment. + for seg in csr_segs { + let mut bm: Option = None; + for (pid, pv) in &targets { + let rows = seg.property_index().rows_eq(*pid, pv); + let acc = match bm.take() { + Some(acc) => acc & rows, + None => rows, + }; + if acc.is_empty() { + bm = Some(acc); + break; + } + bm = Some(acc); + } + let Some(mut bm) = bm else { continue }; + if bm.is_empty() { + continue; + } + if let Some(lid) = label_id { + match seg.label_index().nodes_with_label(lid) { + Some(lbm) => bm &= lbm, + None => continue, + } + } + let metas = seg.node_meta(); + for row in bm { + let Some(meta) = metas.get(row as usize) else { + continue; + }; + if !crate::graph::visibility::is_meta_visible( + meta, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + ) { + continue; + } + keys.push(NodeKey::from(slotmap::KeyData::from_ffi(meta.external_id))); + } + } + + keys +} + +/// Superset-consistent equality between stored and queried property values, +/// mirroring the index's numeric normalization (Int/Float/Bool share one +/// f64 space; String/Bytes compare bytewise). Never narrower than the +/// residual Filter's semantics — a loose match only ADDS candidates. +fn prop_value_loose_eq(have: &PropertyValue, want: &PropertyValue) -> bool { + fn as_num(v: &PropertyValue) -> Option { + match v { + PropertyValue::Int(i) => Some(*i as f64), + PropertyValue::Float(f) => Some(*f), + PropertyValue::Bool(b) => Some(u8::from(*b) as f64), + _ => None, + } + } + fn as_bytes(v: &PropertyValue) -> Option<&[u8]> { + match v { + PropertyValue::String(s) | PropertyValue::Bytes(s) => Some(s.as_ref()), + _ => None, + } + } + match (as_num(have), as_num(want)) { + (Some(a), Some(b)) => a == b, + _ => matches!((as_bytes(have), as_bytes(want)), (Some(a), Some(b)) if a == b), + } +} + +/// Per-hop wall-clock check for multi-hop operators (bounded epoch hold). +/// No-op when the context carries no guard (`Default` / unit tests). +#[inline] +fn guard_check(ctx: &ExecutionContext) -> Result<(), ExecError> { + if let Some(guard) = &ctx.guard { + if let Err(t) = guard.check_timeout() { + return Err(ExecError { + kind: ExecErrorKind::Timeout(t), + partial_mutations: Vec::new(), + }); + } + } + Ok(()) +} + /// Execute a physical plan against a named graph. +/// +/// Builds a fresh `SlotTable` from `plan` on every call. Hot paths that +/// already have a `SlotTable` on hand (e.g. a plan-cache hit, which cached +/// the table alongside the plan at insert time) should call +/// `execute_with_slots` directly instead to skip the rebuild. pub fn execute( graph: &NamedGraph, plan: &PhysicalPlan, params: &HashMap, ctx: &ExecutionContext, +) -> Result { + let slot_table = SlotTable::from_plan(plan); + execute_with_slots(graph, plan, &slot_table, params, ctx) +} + +/// Execute a physical plan against a named graph, using a caller-supplied +/// `SlotTable` instead of rebuilding one from `plan`. +/// +/// Plan-cache hot path (see `PlanCache::get` / `graph_read.rs`): the +/// `SlotTable` is built exactly once, when the plan is first compiled and +/// inserted into the cache, and reused verbatim on every subsequent cache +/// hit -- a `SlotTable` depends only on the plan's bound variables, which +/// never change for a given `PhysicalPlan`. +pub fn execute_with_slots( + graph: &NamedGraph, + plan: &PhysicalPlan, + slot_table: &SlotTable, + params: &HashMap, + ctx: &ExecutionContext, ) -> Result { let start = std::time::Instant::now(); // Seed row: one empty row to bootstrap the pipeline. - let mut rows: Vec = vec![HashMap::new()]; + let empty_table = SlotTable::default(); + let empty_row = Row::seed(&empty_table); + let mut rows: Vec = vec![Row::seed(slot_table)]; let mut columns = Vec::new(); // After Project, rows are converted to positional arrays. let mut projected_rows: Option>> = None; @@ -31,26 +228,42 @@ pub fn execute( PhysicalOp::NodeScan { variable, label } => { let label_id = label.as_ref().map(|l| label_to_id(l.as_bytes())); let committed = roaring::RoaringBitmap::new(); - let mut new_rows = Vec::new(); + // Scan BOTH tiers: the mutable write buffer and frozen CSR + // segments (freeze DRAINS nodes — a memgraph-only scan loses + // every frozen node). + let view = crate::graph::view::MergedNodeView::new(memgraph, csr_segs); + let mut keys = Vec::new(); + view.for_each_visible_node( + label_id, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + |k| keys.push(k), + ); + let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); for row in &rows { - for (key, node) in memgraph.iter_nodes() { - if let Some(lid) = label_id { - if !node.labels.contains(&lid) { - continue; - } - } - // Bi-temporal + MVCC visibility filter - if !crate::graph::visibility::is_node_visible( - node, - ctx.snapshot_lsn, - ctx.my_txn_id, - &committed, - ctx.valid_time_as_of, - ) { - continue; - } + for &key in &keys { let mut new_row = row.clone(); - new_row.insert(variable.clone(), Value::Node(key)); + new_row.insert(variable, Value::Node(key)); + new_rows.push(new_row); + } + } + rows = new_rows; + } + + PhysicalOp::IndexScan { + variable, + label, + prop_eq, + } => { + let keys = + index_scan_keys(memgraph, csr_segs, label.as_ref(), prop_eq, params, ctx); + let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); + for row in &rows { + for &key in &keys { + let mut new_row = row.clone(); + new_row.insert(variable, Value::Node(key)); new_rows.push(new_row); } } @@ -93,6 +306,11 @@ pub fn execute( ); let committed = roaring::RoaringBitmap::new(); + let view = crate::graph::view::MergedNodeView::new(memgraph, csr_segs); + // Scratch reused across every neighbor lookup in this Expand + // (the allocating `neighbors()` built a HashSet+Vec per call). + let mut nb_seen = crate::graph::fasthash::FxHashSet::default(); + let mut nb_buf: Vec = Vec::new(); let mut new_rows = Vec::new(); for row in &rows { let src_key = match row.get(source) { @@ -102,30 +320,31 @@ pub fn execute( if *max_hops <= 1 { // Single-hop expansion via SegmentMergeReader. - for merged in reader.neighbors(src_key) { + reader.neighbors_into(src_key, &mut nb_seen, &mut nb_buf); + for merged in &nb_buf { // Multi-type filter (SegmentMergeReader handles // single-type; we need extra check for multi-type). if type_ids.len() > 1 && !type_ids.contains(&merged.edge_type) { continue; } // Bi-temporal visibility check on target node - if let Some(target_node) = memgraph.get_node(merged.node) { - if !crate::graph::visibility::is_node_visible( - target_node, - ctx.snapshot_lsn, - ctx.my_txn_id, - &committed, - ctx.valid_time_as_of, - ) { - continue; - } + // (merged view — frozen targets get the CSR + // NodeMeta check instead of a free pass). + if !view.is_visible( + merged.node, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + ) { + continue; } let mut new_row = row.clone(); - new_row.insert(target.clone(), Value::Node(merged.node)); + new_row.insert(target, Value::Node(merged.node)); // v0.1.9 CYP-06: bind edge variable for single-hop // expansion so WHERE r.valid_to >= $asof works. if let Some(evar) = edge_variable { - new_row.insert(evar.clone(), Value::Edge(merged.edge)); + new_row.insert(evar, Value::Edge(merged.edge)); } new_rows.push(new_row); } @@ -137,13 +356,15 @@ pub fn execute( let capped_max_hops = (*max_hops).min(MAX_HOPS_LIMIT); let mut frontier = vec![src_key]; - let mut visited = std::collections::HashSet::new(); + let mut visited = crate::graph::fasthash::FxHashSet::default(); visited.insert(src_key); for hop in 1..=capped_max_hops { + guard_check(ctx)?; let mut next_frontier = Vec::new(); for ¤t in &frontier { - for merged in reader.neighbors(current) { + reader.neighbors_into(current, &mut nb_seen, &mut nb_buf); + for merged in &nb_buf { if visited.contains(&merged.node) { continue; } @@ -155,7 +376,7 @@ pub fn execute( if hop >= *min_hops { let mut new_row = row.clone(); - new_row.insert(target.clone(), Value::Node(merged.node)); + new_row.insert(target, Value::Node(merged.node)); new_rows.push(new_row); if new_rows.len() >= MAX_RESULT_ROWS { break; @@ -212,8 +433,10 @@ pub fn execute( .iter() .map(|item| { if matches!(item.expr, Expr::Star) { - let entries: Vec<(String, Value)> = - row.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let entries: Vec<(String, Value)> = row + .iter() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); Value::Map(entries) } else { eval_expr( @@ -304,7 +527,7 @@ pub fn execute( PhysicalOp::Limit { count } => { let n = match eval_expr( count, - &HashMap::new(), + &empty_row, memgraph, params, csr_segs, @@ -324,7 +547,7 @@ pub fn execute( PhysicalOp::Skip { count } => { let n = match eval_expr( count, - &HashMap::new(), + &empty_row, memgraph, params, csr_segs, @@ -362,7 +585,7 @@ pub fn execute( if let Value::List(items) = val { for item in items { let mut new_row = row.clone(); - new_row.insert(alias.clone(), item); + new_row.insert(alias, item); new_rows.push(new_row); } } @@ -434,6 +657,7 @@ pub fn execute( Some(Value::Node(k)) => *k, _ => continue, }; + guard_check(ctx)?; if let Some(path) = super::shortest_path::run_shortest_path( memgraph, csr_segs, @@ -446,7 +670,7 @@ pub fn execute( *max_hops, ) { let mut new_row = row.clone(); - new_row.insert(path_var.clone(), Value::Path(path)); + new_row.insert(path_var, Value::Path(path)); new_rows.push(new_row); } } @@ -460,7 +684,7 @@ pub fn execute( } else { // No Project operator: return all row bindings as columns. if columns.is_empty() && !rows.is_empty() { - columns = rows[0].keys().cloned().collect(); + columns = slot_table.names().to_vec(); columns.sort(); } rows.iter() @@ -493,6 +717,7 @@ pub fn execute( pub(crate) fn op_name(op: &PhysicalOp) -> &'static str { match op { PhysicalOp::NodeScan { .. } => "NodeScan", + PhysicalOp::IndexScan { .. } => "IndexScan", PhysicalOp::Expand { .. } => "Expand", PhysicalOp::Filter { .. } => "Filter", PhysicalOp::Project { .. } => "Project", @@ -522,7 +747,10 @@ pub fn execute_profile( ) -> Result { let start = std::time::Instant::now(); - let mut rows: Vec = vec![HashMap::new()]; + let slot_table = SlotTable::from_plan(plan); + let empty_table = SlotTable::default(); + let empty_row = Row::seed(&empty_table); + let mut rows: Vec = vec![Row::seed(&slot_table)]; let mut columns = Vec::new(); let mut projected_rows: Option>> = None; let nodes_created: u64 = 0; @@ -544,26 +772,40 @@ pub fn execute_profile( PhysicalOp::NodeScan { variable, label } => { let label_id = label.as_ref().map(|l| label_to_id(l.as_bytes())); let committed = roaring::RoaringBitmap::new(); - let mut new_rows = Vec::new(); + // Merged-tier scan (parity with the main executor). + let view = crate::graph::view::MergedNodeView::new(memgraph, csr_segs); + let mut keys = Vec::new(); + view.for_each_visible_node( + label_id, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + |k| keys.push(k), + ); + let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); for row in &rows { - for (key, node) in memgraph.iter_nodes() { - if let Some(lid) = label_id { - if !node.labels.contains(&lid) { - continue; - } - } - // Bi-temporal + MVCC visibility filter - if !crate::graph::visibility::is_node_visible( - node, - ctx.snapshot_lsn, - ctx.my_txn_id, - &committed, - ctx.valid_time_as_of, - ) { - continue; - } + for &key in &keys { + let mut new_row = row.clone(); + new_row.insert(variable, Value::Node(key)); + new_rows.push(new_row); + } + } + rows = new_rows; + } + + PhysicalOp::IndexScan { + variable, + label, + prop_eq, + } => { + let keys = + index_scan_keys(memgraph, csr_segs, label.as_ref(), prop_eq, params, ctx); + let mut new_rows = Vec::with_capacity(rows.len() * keys.len()); + for row in &rows { + for &key in &keys { let mut new_row = row.clone(); - new_row.insert(variable.clone(), Value::Node(key)); + new_row.insert(variable, Value::Node(key)); new_rows.push(new_row); } } @@ -590,7 +832,27 @@ pub fn execute_profile( EdgeDirection::Both => Direction::Both, }; + // Cross-segment expansion (parity with the main executor — + // memgraph-only neighbors miss every frozen CSR edge). + let edge_type_filter = if type_ids.len() == 1 { + Some(type_ids[0]) + } else { + None + }; + let reader = SegmentMergeReader::new( + Some(memgraph), + csr_segs, + dir, + u64::MAX, + edge_type_filter, + ); + let committed = roaring::RoaringBitmap::new(); + let view = crate::graph::view::MergedNodeView::new(memgraph, csr_segs); + // Scratch reused across every neighbor lookup in this Expand + // (the allocating `neighbors()` built a HashSet+Vec per call). + let mut nb_seen = crate::graph::fasthash::FxHashSet::default(); + let mut nb_buf: Vec = Vec::new(); let mut new_rows = Vec::new(); for row in &rows { let src_key = match row.get(source) { @@ -599,32 +861,28 @@ pub fn execute_profile( }; if *max_hops <= 1 { - for (edge_key, neighbor_key) in memgraph.neighbors(src_key, dir, u64::MAX) { - if !type_ids.is_empty() { - if let Some(edge) = memgraph.get_edge(edge_key) { - if !type_ids.contains(&edge.edge_type) { - continue; - } - } + reader.neighbors_into(src_key, &mut nb_seen, &mut nb_buf); + for merged in &nb_buf { + if type_ids.len() > 1 && !type_ids.contains(&merged.edge_type) { + continue; } // Bi-temporal visibility check on target node - if let Some(target_node) = memgraph.get_node(neighbor_key) { - if !crate::graph::visibility::is_node_visible( - target_node, - ctx.snapshot_lsn, - ctx.my_txn_id, - &committed, - ctx.valid_time_as_of, - ) { - continue; - } + // (merged view — parity with main executor). + if !view.is_visible( + merged.node, + ctx.snapshot_lsn, + ctx.my_txn_id, + &committed, + ctx.valid_time_as_of, + ) { + continue; } let mut new_row = row.clone(); - new_row.insert(target.clone(), Value::Node(neighbor_key)); + new_row.insert(target, Value::Node(merged.node)); // v0.1.9 CYP-06: bind edge variable in execute_profile // single-hop path (parity with main executor). if let Some(evar) = edge_variable { - new_row.insert(evar.clone(), Value::Edge(edge_key)); + new_row.insert(evar, Value::Edge(merged.edge)); } new_rows.push(new_row); } @@ -636,31 +894,27 @@ pub fn execute_profile( let capped_max_hops = (*max_hops).min(MAX_HOPS_LIMIT); let mut frontier = vec![src_key]; - let mut visited = std::collections::HashSet::new(); + let mut visited = crate::graph::fasthash::FxHashSet::default(); visited.insert(src_key); for hop in 1..=capped_max_hops { + guard_check(ctx)?; let mut next_frontier = Vec::new(); for ¤t in &frontier { - for (edge_key, neighbor_key) in - memgraph.neighbors(current, dir, u64::MAX) - { - if visited.contains(&neighbor_key) { + reader.neighbors_into(current, &mut nb_seen, &mut nb_buf); + for merged in &nb_buf { + if visited.contains(&merged.node) { continue; } - if !type_ids.is_empty() { - if let Some(edge) = memgraph.get_edge(edge_key) { - if !type_ids.contains(&edge.edge_type) { - continue; - } - } + if type_ids.len() > 1 && !type_ids.contains(&merged.edge_type) { + continue; } - visited.insert(neighbor_key); - next_frontier.push(neighbor_key); + visited.insert(merged.node); + next_frontier.push(merged.node); if hop >= *min_hops { let mut new_row = row.clone(); - new_row.insert(target.clone(), Value::Node(neighbor_key)); + new_row.insert(target, Value::Node(merged.node)); new_rows.push(new_row); if new_rows.len() >= MAX_RESULT_ROWS { break; @@ -717,8 +971,10 @@ pub fn execute_profile( .iter() .map(|item| { if matches!(item.expr, Expr::Star) { - let entries: Vec<(String, Value)> = - row.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let entries: Vec<(String, Value)> = row + .iter() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); Value::Map(entries) } else { eval_expr( @@ -807,7 +1063,7 @@ pub fn execute_profile( PhysicalOp::Limit { count } => { let n = match eval_expr( count, - &HashMap::new(), + &empty_row, memgraph, params, csr_segs, @@ -827,7 +1083,7 @@ pub fn execute_profile( PhysicalOp::Skip { count } => { let n = match eval_expr( count, - &HashMap::new(), + &empty_row, memgraph, params, csr_segs, @@ -865,7 +1121,7 @@ pub fn execute_profile( if let Value::List(items) = val { for item in items { let mut new_row = row.clone(); - new_row.insert(alias.clone(), item); + new_row.insert(alias, item); new_rows.push(new_row); } } @@ -938,6 +1194,7 @@ pub fn execute_profile( Some(Value::Node(k)) => *k, _ => continue, }; + guard_check(ctx)?; if let Some(path) = super::shortest_path::run_shortest_path( memgraph, csr_segs, @@ -950,7 +1207,7 @@ pub fn execute_profile( *max_hops, ) { let mut new_row = row.clone(); - new_row.insert(path_var.clone(), Value::Path(path)); + new_row.insert(path_var, Value::Path(path)); new_rows.push(new_row); } } @@ -975,7 +1232,7 @@ pub fn execute_profile( pr } else { if columns.is_empty() && !rows.is_empty() { - columns = rows[0].keys().cloned().collect(); + columns = slot_table.names().to_vec(); columns.sort(); } rows.iter() diff --git a/src/graph/cypher/executor/write.rs b/src/graph/cypher/executor/write.rs index 311e6f6b..9083230a 100644 --- a/src/graph/cypher/executor/write.rs +++ b/src/graph/cypher/executor/write.rs @@ -14,7 +14,10 @@ pub fn execute_mut( ) -> Result { let start = std::time::Instant::now(); - let mut rows: Vec = vec![HashMap::new()]; + let slot_table = SlotTable::from_plan(plan); + let empty_table = SlotTable::default(); + let empty_row = Row::seed(&empty_table); + let mut rows: Vec = vec![Row::seed(&slot_table)]; let mut columns = Vec::new(); let mut projected_rows: Option>> = None; let mut nodes_created: u64 = 0; @@ -24,7 +27,15 @@ pub fn execute_mut( for op in &plan.operators { match op { - PhysicalOp::NodeScan { variable, label } => { + // The write path matches against the MUTABLE tier only (write + // operators mutate MutableNode in place; SET/DELETE on frozen + // rows is a separate copy-up design — known follow-up). The + // IndexScan arm therefore degrades to the same label scan; the + // residual Filter the planner emits keeps results exact. + PhysicalOp::NodeScan { variable, label } + | PhysicalOp::IndexScan { + variable, label, .. + } => { let label_id = label.as_ref().map(|l| label_to_id(l.as_bytes())); let mut new_rows = Vec::new(); for row in &rows { @@ -35,7 +46,7 @@ pub fn execute_mut( } } let mut new_row = row.clone(); - new_row.insert(variable.clone(), Value::Node(key)); + new_row.insert(variable, Value::Node(key)); new_rows.push(new_row); } } @@ -81,12 +92,12 @@ pub fn execute_mut( } } let mut new_row = row.clone(); - new_row.insert(target.clone(), Value::Node(neighbor_key)); + new_row.insert(target, Value::Node(neighbor_key)); // Phase 174 FIX-01: bind edge variable so DELETE r // can reference it. Previously ignored (`_`), which // made `DELETE r` a silent no-op. if let Some(evar) = edge_variable { - new_row.insert(evar.clone(), Value::Edge(edge_key)); + new_row.insert(evar, Value::Edge(edge_key)); } new_rows.push(new_row); } @@ -122,7 +133,7 @@ pub fn execute_mut( if hop >= *min_hops { let mut new_row = row.clone(); - new_row.insert(target.clone(), Value::Node(neighbor_key)); + new_row.insert(target, Value::Node(neighbor_key)); new_rows.push(new_row); if new_rows.len() >= MAX_RESULT_ROWS { break; @@ -171,8 +182,10 @@ pub fn execute_mut( .iter() .map(|item| { if matches!(item.expr, Expr::Star) { - let entries: Vec<(String, Value)> = - row.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let entries: Vec<(String, Value)> = row + .iter() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); Value::Map(entries) } else { eval_expr( @@ -243,15 +256,7 @@ pub fn execute_mut( } PhysicalOp::Limit { count } => { - let n = match eval_expr( - count, - &HashMap::new(), - &graph.write_buf, - params, - &[], - 0, - None, - ) { + let n = match eval_expr(count, &empty_row, &graph.write_buf, params, &[], 0, None) { Value::Int(n) if n >= 0 => n as usize, _ => 0, }; @@ -263,15 +268,7 @@ pub fn execute_mut( } PhysicalOp::Skip { count } => { - let n = match eval_expr( - count, - &HashMap::new(), - &graph.write_buf, - params, - &[], - 0, - None, - ) { + let n = match eval_expr(count, &empty_row, &graph.write_buf, params, &[], 0, None) { Value::Int(n) if n >= 0 => n as usize, _ => 0, }; @@ -295,7 +292,7 @@ pub fn execute_mut( if let Value::List(items) = val { for item in items { let mut new_row = row.clone(); - new_row.insert(alias.clone(), item); + new_row.insert(alias, item); new_rows.push(new_row); } } @@ -344,7 +341,7 @@ pub fn execute_mut( embedding: None, }); if let Some(ref var) = pn.variable { - new_row.insert(var.clone(), Value::Node(nk)); + new_row.insert(var, Value::Node(nk)); } node_keys.push(nk); } @@ -562,7 +559,7 @@ pub fn execute_mut( if let Some(existing_key) = found { // MATCH path: bind variable and apply on_match. if let Some(ref var) = pn.variable { - new_row.insert(var.clone(), Value::Node(existing_key)); + new_row.insert(var, Value::Node(existing_key)); } apply_set_items( on_match, @@ -586,7 +583,7 @@ pub fn execute_mut( embedding: None, }); if let Some(ref var) = pn.variable { - new_row.insert(var.clone(), Value::Node(nk)); + new_row.insert(var, Value::Node(nk)); } apply_set_items( on_create, @@ -632,10 +629,10 @@ pub fn execute_mut( if edge_exists { // Bind variables. if let Some(ref var) = src_pn.variable { - new_row.insert(var.clone(), Value::Node(sk)); + new_row.insert(var, Value::Node(sk)); } if let Some(ref var) = dst_pn.variable { - new_row.insert(var.clone(), Value::Node(dk)); + new_row.insert(var, Value::Node(dk)); } apply_set_items( on_match, @@ -665,10 +662,10 @@ pub fn execute_mut( }); } if let Some(ref var) = src_pn.variable { - new_row.insert(var.clone(), Value::Node(sk)); + new_row.insert(var, Value::Node(sk)); } if let Some(ref var) = dst_pn.variable { - new_row.insert(var.clone(), Value::Node(dk)); + new_row.insert(var, Value::Node(dk)); } apply_set_items( on_create, @@ -771,10 +768,10 @@ pub fn execute_mut( }); } if let Some(ref var) = src_pn.variable { - new_row.insert(var.clone(), Value::Node(sk)); + new_row.insert(var, Value::Node(sk)); } if let Some(ref var) = dst_pn.variable { - new_row.insert(var.clone(), Value::Node(dk)); + new_row.insert(var, Value::Node(dk)); } apply_set_items( on_create, @@ -826,7 +823,7 @@ pub fn execute_mut( pr } else { if columns.is_empty() && !rows.is_empty() { - columns = rows[0].keys().cloned().collect(); + columns = slot_table.names().to_vec(); columns.sort(); } rows.iter() @@ -859,7 +856,7 @@ pub fn execute_mut( /// created nodes — they are removed entirely by the CreateNode intent). pub(crate) fn apply_set_items( items: &[SetItem], - row: &Row, + row: &Row<'_>, memgraph: &mut crate::graph::memgraph::MemGraph, params: &HashMap, properties_set: &mut u64, @@ -926,7 +923,7 @@ pub(crate) fn apply_set_items( /// Otherwise, search the memgraph for a matching node by labels + properties. pub(crate) fn resolve_or_find_node( pn: &PatternNode, - row: &Row, + row: &Row<'_>, memgraph: &crate::graph::memgraph::MemGraph, params: &HashMap, ) -> Option { diff --git a/src/graph/cypher/mod.rs b/src/graph/cypher/mod.rs index f710076f..751993ff 100644 --- a/src/graph/cypher/mod.rs +++ b/src/graph/cypher/mod.rs @@ -11,6 +11,7 @@ pub mod ast; pub mod executor; pub mod lexer; +pub mod parameterize; pub mod parser; pub mod planner; diff --git a/src/graph/cypher/parameterize.rs b/src/graph/cypher/parameterize.rs new file mode 100644 index 00000000..01bbbc1d --- /dev/null +++ b/src/graph/cypher/parameterize.rs @@ -0,0 +1,248 @@ +//! Literal → auto-parameter rewrite for plan-cache normalization. +//! +//! `MATCH (a:N {id: 3})` and `MATCH (a:N {id: 4})` compile to the same +//! physical plan modulo the literal, but hash to different plan-cache keys, +//! so a point-query workload with varying literals never hits the cache. +//! [`parameterize`] rewrites value literals into synthetic parameters +//! (`$__p0`, `$__p1`, …) at the token level and extracts their values, so +//! the cache keys on the normalized text and the executor resolves the +//! values per run — exactly like a user-supplied `--params` query (which +//! `PhysicalOp::IndexScan` already index-hits on). +//! +//! Structural literals are NOT rewritten, because they are baked into the +//! compiled plan rather than evaluated at runtime: +//! - variable-length hop bounds (`[*1..3]`, `*2`, shortestPath bounds) — +//! parsed by `parse_var_length` into `Expand`/`ShortestPath` `max_hops`; +//! - `LIMIT n` / `SKIP n` — evaluated as `Expr` today, but kept literal +//! conservatively so the plan text pins the bound; +//! - literals with a leading `-`: logos' maximal munch lexes `x-1` as +//! `Ident(x) Integer(-1)`, so rewriting the integer would also swallow a +//! binary minus. +//! +//! The rewrite is fail-open: any surprise (lex error, non-UTF-8 string, +//! unparsable number, pre-existing `$__p` name) returns `None` and the +//! caller falls back to the raw text with raw-hash caching — never wrong +//! results, only a missed cache share. + +use logos::Logos; + +use super::executor::Value; +use super::lexer::Token; + +/// A literal-normalized query plus the extracted literal values. +pub struct ParameterizedQuery { + /// The query text with value literals replaced by `$__pN` parameters. + pub normalized: Vec, + /// Extracted literal values, keyed by generated parameter name WITHOUT + /// the `$` sigil (matching `Expr::Parameter`'s stored name). Merged into + /// the user params map before execution. + pub auto_params: Vec<(String, Value)>, +} + +/// Reserved prefix for auto-generated parameter names. +const AUTO_PREFIX: &[u8] = b"$__p"; + +/// Rewrite value literals into auto-parameters. Returns `None` when nothing +/// was rewritten (no literals, or only structural ones) or when the input is +/// unsuitable for rewriting — callers then use the raw text unchanged. +pub fn parameterize(input: &[u8]) -> Option { + // A user query already naming $__p* could collide with generated names. + if input.windows(AUTO_PREFIX.len()).any(|w| w == AUTO_PREFIX) { + return None; + } + + // Tokenize with spans. Comments stay in the stream (copied verbatim, + // skipped for prev/next significance checks). Any lex error bails: the + // parser skips unrecognized bytes, but a rewrite around them is not + // worth reasoning about. + let mut lexer = Token::lexer(input); + let mut tokens: Vec<(Token, core::ops::Range)> = Vec::new(); + while let Some(result) = lexer.next() { + match result { + Ok(t) => tokens.push((t, lexer.span())), + Err(()) => return None, + } + } + + let significant = |t: &Token| !matches!(t, Token::LineComment | Token::BlockComment); + + let mut normalized = Vec::with_capacity(input.len()); + let mut auto_params: Vec<(String, Value)> = Vec::new(); + let mut last_end = 0usize; + + for i in 0..tokens.len() { + let (tok, span) = &tokens[i]; + let Some(value) = literal_value(tok) else { + continue; + }; + // Structural positions: hop bounds and LIMIT/SKIP stay literal. + let prev = tokens[..i] + .iter() + .rev() + .map(|(t, _)| t) + .find(|t| significant(t)); + let next = tokens[i + 1..] + .iter() + .map(|(t, _)| t) + .find(|t| significant(t)); + if matches!( + prev, + Some(Token::Star | Token::DotDot | Token::Limit | Token::Skip) + ) || matches!(next, Some(Token::DotDot)) + { + continue; + } + + normalized.extend_from_slice(&input[last_end..span.start]); + let name = format!("__p{}", auto_params.len()); + normalized.push(b'$'); + normalized.extend_from_slice(name.as_bytes()); + auto_params.push((name, value)); + last_end = span.end; + } + + if auto_params.is_empty() { + return None; + } + normalized.extend_from_slice(&input[last_end..]); + Some(ParameterizedQuery { + normalized, + auto_params, + }) +} + +/// Decode a rewritable literal token into its runtime [`Value`]. +/// +/// Returns `None` for non-literal tokens AND for literals we refuse to +/// rewrite (leading `-`, unparsable numbers, non-UTF-8 strings) — refusing +/// just leaves that literal in place, which is always correct. +fn literal_value(tok: &Token) -> Option { + match tok { + Token::Integer(s) => { + if s.first() == Some(&b'-') { + return None; + } + let text = core::str::from_utf8(s).ok()?; + text.parse::().ok().map(Value::Int) + } + Token::Float(s) => { + if s.first() == Some(&b'-') { + return None; + } + let text = core::str::from_utf8(s).ok()?; + text.parse::().ok().map(Value::Float) + } + Token::StringLit(s) => { + // Strip the surrounding single quotes. + let inner = &s[1..s.len() - 1]; + core::str::from_utf8(inner) + .ok() + .map(|t| Value::String(t.to_owned())) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn norm_str(pq: &ParameterizedQuery) -> &str { + core::str::from_utf8(&pq.normalized).expect("normalized text is utf8") + } + + #[test] + fn test_int_and_string_literals_rewritten() { + let pq = + parameterize(b"MATCH (a:N {id: 3, name: 'x'}) RETURN a.id").expect("literals present"); + assert_eq!( + norm_str(&pq), + "MATCH (a:N {id: $__p0, name: $__p1}) RETURN a.id" + ); + assert_eq!(pq.auto_params.len(), 2); + assert_eq!(pq.auto_params[0].0, "__p0"); + assert!(matches!(pq.auto_params[0].1, Value::Int(3))); + assert!(matches!(&pq.auto_params[1].1, Value::String(s) if s == "x")); + } + + #[test] + fn test_float_literal_rewritten() { + let pq = parameterize(b"MATCH (a {w: 1.5}) RETURN a").expect("literal present"); + assert_eq!(norm_str(&pq), "MATCH (a {w: $__p0}) RETURN a"); + assert!(matches!(pq.auto_params[0].1, Value::Float(f) if (f - 1.5).abs() < 1e-9)); + } + + #[test] + fn test_literal_variants_normalize_identically() { + let a = parameterize(b"MATCH (a:N {id: 3}) RETURN a").expect("rewrites"); + let b = parameterize(b"MATCH (a:N {id: 44}) RETURN a").expect("rewrites"); + assert_eq!(a.normalized, b.normalized); + } + + #[test] + fn test_var_length_hops_not_rewritten() { + assert!(parameterize(b"MATCH (a)-[*1..3]->(b) RETURN b").is_none()); + assert!(parameterize(b"MATCH (a)-[*2]->(b) RETURN b").is_none()); + assert!(parameterize(b"MATCH (a)-[*..5]->(b) RETURN b").is_none()); + // Mixed: the inline prop rewrites, the hop bounds stay literal. + let pq = parameterize(b"MATCH (a:N {id: 3})-[*1..3]->(b) RETURN b").expect("id rewrites"); + assert_eq!( + norm_str(&pq), + "MATCH (a:N {id: $__p0})-[*1..3]->(b) RETURN b" + ); + assert_eq!(pq.auto_params.len(), 1); + } + + #[test] + fn test_limit_and_skip_not_rewritten() { + assert!(parameterize(b"MATCH (a) RETURN a SKIP 2 LIMIT 5").is_none()); + let pq = + parameterize(b"MATCH (a:N {id: 7}) RETURN a LIMIT 5").expect("inline prop rewrites"); + assert_eq!(norm_str(&pq), "MATCH (a:N {id: $__p0}) RETURN a LIMIT 5"); + } + + #[test] + fn test_negative_literal_not_rewritten() { + // logos lexes `x-1` as Ident(x) Integer(-1); rewriting would swallow + // the binary minus. Leading-minus literals always stay in place. + assert!(parameterize(b"MATCH (a) WHERE a.x = -5 RETURN a").is_none()); + } + + #[test] + fn test_no_literals_returns_none() { + assert!(parameterize(b"MATCH (a:Person) RETURN a").is_none()); + assert!(parameterize(b"MATCH (a {id: $id}) RETURN a").is_none()); + } + + #[test] + fn test_existing_auto_prefix_bails() { + assert!(parameterize(b"MATCH (a {id: $__p0, x: 3}) RETURN a").is_none()); + } + + #[test] + fn test_where_clause_literal_rewritten() { + let pq = parameterize(b"MATCH (a) WHERE a.id = 9 RETURN a").expect("rewrites"); + assert_eq!(norm_str(&pq), "MATCH (a) WHERE a.id = $__p0 RETURN a"); + assert!(matches!(pq.auto_params[0].1, Value::Int(9))); + } + + #[test] + fn test_normalized_text_reparses() { + // The rewrite must produce grammatically valid Cypher. + let inputs: &[&[u8]] = &[ + b"MATCH (a:N {id: 3, name: 'x'}) RETURN a.id", + b"MATCH (a) WHERE a.id = 9 AND a.w > 1.5 RETURN a LIMIT 3", + b"CREATE (:Person {id: 1, name: 'alice'})", + b"MATCH (a:N {id: 3})-[*1..3]->(b) RETURN b", + ]; + for input in inputs { + let pq = parameterize(input).expect("rewrites"); + super::super::parse_cypher(&pq.normalized).unwrap_or_else(|e| { + panic!( + "normalized text must reparse: {e:?} — {:?}", + core::str::from_utf8(&pq.normalized) + ) + }); + } + } +} diff --git a/src/graph/cypher/planner.rs b/src/graph/cypher/planner.rs index a37131db..b6a4b192 100644 --- a/src/graph/cypher/planner.rs +++ b/src/graph/cypher/planner.rs @@ -7,10 +7,11 @@ //! The cost estimator selects between graph-first and vector-first strategies //! based on per-graph `GraphStats` (degree distribution, node/edge counts). -use std::collections::HashMap; use std::sync::Arc; use crate::graph::cypher::ast::*; +use crate::graph::cypher::executor::{SlotTable, Value}; +use crate::graph::fasthash::FxHashMap; use crate::graph::stats::GraphStats; /// A compiled physical plan: a sequence of operators. @@ -27,6 +28,22 @@ pub enum PhysicalOp { variable: String, label: Option, }, + /// Scan nodes via per-segment property indexes: label bitmap ∧ property + /// bitmap per immutable CSR segment, plus a linear mutable-tail check. + /// + /// Emitted instead of `NodeScan` when a pattern node carries inline + /// equality properties whose values are literals or parameters + /// (`(n:L {k: 3})`, `(n:L {k: $v})`). `prop_eq` values are index LOOKUP + /// hints with SUPERSET semantics (string hashes can collide, Bool/Int + /// alias numerically) — the planner always keeps the full residual + /// Filter downstream, so the index only prunes, never decides. + IndexScan { + variable: String, + label: Option, + /// (property name, value expression) equality conjuncts. Expressions + /// are literals or parameters, resolved by the executor per run. + prop_eq: Vec<(String, Expr)>, + }, /// Expand along edges from a source variable. /// /// `edge_variable`: when `Some(name)`, the executor binds the traversed @@ -106,42 +123,187 @@ impl core::fmt::Display for PlanError { } } -/// Plan cache: maps xxhash of Cypher string to compiled plan. +/// A cache hit: the compiled plan, its precomputed `SlotTable`, and (for +/// raw-bytes-hash entries only) the auto-extracted parameter values that +/// belong to this EXACT query text. +#[derive(Clone)] +pub struct CachedPlan { + pub plan: Arc, + /// Built once via `SlotTable::from_plan` at insert time. A cache hit + /// reuses this instead of rebuilding it every execution (`SlotTable` + /// depends only on the plan's bound variables, which are fixed for a + /// given `PhysicalPlan`). + pub slots: Arc, + /// Non-empty ONLY for entries keyed by the raw query-bytes hash: + /// `parameterize()` is a pure function of the exact input bytes, so a + /// raw-hash hit can replay these values without re-lexing. Entries keyed + /// by the literal-normalized hash are shared across many distinct + /// literal values and are always empty here -- callers must re-derive + /// params for that specific query text via `parameterize()`. + pub auto_params: Arc>, +} + +struct CacheEntry { + plan: Arc, + slots: Arc, + auto_params: Arc>, + used: u64, +} + +/// Plan cache: maps a query hash to a compiled plan, with LRU eviction. +/// +/// Two kinds of keys point at the same underlying plan (Fix 2 -- allocation +/// -free exact repeats): +/// - the raw query-bytes hash (`hash_query` on the client's exact text) -- +/// inserted so an exact repeat hits without running `parameterize()` at +/// all, and carries that text's `auto_params` alongside it; +/// - the literal-normalized hash -- the pre-existing key, shared across +/// every raw query that normalizes to the same text (i.e. differs only in +/// literal values), so it MUST NOT carry a fixed set of `auto_params`. +/// +/// `FxHashMap`-keyed (not `SipHash`): the map key is already a fixed-size +/// u64 digest (xxhash64 of query bytes), never raw attacker bytes directly +/// -- see `crate::graph::fasthash` for the DoS-surface contract this relies +/// on -- and the cache is bounded by `max_entries` (LRU-evicted), so even a +/// deliberately hash-flooded cache degrades to O(max_entries) per lookup, +/// not unbounded blowup. +/// +/// Invariant: only READ-ONLY plans may be inserted. A cache hit in +/// `graph_query_or_write` routes straight to the read path without +/// re-parsing, so a cached write plan would mis-route. pub struct PlanCache { - cache: HashMap>, + cache: FxHashMap, max_entries: usize, + /// Monotonic access counter backing LRU eviction. + tick: u64, } impl PlanCache { /// Create a new plan cache with the given maximum size. pub fn new(max_entries: usize) -> Self { Self { - cache: HashMap::new(), + cache: FxHashMap::default(), max_entries, + tick: 0, } } - /// Look up a cached plan by query hash. - pub fn get(&self, hash: u64) -> Option> { - self.cache.get(&hash).cloned() + /// Look up a cached plan by query hash, marking it most-recently-used. + pub fn get(&mut self, hash: u64) -> Option { + self.tick += 1; + let tick = self.tick; + self.cache.get_mut(&hash).map(|e| { + e.used = tick; + CachedPlan { + plan: e.plan.clone(), + slots: e.slots.clone(), + auto_params: e.auto_params.clone(), + } + }) } - /// Insert a plan into the cache. Evicts an arbitrary entry if at capacity. - pub fn insert(&mut self, hash: u64, plan: Arc) { - if self.cache.len() >= self.max_entries { - // Simple eviction: remove arbitrary key (HashMap has no ordering). - if let Some(&first_key) = self.cache.keys().next() { - self.cache.remove(&first_key); + /// Insert a plan under the literal-normalized hash, evicting the + /// least-recently-used entry at capacity. No `auto_params` are cached + /// (see `PlanCache` docs) -- callers re-derive them per query text. + /// + /// Returns the freshly-built `SlotTable` so the caller can execute + /// immediately without rebuilding it a second time. + pub fn insert(&mut self, hash: u64, plan: Arc) -> Arc { + let slots = Arc::new(SlotTable::from_plan(&plan)); + self.put(hash, plan, slots.clone(), Arc::new(Vec::new())); + slots + } + + /// Insert a plan under BOTH the raw query-bytes hash and (if different) + /// the literal-normalized hash, sharing one `Arc`'d plan + `SlotTable`. + /// `raw_auto_params` are the auto-extracted values for THIS raw query + /// text and are cached only on the raw-hash entry. + /// + /// Returns the freshly-built `SlotTable` so the caller can execute + /// immediately without rebuilding it a second time. + pub fn insert_both( + &mut self, + raw_hash: u64, + normalized_hash: u64, + plan: Arc, + raw_auto_params: Vec<(String, Value)>, + ) -> Arc { + let slots = Arc::new(SlotTable::from_plan(&plan)); + self.put( + raw_hash, + plan.clone(), + slots.clone(), + Arc::new(raw_auto_params), + ); + if normalized_hash != raw_hash { + self.put(normalized_hash, plan, slots.clone(), Arc::new(Vec::new())); + } + slots + } + + fn put( + &mut self, + hash: u64, + plan: Arc, + slots: Arc, + auto_params: Arc>, + ) { + self.tick += 1; + if self.cache.len() >= self.max_entries && !self.cache.contains_key(&hash) { + // O(capacity) scan; eviction only fires when a full cache takes a + // brand-new query text, which is rare once the workload warms up. + let lru = self + .cache + .iter() + .min_by_key(|(_, e)| e.used) + .map(|(k, _)| *k); + if let Some(lru) = lru { + self.cache.remove(&lru); } } - self.cache.insert(hash, plan); + self.cache.insert( + hash, + CacheEntry { + plan, + slots, + auto_params, + used: self.tick, + }, + ); } - /// Number of cached plans. + /// Number of hash-key entries in the cache (raw-bytes-hash and + /// literal-normalized-hash keys counted separately). This is what + /// `max_entries` bounds and what LRU eviction operates on. + /// + /// A dual-keyed insert (`insert_both`) can add up to 2 entries for a + /// single distinct plan — use `distinct_plan_count()` if you want "how + /// many different compiled plans are resident" instead. pub fn len(&self) -> usize { self.cache.len() } + /// Number of *distinct* compiled plans currently cached, deduplicated by + /// `Arc` identity. Two hash keys inserted by the same `insert_both` call + /// point at the same `Arc` and count once here, even + /// though they occupy two slots in `len()` / `max_entries`. + /// + /// This is the metric callers actually care about when asserting + /// "queries differing only in literal values share one cached plan": + /// `len()` would (correctly) report 2 for that case (the raw-hash entry + /// for each literal variant, both aliasing one shared normalized-hash + /// entry) -- `distinct_plan_count()` reports 1. + pub fn distinct_plan_count(&self) -> usize { + let mut seen: Vec<*const PhysicalPlan> = Vec::with_capacity(self.cache.len()); + for entry in self.cache.values() { + let ptr = Arc::as_ptr(&entry.plan); + if !seen.contains(&ptr) { + seen.push(ptr); + } + } + seen.len() + } + /// Whether the cache is empty. pub fn is_empty(&self) -> bool { self.cache.is_empty() @@ -369,25 +531,14 @@ fn compile_match(m: &MatchClause, ops: &mut Vec) -> Result<(), PlanE continue; } - // First node becomes a scan. + // First node becomes a scan (index-backed when inline equality + // properties can drive a lookup). let first = &pattern.nodes[0]; let first_var = first .variable .clone() .unwrap_or_else(|| "_anon".to_string()); - ops.push(PhysicalOp::NodeScan { - variable: first_var.clone(), - label: first.labels.first().cloned(), - }); - // Inline node properties `(v {k:e, …})` become an equality-conjunction - // Filter applied right after the op that BINDS the node — here, the - // scan. Without this the predicate is silently dropped and the query - // returns the whole label's edges (the ≈|E| full-scan bug). - if !first.properties.is_empty() { - ops.push(PhysicalOp::Filter { - expr: properties_to_filter(&first_var, &first.properties), - }); - } + push_node_scan(first, &first_var, ops); // Subsequent node+edge pairs become expands. for (i, edge) in pattern.edges.iter().enumerate() { @@ -456,24 +607,8 @@ fn compile_shortest_path_match(sp: &ShortestPathMatchClause, ops: &mut Vec) { + let indexable = !node.properties.is_empty() + && node.properties.iter().all(|(_, e)| { + matches!( + e, + Expr::Integer(_) + | Expr::Float(_) + | Expr::StringLit(_) + | Expr::Bool(_) + | Expr::Parameter(_) + ) + }); + if indexable { + ops.push(PhysicalOp::IndexScan { + variable: var.to_string(), + label: node.labels.first().cloned(), + prop_eq: node.properties.clone(), + }); + } else { + ops.push(PhysicalOp::NodeScan { + variable: var.to_string(), + label: node.labels.first().cloned(), + }); + } + if !node.properties.is_empty() { + ops.push(PhysicalOp::Filter { + expr: properties_to_filter(var, &node.properties), + }); + } +} + /// Build a filter expression `var.k1 = v1 AND var.k2 = v2 ...` from a /// PatternNode's inline property map, expressed as a standalone Filter op. /// Used to apply inline node-property predicates `(v {k:e, …})` in both @@ -572,23 +745,48 @@ mod tests { // immediately AFTER the op that binds each inline-propertied node. RED until the fix. #[test] - fn test_inline_prop_emits_filter_after_nodescan() { - // M3 — `MATCH (a:Person {id:1})-[]->(b)`: a Filter must immediately follow the NodeScan. + fn test_inline_prop_emits_filter_after_scan() { + // M3 — `MATCH (a:Person {id:1})-[]->(b)`: a Filter must immediately + // follow the binding scan. Literal inline props now plan as an + // IndexScan (v0.6 property-index lever); the residual Filter stays. let query = parse_cypher(b"MATCH (a:Person {id:1})-[]->(b) RETURN b").expect("parse failed"); let plan = compile(&query).expect("compile failed"); let ns = plan .operators .iter() - .position(|op| matches!(op, PhysicalOp::NodeScan { .. })) - .expect("a NodeScan is planned"); + .position(|op| matches!(op, PhysicalOp::IndexScan { .. })) + .expect("an IndexScan is planned for literal inline props"); assert!( matches!(plan.operators.get(ns + 1), Some(PhysicalOp::Filter { .. })), - "inline-property Filter must immediately follow the NodeScan; ops = {:?}", + "inline-property Filter must immediately follow the IndexScan; ops = {:?}", plan.operators ); } + #[test] + fn test_inline_literal_props_plan_index_scan() { + // Literal and parameter values are index-eligible. + for q in [ + b"MATCH (a:N {id:3}) RETURN a".as_slice(), + b"MATCH (a:N {id:$p}) RETURN a".as_slice(), + b"MATCH (a:N {name:'x', id: 3}) RETURN a".as_slice(), + ] { + let query = parse_cypher(q).expect("parse failed"); + let plan = compile(&query).expect("compile failed"); + assert!( + matches!(plan.operators[0], PhysicalOp::IndexScan { .. }), + "expected IndexScan for {:?}; ops = {:?}", + core::str::from_utf8(q), + plan.operators + ); + } + // No inline props: plain NodeScan, unchanged. + let query = parse_cypher(b"MATCH (a:N) RETURN a").expect("parse failed"); + let plan = compile(&query).expect("compile failed"); + assert!(matches!(plan.operators[0], PhysicalOp::NodeScan { .. })); + } + #[test] fn test_inline_prop_on_expanded_node_filters_after_expand() { // M2 — `MATCH (a {id:1})-[]->(b {id:3})`: a Filter after the NodeScan AND one after the Expand. @@ -598,11 +796,16 @@ mod tests { let ns = plan .operators .iter() - .position(|op| matches!(op, PhysicalOp::NodeScan { .. })) - .expect("a NodeScan is planned"); + .position(|op| { + matches!( + op, + PhysicalOp::NodeScan { .. } | PhysicalOp::IndexScan { .. } + ) + }) + .expect("a scan is planned"); assert!( matches!(plan.operators.get(ns + 1), Some(PhysicalOp::Filter { .. })), - "scanned node's inline Filter must follow the NodeScan; ops = {:?}", + "scanned node's inline Filter must follow the scan; ops = {:?}", plan.operators ); let ex = plan @@ -688,6 +891,139 @@ mod tests { assert_eq!(cache.len(), 2); } + #[test] + fn test_plan_cache_lru_eviction_order() { + let mut cache = PlanCache::new(2); + let plan = Arc::new(PhysicalPlan { operators: vec![] }); + cache.insert(1, plan.clone()); + cache.insert(2, plan.clone()); + // Touch 1 so 2 becomes the least-recently-used entry. + assert!(cache.get(1).is_some()); + cache.insert(3, plan.clone()); + assert!( + cache.get(1).is_some(), + "recently-used entry must survive eviction" + ); + assert!( + cache.get(2).is_none(), + "least-recently-used entry must be evicted" + ); + assert!(cache.get(3).is_some()); + assert_eq!(cache.len(), 2); + } + + #[test] + fn test_plan_cache_insert_both_shares_plan_and_slots() { + // Fix 2: a raw-bytes-hash entry and its literal-normalized sibling + // must point at the SAME Arc'd plan and SlotTable (no rebuild), and + // ONLY the raw-hash entry carries auto_params. + let mut cache = PlanCache::new(8); + let plan = Arc::new(PhysicalPlan { + operators: vec![PhysicalOp::NodeScan { + variable: "n".to_string(), + label: None, + }], + }); + let raw_hash = 100; + let normalized_hash = 200; + let auto_params = vec![("auto0".to_string(), Value::Int(42))]; + cache.insert_both(raw_hash, normalized_hash, plan.clone(), auto_params.clone()); + assert_eq!(cache.len(), 2, "raw + normalized keys are distinct entries"); + + let raw_hit = cache.get(raw_hash).expect("raw-hash hit"); + let norm_hit = cache.get(normalized_hash).expect("normalized-hash hit"); + + assert!( + Arc::ptr_eq(&raw_hit.plan, &norm_hit.plan), + "both keys must share the identical Arc'd plan (no recompile)" + ); + assert!( + Arc::ptr_eq(&raw_hit.slots, &norm_hit.slots), + "both keys must share the identical Arc'd SlotTable (no rebuild)" + ); + // `Value` has no `PartialEq` impl -- check shape + the one variant + // that matters here (Int) directly. + assert_eq!(raw_hit.auto_params.len(), 1); + assert_eq!(raw_hit.auto_params[0].0, "auto0"); + assert!( + matches!(raw_hit.auto_params[0].1, Value::Int(42)), + "raw-hash entry replays this exact text's auto_params without re-lexing" + ); + assert!( + norm_hit.auto_params.is_empty(), + "normalized-hash entry must NOT carry a fixed set of auto_params \ + (it is shared across many distinct literal values)" + ); + } + + #[test] + fn test_plan_cache_insert_both_same_hash_dedups() { + // A query with no literals to extract normalizes to itself, so + // raw_hash == normalized_hash -- must not create two entries. + let mut cache = PlanCache::new(8); + let plan = Arc::new(PhysicalPlan { operators: vec![] }); + cache.insert_both(50, 50, plan, Vec::new()); + assert_eq!(cache.len(), 1); + } + + #[test] + fn test_plan_cache_raw_hash_hit_avoids_reparse_counter() { + // Simulates the graph_read.rs call site: a raw-hash pre-lookup that + // hits must let the caller skip parameterize() entirely. Modeled + // here via an external "reparse" counter the caller would only + // increment on a raw-hash MISS. + let mut cache = PlanCache::new(8); + let plan = Arc::new(PhysicalPlan { operators: vec![] }); + let raw_hash = hash_query(b"MATCH (n:Person {id: 7}) RETURN n"); + let normalized_hash = hash_query(b"MATCH (n:Person {id: $auto0}) RETURN n"); + + let mut reparse_count = 0u32; + // First call: miss -> caller "parses" (increments counter) then inserts. + if cache.get(raw_hash).is_none() { + reparse_count += 1; + cache.insert_both( + raw_hash, + normalized_hash, + plan.clone(), + vec![("auto0".to_string(), Value::Int(7))], + ); + } + assert_eq!(reparse_count, 1); + + // Second (and third) call: exact repeat hits the raw-hash key -> + // no reparse. + for _ in 0..2 { + if cache.get(raw_hash).is_none() { + reparse_count += 1; + } + } + assert_eq!( + reparse_count, 1, + "exact-repeat queries must hit the raw-hash key without reparsing" + ); + } + + #[test] + fn test_plan_cache_lru_eviction_with_dual_keys() { + // LRU eviction must still function correctly once entries can arrive + // in raw+normalized pairs. + let mut cache = PlanCache::new(2); + let plan = Arc::new(PhysicalPlan { operators: vec![] }); + cache.insert_both(1, 2, plan.clone(), vec![("a".to_string(), Value::Int(1))]); + assert_eq!(cache.len(), 2); + // Cache is now full (max_entries=2). A brand-new key must evict the + // least-recently-used entry (raw key 1, since normalized key 2 was + // inserted after it and is therefore more recently used). + cache.insert(3, plan); + assert_eq!(cache.len(), 2); + assert!( + cache.get(1).is_none(), + "least-recently-used raw-hash entry must be evicted" + ); + assert!(cache.get(2).is_some()); + assert!(cache.get(3).is_some()); + } + #[test] fn test_hash_query_deterministic() { let h1 = hash_query(b"MATCH (n) RETURN n"); diff --git a/src/graph/fasthash.rs b/src/graph/fasthash.rs new file mode 100644 index 00000000..24e21f5f --- /dev/null +++ b/src/graph/fasthash.rs @@ -0,0 +1,105 @@ +//! Fast non-cryptographic hasher for graph-internal keys. +//! +//! `NodeKey`/`EdgeKey` are slotmap keys (u64 POD) that only enter maps and +//! sets we control (visited sets, distance maps, frontier dedup) — SipHash's +//! HashDoS resistance buys nothing there and costs several times more per +//! op. This is the FxHash multiply-fold (rustc's own internal hasher), +//! inlined here to avoid a new dependency. +//! +//! NOT DoS-resistant: never key these maps by attacker-controlled bytes +//! (client key names, query strings). Slotmap keys and label ids only. + +use std::hash::{BuildHasherDefault, Hasher}; + +/// FxHash-style multiplicative hasher (see module docs for the safety +/// contract on key provenance). +#[derive(Default)] +pub struct FxHasher { + hash: u64, +} + +const SEED: u64 = 0x51_7c_c1_b7_27_22_0a_95; + +impl FxHasher { + #[inline] + fn add_to_hash(&mut self, i: u64) { + self.hash = (self.hash.rotate_left(5) ^ i).wrapping_mul(SEED); + } +} + +impl Hasher for FxHasher { + #[inline] + fn write(&mut self, bytes: &[u8]) { + for chunk in bytes.chunks(8) { + let mut buf = [0u8; 8]; + buf[..chunk.len()].copy_from_slice(chunk); + self.add_to_hash(u64::from_le_bytes(buf)); + } + } + #[inline] + fn write_u8(&mut self, i: u8) { + self.add_to_hash(u64::from(i)); + } + #[inline] + fn write_u16(&mut self, i: u16) { + self.add_to_hash(u64::from(i)); + } + #[inline] + fn write_u32(&mut self, i: u32) { + self.add_to_hash(u64::from(i)); + } + #[inline] + fn write_u64(&mut self, i: u64) { + self.add_to_hash(i); + } + #[inline] + fn write_usize(&mut self, i: usize) { + self.add_to_hash(i as u64); + } + #[inline] + fn finish(&self) -> u64 { + self.hash + } +} + +/// BuildHasher for [`FxHasher`]. +pub type FxBuildHasher = BuildHasherDefault; +/// HashMap keyed with [`FxHasher`]. +pub type FxHashMap = std::collections::HashMap; +/// HashSet keyed with [`FxHasher`]. +pub type FxHashSet = std::collections::HashSet; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fx_map_and_set_basic() { + let mut m: FxHashMap = FxHashMap::default(); + for i in 0..1000u64 { + m.insert(i, (i * 2) as u32); + } + assert_eq!(m.len(), 1000); + assert_eq!(m.get(&500), Some(&1000)); + + let mut s: FxHashSet = FxHashSet::default(); + assert!(s.insert(7)); + assert!(!s.insert(7)); + assert!(s.contains(&7)); + } + + #[test] + fn test_fx_hasher_distributes() { + // Sequential keys must not collapse to a few buckets: distinct + // hashes for distinct inputs (sanity, not a statistical test). + use std::hash::{BuildHasher, Hash}; + let bh = FxBuildHasher::default(); + let mut hashes: FxHashSet = FxHashSet::default(); + for i in 0..10_000u64 { + let mut h = bh.build_hasher(); + i.hash(&mut h); + hashes.insert(h.finish()); + } + assert_eq!(hashes.len(), 10_000, "no collisions on 10k sequential u64"); + } +} diff --git a/src/graph/hnsw_bridge.rs b/src/graph/hnsw_bridge.rs new file mode 100644 index 00000000..53aa0a11 --- /dev/null +++ b/src/graph/hnsw_bridge.rs @@ -0,0 +1,476 @@ +//! HNSW bridge for graph-embedded vectors (hybrid HnswPreFilter). +//! +//! `GRAPH.HYBRID`'s strategy selector picks `FilterStrategy::HnswPreFilter` +//! at >= 10K candidates, but until this module the executor silently +//! brute-forced anyway. The bridge builds a real HNSW graph (the vector +//! engine's `HnswBuilder`/`HnswGraph`) over a CSR segment's v5 node +//! embeddings, searched with raw-f32 cosine distances — no TurboQuant +//! codes involved. +//! +//! Lifecycle: built lazily per segment on the FIRST HnswPreFilter query +//! (`CsrStorage::hnsw_bridge`, `OnceLock` — same pattern as the incoming +//! index and the property index) and only when the segment holds at least +//! [`BRIDGE_MIN_VECTORS`] embeddings; below that, brute force wins and the +//! build cost is never paid. In-memory only — never persisted. +//! +//! Search is APPROXIMATE (standard HNSW ef-beam, filter applied to the +//! collected beam). The hybrid caller compensates: a beam that under-fills +//! the requested k falls back to exact brute-force scoring of that +//! candidate group, so results are never silently truncated. + +use crate::graph::csr::CsrStorage; +use crate::graph::fasthash::FxHashMap; +use crate::vector::hnsw::build::HnswBuilder; +use crate::vector::hnsw::graph::{HnswGraph, SENTINEL}; + +/// Minimum embedded-node count before a segment earns an HNSW bridge. +/// Below this, brute-force cosine over the candidates is faster than the +/// build + beam cost (and the strategy selector picks BruteForce for +/// sub-10K candidate sets anyway). +pub const BRIDGE_MIN_VECTORS: usize = 4096; + +/// HNSW upper-layer fanout (layer 0 uses 2*m). +const BRIDGE_M: u8 = 16; +/// Construction beam width. +const BRIDGE_EF_CONSTRUCTION: u16 = 128; +/// Deterministic level-generation seed ("moon"). +const BRIDGE_SEED: u64 = 0x6d6f_6f6e; +/// Floor for the search beam width. +const MIN_EF_SEARCH: usize = 64; + +/// An HNSW index over one CSR segment's node embeddings. +/// +/// Bridge ids are dense `0..len` in insertion (CSR row) order; `rows` maps +/// a bridge id back to its CSR row. Embeddings are copied once at build +/// time and unit-normalized, so cosine similarity is a plain dot product +/// (identical value to `simd::cosine_similarity` on the raw vectors). +pub struct GraphHnsw { + graph: HnswGraph, + /// bridge id -> CSR row. + rows: Vec, + /// CSR row -> bridge id (allow-filter membership). + row_to_id: FxHashMap, + /// Flat unit-normalized embeddings, bridge-id-major. + vecs: Vec, + dim: usize, +} + +impl core::fmt::Debug for GraphHnsw { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("GraphHnsw") + .field("len", &self.rows.len()) + .field("dim", &self.dim) + .finish() + } +} + +impl GraphHnsw { + /// Build an HNSW bridge over a segment's v5 embeddings. + /// + /// Returns `None` when fewer than `min_vectors` usable embeddings exist. + /// A usable embedding has the segment's majority dimension (the first + /// one seen) and a non-zero norm; rows that don't qualify are simply + /// absent from the bridge (`contains_row` false) and the hybrid caller + /// scores them exactly instead. + pub fn build(seg: &CsrStorage, min_vectors: usize) -> Option { + let node_count = seg.node_count(); + let mut rows: Vec = Vec::new(); + let mut vecs: Vec = Vec::new(); + let mut dim = 0usize; + + for row in 0..node_count { + let Some(mut emb) = seg.node_embedding(row) else { + continue; + }; + if emb.is_empty() { + continue; + } + if dim == 0 { + dim = emb.len(); + } + if emb.len() != dim { + continue; + } + let norm: f32 = emb.iter().map(|x| x * x).sum::().sqrt(); + if !(norm.is_finite() && norm > 0.0) { + continue; + } + for x in &mut emb { + *x /= norm; + } + rows.push(row); + vecs.extend_from_slice(&emb); + } + + if rows.is_empty() || rows.len() < min_vectors.max(1) { + return None; + } + + let n = rows.len(); + let start = std::time::Instant::now(); + let mut builder = HnswBuilder::new(BRIDGE_M, BRIDGE_EF_CONSTRUCTION, BRIDGE_SEED); + for _ in 0..n { + // dist(a, b) = cosine distance between two already-inserted + // bridge ids; vectors are unit-normalized so 1 - dot suffices. + builder.insert(|a, b| { + 1.0 - dot( + &vecs[a as usize * dim..(a as usize + 1) * dim], + &vecs[b as usize * dim..(b as usize + 1) * dim], + ) + }); + } + let graph = builder.build((dim * 4) as u32); + tracing::info!( + vectors = n, + dim, + elapsed_ms = start.elapsed().as_millis() as u64, + "graph hnsw_bridge built" + ); + + let mut row_to_id = FxHashMap::default(); + row_to_id.reserve(n); + for (id, &row) in rows.iter().enumerate() { + row_to_id.insert(row, id as u32); + } + + Some(GraphHnsw { + graph, + rows, + row_to_id, + vecs, + dim, + }) + } + + /// Number of indexed embeddings. + pub fn len(&self) -> usize { + self.rows.len() + } + + /// Whether the bridge indexes no embeddings (never true post-build). + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// Embedding dimensionality this bridge was built for. + pub fn dim(&self) -> usize { + self.dim + } + + /// Whether a CSR row is indexed in this bridge. + #[inline] + pub fn contains_row(&self, row: u32) -> bool { + self.row_to_id.contains_key(&row) + } + + /// Resident bytes this bridge pins: the HNSW graph structure plus the + /// bridge's own bookkeeping (`rows`, `row_to_id`, the raw-f32 embedding + /// copy). Always heap-owned -- the bridge is built lazily in-memory and + /// never persisted/mmap'd (see module docs), so unlike `CsrStorage`'s + /// mmap-backed sections there is no "0 for mapped" case here. + pub fn resident_bytes(&self) -> usize { + let rows_bytes = self.rows.len() * std::mem::size_of::(); + // FxHashMap: approximate as capacity * (key + value + + // hashbrown control-byte overhead); exactness isn't the goal here, + // just keeping the elastic memory budget honest about a + // multi-megabyte structure it currently sees as zero. + let row_to_id_bytes = self.row_to_id.capacity() * (std::mem::size_of::() * 2 + 1); + let vecs_bytes = self.vecs.len() * std::mem::size_of::(); + self.graph.resident_bytes() + rows_bytes + row_to_id_bytes + vecs_bytes + } + + #[inline] + fn vec_of(&self, id: u32) -> &[f32] { + &self.vecs[id as usize * self.dim..(id as usize + 1) * self.dim] + } + + #[inline] + fn dist_to_query(&self, q: &[f32], id: u32) -> f32 { + 1.0 - dot(q, self.vec_of(id)) + } + + /// Approximate top-k rows by cosine similarity among rows passing + /// `allow`. Returns `(csr_row, cosine_similarity)` sorted descending; + /// may return FEWER than k when the beam doesn't surface enough allowed + /// rows — the caller decides whether to fall back to exact scoring. + pub fn search(&self, query: &[f32], k: usize, allow: impl Fn(u32) -> bool) -> Vec<(u32, f64)> { + if k == 0 || query.len() != self.dim || self.rows.is_empty() { + return Vec::new(); + } + let norm: f32 = query.iter().map(|x| x * x).sum::().sqrt(); + if !(norm.is_finite() && norm > 0.0) { + return Vec::new(); + } + let mut q = query.to_vec(); + for x in &mut q { + *x /= norm; + } + + let g = &self.graph; + let n = g.num_nodes(); + + // Greedy descent through upper layers (ORIGINAL id space). + let mut cur = g.to_original(g.entry_point()); + let mut cur_dist = self.dist_to_query(&q, cur); + for layer in (1..=g.max_level() as usize).rev() { + loop { + let mut improved = false; + for &nb in g.neighbors_upper(cur, layer) { + if nb == SENTINEL { + continue; + } + let d = self.dist_to_query(&q, nb); + if d < cur_dist { + cur = nb; + cur_dist = d; + improved = true; + } + } + if !improved { + break; + } + } + } + + // Layer-0 ef-beam (BFS-reordered space). The beam ignores the + // filter for NAVIGATION (a disallowed node still routes toward + // allowed ones); allowed nodes are collected from the visited set. + let ef = (k * 4).max(MIN_EF_SEARCH).min(n as usize); + let mut visited = vec![0u64; (n as usize).div_ceil(64)]; + let mark = |bits: &mut [u64], i: u32| -> bool { + let (w, b) = (i as usize / 64, i as usize % 64); + let seen = bits[w] & (1 << b) != 0; + bits[w] |= 1 << b; + !seen + }; + + // candidates: min-heap by distance; beam: max-heap capped at ef. + let mut candidates: std::collections::BinaryHeap> = + std::collections::BinaryHeap::new(); + let mut beam: std::collections::BinaryHeap = std::collections::BinaryHeap::new(); + let entry_bfs = g.to_bfs(cur); + mark(&mut visited, entry_bfs); + candidates.push(std::cmp::Reverse(OrdDist(cur_dist, entry_bfs))); + beam.push(OrdDist(cur_dist, entry_bfs)); + + let mut collected: Vec<(f32, u32)> = Vec::new(); // (dist, bridge id), allowed only + let entry_orig = g.to_original(entry_bfs); + if allow(self.rows[entry_orig as usize]) { + collected.push((cur_dist, entry_orig)); + } + + while let Some(std::cmp::Reverse(OrdDist(c_dist, c_bfs))) = candidates.pop() { + if beam.len() >= ef { + if let Some(&OrdDist(worst, _)) = beam.peek() { + if c_dist > worst { + break; + } + } + } + for &nb in g.neighbors_l0(c_bfs) { + if nb == SENTINEL || !mark(&mut visited, nb) { + continue; + } + let nb_orig = g.to_original(nb); + let d = self.dist_to_query(&q, nb_orig); + let worst = beam.peek().map_or(f32::INFINITY, |o| o.0); + if beam.len() < ef || d < worst { + candidates.push(std::cmp::Reverse(OrdDist(d, nb))); + beam.push(OrdDist(d, nb)); + if beam.len() > ef { + beam.pop(); + } + if allow(self.rows[nb_orig as usize]) { + collected.push((d, nb_orig)); + } + } + } + } + + collected.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + collected.truncate(k); + collected + .into_iter() + .map(|(d, id)| (self.rows[id as usize], f64::from(1.0 - d))) + .collect() + } +} + +/// (distance, node) ordered by distance — max-heap element. +#[derive(Clone, Copy, PartialEq)] +struct OrdDist(f32, u32); + +impl Eq for OrdDist {} + +impl PartialOrd for OrdDist { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for OrdDist { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0 + .partial_cmp(&other.0) + .unwrap_or(std::cmp::Ordering::Equal) + .then(self.1.cmp(&other.1)) + } +} + +#[inline] +fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| x * y).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::graph::csr::CsrSegment; + use crate::graph::memgraph::MemGraph; + use crate::graph::types::NodeKey; + use smallvec::smallvec; + + /// Deterministic pseudo-random embedding (no Math.random in tests). + fn embedding(i: u64, dim: usize) -> Vec { + let mut state = i.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..dim) + .map(|_| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((state >> 33) as f32 / (1u64 << 31) as f32) - 0.5 + }) + .collect() + } + + /// Freeze a graph of `n` embedded nodes into a CsrStorage. + fn frozen_segment(n: usize, dim: usize) -> (CsrStorage, Vec) { + let mut g = MemGraph::new(1_000_000); + let keys: Vec = (0..n) + .map(|i| { + g.add_node( + smallvec![0u16], + smallvec::SmallVec::new(), + Some(embedding(i as u64, dim)), + 1, + ) + }) + .collect(); + // A few edges so the frozen graph is non-degenerate. + for i in 0..n.saturating_sub(1) { + g.add_edge(keys[i], keys[i + 1], 1, 1.0, None, 2) + .expect("edge"); + } + let frozen = g.freeze().expect("freeze"); + let seg = CsrSegment::from_frozen(frozen, 10).expect("csr"); + (CsrStorage::from(seg), keys) + } + + /// Exact top-k (row, cosine) via the segment's embeddings. + fn brute_force_topk(seg: &CsrStorage, query: &[f32], k: usize) -> Vec<(u32, f64)> { + let mut scored: Vec<(u32, f64)> = (0..seg.node_count()) + .filter_map(|row| { + let emb = seg.node_embedding(row)?; + Some((row, crate::graph::simd::cosine_similarity(&emb, query))) + }) + .collect(); + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + scored.truncate(k); + scored + } + + #[test] + fn test_bridge_build_and_unfiltered_recall() { + let (seg, _) = frozen_segment(300, 8); + let bridge = GraphHnsw::build(&seg, 1).expect("bridge"); + assert_eq!(bridge.len(), 300); + assert_eq!(bridge.dim(), 8); + + let query = embedding(9999, 8); + let exact = brute_force_topk(&seg, &query, 10); + let hits = bridge.search(&query, 10, |_| true); + + assert!(!hits.is_empty()); + // Scores must be REAL cosines (parity with simd::cosine_similarity). + for &(row, sim) in &hits { + let emb = seg.node_embedding(row).expect("emb"); + let expect = crate::graph::simd::cosine_similarity(&emb, &query); + assert!( + (sim - expect).abs() < 1e-4, + "row {row}: bridge {sim} vs exact {expect}" + ); + } + // Recall: at least 8/10 of the exact top-10 surfaced. + let exact_rows: std::collections::HashSet = exact.iter().map(|&(r, _)| r).collect(); + let overlap = hits + .iter() + .filter(|&&(r, _)| exact_rows.contains(&r)) + .count(); + assert!(overlap >= 8, "recall too low: {overlap}/10"); + } + + #[test] + fn test_bridge_filtered_search_respects_allow() { + let (seg, _) = frozen_segment(300, 8); + let bridge = GraphHnsw::build(&seg, 1).expect("bridge"); + + let query = embedding(4242, 8); + let hits = bridge.search(&query, 10, |row| row % 2 == 0); + assert!(!hits.is_empty()); + for &(row, _) in &hits { + assert_eq!(row % 2, 0, "disallowed row {row} leaked through filter"); + } + // Descending scores. + for w in hits.windows(2) { + assert!(w[0].1 >= w[1].1); + } + } + + #[test] + fn test_bridge_min_vectors_gate() { + let (seg, _) = frozen_segment(300, 8); + assert!(GraphHnsw::build(&seg, 10_000).is_none()); + assert!(GraphHnsw::build(&seg, 300).is_some()); + } + + #[test] + fn test_bridge_degenerate_queries() { + let (seg, _) = frozen_segment(64, 8); + let bridge = GraphHnsw::build(&seg, 1).expect("bridge"); + // Dimension mismatch and zero-norm queries return empty, not junk. + assert!(bridge.search(&[1.0; 4], 5, |_| true).is_empty()); + assert!(bridge.search(&[0.0; 8], 5, |_| true).is_empty()); + assert!(bridge.search(&embedding(1, 8), 0, |_| true).is_empty()); + } + + #[test] + fn test_bridge_skips_unembedded_nodes() { + let mut g = MemGraph::new(1_000_000); + let mut keys = Vec::new(); + for i in 0..50u64 { + let emb = if i % 2 == 0 { + Some(embedding(i, 8)) + } else { + None + }; + keys.push(g.add_node(smallvec![0u16], smallvec::SmallVec::new(), emb, 1)); + } + for i in 0..49 { + g.add_edge(keys[i], keys[i + 1], 1, 1.0, None, 2) + .expect("edge"); + } + let seg = CsrStorage::from( + CsrSegment::from_frozen(g.freeze().expect("freeze"), 10).expect("csr"), + ); + let bridge = GraphHnsw::build(&seg, 1).expect("bridge"); + assert_eq!(bridge.len(), 25); + // Every indexed row must actually have an embedding. + for row in 0..seg.node_count() { + assert_eq!( + bridge.contains_row(row), + seg.node_embedding(row).is_some(), + "row {row} membership mismatch" + ); + } + } +} diff --git a/src/graph/hybrid.rs b/src/graph/hybrid.rs index 747545b5..f150caee 100644 --- a/src/graph/hybrid.rs +++ b/src/graph/hybrid.rs @@ -6,17 +6,24 @@ //! 3. Vector-guided walk: beam search guided by embedding distance //! 4. Automatic strategy selection based on candidate set size threshold //! -//! All functions take references to MemGraph (graph data + embeddings) and operate -//! without unsafe code or unwrap. The shard command handler passes MemGraph directly -//! since both GraphStore and VectorStore are per-shard, single-owner. +//! All functions operate on BOTH graph tiers — the mutable MemGraph write +//! buffer and the immutable CSR segments (traversal via `SegmentMergeReader`, +//! node existence/embeddings via `MergedNodeView`) — without unsafe code or +//! unwrap. The shard command handler passes the write buffer plus a loaded +//! segment snapshot; both stores are per-shard, single-owner. Pass `&[]` for +//! segments to operate on a bare MemGraph (tests, pre-freeze graphs). use std::cmp::Ordering; use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; use super::simd; +use crate::graph::csr::CsrStorage; use crate::graph::memgraph::MemGraph; +use crate::graph::traversal::SegmentMergeReader; use crate::graph::types::{Direction, NodeKey}; +use crate::graph::view::MergedNodeView; /// Default threshold for switching between brute-force and HNSW pre-filter. pub const DEFAULT_STRATEGY_THRESHOLD: usize = 10_000; @@ -89,9 +96,11 @@ pub struct ContextNode { pub enum FilterStrategy { /// Score all candidates via brute-force cosine similarity. BruteForce, - /// Use HNSW pre-filter (skip non-candidates during search). - /// For now, falls back to brute-force with candidate filtering since - /// the graph MemGraph embeddings are not indexed in HNSW. + /// Route CSR-resident candidates through the per-segment HNSW bridge + /// (`CsrStorage::hnsw_bridge`, built lazily over v5 embeddings) instead + /// of scoring each one. Approximate: a bridge beam that under-fills k + /// falls back to exact scoring of that group. Mutable-tier candidates + /// and rows without a bridge entry are always scored exactly. HnswPreFilter, } @@ -150,19 +159,27 @@ impl GraphFilteredSearch { /// 2. Auto-select strategy (brute-force vs pre-filter) /// 3. Score candidates by cosine similarity to query_vector /// 4. Return top-K results - pub fn execute(&self, memgraph: &MemGraph, lsn: u64) -> Result, HybridError> { + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + lsn: u64, + ) -> Result, HybridError> { if self.query_vector.is_empty() { return Err(HybridError::EmptyQueryVector); } - // Verify start node exists. - if memgraph.get_node(self.start_node).is_none() { + let view = MergedNodeView::new(memgraph, csr_segs); + + // Verify start node exists in EITHER tier. + if !view.contains(self.start_node) { return Err(HybridError::NodeNotFound); } // Step 1: BFS to collect candidates with graph distance. let candidates = bfs_collect( memgraph, + csr_segs, self.start_node, self.hops, self.edge_type_filter, @@ -171,23 +188,34 @@ impl GraphFilteredSearch { )?; // Step 2: Select strategy. - let _strategy = select_strategy(candidates.len(), self.threshold); - // Both strategies score the same way for MemGraph embeddings. - // HNSW pre-filter would be used when an external VectorStore index exists. - // For MemGraph-embedded vectors, brute-force is always used. - - // Step 3: Score candidates by cosine similarity. - let mut scored: Vec = Vec::with_capacity(candidates.len()); - - for (node_key, graph_dist) in &candidates { - let Some(node) = memgraph.get_node(*node_key) else { - continue; - }; - let Some(embedding) = node.embedding.as_ref() else { + let strategy = select_strategy(candidates.len(), self.threshold); + + // Step 3: Score candidates. HnswPreFilter routes CSR-resident + // candidates through each segment's HNSW bridge and leaves the + // rest (mutable tier, bridge-less rows, dim mismatch) in + // `residual` for exact scoring below. + let mut scored: Vec = Vec::with_capacity(candidates.len().min(4096)); + let residual: Vec<(NodeKey, u32)> = match strategy { + FilterStrategy::BruteForce => candidates, + FilterStrategy::HnswPreFilter => hnsw_prefilter_score( + memgraph, + csr_segs, + candidates, + &self.query_vector, + self.k, + &mut scored, + ), + }; + + // Exact scoring for whatever the pre-filter did not cover + // (everything, under BruteForce): embedding resolved from the + // mutable tier or the CSR v5 blob. + for (node_key, graph_dist) in &residual { + let Some(embedding) = view.embedding(*node_key) else { continue; // Skip nodes without embeddings. }; - let sim = simd::cosine_similarity(embedding, &self.query_vector); + let sim = simd::cosine_similarity(&embedding, &self.query_vector); scored.push(HybridResult { node: *node_key, score: sim, @@ -204,6 +232,82 @@ impl GraphFilteredSearch { } } +/// Route candidates through per-segment HNSW bridges (HnswPreFilter). +/// +/// Partitions `candidates` into per-segment groups (a candidate joins a +/// group when its segment has a bridge covering its row at the query's +/// dimension) and searches each bridge with an allow-filter over the +/// group's rows. Bridge hits are pushed to `scored` with their EXACT +/// cosine (bridge vectors are unit-normalized copies of the originals). +/// Returns the residual candidates for exact scoring: mutable-tier nodes, +/// bridge-less rows, dim mismatches, and any whole group whose beam +/// under-filled k (approximate-search rescue — never silently truncate). +fn hnsw_prefilter_score( + memgraph: &MemGraph, + csr_segs: &[Arc], + candidates: Vec<(NodeKey, u32)>, + query: &[f32], + k: usize, + scored: &mut Vec, +) -> Vec<(NodeKey, u32)> { + use crate::graph::fasthash::FxHashMap; + + let mut residual: Vec<(NodeKey, u32)> = Vec::new(); + let mut groups: Vec> = + (0..csr_segs.len()).map(|_| FxHashMap::default()).collect(); + + 'cand: for (key, gdist) in candidates { + // Mutable tier wins (same precedence as MergedNodeView::embedding). + if memgraph.get_node(key).is_some() { + residual.push((key, gdist)); + continue; + } + for (i, seg) in csr_segs.iter().enumerate() { + if let Some(row) = seg.lookup_node(key) { + if let Some(bridge) = seg.hnsw_bridge() { + if bridge.dim() == query.len() && bridge.contains_row(row) { + groups[i].insert(row, (key, gdist)); + continue 'cand; + } + } + // Resident here but not bridge-searchable: score exactly. + residual.push((key, gdist)); + continue 'cand; + } + } + residual.push((key, gdist)); + } + + for (i, group) in groups.iter().enumerate() { + if group.is_empty() { + continue; + } + let Some(bridge) = csr_segs[i].hnsw_bridge() else { + // Unreachable by construction; stay exact if it ever isn't. + residual.extend(group.values().copied()); + continue; + }; + let hits = bridge.search(query, k, |row| group.contains_key(&row)); + if hits.len() < k.min(group.len()) { + // Beam under-filled the ask: rescue with exact scoring. + residual.extend(group.values().copied()); + continue; + } + for (row, sim) in hits { + if let Some(&(key, gdist)) = group.get(&row) { + scored.push(HybridResult { + node: key, + score: sim, + graph_distance: Some(gdist), + context: Vec::new(), + }); + } + } + } + + residual +} + // --------------------------------------------------------------------------- // HYB-02: Vector-to-graph expansion // --------------------------------------------------------------------------- @@ -242,6 +346,7 @@ impl VectorToGraphExpansion { pub fn execute( &self, memgraph: &MemGraph, + csr_segs: &[Arc], candidate_nodes: &[NodeKey], lsn: u64, ) -> Result, HybridError> { @@ -249,21 +354,21 @@ impl VectorToGraphExpansion { return Err(HybridError::EmptyQueryVector); } + let view = MergedNodeView::new(memgraph, csr_segs); + let committed = roaring::RoaringBitmap::new(); + // Step 1: Score all candidate nodes by cosine similarity. let mut scored: Vec<(NodeKey, f64)> = Vec::with_capacity(candidate_nodes.len()); for &node_key in candidate_nodes { - let Some(node) = memgraph.get_node(node_key) else { - continue; - }; - if node.deleted_lsn != u64::MAX { + if !view.is_visible(node_key, 0, 0, &committed, None) { continue; } - let Some(embedding) = node.embedding.as_ref() else { + let Some(embedding) = view.embedding(node_key) else { continue; }; - let sim = simd::cosine_similarity(embedding, &self.query_vector); + let sim = simd::cosine_similarity(&embedding, &self.query_vector); scored.push((node_key, sim)); } @@ -278,6 +383,7 @@ impl VectorToGraphExpansion { let context = if self.expansion_hops > 0 { collect_context( memgraph, + csr_segs, node_key, self.expansion_hops, self.edge_type_filter, @@ -334,12 +440,20 @@ impl VectorGuidedWalk { /// At each step, expand all neighbors of the current beam, score by cosine /// similarity, and keep the top `beam_width` for the next step. Returns the /// walk path: all visited nodes with their cumulative scores. - pub fn execute(&self, memgraph: &MemGraph, lsn: u64) -> Result, HybridError> { + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + lsn: u64, + ) -> Result, HybridError> { if self.query_vector.is_empty() { return Err(HybridError::EmptyQueryVector); } - if memgraph.get_node(self.seed_node).is_none() { + let view = MergedNodeView::new(memgraph, csr_segs); + let reader = SegmentMergeReader::new(Some(memgraph), csr_segs, Direction::Both, lsn, None); + + if !view.contains(self.seed_node) { return Err(HybridError::NodeNotFound); } @@ -347,10 +461,9 @@ impl VectorGuidedWalk { visited.insert(self.seed_node); // Score the seed node. - let seed_score = memgraph - .get_node(self.seed_node) - .and_then(|n| n.embedding.as_ref()) - .map(|emb| simd::cosine_similarity(emb, &self.query_vector)) + let seed_score = view + .embedding(self.seed_node) + .map(|emb| simd::cosine_similarity(&emb, &self.query_vector)) .unwrap_or(0.0); let mut results: Vec = Vec::new(); @@ -368,21 +481,18 @@ impl VectorGuidedWalk { let mut candidates: Vec<(NodeKey, f64)> = Vec::new(); for &(current, _) in &beam { - // Expand neighbors. - for (edge_key, neighbor_key) in memgraph.neighbors(current, Direction::Both, lsn) { + // Expand neighbors across both tiers. + for merged in reader.neighbors(current) { + let neighbor_key = merged.node; if visited.contains(&neighbor_key) { continue; } - let sim = memgraph - .get_node(neighbor_key) - .and_then(|n| n.embedding.as_ref()) - .map(|emb| simd::cosine_similarity(emb, &self.query_vector)) + let sim = view + .embedding(neighbor_key) + .map(|emb| simd::cosine_similarity(&emb, &self.query_vector)) .unwrap_or(0.0); - // Use edge_key to avoid unused variable warning. - let _ = edge_key; - candidates.push((neighbor_key, sim)); } } @@ -478,13 +588,20 @@ impl GraphConstrainedReRanker { /// 3. Iterate ALL nodes with embeddings, compute cosine similarity. /// 4. Combine: `alpha * vector_score + (1-alpha) * 1/(1+graph_dist)`. /// 5. Sort descending, return top-K. - pub fn execute(&self, memgraph: &MemGraph, lsn: u64) -> Result, HybridError> { + pub fn execute( + &self, + memgraph: &MemGraph, + csr_segs: &[Arc], + lsn: u64, + ) -> Result, HybridError> { if self.query_vector.is_empty() { return Err(HybridError::EmptyQueryVector); } - // Validate reference node exists. - if memgraph.get_node(self.reference_node).is_none() { + let view = MergedNodeView::new(memgraph, csr_segs); + + // Validate reference node exists in EITHER tier. + if !view.contains(self.reference_node) { return Err(HybridError::NodeNotFound); } @@ -495,6 +612,7 @@ impl GraphConstrainedReRanker { // Step 1: Single batch BFS from reference node — O(frontier). let bfs_results = bfs_collect( memgraph, + csr_segs, self.reference_node, self.max_hops, None, @@ -509,15 +627,16 @@ impl GraphConstrainedReRanker { distance_map.insert(*node_key, *dist); } - // Step 2: Score ALL nodes with embeddings. + // Step 2: Score ALL nodes with embeddings, across both tiers. + let committed = roaring::RoaringBitmap::new(); let mut scored: Vec = Vec::with_capacity(distance_map.len()); - for (node_key, node) in memgraph.iter_nodes() { - let Some(embedding) = node.embedding.as_ref() else { - continue; // Skip nodes without embeddings. + view.for_each_visible_node(None, 0, 0, &committed, None, |node_key| { + let Some(embedding) = view.embedding(node_key) else { + return; // Skip nodes without embeddings. }; - let vector_score = simd::cosine_similarity(embedding, &self.query_vector); + let vector_score = simd::cosine_similarity(&embedding, &self.query_vector); let graph_dist = distance_map.get(&node_key).copied().unwrap_or(penalty_dist); let graph_score = 1.0 / (1.0 + graph_dist as f64); let combined = alpha * vector_score + (1.0 - alpha) * graph_score; @@ -528,7 +647,7 @@ impl GraphConstrainedReRanker { graph_distance: Some(graph_dist), context: Vec::new(), }); - } + }); // Step 3: Sort descending by combined score, take top-K. scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); @@ -543,15 +662,25 @@ impl GraphConstrainedReRanker { // --------------------------------------------------------------------------- /// BFS from a start node, collecting (NodeKey, graph_distance) pairs. -/// Excludes the start node itself from results. +/// Excludes the start node itself from results. Traverses BOTH tiers via +/// `SegmentMergeReader` (frozen CSR edges + mutable/delta edges). fn bfs_collect( memgraph: &MemGraph, + csr_segs: &[Arc], start: NodeKey, max_depth: u32, edge_type_filter: Option, frontier_cap: usize, lsn: u64, ) -> Result, HybridError> { + let reader = SegmentMergeReader::new( + Some(memgraph), + csr_segs, + Direction::Both, + lsn, + edge_type_filter, + ); + let mut visited: HashSet = HashSet::new(); visited.insert(start); @@ -564,16 +693,8 @@ fn bfs_collect( continue; } - for (edge_key, neighbor_key) in memgraph.neighbors(current, Direction::Both, lsn) { - // Apply edge type filter. - if let Some(filter_type) = edge_type_filter { - if let Some(edge) = memgraph.get_edge(edge_key) { - if edge.edge_type != filter_type { - continue; - } - } - } - + for merged in reader.neighbors(current) { + let neighbor_key = merged.node; if visited.contains(&neighbor_key) { continue; } @@ -592,14 +713,23 @@ fn bfs_collect( Ok(results) } -/// Collect context neighbors for a node via BFS expansion. +/// Collect context neighbors for a node via BFS expansion (both tiers). fn collect_context( memgraph: &MemGraph, + csr_segs: &[Arc], start: NodeKey, max_hops: u32, edge_type_filter: Option, lsn: u64, ) -> Vec { + let reader = SegmentMergeReader::new( + Some(memgraph), + csr_segs, + Direction::Both, + lsn, + edge_type_filter, + ); + let mut visited: HashSet = HashSet::new(); visited.insert(start); @@ -612,19 +742,8 @@ fn collect_context( continue; } - for (edge_key, neighbor_key) in memgraph.neighbors(current, Direction::Both, lsn) { - let edge_type = memgraph - .get_edge(edge_key) - .map(|e| e.edge_type) - .unwrap_or(0); - - // Apply edge type filter. - if let Some(filter_type) = edge_type_filter { - if edge_type != filter_type { - continue; - } - } - + for merged in reader.neighbors(current) { + let neighbor_key = merged.node; if visited.contains(&neighbor_key) { continue; } @@ -633,7 +752,7 @@ fn collect_context( let next_depth = depth + 1; context.push(ContextNode { node: neighbor_key, - edge_type, + edge_type: merged.edge_type, hops: next_depth, }); frontier.push_back((neighbor_key, next_depth)); @@ -769,7 +888,7 @@ mod tests { // Search within 1 hop of A, query vector close to B's embedding. let search = GraphFilteredSearch::new(a, 1, vec![0.8, 0.6, 0.0], 10); - let results = search.execute(&g, u64::MAX - 1).expect("search ok"); + let results = search.execute(&g, &[], u64::MAX - 1).expect("search ok"); // Only B is within 1 hop, and B has embedding [0.8, 0.6, 0.0]. assert_eq!(results.len(), 1); @@ -784,7 +903,7 @@ mod tests { // Search within 2 hops of A, query vector closest to C. let search = GraphFilteredSearch::new(a, 2, vec![0.0, 1.0, 0.0], 10); - let results = search.execute(&g, u64::MAX - 1).expect("search ok"); + let results = search.execute(&g, &[], u64::MAX - 1).expect("search ok"); // B and C are within 2 hops. C should rank first (identical to query). assert_eq!(results.len(), 2); @@ -798,7 +917,7 @@ mod tests { // All spokes are 1 hop from center. Take top 3. let search = GraphFilteredSearch::new(center, 1, vec![1.0, 0.0, 0.0], 3); - let results = search.execute(&g, u64::MAX - 1).expect("search ok"); + let results = search.execute(&g, &[], u64::MAX - 1).expect("search ok"); assert_eq!(results.len(), 3); // Scores should be descending. @@ -811,7 +930,7 @@ mod tests { fn test_graph_filtered_empty_query() { let (g, a, _, _, _) = build_test_graph(); let search = GraphFilteredSearch::new(a, 1, vec![], 10); - let result = search.execute(&g, u64::MAX - 1); + let result = search.execute(&g, &[], u64::MAX - 1); assert!(matches!(result, Err(HybridError::EmptyQueryVector))); } @@ -820,7 +939,7 @@ mod tests { let g = MemGraph::new(100_000); let fake_key: NodeKey = slotmap::KeyData::from_ffi(999).into(); let search = GraphFilteredSearch::new(fake_key, 1, vec![1.0, 0.0], 10); - let result = search.execute(&g, u64::MAX - 1); + let result = search.execute(&g, &[], u64::MAX - 1); assert!(matches!(result, Err(HybridError::NodeNotFound))); } @@ -832,7 +951,7 @@ mod tests { g.add_edge(a, b, 1, 1.0, None, 2).expect("edge"); let search = GraphFilteredSearch::new(a, 1, vec![1.0, 0.0], 10); - let results = search.execute(&g, u64::MAX - 1).expect("ok"); + let results = search.execute(&g, &[], u64::MAX - 1).expect("ok"); assert!(results.is_empty()); // No embeddings -> no results. } @@ -841,7 +960,7 @@ mod tests { let (g, center, _) = build_star_graph(20); let mut search = GraphFilteredSearch::new(center, 1, vec![1.0, 0.0, 0.0], 10); search.frontier_cap = 5; // Very small cap. - let result = search.execute(&g, u64::MAX - 1); + let result = search.execute(&g, &[], u64::MAX - 1); assert!(matches!( result, Err(HybridError::FrontierCapExceeded { .. }) @@ -857,7 +976,9 @@ mod tests { // Query closest to C. Expand 1 hop. let expansion = VectorToGraphExpansion::new(vec![0.0, 1.0, 0.0], 1, 1); - let results = expansion.execute(&g, &all_nodes, u64::MAX - 1).expect("ok"); + let results = expansion + .execute(&g, &[], &all_nodes, u64::MAX - 1) + .expect("ok"); assert_eq!(results.len(), 1); // Top-1 assert_eq!(results[0].node, c); @@ -872,7 +993,9 @@ mod tests { let all_nodes = vec![a, b, c, d]; let expansion = VectorToGraphExpansion::new(vec![0.0, 1.0, 0.0], 3, 1); - let results = expansion.execute(&g, &all_nodes, u64::MAX - 1).expect("ok"); + let results = expansion + .execute(&g, &[], &all_nodes, u64::MAX - 1) + .expect("ok"); assert_eq!(results.len(), 3); // Scores descending. @@ -887,7 +1010,9 @@ mod tests { let all_nodes = vec![a, b, c, d]; let expansion = VectorToGraphExpansion::new(vec![1.0, 0.0, 0.0], 2, 0); - let results = expansion.execute(&g, &all_nodes, u64::MAX - 1).expect("ok"); + let results = expansion + .execute(&g, &[], &all_nodes, u64::MAX - 1) + .expect("ok"); // No expansion: context should be empty. for r in &results { @@ -899,7 +1024,7 @@ mod tests { fn test_vector_expansion_empty_query() { let g = MemGraph::new(100_000); let expansion = VectorToGraphExpansion::new(vec![], 1, 1); - let result = expansion.execute(&g, &[], u64::MAX - 1); + let result = expansion.execute(&g, &[], &[], u64::MAX - 1); assert!(matches!(result, Err(HybridError::EmptyQueryVector))); } @@ -911,7 +1036,7 @@ mod tests { // Walk from A toward [0.0, 1.0, 0.0] (closest to C). let walk = VectorGuidedWalk::new(a, vec![0.0, 1.0, 0.0], 3); - let results = walk.execute(&g, u64::MAX - 1).expect("walk ok"); + let results = walk.execute(&g, &[], u64::MAX - 1).expect("walk ok"); // Should visit A, then expand toward B/C/D based on similarity. assert!(!results.is_empty()); @@ -934,7 +1059,7 @@ mod tests { // High min_similarity: should stop early. let mut walk = VectorGuidedWalk::new(a, vec![0.0, 0.0, 1.0], 10); walk.min_similarity = 0.99; // Very high -- only near-identical passes. - let results = walk.execute(&g, u64::MAX - 1).expect("ok"); + let results = walk.execute(&g, &[], u64::MAX - 1).expect("ok"); // Only seed node (nothing else is similar enough at 0.99). assert_eq!(results.len(), 1); @@ -946,7 +1071,7 @@ mod tests { let mut walk = VectorGuidedWalk::new(center, vec![1.0, 0.0, 0.0], 1); walk.beam_width = 3; - let results = walk.execute(&g, u64::MAX - 1).expect("ok"); + let results = walk.execute(&g, &[], u64::MAX - 1).expect("ok"); // Seed + up to 3 (beam_width) spokes. assert!(results.len() <= 4); @@ -958,7 +1083,7 @@ mod tests { let g = MemGraph::new(100_000); let fake_key: NodeKey = slotmap::KeyData::from_ffi(999).into(); let walk = VectorGuidedWalk::new(fake_key, vec![1.0, 0.0], 3); - let result = walk.execute(&g, u64::MAX - 1); + let result = walk.execute(&g, &[], u64::MAX - 1); assert!(matches!(result, Err(HybridError::NodeNotFound))); } @@ -966,7 +1091,7 @@ mod tests { fn test_vector_walk_empty_query() { let (g, a, _, _, _) = build_test_graph(); let walk = VectorGuidedWalk::new(a, vec![], 3); - let result = walk.execute(&g, u64::MAX - 1); + let result = walk.execute(&g, &[], u64::MAX - 1); assert!(matches!(result, Err(HybridError::EmptyQueryVector))); } @@ -976,7 +1101,7 @@ mod tests { let a = g.add_node(smallvec![0], empty_props(), Some(vec![1.0, 0.0]), 1); // Isolated node. let walk = VectorGuidedWalk::new(a, vec![1.0, 0.0], 3); - let results = walk.execute(&g, u64::MAX - 1).expect("ok"); + let results = walk.execute(&g, &[], u64::MAX - 1).expect("ok"); assert_eq!(results.len(), 1); // Only seed. } @@ -985,7 +1110,7 @@ mod tests { #[test] fn test_bfs_collect_basic() { let (g, a, b, c, _d) = build_test_graph(); - let candidates = bfs_collect(&g, a, 2, None, 100_000, u64::MAX - 1).expect("ok"); + let candidates = bfs_collect(&g, &[], a, 2, None, 100_000, u64::MAX - 1).expect("ok"); // 2 hops from A: B (1 hop), C (2 hops). assert_eq!(candidates.len(), 2); @@ -1005,7 +1130,7 @@ mod tests { g.add_edge(a, c, 2, 1.0, None, 2).expect("edge"); // type 2 // Filter to edge type 1 only. - let candidates = bfs_collect(&g, a, 1, Some(1), 100_000, u64::MAX - 1).expect("ok"); + let candidates = bfs_collect(&g, &[], a, 1, Some(1), 100_000, u64::MAX - 1).expect("ok"); assert_eq!(candidates.len(), 1); assert_eq!(candidates[0].0, b); } @@ -1023,7 +1148,7 @@ mod tests { // C is disconnected from A. Only B should appear. let search = GraphFilteredSearch::new(a, 1, vec![1.0, 0.0], 10); - let results = search.execute(&g, u64::MAX - 1).expect("ok"); + let results = search.execute(&g, &[], u64::MAX - 1).expect("ok"); assert_eq!(results.len(), 1); assert_eq!(results[0].node, b); } @@ -1035,7 +1160,9 @@ mod tests { // Find C, expand 2 hops. let expansion = VectorToGraphExpansion::new(vec![0.0, 1.0, 0.0], 1, 2); - let results = expansion.execute(&g, &all_nodes, u64::MAX - 1).expect("ok"); + let results = expansion + .execute(&g, &[], &all_nodes, u64::MAX - 1) + .expect("ok"); // C is result[0]. Context should include B (1 hop) and D (1 hop), // and A (2 hops from C via B). @@ -1078,7 +1205,7 @@ mod tests { let (g, a, _b, _c, _d, _e) = build_rerank_chain(); let reranker = GraphConstrainedReRanker::new(a, 5, 1.0, vec![1.0, 0.0, 0.0], 5); - let results = reranker.execute(&g, u64::MAX - 1).expect("ok"); + let results = reranker.execute(&g, &[], u64::MAX - 1).expect("ok"); // A has embedding [1,0,0], query is [1,0,0] => score 1.0 (highest). // B has [0.7,0.7,0] => ~0.707 @@ -1099,7 +1226,7 @@ mod tests { let (g, a, b, c, _d, _e) = build_rerank_chain(); let reranker = GraphConstrainedReRanker::new(a, 5, 0.0, vec![1.0, 0.0, 0.0], 5); - let results = reranker.execute(&g, u64::MAX - 1).expect("ok"); + let results = reranker.execute(&g, &[], u64::MAX - 1).expect("ok"); // Graph distances: A=0, B=1, C=2, D=3, E=4. // Graph score = 1/(1+d): A=1.0, B=0.5, C=0.333, D=0.25, E=0.2. @@ -1125,7 +1252,7 @@ mod tests { // Query vector [1,0,0]: A is most similar but 2 hops from C. // B is 1 hop from C and moderately similar. let reranker = GraphConstrainedReRanker::new(c, 2, 0.3, vec![1.0, 0.0, 0.0], 5); - let results = reranker.execute(&g, u64::MAX - 1).expect("ok"); + let results = reranker.execute(&g, &[], u64::MAX - 1).expect("ok"); // A: vector_score ~1.0, graph_dist=2, graph_score=1/3=0.333 // combined = 0.3*1.0 + 0.7*0.333 = 0.300 + 0.233 = 0.533 @@ -1155,7 +1282,7 @@ mod tests { // max_hops=2: A can reach B(1), C(2). D and E are unreachable. let reranker = GraphConstrainedReRanker::new(a, 2, 0.5, vec![1.0, 0.0, 0.0], 10); - let results = reranker.execute(&g, u64::MAX - 1).expect("ok"); + let results = reranker.execute(&g, &[], u64::MAX - 1).expect("ok"); // E should be reachable result with graph_distance = max_hops+1 = 3. let e_result = results.iter().find(|r| r.node == e); @@ -1174,7 +1301,7 @@ mod tests { let a = g.add_node(smallvec![0], empty_props(), None, 1); let reranker = GraphConstrainedReRanker::new(a, 3, 0.5, vec![1.0, 0.0, 0.0], 10); - let results = reranker.execute(&g, u64::MAX - 1).expect("ok"); + let results = reranker.execute(&g, &[], u64::MAX - 1).expect("ok"); assert!(results.is_empty()); } @@ -1183,7 +1310,170 @@ mod tests { let g = MemGraph::new(100_000); let fake_key: NodeKey = slotmap::KeyData::from_ffi(999).into(); let reranker = GraphConstrainedReRanker::new(fake_key, 3, 0.5, vec![1.0, 0.0, 0.0], 10); - let result = reranker.execute(&g, u64::MAX - 1); + let result = reranker.execute(&g, &[], u64::MAX - 1); assert!(matches!(result, Err(HybridError::NodeNotFound))); } + + // --- HnswPreFilter via the per-segment bridge --- + + /// Deterministic pseudo-random embedding (LCG — no RNG in tests). + fn det_embedding(i: u64, dim: usize) -> Vec { + let mut state = i.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..dim) + .map(|_| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((state >> 33) as f32 / (1u64 << 31) as f32) - 0.5 + }) + .collect() + } + + /// Frozen star (center -> 299 embedded spokes) as a CSR segment. + fn frozen_star_segment() -> (MemGraph, Arc, NodeKey) { + let mut g = MemGraph::new(1_000_000); + let center = g.add_node(smallvec![0], empty_props(), Some(det_embedding(0, 8)), 1); + for i in 1..300u64 { + let s = g.add_node(smallvec![0], empty_props(), Some(det_embedding(i, 8)), 1); + g.add_edge(center, s, 1, 1.0, None, 2).expect("edge"); + } + let frozen = g.freeze().expect("freeze"); + let seg = crate::graph::csr::CsrSegment::from_frozen(frozen, 10).expect("csr"); + ( + MemGraph::new(1_000_000), + Arc::new(CsrStorage::from(seg)), + center, + ) + } + + #[test] + fn test_hnsw_prefilter_matches_brute_force() { + let (mg, seg, center) = frozen_star_segment(); + // Pre-build the bridge below the production minimum so the + // HnswPreFilter path actually engages on a 300-node fixture. + assert!(seg.hnsw_bridge_for_test().is_some(), "bridge must build"); + + let query = det_embedding(9999, 8); + let mut hnsw = GraphFilteredSearch::new(center, 1, query.clone(), 5); + hnsw.threshold = 1; // force HnswPreFilter + let mut brute = GraphFilteredSearch::new(center, 1, query.clone(), 5); + brute.threshold = usize::MAX; // force BruteForce + + let segs = vec![seg.clone()]; + let hnsw_res = hnsw.execute(&mg, &segs, u64::MAX - 1).expect("hnsw ok"); + let brute_res = brute.execute(&mg, &segs, u64::MAX - 1).expect("brute ok"); + + assert_eq!(hnsw_res.len(), 5); + assert_eq!(brute_res.len(), 5); + // Every HNSW score must be the node's REAL cosine similarity and + // carry the BFS graph distance. + let view = MergedNodeView::new(&mg, &segs); + for r in &hnsw_res { + let emb = view.embedding(r.node).expect("embedding"); + let exact = simd::cosine_similarity(&emb, &query); + assert!( + (r.score - exact).abs() < 1e-4, + "score {} vs exact {exact}", + r.score + ); + assert_eq!(r.graph_distance, Some(1)); + } + // Top-5 overlap with the exact ranking (approximate search). + let brute_set: HashSet = brute_res.iter().map(|r| r.node).collect(); + let overlap = hnsw_res + .iter() + .filter(|r| brute_set.contains(&r.node)) + .count(); + assert!( + overlap >= 4, + "HNSW/brute top-5 overlap too low: {overlap}/5" + ); + } + + #[test] + fn test_hnsw_prefilter_without_bridge_is_exact() { + // No test bridge pre-built: the production accessor refuses a + // 300-vector segment (min 4096), so every candidate is residual + // and results must EXACTLY equal brute force. + let (mg, seg, center) = frozen_star_segment(); + let query = det_embedding(777, 8); + + let mut hnsw = GraphFilteredSearch::new(center, 1, query.clone(), 5); + hnsw.threshold = 1; + let mut brute = GraphFilteredSearch::new(center, 1, query, 5); + brute.threshold = usize::MAX; + + let segs = vec![seg.clone()]; + let hnsw_res = hnsw.execute(&mg, &segs, u64::MAX - 1).expect("ok"); + let brute_res = brute.execute(&mg, &segs, u64::MAX - 1).expect("ok"); + + let a: Vec<(NodeKey, u64)> = hnsw_res + .iter() + .map(|r| (r.node, r.score.to_bits())) + .collect(); + let b: Vec<(NodeKey, u64)> = brute_res + .iter() + .map(|r| (r.node, r.score.to_bits())) + .collect(); + assert_eq!( + a, b, + "bridge-less HnswPreFilter must be bit-exact brute force" + ); + } + + #[test] + fn test_hnsw_prefilter_partition_routes_tiers() { + // Partition contract of hnsw_prefilter_score: mutable-tier + // candidates land in `residual` (exact scoring), bridge-covered + // frozen candidates surface through `scored` as the group top-k. + let (mut mg, seg, _center) = frozen_star_segment(); + assert!(seg.hnsw_bridge_for_test().is_some(), "bridge must build"); + + // A fresh MemGraph's first key ALIASES a frozen key (same slotmap + // sequence) -- exactly the mutable-tier-wins case the partition + // must route to residual. + let hot = mg.add_node( + smallvec![0], + empty_props(), + Some(vec![1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), + 20, + ); + + // Candidates: the mutable node + every OTHER frozen row. + let mut candidates: Vec<(NodeKey, u32)> = vec![(hot, 1)]; + for meta in seg.node_meta() { + let key: NodeKey = slotmap::KeyData::from_ffi(meta.external_id).into(); + if key != hot { + candidates.push((key, 1)); + } + } + let total = candidates.len(); + + let query = det_embedding(55, 8); + let segs = vec![seg.clone()]; + let mut scored: Vec = Vec::new(); + let residual = hnsw_prefilter_score(&mg, &segs, candidates, &query, 5, &mut scored); + + // Mutable node: residual, never bridge-scored. + assert!(residual.iter().any(|&(k, _)| k == hot)); + assert!(scored.iter().all(|r| r.node != hot)); + assert_eq!( + residual.len(), + 1, + "all frozen candidates are bridge-covered" + ); + // Bridge returned the group's top-k with exact cosines. + assert_eq!(scored.len(), 5); + let view = MergedNodeView::new(&mg, &segs); + for r in &scored { + let emb = view.embedding(r.node).expect("embedding"); + let exact = simd::cosine_similarity(&emb, &query); + assert!((r.score - exact).abs() < 1e-4); + assert_eq!(r.graph_distance, Some(1)); + } + assert!( + total > scored.len() + residual.len(), + "prefilter must prune" + ); + } } diff --git a/src/graph/index.rs b/src/graph/index.rs index 4f6905e4..ffd75977 100644 --- a/src/graph/index.rs +++ b/src/graph/index.rs @@ -316,6 +316,147 @@ impl PropertyIndex { pub fn is_empty(&self) -> bool { self.tree.is_empty() } + + /// Approximate resident bytes: one `OrderedFloat` key plus each + /// bitmap's `serialized_size()` (roaring's own compressed-container + /// estimate -- close enough for the elastic memory budget, which only + /// needs a monotonic signal, not exact byte accounting). + pub fn resident_bytes(&self) -> usize { + self.tree + .values() + .map(|bm| std::mem::size_of::>() + bm.serialized_size()) + .sum() + } +} + +// --------------------------------------------------------------------------- +// SegmentPropertyIndexes +// --------------------------------------------------------------------------- + +/// Per-segment property indexes over CSR rows, built from the v5 +/// node-property blob at first use (segments are immutable, so the index +/// never needs maintenance — this replaces the dead insert-time +/// `NamedGraph.property_indexes`, which indexed the wrong row space and was +/// never read). +/// +/// Numeric values (Int / Float / Bool as 0-1) share one per-property B-tree +/// for equality AND range queries; String / Bytes values index by xxh64 hash +/// for equality. Hash collisions (and Bool-vs-Int aliasing) can only yield +/// SUPERSET candidate sets — callers keep their residual Filter downstream, +/// so a collision costs a re-check, never a wrong row. +/// +/// The build is exhaustive over every row's properties, so an absent +/// (property, value) means NO row matches: lookups return empty bitmaps, +/// not a fall-back-to-scan signal. Pre-v5 segments have an empty blob and +/// genuinely hold no properties, so "empty" is correct there too. +#[derive(Debug, Default)] +pub struct SegmentPropertyIndexes { + /// prop_id -> numeric B-tree (Int/Float/Bool normalized to f64). + numeric: HashMap, + /// prop_id -> xxh64(bytes) -> rows (String/Bytes equality). + strings: HashMap>, +} + +impl SegmentPropertyIndexes { + /// Build from CSR node metadata + the v5 node-property blob. + pub fn build(node_meta: &[NodeMeta], node_props_blob: &[u8]) -> Self { + use crate::graph::types::PropertyValue; + let mut numeric: HashMap = HashMap::new(); + let mut strings: HashMap> = HashMap::new(); + for (row, nm) in node_meta.iter().enumerate() { + if nm.property_offset == 0 { + continue; // no property record for this row + } + let props = + crate::graph::csr::props::decode_node_props(node_props_blob, nm.property_offset); + for (pid, val) in &props { + let num = match val { + PropertyValue::Int(i) => Some(*i as f64), + PropertyValue::Float(f) => Some(*f), + PropertyValue::Bool(b) => Some(u8::from(*b) as f64), + PropertyValue::String(_) | PropertyValue::Bytes(_) => None, + }; + if let Some(v) = num { + numeric + .entry(*pid) + .or_insert_with(|| PropertyIndex::new(*pid)) + .insert(v, row as u32); + } else if let PropertyValue::String(s) | PropertyValue::Bytes(s) = val { + strings + .entry(*pid) + .or_default() + .entry(xxhash_rust::xxh64::xxh64(s, 0)) + .or_default() + .insert(row as u32); + } + } + } + numeric.shrink_to_fit(); + strings.shrink_to_fit(); + Self { numeric, strings } + } + + /// Rows whose property `prop_id` equals `value` (superset semantics for + /// hashed strings / Bool-Int aliasing — see type docs). + pub fn rows_eq( + &self, + prop_id: u16, + value: &crate::graph::types::PropertyValue, + ) -> RoaringBitmap { + use crate::graph::types::PropertyValue; + match value { + PropertyValue::Int(i) => self.numeric_eq(prop_id, *i as f64), + PropertyValue::Float(f) => self.numeric_eq(prop_id, *f), + PropertyValue::Bool(b) => self.numeric_eq(prop_id, u8::from(*b) as f64), + PropertyValue::String(s) | PropertyValue::Bytes(s) => self + .strings + .get(&prop_id) + .and_then(|m| m.get(&xxhash_rust::xxh64::xxh64(s, 0))) + .cloned() + .unwrap_or_default(), + } + } + + /// Rows whose numeric property `prop_id` falls in `[min, max]`. + pub fn rows_range(&self, prop_id: u16, min: f64, max: f64) -> RoaringBitmap { + self.numeric + .get(&prop_id) + .map(|ix| ix.range_query(min, max)) + .unwrap_or_default() + } + + /// True when no property is indexed at all (segment without v5 blob). + pub fn is_empty(&self) -> bool { + self.numeric.is_empty() && self.strings.is_empty() + } + + /// Approximate resident bytes across every numeric B-tree and string + /// hash-bucket bitmap. Always heap-owned: built lazily on first use from + /// the (possibly mmap-backed) property blob, but the index ITSELF is + /// never mmap'd -- see `CsrStorage::resident_bytes`, which is the only + /// caller and where the mmap-vs-heap distinction for the SOURCE blob is + /// applied. + pub fn resident_bytes(&self) -> usize { + let numeric: usize = self + .numeric + .values() + .map(PropertyIndex::resident_bytes) + .sum(); + let strings: usize = self + .strings + .values() + .flat_map(|inner| inner.values()) + .map(|bm| std::mem::size_of::() + bm.serialized_size()) + .sum(); + numeric + strings + } + + fn numeric_eq(&self, prop_id: u16, v: f64) -> RoaringBitmap { + self.numeric + .get(&prop_id) + .map(|ix| ix.range_query(v, v)) + .unwrap_or_default() + } } // --------------------------------------------------------------------------- diff --git a/src/graph/memgraph.rs b/src/graph/memgraph.rs index fdcb7261..f9884821 100644 --- a/src/graph/memgraph.rs +++ b/src/graph/memgraph.rs @@ -3,7 +3,7 @@ //! Absorbs graph writes at O(1) amortized cost per insert. Freezes into a //! `FrozenMemGraph` when the edge threshold is reached, enabling CSR conversion. -use slotmap::SlotMap; +use slotmap::{Key, SlotMap}; use smallvec::SmallVec; use crate::graph::types::{Direction, EdgeKey, MutableEdge, MutableNode, NodeKey, PropertyMap}; @@ -31,6 +31,12 @@ pub struct FrozenMemGraph { pub struct MemGraph { nodes: SlotMap, edges: SlotMap, + /// Adjacency (outgoing / incoming) for cross-tier "delta" edges whose + /// endpoint was frozen into a CSR segment. A frozen endpoint has no + /// `MutableNode` to carry inline adjacency, so its edge keys live here, + /// keyed by the frozen NodeKey. Rebuilt at each freeze (see `freeze`). + ghost_out: std::collections::HashMap>, + ghost_in: std::collections::HashMap>, /// Count of live (non-deleted) nodes. live_node_count: usize, /// Count of live (non-deleted) edges. @@ -38,22 +44,133 @@ pub struct MemGraph { /// Edge count threshold that triggers freeze. edge_threshold: usize, frozen: bool, + /// Index-space watermark added to the raw SlotMap index of every + /// NodeKey this MemGraph hands out or accepts. See `with_id_offset` for + /// the full soundness argument. Zero (the default via `new`) makes the + /// translation the identity function -- the overwhelmingly common case + /// for graphs with no persisted (pre-restart) history. + id_offset: u32, } impl MemGraph { /// Create an empty MemGraph with the given freeze threshold. pub fn new(edge_threshold: usize) -> Self { + Self::with_id_offset(edge_threshold, 0) + } + + /// Create an empty MemGraph whose NodeKeys are all minted `id_offset` + /// above the raw SlotMap index. + /// + /// # Why this exists (soundness argument -- graph NodeKey aliasing, P0) + /// + /// `slotmap::SlotMap` key allocation is fully deterministic and a fresh + /// map's index counter always starts at 0 + /// (`slotmap::basic::SlotMap::insert`, free_head/grow-path). After a + /// restart, WAL replay works against a fresh `MemGraph` + /// (`replay.rs::take_memgraph`), while CSR segments loaded from disk + /// carry `NodeMeta::external_id` values that are the raw + /// `slotmap::KeyData::as_ffi()` bits minted by the PRE-CRASH process's + /// SlotMap -- which also started at index 0. Without an offset, the + /// first node the fresh MemGraph mints gets `(idx=0, version=1)`, + /// bit-for-bit IDENTICAL to the pre-crash process's first-ever node key + /// if that node is still resident in a loaded CSR segment (the common + /// case whenever an AOF fold/WAL checkpoint truncates pre-freeze + /// history so replay only sees post-freeze commands). Every merged read + /// path (`MergedNodeView`) checks the mutable tier first, so the + /// aliasing new node would permanently and silently shadow the real + /// frozen node. + /// + /// ## Fix + /// + /// Shift the INDEX component (low 32 bits of `KeyData::as_ffi()`, see + /// `slotmap::KeyData::{as_ffi, from_ffi}` -- version occupies the high + /// 32 bits) of every key this MemGraph hands out by `id_offset`. The + /// caller (recovery.rs) chooses `id_offset` to be `> ` the largest index + /// component among ALL `external_id`s in the CSR segments it just + /// loaded for this graph: + /// - Outgoing (`add_node` return value, `iter_nodes` keys): raw SlotMap + /// index + `id_offset` (see `to_public_key`). + /// - Incoming (any NodeKey parameter): raw index = public index - + /// `id_offset`; underflow (public index < `id_offset`) means the key + /// was never minted by THIS MemGraph -- it is a CSR-only (or foreign) + /// key -- and is treated as "not resident", exactly the existing + /// ghost / `NodeNotFound` semantics already used for non-resident + /// endpoints (see `to_internal_key`). + /// + /// This costs two `u64` shifts per NodeKey touched (zero when + /// `id_offset == 0`) and precisely **zero** extra permanent memory: + /// the alternative of pre-consuming `id_offset` SlotMap slots via a + /// dummy-insert/remove cycle would permanently pin `id_offset` + /// live-or-vacant slot entries in the SlotMap's backing `Vec` (slotmap + /// never shrinks its storage) -- exactly the O(watermark) leak this + /// design avoids. It is also the only sound option: a dummy-cycle + /// approach only bumps a slot's *generation*, and persisted external_ids + /// may already carry an arbitrarily-bumped generation from pre-crash + /// slot churn, so a fixed number of dummy cycles cannot be proven to + /// out-run every possible persisted generation for a given index. + /// + /// ## Overflow + /// + /// If `idx + id_offset` would exceed `u32::MAX`, the public key + /// saturates at `u32::MAX` instead of wrapping (a wrapped index could + /// re-enter `[0, id_offset)` and alias a persisted `external_id` again). + /// This is an astronomical corner case (>4 billion prior nodes on one + /// graph) and degrades to "new inserts stop being independently + /// addressable" rather than corrupting existing data. + pub fn with_id_offset(edge_threshold: usize, id_offset: u32) -> Self { Self { nodes: SlotMap::with_key(), edges: SlotMap::with_key(), + ghost_out: std::collections::HashMap::new(), + ghost_in: std::collections::HashMap::new(), live_node_count: 0, live_edge_count: 0, edge_threshold, frozen: false, + id_offset, } } - /// Insert a new node. Returns the generational key. + /// Translate a raw internal SlotMap `NodeKey` to the PUBLIC key handed + /// to callers (index + `id_offset`, version unchanged). Identity when + /// `id_offset == 0`. See `with_id_offset` for the soundness argument. + #[inline] + fn to_public_key(id_offset: u32, key: NodeKey) -> NodeKey { + if id_offset == 0 { + return key; + } + let ffi = key.data().as_ffi(); + let idx = ffi as u32; + let version = ffi >> 32; + // Saturate rather than wrap: a wrapped index could re-enter + // [0, id_offset) and alias a persisted external_id again. + let public_idx = idx.saturating_add(id_offset); + NodeKey::from(slotmap::KeyData::from_ffi( + (version << 32) | u64::from(public_idx), + )) + } + + /// Translate a PUBLIC `NodeKey` (offset applied) to the raw internal + /// SlotMap key used to index `self.nodes`. Returns `None` if the public + /// index is below `id_offset` -- such a key was never minted by this + /// MemGraph (it belongs to a CSR segment or a foreign graph) and must be + /// treated as non-resident, matching the existing ghost / NodeNotFound + /// semantics for non-resident endpoints. Identity when `id_offset == 0`. + #[inline] + fn to_internal_key(id_offset: u32, key: NodeKey) -> Option { + if id_offset == 0 { + return Some(key); + } + let ffi = key.data().as_ffi(); + let idx = ffi as u32; + let version = ffi >> 32; + let internal_idx = idx.checked_sub(id_offset)?; + Some(NodeKey::from(slotmap::KeyData::from_ffi( + (version << 32) | u64::from(internal_idx), + ))) + } + + /// Insert a new node. Returns the generational (public) key. pub fn add_node( &mut self, labels: SmallVec<[u16; 4]>, @@ -74,7 +191,7 @@ impl MemGraph { valid_to: i64::MAX, }); self.live_node_count += 1; - key + Self::to_public_key(self.id_offset, key) } /// Insert a new edge between `src` and `dst`. Validates both exist and are alive. @@ -93,20 +210,33 @@ impl MemGraph { if src == dst { return Err(GraphError::SelfLoop); } + // Translate PUBLIC keys to raw internal SlotMap keys. Translation + // failure (offset underflow) means the key was never minted by this + // MemGraph -- treat exactly like "not resident" (`add_edge` does not + // support cross-tier endpoints; use `add_edge_across_tiers`). + let Some(src_i) = Self::to_internal_key(self.id_offset, src) else { + return Err(GraphError::NodeNotFound); + }; + let Some(dst_i) = Self::to_internal_key(self.id_offset, dst) else { + return Err(GraphError::NodeNotFound); + }; // Validate both nodes exist and are alive. let src_alive = self .nodes - .get(src) + .get(src_i) .map_or(false, |n| n.deleted_lsn == u64::MAX); let dst_alive = self .nodes - .get(dst) + .get(dst_i) .map_or(false, |n| n.deleted_lsn == u64::MAX); if !src_alive || !dst_alive { return Err(GraphError::NodeNotFound); } let ek = self.edges.insert(MutableEdge { + // Stored as PUBLIC keys: `MutableEdge.src/dst` are the identity + // callers (freeze(), neighbors()) compare against, matching + // `add_node`'s return value and CSR `external_id`s. src, dst, edge_type, @@ -124,19 +254,86 @@ impl MemGraph { // Push edge key into src.outgoing and dst.incoming. // Both are validated alive above, so get_mut is safe. - if let Some(src_node) = self.nodes.get_mut(src) { + if let Some(src_node) = self.nodes.get_mut(src_i) { src_node.outgoing.push(ek); } - if let Some(dst_node) = self.nodes.get_mut(dst) { + if let Some(dst_node) = self.nodes.get_mut(dst_i) { dst_node.incoming.push(ek); } self.live_edge_count += 1; Ok(ek) } + /// Insert an edge whose endpoints may live in the frozen CSR tier + /// (a "delta" edge). Resident endpoints are validated alive and get + /// inline adjacency; non-resident endpoints get ghost adjacency. + /// + /// The CALLER is responsible for having verified that each non-resident + /// endpoint exists and is alive in an immutable segment (e.g. via + /// `MergedNodeView::is_visible`) — MemGraph cannot see the CSR tier. + pub fn add_edge_across_tiers( + &mut self, + src: NodeKey, + dst: NodeKey, + edge_type: u16, + weight: f64, + properties: Option, + lsn: u64, + ) -> Result { + if self.frozen { + return Err(GraphError::AlreadyFrozen); + } + if src == dst { + return Err(GraphError::SelfLoop); + } + // Translate PUBLIC keys to internal SlotMap keys where possible. + // `None` (translation underflow, or a key genuinely absent from this + // MemGraph) means non-resident -- caller-verified via the CSR tier. + let src_i = Self::to_internal_key(self.id_offset, src); + let dst_i = Self::to_internal_key(self.id_offset, dst); + // Resident endpoints must be alive; non-resident are caller-verified. + for key_i in [src_i, dst_i].into_iter().flatten() { + if let Some(n) = self.nodes.get(key_i) { + if n.deleted_lsn != u64::MAX { + return Err(GraphError::NodeNotFound); + } + } + } + + let ek = self.edges.insert(MutableEdge { + // PUBLIC keys: ghost_out/ghost_in are keyed by public identity + // too (a non-resident endpoint has no internal key at all). + src, + dst, + edge_type, + weight, + properties, + created_lsn: lsn, + deleted_lsn: u64::MAX, + txn_id: 0, + valid_from: 0, + valid_to: i64::MAX, + created_ms: crate::storage::entry::current_time_ms(), + }); + + match src_i.and_then(|k| self.nodes.get_mut(k)) { + Some(src_node) => src_node.outgoing.push(ek), + None => self.ghost_out.entry(src).or_default().push(ek), + } + match dst_i.and_then(|k| self.nodes.get_mut(k)) { + Some(dst_node) => dst_node.incoming.push(ek), + None => self.ghost_in.entry(dst).or_default().push(ek), + } + self.live_edge_count += 1; + Ok(ek) + } + /// Soft-delete a node and all its incident edges. pub fn remove_node(&mut self, key: NodeKey, lsn: u64) -> bool { - let Some(node) = self.nodes.get_mut(key) else { + let Some(internal) = Self::to_internal_key(self.id_offset, key) else { + return false; + }; + let Some(node) = self.nodes.get_mut(internal) else { return false; }; if node.deleted_lsn != u64::MAX { @@ -189,12 +386,14 @@ impl MemGraph { /// O(1) node lookup by key. pub fn get_node(&self, key: NodeKey) -> Option<&MutableNode> { - self.nodes.get(key) + let internal = Self::to_internal_key(self.id_offset, key)?; + self.nodes.get(internal) } /// O(1) mutable node lookup by key. pub fn get_node_mut(&mut self, key: NodeKey) -> Option<&mut MutableNode> { - self.nodes.get_mut(key) + let internal = Self::to_internal_key(self.id_offset, key)?; + self.nodes.get_mut(internal) } /// O(1) edge lookup by key. @@ -212,11 +411,30 @@ impl MemGraph { /// Yields `(EdgeKey, NodeKey)` pairs -- the edge and the neighbor node. /// No heap allocation: iterates over borrowed SmallVec adjacency lists. pub fn neighbors(&self, node: NodeKey, direction: Direction, lsn: u64) -> NeighborIter<'_> { - let Some(n) = self.nodes.get(node) else { + let internal = Self::to_internal_key(self.id_offset, node); + let Some(n) = internal.and_then(|k| self.nodes.get(k)) else { + // Non-resident (frozen) node: serve delta-edge adjacency from the + // ghost maps so cross-tier edges are traversable from BOTH ends. + let ghost_out = match direction { + Direction::Outgoing | Direction::Both => self + .ghost_out + .get(&node) + .map(|v| v.as_slice()) + .unwrap_or(&[]), + Direction::Incoming => &[], + }; + let ghost_in = match direction { + Direction::Incoming | Direction::Both => self + .ghost_in + .get(&node) + .map(|v| v.as_slice()) + .unwrap_or(&[]), + Direction::Outgoing => &[], + }; return NeighborIter { edges: &self.edges, - out_iter: [].iter(), - in_iter: [].iter(), + out_iter: ghost_out.iter(), + in_iter: ghost_in.iter(), lsn, source: node, }; @@ -237,9 +455,14 @@ impl MemGraph { } } - /// Iterate over all live (non-deleted) nodes. Yields `(NodeKey, &MutableNode)`. + /// Iterate over all live (non-deleted) nodes. Yields `(NodeKey, &MutableNode)` + /// with PUBLIC keys (offset applied) -- matching `add_node`'s return value. pub fn iter_nodes(&self) -> impl Iterator { - self.nodes.iter().filter(|(_, n)| n.deleted_lsn == u64::MAX) + let id_offset = self.id_offset; + self.nodes + .iter() + .filter(|(_, n)| n.deleted_lsn == u64::MAX) + .map(move |(k, n)| (Self::to_public_key(id_offset, k), n)) } /// Iterate over all live (non-deleted) edges. Yields `(EdgeKey, &MutableEdge)`. @@ -275,7 +498,10 @@ impl MemGraph { /// were cascade-deleted at `lsn` by `remove_node`. Restores `deleted_lsn` /// to `u64::MAX` and increments `live_edge_count` for each restored edge. pub fn undelete_edges_at_lsn(&mut self, node: NodeKey, lsn: u64) { - let Some(n) = self.nodes.get(node) else { + let Some(internal) = Self::to_internal_key(self.id_offset, node) else { + return; + }; + let Some(n) = self.nodes.get(internal) else { return; }; let edge_keys: SmallVec<[EdgeKey; 16]> = n @@ -309,26 +535,84 @@ impl MemGraph { /// Freeze the MemGraph, returning a FrozenMemGraph with all data for CSR conversion. /// Only includes live (non-deleted) nodes and edges. + /// + /// Cross-tier "delta" edges — edges with at least one endpoint that was + /// frozen into an EARLIER segment — are RETAINED in the mutable tier + /// (with their EdgeKeys intact) instead of being handed to CSR + /// conversion: `CsrSegment::from_frozen` can only encode edges whose + /// endpoint rows exist in the segment being built, and would silently + /// drop them. Their ghost adjacency is rebuilt for the post-freeze world + /// (every endpoint is non-resident once the nodes drain). pub fn freeze(&mut self) -> Result { if self.frozen { return Err(GraphError::AlreadyFrozen); } self.frozen = true; + // Partition edges BEFORE draining nodes (residency check needs the + // slot map): live + both endpoints resident → freeze into CSR; + // live + any frozen-elsewhere endpoint → retain as delta; + // dead → drop. + let mut freeze_keys: Vec = Vec::new(); + let mut dead_keys: Vec = Vec::new(); + let id_offset = self.id_offset; + // `e.src`/`e.dst` are PUBLIC keys; translate to internal before + // checking slot-map residency. + let is_resident = |nodes: &SlotMap, key: NodeKey| { + Self::to_internal_key(id_offset, key).is_some_and(|k| nodes.contains_key(k)) + }; + for (ek, e) in self.edges.iter() { + if e.deleted_lsn != u64::MAX { + dead_keys.push(ek); + } else if is_resident(&self.nodes, e.src) && is_resident(&self.nodes, e.dst) { + freeze_keys.push(ek); + } + } + for ek in dead_keys { + self.edges.remove(ek); + } + let edges: Vec<(EdgeKey, MutableEdge)> = freeze_keys + .into_iter() + .filter_map(|ek| self.edges.remove(ek).map(|e| (ek, e))) + .collect(); + + // `drain()` yields raw internal keys -- translate to PUBLIC before + // handing them to CSR conversion (they become `NodeMeta::external_id` + // and must match the identity every other reference to this node + // uses: node_map, ghost adjacency, edge endpoints). let nodes: Vec<(NodeKey, MutableNode)> = self .nodes .drain() .filter(|(_, n)| n.deleted_lsn == u64::MAX) + .map(|(k, n)| (Self::to_public_key(id_offset, k), n)) .collect(); - let edges: Vec<(EdgeKey, MutableEdge)> = self - .edges - .drain() - .filter(|(_, e)| e.deleted_lsn == u64::MAX) - .collect(); + // Rebuild ghost adjacency for the retained delta edges: with the + // node slot map drained, EVERY endpoint is now non-resident. + self.ghost_out.clear(); + self.ghost_in.clear(); + for (ek, e) in self.edges.iter() { + self.ghost_out.entry(e.src).or_default().push(ek); + self.ghost_in.entry(e.dst).or_default().push(ek); + } Ok(FrozenMemGraph { nodes, edges }) } + + /// Re-arm a drained MemGraph for writes after a successful freeze. + /// + /// Keeps the SAME slot maps (drain bumps each vacated slot's generation, + /// so keys handed out after thaw can never collide with the external_ids + /// of frozen CSR rows). Replacing the MemGraph with a fresh one instead + /// would restart SlotMap allocation at the same (index, generation) pairs + /// and silently alias new nodes onto frozen rows. + pub fn thaw(&mut self) { + self.frozen = false; + // Post-freeze contents: zero nodes, retained delta edges (all live — + // freeze removed dead ones). + self.live_node_count = 0; + self.live_edge_count = self.edges.len(); + } } /// Zero-allocation neighbor iterator. Borrows from MemGraph's SmallVec adjacency lists. @@ -520,6 +804,72 @@ mod tests { assert_eq!(g.freeze().unwrap_err(), GraphError::AlreadyFrozen); } + #[test] + fn test_delta_edge_across_freeze_traversable_both_ends() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], empty_props(), None, 1); + let b = g.add_node(smallvec![0], empty_props(), None, 1); + g.add_edge(a, b, 1, 1.0, None, 2).expect("ok"); + let frozen = g.freeze().expect("freeze"); + assert_eq!(frozen.nodes.len(), 2); + g.thaw(); + + // Plain add_edge between frozen endpoints must still refuse (caller + // hasn't verified segment existence)... + assert_eq!( + g.add_edge(a, b, 2, 1.0, None, 3).unwrap_err(), + GraphError::NodeNotFound + ); + // ...while the cross-tier insert succeeds and is traversable from + // BOTH frozen endpoints via ghost adjacency. + let ek = g + .add_edge_across_tiers(a, b, 2, 1.0, None, 3) + .expect("delta edge"); + let out: Vec<_> = g.neighbors(a, Direction::Outgoing, u64::MAX).collect(); + assert_eq!(out, vec![(ek, b)]); + let inc: Vec<_> = g.neighbors(b, Direction::Incoming, u64::MAX).collect(); + assert_eq!(inc, vec![(ek, a)]); + assert!( + g.neighbors(a, Direction::Incoming, u64::MAX) + .next() + .is_none() + ); + } + + #[test] + fn test_freeze_retains_delta_edges_with_stable_keys() { + let mut g = MemGraph::new(1000); + let a = g.add_node(smallvec![0], empty_props(), None, 1); + let b = g.add_node(smallvec![0], empty_props(), None, 1); + g.add_edge(a, b, 1, 1.0, None, 2).expect("ok"); + g.freeze().expect("freeze 1"); + g.thaw(); + + // Delta edge between frozen endpoints + a fresh resident pair. + let ek_delta = g + .add_edge_across_tiers(a, b, 2, 1.5, None, 3) + .expect("delta"); + let c = g.add_node(smallvec![0], empty_props(), None, 4); + let d = g.add_node(smallvec![0], empty_props(), None, 4); + let ek_res = g.add_edge(c, d, 1, 1.0, None, 5).expect("resident"); + + // Second freeze: the resident pair + edge freezes; the delta edge is + // RETAINED (CSR cannot host an edge without its endpoint rows) with + // the SAME EdgeKey, and stays traversable. + let frozen = g.freeze().expect("freeze 2"); + g.thaw(); + assert_eq!(frozen.nodes.len(), 2, "c and d freeze"); + assert_eq!(frozen.edges.len(), 1, "only the resident edge freezes"); + assert_eq!(frozen.edges[0].0, ek_res); + + assert!(g.get_edge(ek_delta).is_some(), "delta edge key stable"); + let out: Vec<_> = g.neighbors(a, Direction::Outgoing, u64::MAX).collect(); + assert_eq!(out, vec![(ek_delta, b)]); + // c/d froze normally: no delta adjacency left behind for them. + assert!(g.neighbors(c, Direction::Both, u64::MAX).next().is_none()); + assert_eq!(g.edge_count(), 1, "live count = retained delta edge"); + } + #[test] fn test_self_loop_rejected() { let mut g = MemGraph::new(1000); diff --git a/src/graph/mod.rs b/src/graph/mod.rs index 8fbc5dfc..b78073f1 100644 --- a/src/graph/mod.rs +++ b/src/graph/mod.rs @@ -6,12 +6,15 @@ pub mod compaction; pub mod cross_shard; pub mod csr; pub mod cypher; +pub mod fasthash; +pub mod hnsw_bridge; pub mod hybrid; pub mod index; pub mod manifest; pub mod memgraph; pub mod recovery; pub mod replay; +pub mod row_bfs; pub mod scoring; pub mod segment; pub mod simd; @@ -20,6 +23,7 @@ pub mod store; pub mod traversal; pub mod traversal_guard; pub mod types; +pub mod view; pub mod visibility; pub mod wal; @@ -41,3 +45,4 @@ pub use stats::GraphStats; pub use store::GraphStore; pub use traversal::{BoundedBfs, BoundedDfs, DijkstraTraversal, ParallelBfs, SegmentMergeReader}; pub use types::{Direction, EdgeKey, NodeKey, PropertyMap, PropertyValue}; +pub use view::MergedNodeView; diff --git a/src/graph/recovery.rs b/src/graph/recovery.rs index bea19530..586f3df7 100644 --- a/src/graph/recovery.rs +++ b/src/graph/recovery.rs @@ -15,9 +15,29 @@ use std::sync::Arc; use crate::graph::csr::{CsrError, CsrStorage}; use crate::graph::manifest::GraphManifest; +use crate::graph::memgraph::MemGraph; use crate::graph::segment::GraphSegmentList; use crate::graph::store::GraphStore; +/// Compute the NodeKey index-space watermark for a set of loaded CSR +/// segments: one past the largest SlotMap index component among all +/// persisted `NodeMeta::external_id`s. Passed to `MemGraph::with_id_offset` +/// so that replay (or any post-recovery write) can never mint a NodeKey +/// that aliases a persisted external_id -- see `MemGraph::with_id_offset` +/// for the full soundness argument. Zero (no offset) when there are no +/// loaded segments -- there is nothing to alias against. +fn node_id_watermark(segments: &[Arc]) -> u32 { + segments + .iter() + .flat_map(|seg| seg.node_meta().iter()) + // KeyData::as_ffi() packs the SlotMap index in the low 32 bits + // (version occupies the high 32 bits) -- `as u32` truncation + // extracts exactly that index component. + .map(|meta| meta.external_id as u32) + .max() + .map_or(0, |max_idx| max_idx.saturating_add(1)) +} + /// Result of graph recovery for a single shard. pub struct GraphRecoveryResult { /// The recovered GraphStore with loaded segments. @@ -147,11 +167,19 @@ pub fn recover_graph_store( } } - // Inject loaded segments into the graph's segment holder. + // Inject loaded segments into the graph's segment holder, and + // fast-forward the mutable tier's key allocator (P0 fix: graph + // NodeKey aliasing across restart). At this point `graph.segments` + // still holds the pristine fresh MemGraph `create_graph` built via + // `GraphStore::load_metadata` -- nothing has written to it yet, so + // it is safe to replace outright rather than merge. if let Some(graph) = store.get_graph_mut(graph_name.as_bytes()) { - let current = graph.segments.load(); + let watermark = node_id_watermark(&loaded_segments); graph.segments.swap(GraphSegmentList { - mutable: current.mutable.clone(), + mutable: Some(Arc::new(MemGraph::with_id_offset( + graph.edge_threshold, + watermark, + ))), immutable: loaded_segments, }); } diff --git a/src/graph/replay.rs b/src/graph/replay.rs index 79a1b7fb..290fc906 100644 --- a/src/graph/replay.rs +++ b/src/graph/replay.rs @@ -480,10 +480,37 @@ impl GraphReplayCollector { .. } = &self.commands[idx] { - let nk = - mg.add_node(labels.clone(), properties.clone(), embedding.clone(), 0); - node_map.insert(*node_id, nk); - // Track _key properties for registration after memgraph is returned. + // Dedup skip (P0 fix: graph NodeKey aliasing across + // restart). `node_map` is seeded from loaded CSR + // segments above; if `node_id` is already present, + // this AddNode is a full-history replay of a node + // that is ALREADY resident in the frozen tier (the + // pre-crash WAL was never truncated for it). Calling + // `mg.add_node` again would duplicate the node in + // the mutable tier (double enumeration via + // `for_each_visible_node`, which has no dedup) even + // though the fresh-vs-persisted key collision itself + // is now impossible (see `node_id_watermark` / + // `MemGraph::with_id_offset`). Keep the CSR-seeded + // identity and skip the insert. + let nk = if let Some(&existing) = node_map.get(node_id) { + existing + } else { + let nk = mg.add_node( + labels.clone(), + properties.clone(), + embedding.clone(), + 0, + ); + node_map.insert(*node_id, nk); + nk + }; + // Track _key properties for registration after memgraph is + // returned. Re-derived even for dedup-skipped (CSR-resident) + // nodes: `key_to_node` is in-memory only and is not itself + // reloaded from CSR, so a node's `_key` mapping would + // otherwise be lost whenever its AddNode is replayed against + // an already-seeded id. for (prop_id, prop_val) in properties { if *prop_id == key_prop_id { if let PropertyValue::String(s) = prop_val { @@ -509,8 +536,20 @@ impl GraphReplayCollector { let src_key = node_map.get(src_id).copied(); let dst_key = node_map.get(dst_id).copied(); if let (Some(src), Some(dst)) = (src_key, dst_key) { + // Cross-tier aware: endpoints seeded from CSR + // segments are non-resident in `mg` — a plain + // add_edge would silently drop the edge on + // replay. node_map membership IS the existence + // proof (replayed node or CSR row). if mg - .add_edge(src, dst, *edge_type, *weight, properties.clone(), 0) + .add_edge_across_tiers( + src, + dst, + *edge_type, + *weight, + properties.clone(), + 0, + ) .is_ok() { *replayed += 1; diff --git a/src/graph/row_bfs.rs b/src/graph/row_bfs.rs new file mode 100644 index 00000000..fb036b85 --- /dev/null +++ b/src/graph/row_bfs.rs @@ -0,0 +1,735 @@ +//! CSR row-space BFS fast path. +//! +//! When a graph is fully frozen into a SINGLE CSR segment (steady state +//! after compaction) and the mutable tier is empty, BFS does not need +//! NodeKey hashing at all: rows are dense `u32`s, so the visited set is a +//! bitmap, adjacency is two slice lookups (`row_offsets`/`col_indices`), +//! and NodeKey materialization happens once per RESULT row instead of once +//! per EDGE probe. On top of the row space this module adds what the old +//! "ParallelBfs" could not (its reader borrowed `!Send` MemGraph, so +//! neighbor expansion ran sequentially and threads only merged sets): +//! +//! - **True parallel expansion**: `CsrStorage` is `Send + Sync`, so +//! frontier chunks expand on worker threads against a shared +//! `AtomicU64` visited bitmap (`fetch_or` test-and-set). +//! - **Direction-optimizing BFS** (Beamer α/β heuristic): when the +//! frontier's out-edge count dwarfs the unexplored remainder, a level +//! switches from top-down push to bottom-up pull over the v3-2 +//! IncomingIndex (each unvisited row scans its in-edges for a frontier +//! parent). Outgoing direction only — pull for Incoming/Both would +//! need the transposed heuristic and buys little. +//! +//! Semantics parity with `SegmentMergeReader`-based BFS (the fallback): +//! same edge-validity bitmap, same edge-type filter, same node visibility +//! (`deleted_lsn <= snapshot`), same `MergedNeighbor` materialization for +//! CSR edges (placeholder edge key, weight 1.0, segment-LSN timestamp). +//! Within a level, PARALLEL discovery order is unspecified (the visited +//! set and depths are deterministic; sequential levels keep FIFO order). + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::graph::csr::CsrStorage; +use crate::graph::memgraph::MemGraph; +use crate::graph::traversal::{BfsResult, MergedNeighbor, TraversalError}; +use crate::graph::types::{Direction, EdgeMeta, NodeKey}; + +/// Frontier size at which a level's expansion goes parallel. +const PARALLEL_LEVEL_THRESHOLD: usize = 256; +/// Beamer top-down → bottom-up switch: pull when m_f > m_u / ALPHA. +const ALPHA: u64 = 14; +/// Beamer bottom-up → top-down switch: push when n_f < n / BETA. +const BETA: usize = 24; +/// Upper bound on worker threads for a parallel level. +const MAX_WORKERS: usize = 8; + +/// Push/pull selection, exposed for tests (`Auto` in production). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum BfsMode { + /// Beamer heuristic per level. + Auto, + /// Always top-down (frontier expands its out-edges). + ForcePush, + /// Always bottom-up (unvisited rows scan in-edges for frontier parents). + ForcePull, +} + +/// Row-space BFS if the fast-path gate holds, else `None` (caller falls +/// back to the `SegmentMergeReader` path). +/// +/// Gate: mutable tier absent-or-empty ∧ exactly one CSR segment ∧ segment +/// visible at `snapshot_lsn`. Each condition the reader path handles +/// generically (delta edges, cross-segment dedup, future segments) is a +/// reason NOT to take the fast path — never a behavior fork. +#[allow(clippy::too_many_arguments)] +pub fn try_row_bfs( + memgraph: Option<&MemGraph>, + csr_segments: &[Arc], + direction: Direction, + snapshot_lsn: u64, + edge_type_filter: Option, + start: NodeKey, + depth_limit: u32, + frontier_cap: usize, +) -> Option> { + if let Some(mg) = memgraph { + if mg.node_count() != 0 || mg.edge_count() != 0 { + return None; + } + } + let [seg] = csr_segments else { + return None; + }; + if seg.created_lsn() > snapshot_lsn { + return None; + } + Some(row_bfs( + seg, + direction, + snapshot_lsn, + edge_type_filter, + start, + depth_limit, + frontier_cap, + BfsMode::Auto, + )) +} + +/// A node discovered during one level: (row, discovery edge, edge wall-ms). +type Discovery = (u32, EdgeMeta, u64); + +#[allow(clippy::too_many_arguments)] +pub(crate) fn row_bfs( + seg: &CsrStorage, + direction: Direction, + snapshot_lsn: u64, + edge_type_filter: Option, + start: NodeKey, + depth_limit: u32, + frontier_cap: usize, + mode: BfsMode, +) -> Result { + let Some(start_row) = seg.lookup_node(start) else { + return Err(TraversalError::NodeNotFound); + }; + seg.madvise_sequential(); + + let n = seg.node_count() as usize; + let node_meta = seg.node_meta(); + let total_edges = seg.edge_count() as u64; + + let visited: Vec = (0..n.div_ceil(64)).map(|_| AtomicU64::new(0)).collect(); + test_and_set(&visited, start_row); + + let mut result: Vec<(NodeKey, u32, Option)> = vec![(start, 0, None)]; + let mut frontier: Vec = vec![start_row]; + let mut visited_count: usize = 1; + let mut edges_explored: u64 = 0; + let mut depth: u32 = 0; + + while !frontier.is_empty() && depth < depth_limit { + let next_depth = depth + 1; + + // Direction-optimizing choice (Outgoing only; see module docs). + let use_pull = match mode { + BfsMode::ForcePush => false, + BfsMode::ForcePull => direction == Direction::Outgoing, + BfsMode::Auto => { + direction == Direction::Outgoing && frontier.len() >= n / BETA.max(1) && { + let ro = seg.row_offsets(); + let m_f: u64 = frontier + .iter() + .map(|&r| u64::from(ro[r as usize + 1] - ro[r as usize])) + .sum(); + let m_u = total_edges.saturating_sub(edges_explored); + m_f > m_u / ALPHA + } + } + }; + + let discovered: Vec = if use_pull { + expand_pull(seg, &frontier, &visited, snapshot_lsn, edge_type_filter, n) + } else { + let ro = seg.row_offsets(); + edges_explored += frontier + .iter() + .map(|&r| u64::from(ro[r as usize + 1] - ro[r as usize])) + .sum::(); + expand_push( + seg, + &frontier, + &visited, + direction, + snapshot_lsn, + edge_type_filter, + ) + }; + + frontier = Vec::with_capacity(discovered.len()); + for (row, meta, created_ms) in discovered { + let meta_n = &node_meta[row as usize]; + let key: NodeKey = slotmap::KeyData::from_ffi(meta_n.external_id).into(); + result.push(( + key, + next_depth, + Some(MergedNeighbor { + node: key, + edge: slotmap::KeyData::from_ffi(0).into(), + edge_type: meta.edge_type, + weight: 1.0, + timestamp: seg.created_lsn(), + created_ms, + }), + )); + frontier.push(row); + visited_count += 1; + if visited_count >= frontier_cap { + return Err(TraversalError::FrontierCapExceeded { + cap: frontier_cap, + depth: next_depth, + }); + } + } + + depth = next_depth; + } + + Ok(BfsResult { visited: result }) +} + +/// Whether a CSR row is visible at the snapshot (mirrors the reader path's +/// deleted-node check exactly). +#[inline] +fn row_visible(meta: &crate::graph::types::NodeMeta, snapshot_lsn: u64) -> bool { + !(meta.deleted_lsn != u64::MAX && meta.deleted_lsn <= snapshot_lsn) +} + +/// Atomically set a row's visited bit; true iff it was previously clear. +#[inline] +fn test_and_set(bits: &[AtomicU64], row: u32) -> bool { + let mask = 1u64 << (row % 64); + bits[(row / 64) as usize].fetch_or(mask, Ordering::Relaxed) & mask == 0 +} + +#[inline] +fn is_set(bits: &[AtomicU64], row: u32) -> bool { + bits[(row / 64) as usize].load(Ordering::Relaxed) & (1u64 << (row % 64)) != 0 +} + +/// Top-down level: every frontier row expands its (direction-appropriate) +/// edges; targets that pass the filter + visibility and win the visited +/// test-and-set are discovered. Parallel over frontier chunks when large. +fn expand_push( + seg: &CsrStorage, + frontier: &[u32], + visited: &[AtomicU64], + direction: Direction, + snapshot_lsn: u64, + edge_type_filter: Option, +) -> Vec { + let expand_chunk = |chunk: &[u32], out: &mut Vec| { + let node_meta = seg.node_meta(); + for &row in chunk { + let mut on_edge = |other_row: u32, meta: EdgeMeta, created_ms: u64| { + if let Some(filter) = edge_type_filter { + if meta.edge_type != filter { + return; + } + } + let Some(meta_n) = node_meta.get(other_row as usize) else { + return; + }; + if !row_visible(meta_n, snapshot_lsn) { + return; + } + if test_and_set(visited, other_row) { + out.push((other_row, meta, created_ms)); + } + }; + if direction != Direction::Incoming { + seg.for_each_neighbor_edge_ms(row, &mut on_edge); + } + if direction != Direction::Outgoing { + seg.for_each_incoming_edge_ms(row, &mut on_edge); + } + } + }; + + if frontier.len() < PARALLEL_LEVEL_THRESHOLD { + let mut out = Vec::new(); + expand_chunk(frontier, &mut out); + return out; + } + + // Warm the lazy incoming index OUTSIDE the worker threads so the + // OnceLock build isn't serialized inside the scope. + if direction != Direction::Outgoing { + seg.for_each_incoming_edge_ms(0, |_, _, _| {}); + } + + let workers = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1) + .clamp(1, MAX_WORKERS); + let chunk_size = frontier.len().div_ceil(workers); + let expand_chunk = &expand_chunk; + let mut outs: Vec> = Vec::new(); + std::thread::scope(|s| { + let handles: Vec<_> = frontier + .chunks(chunk_size) + .map(|chunk| { + s.spawn(move || { + let mut out = Vec::new(); + expand_chunk(chunk, &mut out); + out + }) + }) + .collect(); + for h in handles { + // A worker panicking is a bug in expand_chunk itself; surface it. + if let Ok(out) = h.join() { + outs.push(out); + } + } + }); + outs.concat() +} + +/// Bottom-up level (Outgoing only): every UNVISITED visible row scans its +/// in-edges for a frontier parent; first passing parent wins. Rows are +/// partitioned across workers, so discovery is per-row disjoint. +fn expand_pull( + seg: &CsrStorage, + frontier: &[u32], + visited: &[AtomicU64], + snapshot_lsn: u64, + edge_type_filter: Option, + n: usize, +) -> Vec { + // Frontier membership bitmap for O(1) parent tests. + let mut frontier_bits = vec![0u64; n.div_ceil(64)]; + for &row in frontier { + frontier_bits[(row / 64) as usize] |= 1u64 << (row % 64); + } + let frontier_bits = &frontier_bits; + + let scan_range = |range: std::ops::Range, out: &mut Vec| { + let node_meta = seg.node_meta(); + for row in range { + let row = row as u32; + if is_set(visited, row) { + continue; + } + if !row_visible(&node_meta[row as usize], snapshot_lsn) { + continue; + } + let mut found: Option<(EdgeMeta, u64)> = None; + seg.for_each_incoming_edge_ms(row, |src_row, meta, created_ms| { + if found.is_some() { + return; + } + if let Some(filter) = edge_type_filter { + if meta.edge_type != filter { + return; + } + } + if frontier_bits[(src_row / 64) as usize] & (1u64 << (src_row % 64)) != 0 { + found = Some((meta, created_ms)); + } + }); + if let Some((meta, created_ms)) = found { + // Rows are partitioned per worker — the set cannot race. + test_and_set(visited, row); + out.push((row, meta, created_ms)); + } + } + }; + + // Warm the lazy incoming index before any parallel section. + seg.for_each_incoming_edge_ms(0, |_, _, _| {}); + + if n < PARALLEL_LEVEL_THRESHOLD { + let mut out = Vec::new(); + scan_range(0..n, &mut out); + return out; + } + + let workers = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1) + .clamp(1, MAX_WORKERS); + let chunk = n.div_ceil(workers); + let scan_range = &scan_range; + let mut outs: Vec> = Vec::new(); + std::thread::scope(|s| { + let handles: Vec<_> = (0..workers) + .map(|w| { + let range = (w * chunk).min(n)..((w + 1) * chunk).min(n); + s.spawn(move || { + let mut out = Vec::new(); + scan_range(range, &mut out); + out + }) + }) + .collect(); + for h in handles { + if let Ok(out) = h.join() { + outs.push(out); + } + } + }); + outs.concat() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::graph::csr::CsrSegment; + use crate::graph::traversal::SegmentMergeReader; + use smallvec::SmallVec; + use std::collections::HashMap; + + /// Deterministic pseudo-random frozen graph: `n` nodes, each node i + /// gets out-edges to (i*7+k*13+1) % n for k in 0..deg(i). + fn frozen_storage(n: u64, max_deg: u64) -> (CsrStorage, Vec) { + let mut mg = MemGraph::new(usize::MAX >> 1); + let mut keys = Vec::new(); + for _ in 0..n { + keys.push(mg.add_node(SmallVec::from_elem(1u16, 1), SmallVec::new(), None, 1)); + } + for i in 0..n { + let deg = 1 + (i % max_deg); + for k in 0..deg { + let j = (i * 7 + k * 13 + 1) % n; + if j != i { + let etype = if k % 2 == 0 { 1u16 } else { 2u16 }; + mg.add_edge(keys[i as usize], keys[j as usize], etype, 1.0, None, 2) + .expect("edge ok"); + } + } + } + let frozen = mg.freeze().expect("freeze"); + let seg = CsrSegment::from_frozen(frozen, 3).expect("csr"); + (CsrStorage::from(seg), keys) + } + + fn frozen_graph(n: u64, max_deg: u64) -> (Arc, Vec) { + let (storage, keys) = frozen_storage(n, max_deg); + (Arc::new(storage), keys) + } + + /// Oracle: reader-path BFS (the code path row_bfs replaces). + fn reader_bfs( + seg: &Arc, + start: NodeKey, + direction: Direction, + depth_limit: u32, + filter: Option, + ) -> HashMap { + let segs = vec![seg.clone()]; + let reader = SegmentMergeReader::new(None, &segs, direction, u64::MAX - 1, filter); + // Hand-rolled reader-path BFS: BoundedBfs::execute takes the fast + // path for this fixture, so it cannot serve as its own oracle. + let mut depths = HashMap::new(); + depths.insert(start, 0u32); + let mut frontier = vec![start]; + let mut d = 0; + while !frontier.is_empty() && d < depth_limit { + let mut next = Vec::new(); + for &node in &frontier { + for nb in reader.neighbors(node) { + if let std::collections::hash_map::Entry::Vacant(e) = depths.entry(nb.node) { + e.insert(d + 1); + next.push(nb.node); + } + } + } + frontier = next; + d += 1; + } + depths + } + + fn depth_map(result: &BfsResult) -> HashMap { + result.visited.iter().map(|&(k, d, _)| (k, d)).collect() + } + + #[test] + fn test_row_bfs_matches_reader_path() { + let (seg, keys) = frozen_graph(300, 5); + for direction in [Direction::Outgoing, Direction::Incoming, Direction::Both] { + for filter in [None, Some(1u16)] { + let got = row_bfs( + &seg, + direction, + u64::MAX - 1, + filter, + keys[0], + 4, + usize::MAX, + BfsMode::ForcePush, + ) + .expect("bfs ok"); + let want = reader_bfs(&seg, keys[0], direction, 4, filter); + assert_eq!( + depth_map(&got), + want, + "push parity failed dir={direction:?} filter={filter:?}" + ); + } + } + } + + #[test] + fn test_pull_matches_push() { + let (seg, keys) = frozen_graph(500, 8); + let push = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[3], + 5, + usize::MAX, + BfsMode::ForcePush, + ) + .expect("push ok"); + let pull = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[3], + 5, + usize::MAX, + BfsMode::ForcePull, + ) + .expect("pull ok"); + assert_eq!(depth_map(&push), depth_map(&pull), "pull/push must agree"); + + // With an edge-type filter too (pull checks the filter per parent). + let push_f = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + Some(2), + keys[3], + 5, + usize::MAX, + BfsMode::ForcePush, + ) + .expect("push ok"); + let pull_f = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + Some(2), + keys[3], + 5, + usize::MAX, + BfsMode::ForcePull, + ) + .expect("pull ok"); + assert_eq!(depth_map(&push_f), depth_map(&pull_f)); + } + + #[test] + fn test_parallel_level_matches_sequential() { + // Star head fans out to >PARALLEL_LEVEL_THRESHOLD nodes → level 2 + // expands in parallel. Compare against the reader-path oracle. + let n = 2_000u64; + let mut mg = MemGraph::new(usize::MAX >> 1); + let mut keys = Vec::new(); + for _ in 0..n { + keys.push(mg.add_node(SmallVec::from_elem(1u16, 1), SmallVec::new(), None, 1)); + } + for i in 1..600u64 { + mg.add_edge(keys[0], keys[i as usize], 1, 1.0, None, 2) + .expect("edge"); + // Second hop: each fan-out node points at a distinct tail node. + let tail = 600 + (i % (n - 600)); + mg.add_edge(keys[i as usize], keys[tail as usize], 1, 1.0, None, 2) + .expect("edge"); + } + let frozen = mg.freeze().expect("freeze"); + let seg = Arc::new(CsrStorage::from( + CsrSegment::from_frozen(frozen, 3).expect("csr"), + )); + + let got = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 3, + usize::MAX, + BfsMode::ForcePush, + ) + .expect("bfs ok"); + let want = reader_bfs(&seg, keys[0], Direction::Outgoing, 3, None); + assert_eq!(depth_map(&got), want); + } + + #[test] + fn test_gate_falls_back_when_mutable_tier_nonempty_or_multiseg() { + let (seg, keys) = frozen_graph(50, 3); + + // Empty mutable tier + single segment → fast path taken. + let empty = MemGraph::new(1 << 20); + let segs = vec![seg.clone()]; + assert!( + try_row_bfs( + Some(&empty), + &segs, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 3, + usize::MAX + ) + .is_some() + ); + + // Non-empty mutable tier → fall back. + let mut busy = MemGraph::new(1 << 20); + busy.add_node(SmallVec::new(), SmallVec::new(), None, 9); + assert!( + try_row_bfs( + Some(&busy), + &segs, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 3, + usize::MAX + ) + .is_none() + ); + + // Two segments → fall back. + let two = vec![seg.clone(), seg.clone()]; + assert!( + try_row_bfs( + None, + &two, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 3, + usize::MAX + ) + .is_none() + ); + + // Segment newer than the snapshot → fall back. + assert!( + try_row_bfs( + None, + &segs, + Direction::Outgoing, + 1, // snapshot before seg lsn 3 + None, + keys[0], + 3, + usize::MAX + ) + .is_none() + ); + } + + #[test] + fn test_deleted_nodes_excluded() { + // Soft-delete a node pre-freeze: its row carries deleted_lsn into + // the CSR NodeMeta, and BFS must agree with the reader path (which + // applies the same deleted_lsn visibility check on every probe). + let mut mg = MemGraph::new(usize::MAX >> 1); + let mut keys = Vec::new(); + for _ in 0..40u64 { + keys.push(mg.add_node(SmallVec::from_elem(1u16, 1), SmallVec::new(), None, 1)); + } + for i in 0..39usize { + mg.add_edge(keys[i], keys[i + 1], 1, 1.0, None, 2) + .expect("edge"); + } + assert!(mg.remove_node(keys[8], 5), "victim soft-deleted"); + let frozen = mg.freeze().expect("freeze"); + let seg = Arc::new(CsrStorage::from( + CsrSegment::from_frozen(frozen, 6).expect("csr"), + )); + + let got = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 50, + usize::MAX, + BfsMode::ForcePush, + ) + .expect("bfs ok"); + assert!( + !depth_map(&got).contains_key(&keys[8]), + "deleted node must not be discovered" + ); + let want = reader_bfs(&seg, keys[0], Direction::Outgoing, 50, None); + assert_eq!(depth_map(&got), want); + } + + #[test] + fn test_depth_limit_and_frontier_cap() { + let (seg, keys) = frozen_graph(200, 4); + let d1 = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 1, + usize::MAX, + BfsMode::ForcePush, + ) + .expect("bfs ok"); + assert!(depth_map(&d1).values().all(|&d| d <= 1)); + + let err = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + keys[0], + 10, + 3, // tiny cap + BfsMode::ForcePush, + ) + .expect_err("cap must trip"); + assert!(matches!( + err, + TraversalError::FrontierCapExceeded { cap: 3, .. } + )); + } + + #[test] + fn test_missing_start_is_node_not_found() { + let (seg, _keys) = frozen_graph(10, 2); + // A key from a fresh MemGraph would ALIAS the frozen rows (SlotMap + // sequences restart) — use a synthetic key far outside the segment. + let ghost: NodeKey = slotmap::KeyData::from_ffi((1u64 << 32) | 999_999).into(); + let err = row_bfs( + &seg, + Direction::Outgoing, + u64::MAX - 1, + None, + ghost, + 3, + usize::MAX, + BfsMode::ForcePush, + ) + .expect_err("unknown start"); + assert!(matches!(err, TraversalError::NodeNotFound)); + } +} diff --git a/src/graph/store.rs b/src/graph/store.rs index dae8e505..dbfac9c6 100644 --- a/src/graph/store.rs +++ b/src/graph/store.rs @@ -12,7 +12,6 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use crate::graph::csr::CsrSegment; -use crate::graph::index::PropertyIndex; use crate::graph::segment::GraphSegmentHolder; use crate::graph::stats::GraphStats; @@ -39,9 +38,6 @@ pub struct NamedGraph { pub edge_threshold: usize, /// LSN at which this graph was created. pub created_lsn: u64, - /// Optional property indexes for cross-segment numeric range queries. - /// Key is the property name dictionary ID. - pub property_indexes: HashMap, /// Per-graph statistics for cost-based query planning. /// Updated incrementally on node/edge insert/delete. pub stats: GraphStats, @@ -76,7 +72,8 @@ impl NamedGraph { } /// Freeze the current MemGraph, convert to CSR, and push the new immutable - /// segment into the segment holder. Replaces write_buf with a fresh MemGraph. + /// segment into the segment holder. Thaws write_buf in place afterwards + /// (same slot maps — see key-uniqueness note below). /// /// Returns `true` if compaction succeeded, `false` if freeze/CSR build failed. pub fn freeze_and_compact(&mut self, lsn: u64) -> bool { @@ -86,21 +83,31 @@ impl NamedGraph { Err(_) => return false, }; - // Convert frozen MemGraph to a CSR segment. - match CsrSegment::from_frozen(frozen, lsn) { + // Nothing freezable (e.g. the buffer holds only cross-tier delta + // edges, which freeze() retains): skip — pushing an empty segment + // per write attempt would churn the segment list. + if frozen.nodes.is_empty() && frozen.edges.is_empty() { + self.write_buf.thaw(); + return false; + } + + // Convert frozen MemGraph to a CSR segment. Either way, thaw the + // drained write_buf IN PLACE — reusing the slot maps keeps SlotMap + // generation continuity, so post-freeze NodeKeys can never collide + // with the external_ids of frozen CSR rows (a fresh MemGraph would + // restart allocation at the same keys and alias new nodes onto + // frozen rows). + let ok = match CsrSegment::from_frozen(frozen, lsn) { Ok(csr) => { self.segments.add_immutable(csr); - // Replace write_buf with a fresh empty MemGraph. - self.write_buf = crate::graph::memgraph::MemGraph::new(self.edge_threshold); true } - Err(_) => { - // CSR build failed -- write_buf is already frozen/drained. - // Replace with fresh MemGraph so writes can continue. - self.write_buf = crate::graph::memgraph::MemGraph::new(self.edge_threshold); - false - } - } + // CSR build failed — write_buf is already drained; thaw so + // writes can continue. + Err(_) => false, + }; + self.write_buf.thaw(); + ok } } @@ -195,7 +202,6 @@ impl GraphStore { write_buf: crate::graph::memgraph::MemGraph::new(edge_threshold), edge_threshold, created_lsn: lsn, - property_indexes: HashMap::new(), stats: GraphStats::new(), plan_cache: parking_lot::Mutex::new(crate::graph::cypher::planner::PlanCache::new( 1024, @@ -472,6 +478,44 @@ mod tests { assert!(result.is_err()); } + #[test] + fn test_freeze_and_compact_preserves_key_uniqueness() { + use smallvec::smallvec; + let mut store = GraphStore::new(); + store + .create_graph(Bytes::from_static(b"g"), 4, 1) + .expect("ok"); + let g = store.get_graph_mut(b"g").expect("g"); + + let mut keys = Vec::new(); + for i in 0..5u64 { + keys.push(g.write_buf.add_node(smallvec![0u16], smallvec![], None, i)); + } + for w in keys.windows(2) { + g.write_buf + .add_edge(w[0], w[1], 0, 1.0, None, 10) + .expect("edge"); + } + assert!(g.freeze_and_compact(20), "compact succeeds"); + + // A node added AFTER the freeze must get a key distinct from every + // frozen key. A fresh SlotMap restarts allocation at the same + // (index, generation) pairs, silently aliasing new nodes onto frozen + // CSR rows (external_id collision across tiers). + let new_key = g.write_buf.add_node(smallvec![0u16], smallvec![], None, 21); + assert!( + !keys.contains(&new_key), + "post-freeze NodeKey collides with a frozen node's key" + ); + let snap = g.segments.load(); + for seg in &snap.immutable { + assert!( + seg.lookup_node(new_key).is_none(), + "frozen segment resolves a post-freeze key" + ); + } + } + #[test] fn test_allocate_lsn_monotonic() { let mut store = GraphStore::new(); diff --git a/src/graph/traversal.rs b/src/graph/traversal.rs index d55c29ae..604c9c4a 100644 --- a/src/graph/traversal.rs +++ b/src/graph/traversal.rs @@ -9,13 +9,14 @@ //! Dijkstra finds shortest weighted path using composite cost (TRAV-04). use std::cmp::Ordering; -use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque}; +use std::collections::{BinaryHeap, VecDeque}; use std::sync::Arc; use dashmap::DashSet; use parking_lot::Mutex; use crate::graph::csr::CsrStorage; +use crate::graph::fasthash::{FxHashMap, FxHashSet}; use crate::graph::memgraph::MemGraph; use crate::graph::scoring::WeightedCostFn; use crate::graph::types::{Direction, EdgeKey, NodeKey}; @@ -122,7 +123,7 @@ impl<'a> SegmentMergeReader<'a> { /// Allocates per call. For BFS/DFS hot loops, prefer [`neighbors_into`] /// which reuses caller-provided scratch buffers. pub fn neighbors(&self, node: NodeKey) -> Vec { - let mut seen: HashSet = HashSet::new(); + let mut seen: FxHashSet = FxHashSet::default(); let mut result: Vec = Vec::new(); self.neighbors_inner(node, &mut seen, &mut result); result @@ -136,7 +137,7 @@ impl<'a> SegmentMergeReader<'a> { pub fn neighbors_into( &self, node: NodeKey, - seen: &mut HashSet, + seen: &mut FxHashSet, out: &mut Vec, ) { seen.clear(); @@ -148,7 +149,7 @@ impl<'a> SegmentMergeReader<'a> { fn neighbors_inner( &self, node: NodeKey, - seen: &mut HashSet, + seen: &mut FxHashSet, result: &mut Vec, ) { // 1. Read from mutable MemGraph (highest priority -- newest data). @@ -315,6 +316,22 @@ impl BoundedBfs { reader: &SegmentMergeReader<'_>, start: NodeKey, ) -> Result { + // CSR row-space fast path: a fully-frozen single-segment graph + // expands in u32 row space (dense visited bitmap, parallel levels, + // direction-optimizing pull) — see `row_bfs` module docs. + if let Some(res) = crate::graph::row_bfs::try_row_bfs( + reader.memgraph, + reader.csr_segments, + reader.direction, + reader.snapshot_lsn, + reader.edge_type_filter, + start, + self.depth_limit, + self.frontier_cap, + ) { + return res; + } + if !reader.node_exists(start) { return Err(TraversalError::NodeNotFound); } @@ -322,7 +339,7 @@ impl BoundedBfs { // Hint the OS for sequential page access on mmap'd CSR segments. reader.madvise_sequential(); - let mut visited_set: HashSet = HashSet::new(); + let mut visited_set: FxHashSet = FxHashSet::default(); visited_set.insert(start); let mut result = Vec::new(); @@ -332,7 +349,7 @@ impl BoundedBfs { frontier.push_back((start, 0)); // Scratch buffers reused across all neighbor lookups (avoids per-node alloc). - let mut nb_seen: HashSet = HashSet::new(); + let mut nb_seen: FxHashSet = FxHashSet::default(); let mut nb_buf: Vec = Vec::new(); while let Some((current, depth)) = frontier.pop_front() { @@ -413,13 +430,29 @@ impl ParallelBfs { reader: &SegmentMergeReader<'_>, start: NodeKey, ) -> Result { + // CSR row-space fast path: a fully-frozen single-segment graph + // expands in u32 row space (dense visited bitmap, parallel levels, + // direction-optimizing pull) — see `row_bfs` module docs. + if let Some(res) = crate::graph::row_bfs::try_row_bfs( + reader.memgraph, + reader.csr_segments, + reader.direction, + reader.snapshot_lsn, + reader.edge_type_filter, + start, + self.depth_limit, + self.frontier_cap, + ) { + return res; + } + if !reader.node_exists(start) { return Err(TraversalError::NodeNotFound); } reader.madvise_sequential(); - let mut visited_set: HashSet = HashSet::new(); + let mut visited_set: FxHashSet = FxHashSet::default(); visited_set.insert(start); let mut result: Vec<(NodeKey, u32, Option)> = vec![(start, 0, None)]; @@ -427,7 +460,7 @@ impl ParallelBfs { let mut depth: u32 = 0; // Scratch buffers reused across all neighbor lookups. - let mut nb_seen: HashSet = HashSet::new(); + let mut nb_seen: FxHashSet = FxHashSet::default(); let mut nb_buf: Vec = Vec::new(); while !frontier.is_empty() { @@ -554,7 +587,7 @@ impl BoundedDfs { return Err(TraversalError::NodeNotFound); } - let mut visited_set: HashSet = HashSet::new(); + let mut visited_set: FxHashSet = FxHashSet::default(); let mut result = Vec::new(); // Stack: (node, depth, cumulative_cost) @@ -562,7 +595,7 @@ impl BoundedDfs { stack.push((start, 0, 0.0)); // Scratch buffers reused across all neighbor lookups. - let mut nb_seen: HashSet = HashSet::new(); + let mut nb_seen: FxHashSet = FxHashSet::default(); let mut nb_buf: Vec = Vec::new(); while let Some((current, depth, cum_cost)) = stack.pop() { @@ -651,6 +684,14 @@ impl Ord for DijkstraEntry { } } +/// Per-node Dijkstra state: the former dist/prev/depth map triple merged +/// into one entry so each relaxation costs one hash lookup instead of three. +struct DijkstraNodeState { + dist: f64, + prev: Option, + depth: u32, +} + /// Dijkstra shortest weighted path traversal. pub struct DijkstraTraversal { pub cost_fn: WeightedCostFn, @@ -676,40 +717,45 @@ impl DijkstraTraversal { return Err(TraversalError::NodeNotFound); } - let mut dist: HashMap = HashMap::new(); - let mut prev: HashMap = HashMap::new(); - let mut depth_map: HashMap = HashMap::new(); + let mut state: FxHashMap = FxHashMap::default(); let mut heap = BinaryHeap::new(); - dist.insert(start, 0.0); - depth_map.insert(start, 0); + state.insert( + start, + DijkstraNodeState { + dist: 0.0, + prev: None, + depth: 0, + }, + ); heap.push(DijkstraEntry { node: start, cost: 0.0, }); // Scratch buffers reused across all neighbor lookups. - let mut nb_seen: HashSet = HashSet::new(); + let mut nb_seen: FxHashSet = FxHashSet::default(); let mut nb_buf: Vec = Vec::new(); while let Some(DijkstraEntry { node, cost }) = heap.pop() { // Found target -- reconstruct path. if node == target { - let path = self.reconstruct_path(&prev, start, target); + let path = self.reconstruct_path(&state, start, target); return Ok(Some(DijkstraResult { total_cost: cost, path, })); } + // One lookup where the dist/prev/depth triple used to cost three. + let (best, current_depth) = match state.get(&node) { + Some(s) => (s.dist, s.depth), + None => (f64::INFINITY, 0), + }; // Skip if we've already found a better path. - if let Some(&best) = dist.get(&node) { - if cost > best { - continue; - } + if cost > best { + continue; } - - let current_depth = depth_map.get(&node).copied().unwrap_or(0); if current_depth >= self.max_depth { continue; } @@ -719,15 +765,20 @@ impl DijkstraTraversal { let edge_cost = self.cost_fn.cost_ms(neighbor.created_ms, neighbor.weight); let new_cost = cost + edge_cost; - let is_better = match dist.get(&neighbor.node) { - Some(&existing) => new_cost < existing, + let is_better = match state.get(&neighbor.node) { + Some(s) => new_cost < s.dist, None => true, }; if is_better { - dist.insert(neighbor.node, new_cost); - prev.insert(neighbor.node, node); - depth_map.insert(neighbor.node, current_depth + 1); + state.insert( + neighbor.node, + DijkstraNodeState { + dist: new_cost, + prev: Some(node), + depth: current_depth + 1, + }, + ); heap.push(DijkstraEntry { node: neighbor.node, cost: new_cost, @@ -739,10 +790,10 @@ impl DijkstraTraversal { Ok(None) // No path found } - /// Reconstruct path from prev-pointers. + /// Reconstruct path from the settled states' prev-pointers. fn reconstruct_path( &self, - prev: &HashMap, + state: &FxHashMap, start: NodeKey, target: NodeKey, ) -> Vec { @@ -750,8 +801,8 @@ impl DijkstraTraversal { let mut current = target; path.push(current); while current != start { - match prev.get(¤t) { - Some(&p) => { + match state.get(¤t).and_then(|s| s.prev) { + Some(p) => { current = p; path.push(current); } diff --git a/src/graph/traversal_guard.rs b/src/graph/traversal_guard.rs index 45c6837e..5f5a0c7a 100644 --- a/src/graph/traversal_guard.rs +++ b/src/graph/traversal_guard.rs @@ -37,6 +37,7 @@ impl core::fmt::Display for TraversalTimeout { /// for all hops, ensuring consistent graph visibility across the entire traversal. /// Each hop should call `check_timeout()` to verify the traversal has not exceeded /// its time budget. +#[derive(Debug, Clone, Copy)] pub struct TraversalGuard { snapshot_lsn: u64, start: Instant, diff --git a/src/graph/types.rs b/src/graph/types.rs index 94fde28f..f28ed23c 100644 --- a/src/graph/types.rs +++ b/src/graph/types.rs @@ -96,9 +96,14 @@ pub struct MutableEdge { /// a construction site stamping a stale literal is exactly the bug class that /// made vacuumed segments misparse on reload (v1 stamp on v2 records). /// -/// Parse-side gates stay numeric (`version >= 2`, `version >= 3`, `version >= 4`): -/// they encode the version a feature was INTRODUCED at and must not track this constant. -pub const CSR_CURRENT_VERSION: u32 = 4; +/// Parse-side gates stay numeric (`version >= 2`, `version >= 3`, `version >= 4`, +/// `version >= 5`): they encode the version a feature was INTRODUCED at and must +/// not track this constant. +/// +/// v5 adds two trailing variable-length sections (node/edge property blobs — +/// see `csr::props`) so properties, embeddings, and edge weights survive the +/// freeze boundary instead of being discarded. +pub const CSR_CURRENT_VERSION: u32 = 5; /// On-disk CSR segment header -- cache-line aligned, zero-copy mmap. #[derive(Debug)] diff --git a/src/graph/view.rs b/src/graph/view.rs new file mode 100644 index 00000000..5452d29d --- /dev/null +++ b/src/graph/view.rs @@ -0,0 +1,277 @@ +//! MergedNodeView — unified node resolution across the mutable MemGraph tier +//! and the immutable CSR segment tier. +//! +//! `MemGraph::freeze` DRAINS nodes into a CSR segment (it does not copy), so +//! any read path that consults only the mutable tier silently loses every +//! frozen node. This view is the single seam read paths (Cypher NodeScan / +//! property eval, GRAPH.NEIGHBORS existence, GRAPH.HYBRID) go through to see +//! the whole graph. +//! +//! Tier invariant: a NodeKey lives in EXACTLY ONE tier, so lookups here need +//! no dedup -- but this is actively ENFORCED, not automatic, across the two +//! ways a mutable-tier `MemGraph` comes into being. +//! +//! In-process freeze (`NamedGraph::freeze_and_compact`) needs no extra work: +//! `freeze()` drains keys out of the slotmap and thaws `write_buf` IN PLACE +//! (same underlying slot maps), so slot reuse bumps the generation and a +//! drained key can never reappear in the mutable tier; `compact_segments` +//! swaps the segment list atomically, so within one loaded snapshot the +//! segments are pairwise disjoint too. +//! +//! Restart (`recover_graph_store` + WAL replay, `src/graph/recovery.rs` + +//! `src/graph/replay.rs`) is different: a FRESH `MemGraph` has no history +//! with the loaded CSR segments' generations, so its deterministic SlotMap +//! allocation (index 0, generation 1 first) could otherwise numerically +//! alias a persisted `external_id` -- silently shadowing a frozen node, +//! since lookups here check the mutable tier first. The invariant is +//! restored by two restart-only mechanisms: `recover_graph_store` seeds the +//! post-recovery mutable tier via `MemGraph::with_id_offset` with a +//! watermark one past the largest persisted `external_id` (so fresh keys +//! can never collide), and `replay.rs`'s AddNode replay loop skips +//! re-inserting any `node_id` already resident in a loaded segment (avoiding +//! double enumeration when un-truncated WAL history replays a node that is +//! already frozen). See `tests/graph_restart_id_aliasing.rs` for the +//! regression coverage. + +use std::sync::Arc; + +use roaring::RoaringBitmap; +use smallvec::SmallVec; + +use crate::graph::csr::CsrStorage; +use crate::graph::memgraph::MemGraph; +use crate::graph::types::{NodeKey, PropertyMap, PropertyValue}; +use crate::graph::visibility::{is_meta_visible, is_node_visible}; + +/// Borrowed view over both graph tiers. Cheap to construct (two references); +/// build one per query from `NamedGraph::write_buf` + a loaded segment guard. +pub struct MergedNodeView<'a> { + memgraph: &'a MemGraph, + segments: &'a [Arc], +} + +impl<'a> MergedNodeView<'a> { + pub fn new(memgraph: &'a MemGraph, segments: &'a [Arc]) -> Self { + Self { memgraph, segments } + } + + /// Locate a frozen node: `(segment, CSR row)`. Mutable-tier nodes return + /// `None` — callers that care about both tiers check `memgraph` first. + /// Newest segment wins on the (impossible-by-invariant) overlap. + pub fn segment_row(&self, key: NodeKey) -> Option<(&'a CsrStorage, u32)> { + for seg in self.segments.iter().rev() { + if let Some(row) = seg.lookup_node(key) { + return Some((seg.as_ref(), row)); + } + } + None + } + + /// Does the node exist in ANY tier (ignoring MVCC/valid-time visibility)? + pub fn contains(&self, key: NodeKey) -> bool { + self.memgraph.get_node(key).is_some() || self.segment_row(key).is_some() + } + + /// Node visibility across tiers. + pub fn is_visible( + &self, + key: NodeKey, + snapshot_lsn: u64, + my_txn_id: u64, + committed: &RoaringBitmap, + valid_at: Option, + ) -> bool { + if let Some(node) = self.memgraph.get_node(key) { + return is_node_visible(node, snapshot_lsn, my_txn_id, committed, valid_at); + } + if let Some((seg, row)) = self.segment_row(key) { + if let Some(meta) = seg.node_meta().get(row as usize) { + return is_meta_visible(meta, snapshot_lsn, my_txn_id, committed, valid_at); + } + } + false + } + + /// Single node property lookup across tiers (frozen side decodes one + /// record from the v5 blob without materializing the whole map). + pub fn property(&self, key: NodeKey, prop_id: u16) -> Option { + if let Some(node) = self.memgraph.get_node(key) { + return node + .properties + .iter() + .find(|(pid, _)| *pid == prop_id) + .map(|(_, v)| v.clone()); + } + let (seg, row) = self.segment_row(key)?; + seg.node_property(row, prop_id) + } + + /// All node properties across tiers. `None` = node not found in any tier; + /// an existing node with no properties yields an empty map. + pub fn properties(&self, key: NodeKey) -> Option { + if let Some(node) = self.memgraph.get_node(key) { + return Some(node.properties.clone()); + } + let (seg, row) = self.segment_row(key)?; + Some(seg.node_properties(row)) + } + + /// Node embedding across tiers (`None` = node absent or no embedding). + pub fn embedding(&self, key: NodeKey) -> Option> { + if let Some(node) = self.memgraph.get_node(key) { + return node.embedding.clone(); + } + let (seg, row) = self.segment_row(key)?; + seg.node_embedding(row) + } + + /// Node labels across tiers. Frozen rows decode `label_bitmap` (ids 0-31) + /// plus the sparse overflow map (ids >= 32). + pub fn labels(&self, key: NodeKey) -> Option> { + if let Some(node) = self.memgraph.get_node(key) { + return Some(node.labels.clone()); + } + let (seg, row) = self.segment_row(key)?; + let meta = seg.node_meta().get(row as usize)?; + let mut out: SmallVec<[u16; 4]> = SmallVec::new(); + let mut bm = meta.label_bitmap; + while bm != 0 { + out.push(bm.trailing_zeros() as u16); + bm &= bm - 1; + } + if let Some(extra) = seg.label_overflow().get(&row) { + out.extend_from_slice(extra); + } + Some(out) + } + + /// Enumerate every VISIBLE node key across both tiers, optionally + /// restricted to a label. Frozen rows use the per-segment `LabelIndex` + /// (which already folds in overflow labels); the mutable tier checks + /// `labels` inline. Callback order: mutable tier first, then segments + /// oldest-to-newest. + pub fn for_each_visible_node( + &self, + label: Option, + snapshot_lsn: u64, + my_txn_id: u64, + committed: &RoaringBitmap, + valid_at: Option, + mut f: impl FnMut(NodeKey), + ) { + for (key, node) in self.memgraph.iter_nodes() { + if let Some(lid) = label { + if !node.labels.contains(&lid) { + continue; + } + } + if !is_node_visible(node, snapshot_lsn, my_txn_id, committed, valid_at) { + continue; + } + f(key); + } + for seg in self.segments { + let metas = seg.node_meta(); + match label { + Some(lid) => { + let Some(bm) = seg.label_index().nodes_with_label(lid) else { + continue; + }; + for row in bm { + let Some(meta) = metas.get(row as usize) else { + continue; + }; + if !is_meta_visible(meta, snapshot_lsn, my_txn_id, committed, valid_at) { + continue; + } + f(NodeKey::from(slotmap::KeyData::from_ffi(meta.external_id))); + } + } + None => { + for meta in metas { + if !is_meta_visible(meta, snapshot_lsn, my_txn_id, committed, valid_at) { + continue; + } + f(NodeKey::from(slotmap::KeyData::from_ffi(meta.external_id))); + } + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use smallvec::smallvec; + + use crate::graph::csr::CsrSegment; + use crate::graph::memgraph::MemGraph; + use crate::graph::types::PropertyValue; + + /// Two-tier fixture: nodes A,B frozen into a CSR segment, node C live in + /// a fresh MemGraph. A carries label 1 + a property + an embedding. + fn fixture() -> (MemGraph, Vec>, NodeKey, NodeKey, NodeKey) { + let mut mg = MemGraph::new(64); + let a = mg.add_node( + smallvec![1u16], + smallvec![(7u16, PropertyValue::Int(42))], + Some(vec![1.0, 2.0]), + 1, + ); + let b = mg.add_node(smallvec![2u16], SmallVec::new(), None, 2); + mg.add_edge(a, b, 0, 1.0, None, 3).expect("edge"); + let seg = CsrSegment::from_frozen(mg.freeze().expect("freeze"), 3).expect("csr"); + mg.thaw(); + assert_eq!(mg.node_count(), 0, "freeze drains"); + let c = mg.add_node(smallvec![1u16], SmallVec::new(), None, 4); + let segs: Vec> = vec![Arc::new(CsrStorage::Heap(seg))]; + (mg, segs, a, b, c) + } + + #[test] + fn test_view_contains_both_tiers() { + let (mg, segs, a, b, c) = fixture(); + let view = MergedNodeView::new(&mg, &segs); + assert!(view.contains(a)); + assert!(view.contains(b)); + assert!(view.contains(c)); + assert!(!view.contains(NodeKey::from(slotmap::KeyData::from_ffi(u64::MAX)))); + } + + #[test] + fn test_view_property_and_embedding_from_frozen() { + let (mg, segs, a, b, _c) = fixture(); + let view = MergedNodeView::new(&mg, &segs); + assert_eq!(view.property(a, 7), Some(PropertyValue::Int(42))); + assert_eq!(view.property(a, 8), None); + assert_eq!(view.property(b, 7), None); + assert_eq!(view.embedding(a), Some(vec![1.0, 2.0])); + assert_eq!(view.embedding(b), None); + } + + #[test] + fn test_view_labels_from_frozen() { + let (mg, segs, a, _b, c) = fixture(); + let view = MergedNodeView::new(&mg, &segs); + assert_eq!(view.labels(a).expect("frozen labels").as_slice(), &[1u16]); + assert_eq!(view.labels(c).expect("live labels").as_slice(), &[1u16]); + } + + #[test] + fn test_view_scan_merges_tiers_with_label_filter() { + let (mg, segs, a, b, c) = fixture(); + let view = MergedNodeView::new(&mg, &segs); + let committed = RoaringBitmap::new(); + + let mut all = Vec::new(); + view.for_each_visible_node(None, 0, 0, &committed, None, |k| all.push(k)); + assert_eq!(all.len(), 3); + assert!(all.contains(&a) && all.contains(&b) && all.contains(&c)); + + let mut labeled = Vec::new(); + view.for_each_visible_node(Some(1), 0, 0, &committed, None, |k| labeled.push(k)); + assert_eq!(labeled.len(), 2); + assert!(labeled.contains(&a) && labeled.contains(&c)); + } +} diff --git a/src/graph/visibility.rs b/src/graph/visibility.rs index 673a7d63..ce55d26d 100644 --- a/src/graph/visibility.rs +++ b/src/graph/visibility.rs @@ -6,7 +6,38 @@ use roaring::RoaringBitmap; -use crate::graph::types::{MutableEdge, MutableNode}; +use crate::graph::types::{MutableEdge, MutableNode, NodeMeta}; + +/// Check if a frozen (CSR-resident) node is visible at the given snapshot. +/// +/// Same rule as [`is_node_visible`] applied to CSR [`NodeMeta`]. Frozen +/// segments hold only committed data — `freeze()` drains the committed +/// mutable tier — so the transaction-ownership check runs with `txn_id = 0`. +#[inline(always)] +pub fn is_meta_visible( + meta: &NodeMeta, + snapshot_lsn: u64, + my_txn_id: u64, + committed: &RoaringBitmap, + valid_at: Option, +) -> bool { + if !is_entity_visible( + meta.created_lsn, + meta.deleted_lsn, + 0, + snapshot_lsn, + my_txn_id, + committed, + ) { + return false; + } + if let Some(t) = valid_at { + if t < meta.valid_from || t > meta.valid_to { + return false; + } + } + true +} /// Check if a node is visible at the given snapshot. /// diff --git a/src/graph/wal.rs b/src/graph/wal.rs index 2c4bc6b9..3006ec2f 100644 --- a/src/graph/wal.rs +++ b/src/graph/wal.rs @@ -14,13 +14,6 @@ use crate::graph::types::{PropertyMap, PropertyValue}; -/// Format an f64 as a string. Uses Display formatting (no external crate needed). -/// This is only called during WAL serialization (cold path), so allocation is acceptable. -#[inline] -fn format_f64(val: f64) -> String { - format!("{val}") -} - /// Write a RESP bulk string element: `$\r\n\r\n` fn write_bulk(buf: &mut Vec, data: &[u8]) { buf.push(b'$'); @@ -61,8 +54,7 @@ pub fn serialize_remove_node(graph_name: &[u8], node_id: u64) -> Vec { write_array_header(&mut buf, 3); write_bulk(&mut buf, b"GRAPH.REMOVENODE"); write_bulk(&mut buf, graph_name); - let id_str = itoa::Buffer::new().format(node_id).to_owned(); - write_bulk(&mut buf, id_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(node_id).as_bytes()); buf } @@ -72,8 +64,7 @@ pub fn serialize_remove_edge(graph_name: &[u8], edge_id: u64) -> Vec { write_array_header(&mut buf, 3); write_bulk(&mut buf, b"GRAPH.REMOVEEDGE"); write_bulk(&mut buf, graph_name); - let id_str = itoa::Buffer::new().format(edge_id).to_owned(); - write_bulk(&mut buf, id_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(edge_id).as_bytes()); buf } @@ -100,31 +91,28 @@ pub fn serialize_add_node( write_bulk(&mut buf, graph_name); // Node ID - let id_str = itoa::Buffer::new().format(node_id).to_owned(); - write_bulk(&mut buf, id_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(node_id).as_bytes()); // Labels - let nlabels_str = itoa::Buffer::new().format(labels.len()).to_owned(); - write_bulk(&mut buf, nlabels_str.as_bytes()); + write_bulk( + &mut buf, + itoa::Buffer::new().format(labels.len()).as_bytes(), + ); for &label in labels { - let label_str = itoa::Buffer::new().format(label).to_owned(); - write_bulk(&mut buf, label_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(label).as_bytes()); } // Properties - let nprops_str = itoa::Buffer::new().format(props.len()).to_owned(); - write_bulk(&mut buf, nprops_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(props.len()).as_bytes()); for (key, val) in props.iter() { - let key_str = itoa::Buffer::new().format(*key).to_owned(); - write_bulk(&mut buf, key_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(*key).as_bytes()); serialize_property_value(&mut buf, val); } // Optional embedding if let Some(embed) = embedding { write_bulk(&mut buf, b"VECTOR"); - let dim_str = itoa::Buffer::new().format(embed.len()).to_owned(); - write_bulk(&mut buf, dim_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(embed.len()).as_bytes()); // Encode as raw f32 bytes (little-endian) let bytes: Vec = embed.iter().flat_map(|f| f.to_le_bytes()).collect(); write_bulk(&mut buf, &bytes); @@ -154,29 +142,22 @@ pub fn serialize_add_edge( write_bulk(&mut buf, b"GRAPH.ADDEDGE"); write_bulk(&mut buf, graph_name); - let edge_id_str = itoa::Buffer::new().format(edge_id).to_owned(); - write_bulk(&mut buf, edge_id_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(edge_id).as_bytes()); - let src_str = itoa::Buffer::new().format(src_id).to_owned(); - write_bulk(&mut buf, src_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(src_id).as_bytes()); - let dst_str = itoa::Buffer::new().format(dst_id).to_owned(); - write_bulk(&mut buf, dst_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(dst_id).as_bytes()); - let type_str = itoa::Buffer::new().format(edge_type).to_owned(); - write_bulk(&mut buf, type_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(edge_type).as_bytes()); - // Weight as string representation - let weight_str = format_f64(weight); - write_bulk(&mut buf, weight_str.as_bytes()); + // Weight as string representation (ryu: no intermediate String) + write_bulk(&mut buf, ryu::Buffer::new().format(weight).as_bytes()); - let nprops_str = itoa::Buffer::new().format(prop_count).to_owned(); - write_bulk(&mut buf, nprops_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(prop_count).as_bytes()); if let Some(props) = props { for (key, val) in props.iter() { - let key_str = itoa::Buffer::new().format(*key).to_owned(); - write_bulk(&mut buf, key_str.as_bytes()); + write_bulk(&mut buf, itoa::Buffer::new().format(*key).as_bytes()); serialize_property_value(&mut buf, val); } } @@ -189,13 +170,11 @@ fn serialize_property_value(buf: &mut Vec, val: &PropertyValue) { match val { PropertyValue::Int(i) => { write_bulk(buf, b"i"); - let s = itoa::Buffer::new().format(*i).to_owned(); - write_bulk(buf, s.as_bytes()); + write_bulk(buf, itoa::Buffer::new().format(*i).as_bytes()); } PropertyValue::Float(f) => { write_bulk(buf, b"f"); - let s = format_f64(*f); - write_bulk(buf, s.as_bytes()); + write_bulk(buf, ryu::Buffer::new().format(*f).as_bytes()); } PropertyValue::String(s) => { write_bulk(buf, b"s"); diff --git a/tests/graph_freeze_boundary.rs b/tests/graph_freeze_boundary.rs new file mode 100644 index 00000000..08e8b8cf --- /dev/null +++ b/tests/graph_freeze_boundary.rs @@ -0,0 +1,362 @@ +//! Phase-0 freeze-boundary correctness suite (2026-07 graph deep review). +//! +//! `MemGraph::freeze()` DRAINS all live nodes/edges into a CSR segment when a +//! graph crosses `edge_threshold` (production trigger: GRAPH.ADDEDGE → +//! `NamedGraph::freeze_and_compact`). Every test here drives the REAL command +//! handlers with a tiny threshold so the freeze fires after a handful of +//! inserts, then asserts that compacted data remains fully queryable. +//! +//! Written red-first: at review time (tmp/GRAPH-DEEP-REVIEW-2026-07.md §2/P0-1) +//! MATCH scans, property access, GRAPH.NEIGHBORS, GRAPH.ADDEDGE-to-frozen-node, +//! GRAPH.PROFILE, and GRAPH.HYBRID all silently lose or reject compacted data. +#![cfg(feature = "graph")] + +use bytes::Bytes; +use moon::command::graph::graph_read::{graph_hybrid, graph_neighbors, graph_profile, graph_query}; +use moon::command::graph::graph_write::{graph_addedge, graph_addnode}; +use moon::graph::store::GraphStore; +use moon::protocol::Frame; + +/// Freeze after 8 live edges (production default is 64_000). +const THRESHOLD: usize = 8; +const GRAPH: &str = "g"; + +fn bs(s: &str) -> Frame { + Frame::BulkString(Bytes::copy_from_slice(s.as_bytes())) +} + +fn blob(bytes: Vec) -> Frame { + Frame::BulkString(Bytes::from(bytes)) +} + +/// f32 slice → little-endian byte blob (GRAPH.ADDNODE VECTOR format). +fn vec_blob(v: &[f32]) -> Frame { + let mut out = Vec::with_capacity(v.len() * 4); + for f in v { + out.extend_from_slice(&f.to_le_bytes()); + } + blob(out) +} + +fn setup() -> GraphStore { + let mut store = GraphStore::new(); + let lsn = store.allocate_lsn(); + store + .create_graph(Bytes::from_static(GRAPH.as_bytes()), THRESHOLD, lsn) + .expect("create graph"); + store +} + +/// ADDNODE with an `id` property and a 4-dim embedding; returns the external id. +fn add_node(store: &mut GraphStore, id: u64) -> u64 { + let args = vec![ + bs(GRAPH), + bs("N"), + bs("id"), + bs(&id.to_string()), + bs("VECTOR"), + bs("emb"), + vec_blob(&[id as f32, 1.0, 0.0, 0.0]), + ]; + match graph_addnode(store, &args) { + Frame::Integer(ext) => ext as u64, + other => panic!("ADDNODE failed: {other:?}"), + } +} + +fn add_edge(store: &mut GraphStore, src: u64, dst: u64) -> Frame { + graph_addedge( + store, + &[ + bs(GRAPH), + bs(&src.to_string()), + bs(&dst.to_string()), + bs("E"), + ], + ) +} + +fn immutable_segment_count(store: &GraphStore) -> usize { + store + .get_graph(GRAPH.as_bytes()) + .expect("graph exists") + .segments + .load() + .immutable + .len() +} + +/// Seed 6 nodes (ids 0..=5, `id` property + embedding) in a ring plus two +/// chords (8 edges total) so the freeze fires on the 8th ADDEDGE. Returns the +/// external handles indexed by logical id. +/// +/// Post-seed invariant (asserted): exactly one immutable CSR segment exists and +/// the mutable write buffer has been drained. +fn seed_across_freeze(store: &mut GraphStore) -> Vec { + let handles: Vec = (0..6).map(|i| add_node(store, i)).collect(); + let edges: [(u64, u64); 8] = [ + (0, 1), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (5, 0), + (0, 2), + (0, 3), + ]; + for (s, d) in edges { + let r = add_edge(store, handles[s as usize], handles[d as usize]); + assert!( + matches!(r, Frame::Integer(_)), + "seed ADDEDGE {s}->{d} failed: {r:?}" + ); + } + assert_eq!( + immutable_segment_count(store), + 1, + "freeze must fire at edge_threshold={THRESHOLD}" + ); + let g = store.get_graph(GRAPH.as_bytes()).expect("graph"); + assert_eq!( + g.write_buf.node_count(), + 0, + "freeze() drains the mutable write buffer" + ); + handles +} + +/// Extract the rows array from a GRAPH.QUERY reply: Array[headers, rows, stats]. +fn query_rows(frame: &Frame) -> Vec { + match frame { + Frame::Array(items) => match items.get(1) { + Some(Frame::Array(rows)) => rows.iter().cloned().collect(), + other => panic!("malformed query reply, rows slot = {other:?}"), + }, + Frame::Error(e) => panic!("query returned error: {}", String::from_utf8_lossy(e)), + other => panic!("malformed query reply: {other:?}"), + } +} + +fn run_query(store: &GraphStore, cypher: &str) -> Vec { + query_rows(&graph_query(store, &[bs(GRAPH), bs(cypher)])) +} + +/// Flatten a single-column row set into cell frames. +fn single_cells(rows: &[Frame]) -> Vec { + rows.iter() + .map(|r| match r { + Frame::Array(cells) => cells.first().cloned().expect("row has a cell"), + other => panic!("malformed row: {other:?}"), + }) + .collect() +} + +// --------------------------------------------------------------------------- +// 1. Mechanism sanity (documents the drain — passes before and after Phase 0) +// --------------------------------------------------------------------------- + +#[test] +fn freeze_fires_and_drains_write_buffer() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); +} + +// --------------------------------------------------------------------------- +// 2. Cypher visibility across the freeze boundary +// --------------------------------------------------------------------------- + +#[test] +fn match_label_scan_sees_frozen_nodes() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); + let rows = run_query(&store, "MATCH (n:N) RETURN n.id"); + assert_eq!( + rows.len(), + 6, + "label scan must see all 6 compacted nodes, got {} rows", + rows.len() + ); +} + +#[test] +fn match_property_filter_finds_frozen_node() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); + let rows = run_query(&store, "MATCH (a:N {id:3}) RETURN a.id"); + assert_eq!(rows.len(), 1, "point query must find the compacted node"); +} + +#[test] +fn return_property_value_from_frozen_node() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); + let rows = run_query(&store, "MATCH (a:N {id:3}) RETURN a.id"); + let cells = single_cells(&rows); + assert_eq!(cells.len(), 1); + match &cells[0] { + Frame::BulkString(b) => assert_eq!(&b[..], b"3", "property value must survive freeze"), + Frame::Integer(i) => assert_eq!(*i, 3, "property value must survive freeze"), + other => panic!("property lost at freeze (got {other:?})"), + } +} + +#[test] +fn one_hop_expand_from_frozen_point_query() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); + // Node 0 has out-edges to 1, 2, 3. + let rows = run_query(&store, "MATCH (a:N {id:0})-[:E]->(b) RETURN b.id"); + assert_eq!( + rows.len(), + 3, + "expand from compacted node must see CSR edges" + ); +} + +#[test] +fn mixed_tier_match_sees_frozen_and_live_nodes() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); + // One post-freeze node lives in the fresh write buffer. + let _ = add_node(&mut store, 6); + let rows = run_query(&store, "MATCH (n:N) RETURN n.id"); + assert_eq!( + rows.len(), + 7, + "scan must union the mutable tier and CSR segments" + ); +} + +// --------------------------------------------------------------------------- +// 3. Native command surface across the boundary +// --------------------------------------------------------------------------- + +#[test] +fn neighbors_works_on_frozen_node() { + let mut store = setup(); + let handles = seed_across_freeze(&mut store); + let reply = graph_neighbors(&store, &[bs(GRAPH), bs(&handles[0].to_string())]); + match reply { + Frame::Error(e) => panic!( + "GRAPH.NEIGHBORS rejected a compacted node: {}", + String::from_utf8_lossy(&e) + ), + Frame::Array(items) => assert!( + !items.is_empty(), + "compacted node 0 has degree 5 (out 1,2,3 + in 5; Both), got empty reply" + ), + other => panic!("unexpected NEIGHBORS reply: {other:?}"), + } +} + +#[test] +fn neighbors_direction_arg_is_honored() { + let mut store = setup(); + let handles = seed_across_freeze(&mut store); + // Reply layout: one edge frame + one node frame per neighbor. + let count = |dir: &str| -> usize { + let reply = graph_neighbors( + &store, + &[ + bs(GRAPH), + bs(&handles[0].to_string()), + bs("DIRECTION"), + bs(dir), + ], + ); + match reply { + Frame::Array(items) => items.len() / 2, + other => panic!("NEIGHBORS DIRECTION {dir} failed: {other:?}"), + } + }; + assert_eq!(count("OUT"), 3, "node 0 out-neighbors: 1, 2, 3"); + assert_eq!(count("IN"), 1, "node 0 in-neighbor: 5"); + assert_eq!(count("BOTH"), 4, "union of both directions"); + let bad = graph_neighbors( + &store, + &[ + bs(GRAPH), + bs(&handles[0].to_string()), + bs("DIRECTION"), + bs("SIDEWAYS"), + ], + ); + assert!( + matches!(bad, Frame::Error(_)), + "invalid DIRECTION must error, got {bad:?}" + ); +} + +#[test] +fn addedge_between_frozen_nodes_succeeds() { + let mut store = setup(); + let handles = seed_across_freeze(&mut store); + // Both endpoints were drained into the CSR segment. + let reply = add_edge(&mut store, handles[4], handles[1]); + assert!( + matches!(reply, Frame::Integer(_)), + "ADDEDGE between compacted endpoints must succeed, got {reply:?}" + ); + // And the new edge must be immediately traversable. + let rows = run_query(&store, "MATCH (a:N {id:4})-[:E]->(b) RETURN b.id"); + assert_eq!(rows.len(), 2, "old CSR edge 4->5 plus new edge 4->1"); +} + +// --------------------------------------------------------------------------- +// 4. PROFILE parity across the boundary +// --------------------------------------------------------------------------- + +#[test] +fn profile_rows_correct_post_freeze() { + let mut store = setup(); + let _ = seed_across_freeze(&mut store); + let reply = graph_profile( + &store, + &[bs(GRAPH), bs("MATCH (a:N {id:0})-[:E]->(b) RETURN b.id")], + ); + // ProfileResult frame = Array[exec_result_frame, operator_profiles]. + let exec_frame = match &reply { + Frame::Array(items) => items.first().cloned().expect("profile has exec result"), + Frame::Error(e) => panic!("PROFILE errored: {}", String::from_utf8_lossy(e)), + other => panic!("malformed PROFILE reply: {other:?}"), + }; + let rows = query_rows(&exec_frame); + assert_eq!( + rows.len(), + 3, + "PROFILE must execute against CSR segments like GRAPH.QUERY does" + ); +} + +// --------------------------------------------------------------------------- +// 5. Hybrid graph+vector across the boundary +// --------------------------------------------------------------------------- + +#[test] +fn hybrid_filter_sees_frozen_candidates() { + let mut store = setup(); + let handles = seed_across_freeze(&mut store); + // GRAPH.HYBRID g FILTER + let reply = graph_hybrid( + &store, + &[ + bs(GRAPH), + bs("FILTER"), + bs(&handles[0].to_string()), + bs("2"), + bs("5"), + vec_blob(&[0.0, 1.0, 0.0, 0.0]), + ], + ); + match reply { + Frame::Error(e) => panic!( + "GRAPH.HYBRID rejected/lost compacted data: {}", + String::from_utf8_lossy(&e) + ), + Frame::Array(items) => assert!( + !items.is_empty(), + "hybrid FILTER from compacted node 0 must score its 2-hop neighborhood" + ), + other => panic!("unexpected HYBRID reply: {other:?}"), + } +} diff --git a/tests/graph_restart_id_aliasing.rs b/tests/graph_restart_id_aliasing.rs new file mode 100644 index 00000000..28c876d2 --- /dev/null +++ b/tests/graph_restart_id_aliasing.rs @@ -0,0 +1,278 @@ +//! RED-first regression test: graph `NodeKey` aliasing across restart +//! (2026-07 PR #231 pre-merge review, Fix 1 / P0, data-corruption class). +//! +//! ## The hazard +//! +//! `slotmap::SlotMap` key allocation is deterministic: a fresh `MemGraph`'s +//! first `add_node()` always mints SlotMap index 0 / generation 1. CSR +//! segments persist `NodeMeta::external_id` = the pre-crash process's raw +//! `NodeKey` FFI bits (also index-0-based for a segment built from a fresh +//! `MemGraph`, as production commonly does at the first freeze). +//! +//! Historically, `recover_graph_store` re-seeded the post-recovery mutable +//! tier with ANOTHER fresh, unoffset `MemGraph::new(..)`, and `replay.rs`'s +//! AddNode replay loop called `mg.add_node()` unconditionally for every +//! replayed command. The first brand-new node replayed after a restart +//! could then mint the EXACT SAME SlotMap key as a loaded CSR segment's +//! first frozen node. `MergedNodeView` (`src/graph/view.rs`) checks the +//! mutable tier first, so the frozen node's properties would be silently +//! and permanently shadowed by the replayed node's properties -- a +//! same-key, different-identity collision, not a mere duplicate. +//! +//! ## The fix +//! +//! 1. `MemGraph::with_id_offset` (`src/graph/memgraph.rs`) -- `recover_graph_store` +//! (`src/graph/recovery.rs`) seeds the post-recovery mutable tier with a +//! watermark one past the largest persisted `external_id` across all +//! loaded segments, so freshly-minted keys can never numerically alias a +//! loaded segment's rows. +//! 2. A dedup skip in the AddNode replay loop (`src/graph/replay.rs`) -- if a +//! replayed `node_id` is already resident in a loaded CSR segment (full +//! history replay after AOF fold / WAL truncation of pre-freeze history +//! left the record in place), reuse the CSR-derived identity instead of +//! minting a new key, avoiding double enumeration of the same logical node. +//! +//! ## What this test proves +//! +//! (a) A brand-new node replayed after restart never receives a key that +//! equals a loaded segment's `external_id`. +//! (b) `MergedNodeView` reads for the frozen node's id return the frozen +//! node's own properties -- no shadowing by the replayed node. +//! (c) Replaying the frozen node's OWN AddNode record again (simulating +//! un-truncated pre-freeze WAL history) does not double-enumerate it: +//! `for_each_visible_node` still reports exactly the same node set. +//! +//! ## Confirming this is RED without the fix +//! +//! Reverting `MemGraph::with_id_offset` (memgraph.rs), the watermark-based +//! `graph.segments.swap(..)` in `recover_graph_store` (recovery.rs), and the +//! `node_map`-dedup skip in the AddNode replay loop (replay.rs) back to: +//! - `recover_graph_store` seeding `mutable: Some(Arc::new(MemGraph::new(graph.edge_threshold)))` +//! (no offset), and +//! - the replay loop unconditionally calling +//! `mg.add_node(labels.clone(), properties.clone(), embedding.clone(), 0)` +//! for every AddNode record (no `node_map.get(node_id)` dedup check) +//! reproduces exactly this failure: `test_restart_replay_does_not_alias_frozen_node` +//! fails with +//! `assertion failed: replayed key must not equal the frozen node's key` +//! because the freshly-minted replay key (SlotMap index 0, generation 1) is +//! bit-for-bit identical to the frozen node's `external_id`, and the +//! subsequent property read for the frozen key returns the REPLAYED node's +//! property (222) instead of the frozen node's (111) -- i.e. shadowing. +#![cfg(feature = "graph")] + +use bytes::Bytes; +use moon::graph::memgraph::MemGraph; +use moon::graph::recovery::{recover_graph_store, save_graph_store}; +use moon::graph::replay::GraphReplayCollector; +use moon::graph::types::{NodeKey, PropertyValue}; +use moon::graph::view::MergedNodeView; +use roaring::RoaringBitmap; +use slotmap::Key; +use smallvec::smallvec; +use tempfile::TempDir; + +const GRAPH: &str = "g"; +/// Marker property key used to distinguish the frozen node from replayed ones. +const MARKER_PROP: u16 = 50; + +/// Build and freeze a single-node CSR segment. Using a brand-new +/// `MemGraph::new(..)` means the frozen node's `external_id` is guaranteed +/// to be the lowest possible SlotMap key (index 0, generation 1) -- exactly +/// the key a second, independently-fresh `MemGraph` would mint first. This +/// is the realistic worst case: production's very first freeze starts from +/// an empty `MemGraph` the same way. +fn freeze_one_node(marker_value: i64) -> moon::graph::csr::CsrSegment { + let mut mg = MemGraph::new(100); + let props = smallvec![(MARKER_PROP, PropertyValue::Int(marker_value))]; + mg.add_node(smallvec![9u16], props, None, 1); + let frozen = mg.freeze().expect("freeze ok"); + moon::graph::csr::CsrSegment::from_frozen(frozen, 100).expect("csr ok") +} + +fn addnode_args(node_id: u64, label: u16, marker_value: i64) -> Vec> { + vec![ + GRAPH.as_bytes().to_vec(), + node_id.to_string().into_bytes(), + b"1".to_vec(), + label.to_string().into_bytes(), + b"1".to_vec(), + MARKER_PROP.to_string().into_bytes(), + b"i".to_vec(), + marker_value.to_string().into_bytes(), + ] +} + +fn as_slices(args: &[Vec]) -> Vec<&[u8]> { + args.iter().map(Vec::as_slice).collect() +} + +fn empty_committed() -> RoaringBitmap { + RoaringBitmap::new() +} + +/// Enumerate every visible node key (mutable + immutable tiers) via +/// `MergedNodeView`, alongside its `MARKER_PROP` value. +fn visible_marked_nodes(view: &MergedNodeView) -> Vec<(NodeKey, Option)> { + let committed = empty_committed(); + let mut out = Vec::new(); + view.for_each_visible_node(None, 0, 0, &committed, None, |key| { + let marker = match view.property(key, MARKER_PROP) { + Some(PropertyValue::Int(v)) => Some(v), + _ => None, + }; + out.push((key, marker)); + }); + out +} + +#[test] +fn test_restart_replay_does_not_alias_frozen_node() { + let dir = TempDir::new().expect("tmpdir"); + let shard_id = 0; + + // 1. Build + persist a store with one graph holding one frozen CSR + // segment (one node, marker=111). + let mut store = moon::graph::store::GraphStore::new(); + store + .create_graph(Bytes::from_static(GRAPH.as_bytes()), 64_000, 10) + .expect("create graph"); + let csr = freeze_one_node(111); + let frozen_external_id = csr.node_meta[0].external_id; + store + .get_graph_mut(GRAPH.as_bytes()) + .expect("graph exists") + .segments + .add_immutable(csr); + save_graph_store(&store, dir.path(), shard_id).expect("save ok"); + + // 2. "Restart": recover fresh from persistence. This is where + // `recover_graph_store` seeds the post-recovery mutable tier's key + // allocator watermark. + let recovered = recover_graph_store(dir.path(), shard_id) + .expect("io ok") + .expect("result exists"); + let mut store = recovered.store; + assert_eq!(recovered.segments_loaded, 1); + + // 3. Replay one brand-new AddNode record (node_id=999, marker=222) -- + // simulating a WAL record for a node created after the last freeze, + // before the crash. This is the replay path that must never mint a + // key aliasing `frozen_external_id`. + let mut collector = GraphReplayCollector::new(); + let args = addnode_args(999, 3, 222); + assert!(collector.collect_command(b"GRAPH.ADDNODE", &as_slices(&args))); + let replayed = collector.replay_into(&mut store); + assert_eq!(replayed, 1, "exactly one AddNode command must replay"); + + // 4. Inspect the post-replay mutable + immutable tiers directly (the + // subsystem under test: `recovery.rs` + `replay.rs`'s own segment + // holder, matching their existing self-consistent test contract -- + // see `recovery.rs` / `replay.rs` unit tests, which assert against + // `graph.segments.load()` the same way). + let graph = store.get_graph(GRAPH.as_bytes()).expect("graph exists"); + let segs = graph.segments.load(); + let mutable = segs.mutable.as_ref().expect("mutable tier present"); + let view = MergedNodeView::new(mutable, &segs.immutable); + + // (a) No replayed key equals the frozen segment's external_id. + let replayed_keys: Vec = mutable.iter_nodes().map(|(k, _)| k).collect(); + assert_eq!( + replayed_keys.len(), + 1, + "exactly one node in the mutable tier" + ); + let replayed_key = replayed_keys[0]; + let frozen_key = NodeKey::from(slotmap::KeyData::from_ffi(frozen_external_id)); + assert_ne!( + replayed_key.data().as_ffi(), + frozen_external_id, + "replayed key must not equal the frozen node's key (the aliasing hazard)" + ); + + // (b) The frozen node's id still resolves to the FROZEN properties -- + // not shadowed by the replayed node. + assert_eq!( + view.property(frozen_key, MARKER_PROP), + Some(PropertyValue::Int(111)), + "frozen node's properties must not be shadowed by the replayed node" + ); + assert_eq!( + view.property(replayed_key, MARKER_PROP), + Some(PropertyValue::Int(222)), + "replayed node keeps its own properties" + ); + + // Both nodes visible, distinct, correctly attributed. + let mut marks: Vec> = visible_marked_nodes(&view) + .into_iter() + .map(|(_, m)| m) + .collect(); + marks.sort(); + assert_eq!( + marks, + vec![Some(111), Some(222)], + "both the frozen and replayed nodes must be visible with their own markers" + ); +} + +#[test] +fn test_restart_replay_of_already_frozen_node_does_not_double_enumerate() { + // (c) Full-history replay: the WAL still contains the AddNode record + // for a node that is ALREADY resident in a loaded CSR segment (e.g. + // AOF fold didn't truncate pre-freeze history). Replaying it again must + // not create a second, duplicate live-tier entry. + let dir = TempDir::new().expect("tmpdir"); + let shard_id = 0; + + let mut store = moon::graph::store::GraphStore::new(); + store + .create_graph(Bytes::from_static(GRAPH.as_bytes()), 64_000, 10) + .expect("create graph"); + let csr = freeze_one_node(111); + // The frozen node's logical id, as it would appear in the WAL's + // original AddNode record: the raw external_id (production encodes + // GRAPH.ADDNODE's `node_id` field as the NodeKey's FFI bits). + let frozen_external_id = csr.node_meta[0].external_id; + store + .get_graph_mut(GRAPH.as_bytes()) + .expect("graph exists") + .segments + .add_immutable(csr); + save_graph_store(&store, dir.path(), shard_id).expect("save ok"); + + let recovered = recover_graph_store(dir.path(), shard_id) + .expect("io ok") + .expect("result exists"); + let mut store = recovered.store; + + // Replay the SAME node's AddNode record again (un-truncated history), + // using its persisted external_id as the WAL `node_id`. + let mut collector = GraphReplayCollector::new(); + let args = addnode_args(frozen_external_id, 9, 111); + assert!(collector.collect_command(b"GRAPH.ADDNODE", &as_slices(&args))); + let replayed = collector.replay_into(&mut store); + assert_eq!( + replayed, 1, + "the dedup-skipped AddNode still counts as replayed (identity preserved)" + ); + + let graph = store.get_graph(GRAPH.as_bytes()).expect("graph exists"); + let segs = graph.segments.load(); + let mutable = segs.mutable.as_ref().expect("mutable tier present"); + + assert_eq!( + mutable.node_count(), + 0, + "dedup skip must not insert an already-frozen node into the mutable tier" + ); + + let view = MergedNodeView::new(mutable, &segs.immutable); + let marks = visible_marked_nodes(&view); + assert_eq!( + marks.len(), + 1, + "MATCH-scan count must equal distinct nodes -- no double enumeration" + ); + assert_eq!(marks[0].1, Some(111)); +}