Resolve lookup-refs via bounded AVET seeks, not full-index scans

docs/upstream_differences.md

@@ -105,14 +105,12 @@ through them.
 
 ### DataScript: Direct Lookup Refs Scan Visible Datoms
 
-Some unique-value checks already use the `AVET` index, including transaction
-conflict checks. Direct `entid` and lookup-ref resolution still go through
-`visible_datoms` and then scan a list for the matching unique attribute/value.
-
-That preserves result behavior, but it is an eager path where upstream can use
-indexed lookup. Exact parity should route direct lookup-ref resolution through a
-bounded `AVET` lookup, while keeping transaction-local datom-list paths for
-staged transaction state.
+Resolved. Direct `entid` and lookup-ref resolution now go through `find_avet_exact`,
+a bounded `AVET` seek on the attribute/value slice, instead of materializing
+`visible_datoms` and scanning a list. Transaction-local datom-list paths remain
+for staged transaction state (`entity_id_in_datoms` keeps its list semantics), and
+`find_avet_exact` applies `filter_pred` so filtered-db results match the old
+`visible_datoms` path.
 
 ### DataScript: Public Options Are Narrower Than Upstream
 
@@ -250,9 +248,9 @@ evidence shows parity or there is no upstream surface in the checked-out source:
    values by default to match upstream API shape more closely.
 5. Add lazy seek/range APIs for `seek_datoms`, `rseek_datoms`, and
    `index_range`, then route compatibility layers through those APIs.
-6. Route direct lookup-ref resolution through bounded `AVET` lookup instead of
+6. ~~Route direct lookup-ref resolution through bounded `AVET` lookup instead of
    scanning `visible_datoms`, while preserving transaction-local datom-list
-   behavior.
+   behavior.~~ Done.
 7. ~~Rework entity values into lazy views so constructing an entity does not
    materialize all forward and reverse attributes before `entity_attr`
    access.~~ Done: `entity` construction is O(1) (db + eid plus a bounded

impl/datascript.ml

@@ -92,8 +92,6 @@ let empty db = Db_impl.empty db_core_context db
 let init_db ?(schema = []) ?storage datoms =
   Db_impl.init_db db_core_context ~schema ?storage datoms
 
-let visible_datoms = Db_impl.visible_datoms
-
 let is_filtered = Db_impl.is_filtered
 
 let unfiltered_db db = Db_impl.unfiltered db_core_context db
@@ -205,7 +203,6 @@ module Transact_datoms_impl = Transact_datoms.Make (struct
   let normalize_value = normalize_value
   let validate_entity_id = validate_entity_id
   let max_allocatable_entity_id = max_allocatable_entity_id
-  let visible_datoms = visible_datoms
 end)
 
 let normalize_entity_attr_value = Transact_datoms_impl.normalize_entity_attr_value
@@ -251,20 +248,6 @@ let cas_expected_value_string = function
   | None -> "nil"
   | Some value -> edn_string_of_value value
 
-let lookup_refs_context : Lookup_refs.context =
-  { is_unique
-  ; entid_in_datoms
-  ; visible_datoms
-  ; value_to_string = edn_string_of_value
-  }
-
-let unresolved_lookup_ref_message attr value =
-  Lookup_refs.unresolved_message lookup_refs_context attr value
-
-let unresolved_entity_ref_message = function
-  | Lookup_ref (attr, value) -> unresolved_lookup_ref_message attr value
-  | _ -> "lookup ref did not resolve"
-
 let find_avet_exact db attr value =
   let bound = datom ~e:0 ~a:attr ~v:value () in
   let compare_prefix left right =
@@ -276,15 +259,15 @@ let find_avet_exact db attr value =
     else if left == bound then -compare_prefix right left
     else Util.compare_datom Avet left right
   in
-  match
-    PSet.slice ~from_:bound ~to_:bound ~cmp db.avet_index
-    @ List.filter
-        (fun datom -> datom.a = attr && value_equal datom.v value)
-        (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[])
-    |> List.sort (Util.compare_datom Avet)
-  with
-  | datom :: _ -> Some datom
-  | [] -> None
+  PSet.slice ~from_:bound ~to_:bound ~cmp db.avet_index
+  @ List.filter
+      (fun datom -> datom.a = attr && value_equal datom.v value)
+      (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[])
+  |> List.sort (Util.compare_datom Avet)
+  |> List.find_opt (fun datom ->
+    match db.filter_pred with
+    | Some pred -> pred datom
+    | None -> true)
 
 let find_eavt_exact db entity_id attr value =
   let bound = datom ~e:entity_id ~a:attr ~v:value () in
@@ -357,6 +340,22 @@ and entid_db db attr value =
   else
     None
 
+let entid = entid_db
+
+let lookup_refs_context : Lookup_refs.context =
+  { is_unique
+  ; entid = entid_db
+  ; entid_in_datoms
+  ; value_to_string = edn_string_of_value
+  }
+
+let unresolved_lookup_ref_message attr value =
+  Lookup_refs.unresolved_message lookup_refs_context attr value
+
+let unresolved_entity_ref_message = function
+  | Lookup_ref (attr, value) -> unresolved_lookup_ref_message attr value
+  | _ -> "lookup ref did not resolve"
+
 let lookup_ref_entity_id_db ?(strict_missing = false) db attr value =
   if not (is_unique db attr) then
     invalid_arg (Lookup_refs.non_unique_message lookup_refs_context attr value);
@@ -368,8 +367,6 @@ let lookup_ref_entity_id_db ?(strict_missing = false) db attr value =
     else
       None
 
-let entid = entid_db
-
 module Transact_impl = Transact
 
 let transact_resolve_context : Transact_impl.context =

impl/datascript.mli

@@ -198,8 +198,8 @@ end
 module Lookup_refs : sig
   type context =
     { is_unique : db -> attr -> bool
+    ; entid : db -> attr -> value -> entity_id option
     ; entid_in_datoms : db -> datom list -> attr -> value -> entity_id option
-    ; visible_datoms : db -> datom list
     ; value_to_string : value -> string
     }
 

impl/lookup_refs.ml

@@ -2,8 +2,8 @@ open Datascript_types
 
 type context =
   { is_unique : db -> attr -> bool
+  ; entid : db -> attr -> value -> entity_id option
   ; entid_in_datoms : db -> datom list -> attr -> value -> entity_id option
-  ; visible_datoms : db -> datom list
   ; value_to_string : value -> string
   }
 
@@ -17,16 +17,25 @@ let non_unique_message context attr value =
   ^ context.value_to_string value
   ^ "]"
 
-let entity_id_in_datoms ?(strict_missing = false) context db datoms attr value =
+let entity_id_of_resolution ?(strict_missing = false) context db attr value resolved =
   if not (context.is_unique db attr) then
     invalid_arg (non_unique_message context attr value);
-  match context.entid_in_datoms db datoms attr value with
+  match resolved with
   | Some entity_id -> Some entity_id
   | None ->
     if strict_missing then
       invalid_arg (unresolved_message context attr value)
     else
       None
 
+let entity_id_in_datoms ?strict_missing context db datoms attr value =
+  entity_id_of_resolution
+    ?strict_missing
+    context
+    db
+    attr
+    value
+    (context.entid_in_datoms db datoms attr value)
+
 let entity_id ?strict_missing context db attr value =
-  entity_id_in_datoms ?strict_missing context db (context.visible_datoms db) attr value
+  entity_id_of_resolution ?strict_missing context db attr value (context.entid db attr value)

impl/lookup_refs.mli

@@ -2,8 +2,8 @@ open Datascript_types
 
 type context =
   { is_unique : db -> attr -> bool
+  ; entid : db -> attr -> value -> entity_id option
   ; entid_in_datoms : db -> datom list -> attr -> value -> entity_id option
-  ; visible_datoms : db -> datom list
   ; value_to_string : value -> string
   }
 

impl/transact_datoms.ml

@@ -17,7 +17,6 @@ module Make (Context : sig
   val normalize_value : value -> value
   val validate_entity_id : int -> entity_id
   val max_allocatable_entity_id : int
-  val visible_datoms : db -> datom list
 end) = struct
   open Context
 
@@ -315,6 +314,4 @@ end) = struct
     else
       None
 
-  let entid db attr value = entid_in_datoms db (visible_datoms db) attr value
-
 end