fix: CommitmentTreeInsert under-costed in estimated-cost paths (issue #812)

grovedb-version/src/version/grovedb_versions.rs

@@ -128,6 +128,23 @@ pub struct GroveDBApplyBatchVersions {
     /// not cost: a non-empty indexed replacement that would be accepted
     /// blind on V1..V3 is refused on V4+.
     pub overwrite_indexed_cleanup_inspection: FeatureVersion,
+    /// Whether keyless append-only ops (`CommitmentTreeInsert`,
+    /// `MmrTreeAppend`, `BulkAppend`, `DenseTreeInsert`) reach the cost
+    /// dispatch in the estimated-cost batch structure.
+    ///
+    /// - `0` (V1..V3): keyless ops are silently skipped when building the
+    ///   batch structure, so in the estimated-cost paths the append
+    ///   contributes ZERO — the under-estimate behind issue #812's
+    ///   admission-control bypass. Preserved for replay: historical blocks
+    ///   were admitted under this estimate and must evaluate identically.
+    /// - `1` (V4+): the tree key is split off the op's path and the op is
+    ///   filed under a unique synthetic key, so every append reaches the
+    ///   cost arms and is charged individually.
+    ///
+    /// The apply path is unaffected on every version: preprocessing
+    /// rewrites keyless ops into keyed ops before the batch structure is
+    /// built.
+    pub keyless_op_cost_dispatch: FeatureVersion,
 }
 
 #[derive(Clone, Debug, Default)]
@@ -291,6 +308,16 @@ pub struct GroveDBOperationsAverageCaseVersions {
     pub add_average_case_get_raw_cost: FeatureVersion,
     pub add_average_case_get_raw_tree_cost: FeatureVersion,
     pub add_average_case_get_cost: FeatureVersion,
+    /// Cost model for the `CommitmentTreeInsert` estimation arm.
+    ///
+    /// - `0` (V1..V3): the legacy average-case constants (33 Sinsemilla
+    ///   hashes, 554-byte frontier, 1 blake3, frontier charged as replaced
+    ///   bytes). Preserved for replay of historical admission decisions.
+    /// - `1` (V4+): the depth-derived upper-bound model shared with the
+    ///   worst-case arm (`commitment_tree_insert_op_cost`), covering the
+    ///   full ommer cascade, dense-buffer recompute, and epoch compaction
+    ///   (issue #812).
+    pub average_case_commitment_tree_insert: FeatureVersion,
 }
 
 #[derive(Clone, Debug, Default)]
@@ -307,6 +334,17 @@ pub struct GroveDBOperationsWorstCaseVersions {
     pub add_worst_case_get_raw_tree_cost: FeatureVersion,
     pub add_worst_case_get_raw_cost: FeatureVersion,
     pub add_worst_case_get_cost: FeatureVersion,
+    /// Cost model for the `CommitmentTreeInsert` estimation arm.
+    ///
+    /// - `0` (V1..V3): the legacy flat model (64 Sinsemilla hashes and a
+    ///   1066-byte frontier, but only 3 seeks, 1 blake3, no dense-buffer
+    ///   recompute or epoch compaction, frontier charged as replaced
+    ///   bytes). Preserved for replay of historical admission decisions.
+    /// - `1` (V4+): the depth-derived upper-bound model shared with the
+    ///   average-case arm (`commitment_tree_insert_op_cost`), covering the
+    ///   full ommer cascade, dense-buffer recompute, and epoch compaction
+    ///   (issue #812).
+    pub worst_case_commitment_tree_insert: FeatureVersion,
 }
 
 #[derive(Clone, Debug, Default)]

grovedb-version/src/version/v1.rs

@@ -32,6 +32,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
             estimated_case_operations_for_batch: 0,
             delete_tree_cleanup_type_source: 0,
             overwrite_indexed_cleanup_inspection: 0,
+            keyless_op_cost_dispatch: 0,
         },
         element: GroveDBElementMethodVersions {
             delete: 0,
@@ -190,6 +191,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
                 add_average_case_get_raw_cost: 0,
                 add_average_case_get_raw_tree_cost: 0,
                 add_average_case_get_cost: 0,
+                average_case_commitment_tree_insert: 0,
             },
             worst_case: GroveDBOperationsWorstCaseVersions {
                 add_worst_case_get_merk_at_path: 0,
@@ -204,6 +206,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
                 add_worst_case_get_raw_tree_cost: 0,
                 add_worst_case_get_raw_cost: 0,
                 add_worst_case_get_cost: 0,
+                worst_case_commitment_tree_insert: 0,
             },
         },
         aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 },

grovedb-version/src/version/v2.rs

@@ -32,6 +32,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion {
             estimated_case_operations_for_batch: 0,
             delete_tree_cleanup_type_source: 0,
             overwrite_indexed_cleanup_inspection: 0,
+            keyless_op_cost_dispatch: 0,
         },
         element: GroveDBElementMethodVersions {
             delete: 0,
@@ -190,6 +191,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion {
                 add_average_case_get_raw_cost: 0,
                 add_average_case_get_raw_tree_cost: 0,
                 add_average_case_get_cost: 0,
+                average_case_commitment_tree_insert: 0,
             },
             worst_case: GroveDBOperationsWorstCaseVersions {
                 add_worst_case_get_merk_at_path: 0,
@@ -204,6 +206,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion {
                 add_worst_case_get_raw_tree_cost: 0,
                 add_worst_case_get_raw_cost: 0,
                 add_worst_case_get_cost: 0,
+                worst_case_commitment_tree_insert: 0,
             },
         },
         aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 },

grovedb-version/src/version/v3.rs

@@ -32,6 +32,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion {
             estimated_case_operations_for_batch: 0,
             delete_tree_cleanup_type_source: 0,
             overwrite_indexed_cleanup_inspection: 0,
+            keyless_op_cost_dispatch: 0,
         },
         element: GroveDBElementMethodVersions {
             delete: 0,
@@ -194,6 +195,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion {
                 add_average_case_get_raw_cost: 0,
                 add_average_case_get_raw_tree_cost: 0,
                 add_average_case_get_cost: 0,
+                average_case_commitment_tree_insert: 0,
             },
             worst_case: GroveDBOperationsWorstCaseVersions {
                 add_worst_case_get_merk_at_path: 0,
@@ -208,6 +210,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion {
                 add_worst_case_get_raw_tree_cost: 0,
                 add_worst_case_get_raw_cost: 0,
                 add_worst_case_get_cost: 0,
+                worst_case_commitment_tree_insert: 0,
             },
         },
         aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 },

grovedb-version/src/version/v4.rs

@@ -62,6 +62,27 @@
 //!   the version-2 `Query` wire encoding outright, so the slot's `0` value
 //!   is the in-process mirror of that fail-closed decode.
 //!
+//! - `apply_batch.keyless_op_cost_dispatch: 1` — keyless append-only ops
+//!   (`CommitmentTreeInsert`, `MmrTreeAppend`, `BulkAppend`,
+//!   `DenseTreeInsert`) reach the cost dispatch in the estimated-cost batch
+//!   structure, filed under unique synthetic keys so every append is
+//!   charged. V1..V3 silently skip them — the append estimates as free,
+//!   the under-estimate behind issue #812's admission-control bypass —
+//!   preserved so historical admission decisions replay identically. The
+//!   apply path is unaffected on every version (preprocessing rewrites
+//!   keyless ops before the batch structure is built).
+//!
+//! - `operations.average_case.average_case_commitment_tree_insert: 1` and
+//!   `operations.worst_case.worst_case_commitment_tree_insert: 1` — the
+//!   `CommitmentTreeInsert` estimation arms charge the depth-derived
+//!   upper-bound model (full ommer cascade, dense-buffer recompute, epoch
+//!   compaction, flags-bounded element load). V1..V3 keep the legacy
+//!   constants (average-case 33 Sinsemilla / 554-byte frontier; worst-case
+//!   64 / 1066 but no compaction), which are NOT upper bounds — preserved
+//!   for replay only. Gated because downstream the estimate is the
+//!   admission bound: raising it ungated would make already-committed
+//!   shield transitions re-validate as under-funded and brick sync.
+//!
 //! Note that `GroveVersion::latest()` resolves to this version, so anything
 //! defaulting to "latest" — tests, benchmarks, tools — exercises every gate
 //! listed above rather than V3 behaviour.
@@ -112,6 +133,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion {
             estimated_case_operations_for_batch: 0,
             delete_tree_cleanup_type_source: 1,
             overwrite_indexed_cleanup_inspection: 1,
+            keyless_op_cost_dispatch: 1,
         },
         element: GroveDBElementMethodVersions {
             delete: 0,
@@ -274,6 +296,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion {
                 add_average_case_get_raw_cost: 0,
                 add_average_case_get_raw_tree_cost: 0,
                 add_average_case_get_cost: 0,
+                average_case_commitment_tree_insert: 1,
             },
             worst_case: GroveDBOperationsWorstCaseVersions {
                 add_worst_case_get_merk_at_path: 0,
@@ -288,6 +311,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion {
                 add_worst_case_get_raw_tree_cost: 0,
                 add_worst_case_get_raw_cost: 0,
                 add_worst_case_get_cost: 0,
+                worst_case_commitment_tree_insert: 1,
             },
         },
         aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 },

grovedb/src/batch/batch_structure.rs

@@ -11,6 +11,10 @@ use grovedb_costs::{
 };
 use grovedb_merk::element::tree_type::ElementTreeTypeExtensions;
 #[cfg(feature = "minimal")]
+use grovedb_storage::worst_case_costs::WorstKeyLength;
+#[cfg(feature = "minimal")]
+use grovedb_version::version::GroveVersion;
+#[cfg(feature = "minimal")]
 use grovedb_visualize::{DebugByteVectors, DebugBytes};
 #[cfg(feature = "minimal")]
 use intmap::IntMap;
@@ -28,6 +32,35 @@ pub type OpsByPath = BTreeMap<KeyInfoPath, BTreeMap<KeyInfo, GroveOp>>;
 #[cfg(feature = "minimal")]
 pub type OpsByLevelPath = IntMap<u32, OpsByPath>;
 
+/// Build the synthetic key under which a keyless append-only op is filed.
+///
+/// The `MaxKeySize` variant sizes estimates with the real tree-key length,
+/// while the 8-byte big-endian op-index prefix in `unique_id` keeps several
+/// appends to the same tree from collapsing into a single `BTreeMap` entry
+/// (each append must be charged). [`keyless_op_tree_key`] is the inverse.
+#[cfg(feature = "minimal")]
+pub(in crate::batch) fn keyless_op_synthetic_key(op_index: usize, tree_key: &KeyInfo) -> KeyInfo {
+    let mut unique_id = (op_index as u64).to_be_bytes().to_vec();
+    unique_id.extend_from_slice(tree_key.as_slice());
+    KeyInfo::MaxKeySize {
+        unique_id,
+        max_size: tree_key.max_length(),
+    }
+}
+
+/// Recover the real tree-key bytes from a [`keyless_op_synthetic_key`].
+///
+/// Only meaningful for keys of ops that arrive keyless (the append-only tree
+/// ops) — a user-supplied `MaxKeySize` key on a keyed op has no such
+/// structure, so callers must check the op type before trusting the result.
+#[cfg(feature = "minimal")]
+pub(in crate::batch) fn keyless_op_tree_key(key: &KeyInfo) -> Option<&[u8]> {
+    match key {
+        KeyInfo::MaxKeySize { unique_id, .. } => unique_id.get(8..),
+        KeyInfo::KnownKey(_) => None,
+    }
+}
+
 /// Batch structure
 #[cfg(feature = "minimal")]
 pub(super) struct BatchStructure<C, F, SR> {
@@ -90,13 +123,15 @@ where
         update_element_flags_function: F,
         split_remove_bytes_function: SR,
         merk_tree_cache: C,
+        grove_version: &GroveVersion,
     ) -> CostResult<BatchStructure<C, F, SR>, Error> {
         Self::continue_from_ops(
             None,
             ops,
             update_element_flags_function,
             split_remove_bytes_function,
             merk_tree_cache,
+            grove_version,
         )
     }
 
@@ -107,7 +142,13 @@ where
         update_element_flags_function: F,
         split_remove_bytes_function: SR,
         mut merk_tree_cache: C,
+        grove_version: &GroveVersion,
     ) -> CostResult<BatchStructure<C, F, SR>, Error> {
+        let keyless_ops_reach_cost_dispatch = grove_version
+            .grovedb_versions
+            .apply_batch
+            .keyless_op_cost_dispatch
+            >= 1;
         let mut cost = OperationCost::default();
 
         let mut ops_by_level_paths: OpsByLevelPath = previous_ops.unwrap_or_default();
@@ -117,18 +158,46 @@ where
         // qualified paths meaning path + key
         let mut ops_by_qualified_paths: BTreeMap<Vec<Vec<u8>>, GroveOp> = BTreeMap::new();
 
-        for op in ops.into_iter() {
+        for (op_index, op) in ops.into_iter().enumerate() {
             let QualifiedGroveDbOp {
                 path: op_path,
                 key: op_key,
                 op: grove_op,
             } = op;
 
-            // Keyless ops (append-only tree ops) are handled by preprocessing.
-            // In estimated-cost paths they have no cost model yet — skip.
-            let key = match op_key {
-                Some(k) => k,
-                None => continue,
+            // Keyless ops (append-only tree ops: CommitmentTreeInsert,
+            // MmrTreeAppend, BulkAppend, DenseTreeInsert) carry the tree key
+            // as the last segment of `path`. In the apply path they are
+            // rewritten into keyed ops by preprocessing before reaching here;
+            // in the estimated-cost paths there is no preprocessing, so split
+            // the tree key off the path and let the op flow to the cost
+            // dispatch. Silently dropping them (as V1..V3 do below) makes
+            // every append estimate as free — see issue #812. The old skip
+            // is version-gated, not deleted: downstream the estimate is an
+            // admission bound, and historical blocks admitted under the old
+            // under-estimate must re-validate identically on replay.
+            //
+            // The synthetic key (see `keyless_op_synthetic_key`) sizes
+            // estimates with the real tree-key length while keeping one map
+            // entry per op, so each append is charged. If such an op ever
+            // reaches real execution, `execute_ops_on_path` rejects it with
+            // "should have been preprocessed" — a loud failure instead of a
+            // silent drop.
+            let (op_path, key, is_keyless_append) = match op_key {
+                Some(k) => (op_path, k, false),
+                None if !keyless_ops_reach_cost_dispatch => continue,
+                None => {
+                    let mut path = op_path;
+                    let Some(tree_key) = path.0.pop() else {
+                        return Err(Error::InvalidBatchOperation(
+                            "keyless append-only op must have the tree key as its path's last \
+                             segment",
+                        ))
+                        .wrap_with_cost(cost);
+                    };
+                    let key = keyless_op_synthetic_key(op_index, &tree_key);
+                    (path, key, true)
+                }
             };
 
             // Validate key length: Merk link encoding stores key length as a
@@ -141,10 +210,15 @@ where
                     .wrap_with_cost(cost);
             }
 
-            // Build qualified path (path + key) for reference lookups
-            let mut qualified_path = op_path.clone();
-            qualified_path.push(key.clone());
-            ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone());
+            // Build qualified path (path + key) for reference lookups.
+            // Keyless append ops are skipped: they are not elements a
+            // reference can target, and their synthetic keys must not
+            // shadow the tree element itself.
+            if !is_keyless_append {
+                let mut qualified_path = op_path.clone();
+                qualified_path.push(key.clone());
+                ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone());
+            }
 
             let op_cost = OperationCost::default();
             let op_result = match &grove_op {