diff --git a/book/src/drive/index-only-document-types.md b/book/src/drive/index-only-document-types.md index 7c5565e68b..8843cb6a2b 100644 --- a/book/src/drive/index-only-document-types.md +++ b/book/src/drive/index-only-document-types.md @@ -41,6 +41,20 @@ types unchanged: "the five most-liked posts in `#dash`" is an O(log n + k) read with an O(log n + k) proof, and Items count in count/ranked trees exactly as References do. +**`timeRange` buckets** compose too: a bucketed indexOnly index writes +one commitment entry per containing bucket under the grid-qualified +level, exactly as stored types do — the walkers' bucket fan-out, the +probes' path derivation (`entry_keys_for_raw`, shared so probe and write +paths cannot drift), and the `IN_TIME_RANGE` count aggregates are all the +same machinery ("how many likes under `#dash` this hour"). The source can +only be `$createdAt` (the prefix rule admits no other timestamp), which +`required` must carry, so a delete's values reproduce the exact bucket +set. A bucketed index involves `$createdAt` and therefore never serves as +the proof index; and document synthesis over bucketed entries is refused +with guidance — the bucket level carries bucket-start granularity, not +the document's timestamp, and the raw entries are served by the type's +non-bucketed indexes. + The **sum axes** compose the same way: a `summable: ""` index stores `ItemWithSumItem(, )` terminals — the same commitment payload, plus the summed property's value — so entries @@ -85,7 +99,7 @@ aggregate keywords follow: | terminal is `$ownerId` or a single-id refersTo property | the member key must alone be a referable entity id (`identityPublicKey` is compound and rejected) | | indexed `$createdAt` requires `$createdAt` in `required` | creation only assigns timestamps for required system times | | `documentsMutable: false`, no transfers/trading/history/transient | no stored row, no revision | -| non-unique, non-contested, `nullSearchable` default, no `timeRange` | v1 scope; buckets are a follow-up | +| non-unique, non-contested, `nullSearchable` default | v1 scope | `indexOnly` and the index set (terminals included) are immutable across contract updates — a later-added index could never be backfilled. diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs index e14daff5b9..34ba0e9236 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs @@ -2040,13 +2040,20 @@ pub(super) fn apply_index_only( index_name, name, ))); } - if index.time_range.is_some() { - return Err(structure_error(format!( - "index \"{}\" on indexOnly document type \"{}\" cannot declare a timeRange: \ - bucketed indexOnly indexes are not yet supported", - index_name, name, - ))); - } + // `timeRange` is admitted: a bucketed indexOnly index writes one + // entry per containing bucket, exactly as stored types do (the + // walkers' bucket fan-out is shared). No indexOnly-specific + // source rule is needed — the transform's source must be a + // system timestamp (the shared timeRange rules), it must be the + // index's first property, and the prefix rule below admits only + // `$ownerId` and `$createdAt` as system properties, which pins + // the source to `$createdAt` (the only timestamp an immutable, + // create-once document carries). Delete-by-values stays + // deterministic: `$createdAt` is forced into `required` (rule + // below), so the carried value reproduces the exact bucket set + // the create wrote. A bucketed index involves `$createdAt` and + // therefore never counts as the required `$createdAt`-free + // proof index. // The sum axes (summable / rangeSummable / rankedSummable / // rankedAverageable / the averageable sugar) are admitted: a // summable index's terminal entry is an diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/index_only_tests.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/index_only_tests.rs index 25e51e2286..492c60cc2a 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/index_only_tests.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/index_only_tests.rs @@ -344,6 +344,100 @@ fn rejects_summable_naming_non_integer_property() { ); } +#[test] +fn accepts_time_range_bucketed_index() { + // A bucketed indexOnly index writes one entry per containing bucket, + // sharing the stored types' walker fan-out. `$createdAt` must be the + // transform source (the prefix rule admits no other system timestamp) + // and must be required (shared timeRange rule). + let mut schema = likes_schema(); + schema + .set_value( + "required", + platform_value!(["hashtag", "postId", "$createdAt"]), + ) + .expect("required applies"); + schema + .get_mut("indices") + .expect("indices accessible") + .expect("indices present") + .as_array_mut() + .expect("indices is an array") + .push(platform_value!({ + "name": "byHourHashtag", + "properties": [{ "$createdAt": "asc" }, { "hashtag": "asc" }], + "terminal": "$ownerId", + "timeRange": { "on": "$createdAt", "range": 3600u64, "step": 900u64 }, + "countable": true, + "rangeCountable": true + })); + let document_type = parse_with(schema, PlatformVersion::latest(), false) + .expect("bucketed indexOnly index admitted"); + let bucketed = document_type + .indices + .values() + .find(|index| index.time_range.is_some()) + .expect("the bucketed index parsed"); + assert_eq!(bucketed.time_range.as_ref().unwrap().overlap_factor(), 4); +} + +#[test] +fn rejects_time_range_bucketed_index_without_required_created_at() { + // Same shape, but $createdAt missing from `required` — the indexOnly + // indexed-$createdAt rule fires (creation only assigns the timestamp + // for required system times, and an entry cannot represent a missing + // value). + let mut schema = likes_schema(); + schema + .get_mut("indices") + .expect("indices accessible") + .expect("indices present") + .as_array_mut() + .expect("indices is an array") + .push(platform_value!({ + "name": "byHourHashtag", + "properties": [{ "$createdAt": "asc" }, { "hashtag": "asc" }], + "terminal": "$ownerId", + "timeRange": { "on": "$createdAt", "range": 3600u64, "step": 900u64 } + })); + expect_structure_error( + parse_with(schema, PlatformVersion::latest(), false), + "must be listed in `required`", + ); +} + +#[test] +fn rejects_only_bucketed_indexes() { + // A bucketed index involves $createdAt, so a doctype whose every index + // is bucketed has no $createdAt-free proof index and stays refused. + let mut schema = likes_schema(); + schema + .set_value( + "required", + platform_value!(["hashtag", "postId", "$createdAt"]), + ) + .expect("required applies"); + schema + .set_value( + "indices", + platform_value!([{ + "name": "byHourHashtagPost", + "properties": [ + { "$createdAt": "asc" }, + { "hashtag": "asc" }, + { "postId": "asc" } + ], + "terminal": "$ownerId", + "timeRange": { "on": "$createdAt", "range": 3600u64, "step": 900u64 } + }]), + ) + .expect("indices apply"); + expect_structure_error( + parse_with(schema, PlatformVersion::latest(), false), + "does not involve $createdAt", + ); +} + #[test] fn rejects_terminal_repeating_an_index_property() { // byLiker's prefix is [$ownerId]; making $ownerId its terminal too diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/index_only.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/index_only.rs index 01843525a3..753dc19892 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/index_only.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/index_only.rs @@ -1156,6 +1156,197 @@ mod index_only_executed_proof_tests { ); } + /// A signed `beat` create — the bucketed-type counterpart of + /// `signed_mark_create`. The `beat` doctype's `byHourHashtag` index + /// buckets `$createdAt` on a 3600s/900s grid, so a create fans out one + /// entry per containing bucket, while the plain `byHashtag` index is + /// the `$createdAt`-free proof index the executed proofs run against. + async fn signed_beat_create( + contract: &DataContract, + owner: Identifier, + hashtag: &str, + nonce: u64, + key: &dpp::identity::IdentityPublicKey, + signer: &SimpleSigner, + rng: &mut StdRng, + platform_version: &PlatformVersion, + ) -> (StateTransition, Document) { + use dpp::document::DocumentV0Setters; + let beat_type = contract + .document_type_for_name("beat") + .expect("beat doctype exists"); + let entropy = Bytes32::random_with_rng(rng); + let mut beat = beat_type + .random_document_with_identifier_and_entropy( + rng, + owner, + entropy, + DocumentFieldFillType::FillIfNotRequired, + DocumentFieldFillSize::AnyDocumentFillSize, + platform_version, + ) + .expect("expected a random beat"); + beat.set("hashtag", hashtag.into()); + // Consensus assigns `$createdAt` from the block time at create + // (BlockInfo::default() in this suite), and the delete-by-values + // must carry that committed value to reproduce the bucket set. + beat.set_created_at(Some(0)); + let create = BatchTransition::new_document_creation_transition_from_document( + beat.clone(), + beat_type, + entropy.0, + key, + nonce, + 0, + None, + signer, + platform_version, + None, + ) + .await + .expect("expected the create transition"); + (create, beat) + } + + /// The bucketed lifecycle through the pipeline: a `beat` create fans + /// out per bucket and executes, its executed proof verifies against + /// the non-bucketed proof index, a duplicate collides on the probes, + /// and the delete removes every bucket entry and proves absence. + #[tokio::test] + async fn test_bucketed_beat_lifecycle_and_executed_proofs() { + let platform_version = PlatformVersion::latest(); + let mut platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let platform_state = platform.state.load(); + let mut rng = StdRng::seed_from_u64(9099); + + let (alice, alice_signer, alice_key) = + setup_identity(&mut platform, 958, dash_to_credits!(1.0)); + let contract = register_likes(&platform, alice.id(), platform_version); + let contract_arc = Arc::new(contract.clone()); + + let (create, beat) = signed_beat_create( + &contract, + alice.id(), + "dash", + 2, + &alice_key, + &alice_signer, + &mut rng, + platform_version, + ) + .await; + let result = process_and_commit(&platform, &platform_state, &create, platform_version); + assert_eq!( + result.valid_count(), + 1, + "the beat create must execute: {:?}", + result.execution_results() + ); + + // ── executed-create proof (via the non-bucketed proof index) ─── + let proof = platform + .drive + .prove_state_transition(&create, None, platform_version) + .expect("expected to prove the executed create") + .into_data() + .expect("expected proof bytes"); + let lookup = |_id: &dpp::identifier::Identifier| Ok(Some(Arc::clone(&contract_arc))); + let (root_hash, outcome) = Drive::verify_state_transition_was_executed_with_proof( + &create, + &BlockInfo::default(), + proof.as_slice(), + &lookup, + platform_version, + ) + .expect("expected the executed bucketed create proof to verify"); + assert_ne!(root_hash, [0u8; 32]); + let StateTransitionProofResult::VerifiedDocuments(documents) = outcome.into_result() else { + panic!("expected verified documents"); + }; + assert!( + documents + .into_iter() + .next() + .expect("one document") + .1 + .is_some(), + "the created beat is present" + ); + + // ── a duplicate collides on the probes ───────────────────────── + let (duplicate, _) = signed_beat_create( + &contract, + alice.id(), + "dash", + 3, + &alice_key, + &alice_signer, + &mut rng, + platform_version, + ) + .await; + let result = process_and_commit(&platform, &platform_state, &duplicate, platform_version); + assert_eq!(result.invalid_paid_count(), 1); + assert_matches!( + result.execution_results().as_slice(), + [StateTransitionExecutionResult::PaidConsensusError { + error: ConsensusError::StateError(StateError::DuplicateUniqueIndexError(_)), + .. + }], + "a second beat by the same owner under the same hashtag must collide" + ); + + // ── delete removes every bucket entry and proves absence ─────── + let beat_type = contract + .document_type_for_name("beat") + .expect("beat doctype exists"); + let delete = BatchTransition::new_document_deletion_transition_from_document( + beat, + beat_type, + &alice_key, + 4, + 0, + None, + &alice_signer, + platform_version, + None, + ) + .await + .expect("expected the delete transition"); + let result = process_and_commit(&platform, &platform_state, &delete, platform_version); + assert_eq!( + result.valid_count(), + 1, + "the beat delete must execute: {:?}", + result.execution_results() + ); + + let proof = platform + .drive + .prove_state_transition(&delete, None, platform_version) + .expect("expected to prove the executed delete") + .into_data() + .expect("expected proof bytes"); + let (root_hash, outcome) = Drive::verify_state_transition_was_executed_with_proof( + &delete, + &BlockInfo::default(), + proof.as_slice(), + &lookup, + platform_version, + ) + .expect("expected the executed bucketed delete proof to verify"); + assert_ne!(root_hash, [0u8; 32]); + let StateTransitionProofResult::VerifiedDocuments(documents) = outcome.into_result() else { + panic!("expected verified documents"); + }; + assert!( + documents.into_iter().next().expect("one entry").1.is_none(), + "the deleted beat must be proven absent" + ); + } + /// A signed `tip` create for the given amount — the summable-type /// counterpart of `signed_mark_create`. The `tip` doctype's proof index /// (`byPost`) is summable, so its entries are diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs index 7b1a77ea7e..9bebf3f070 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs @@ -15,9 +15,9 @@ //! | `byPost` | `[postId]` | `$ownerId` | countable + range + ranked | //! | `byLiker` | `[$ownerId]` | `postId` | none | //! -//! plus the `tip` doctype (sum axes — see the sum-axes section at the -//! bottom) and the `mark` doctype (two single-property indexes, the -//! splice-prone shape). +//! plus the `tip` doctype (sum axes), the `beat` doctype (timeRange +//! buckets) — see their sections at the bottom — and the `mark` doctype +//! (two single-property indexes, the splice-prone shape). //! //! Three layers are pinned. The *registration shape*: no `[0]` primary-key //! tree is created for the doctype, while the top-level property-name trees @@ -1935,3 +1935,481 @@ fn tip_estimated_fees_upper_bound_actual_fees() { assert_grovedb_is_consistent(&drive); } + +// --------------------------------------------------------------------------- +// timeRange buckets: bucketed entries (the `beat` doctype) +// --------------------------------------------------------------------------- +// +// The `beat` doctype pins the bucketed storage mode: `byHourHashtag` — +// `[$createdAt, hashtag] → $ownerId` with `timeRange { on: $createdAt, +// range: 3600, step: 900 }` (overlap factor 4) plus the count axes — writes +// one commitment entry per containing bucket under the grid-qualified +// level, while `byHashtag` — `[hashtag] → $ownerId`, no transform — is the +// $createdAt-free proof index. Bucketed entries serve the count aggregate +// surfaces only; document synthesis over them is refused. + +const BEAT_DOCTYPE: &str = "beat"; + +/// A real timestamp and the four bucket starts (ms) containing it on the +/// 3600s/900s grid — pinned by arithmetic, independent of +/// `containing_buckets`. +const BEAT_T_MS: u64 = 1_700_000_000_000; +const BEAT_BUCKET_STARTS_MS: [u64; 4] = [ + 1_699_996_500_000, + 1_699_997_400_000, + 1_699_998_300_000, + 1_699_999_200_000, +]; +/// `TimeRangeTransform::storage_key("$createdAt")` for the fixture grid. +const BEAT_GRID_LEVEL: &[u8] = b"$createdAt#3600#900"; + +fn beat_doctype_path(contract: &DataContract) -> Vec> { + vec![ + vec![crate::drive::RootTree::DataContractDocuments as u8], + contract.id().as_bytes().to_vec(), + vec![1], + BEAT_DOCTYPE.as_bytes().to_vec(), + ] +} + +fn encode_timestamp(ms: u64) -> Vec { + dpp::data_contract::document_type::DocumentPropertyType::encode_date_timestamp(ms) +} + +/// A beat under `hashtag` by `owner` created at `created_at_ms`. +fn build_beat( + contract: &DataContract, + hashtag: &str, + owner: [u8; 32], + created_at_ms: u64, + seed: u64, +) -> Document { + use dpp::document::DocumentV0Setters; + let pv = platform_version(); + let document_type = contract + .document_type_for_name(BEAT_DOCTYPE) + .expect("beat doctype exists"); + let mut doc = document_type + .random_document(Some(seed), pv) + .expect("random document"); + let mut props = std::collections::BTreeMap::new(); + props.insert("hashtag".to_string(), Value::Text(hashtag.to_string())); + doc.set_properties(props); + doc.set_owner_id(Identifier::from(owner)); + doc.set_created_at(Some(created_at_ms)); + doc +} + +fn insert_beat( + drive: &Drive, + contract: &DataContract, + doc: &Document, + apply: bool, +) -> Result { + let pv = platform_version(); + let document_type = contract + .document_type_for_name(BEAT_DOCTYPE) + .expect("beat doctype exists"); + drive.add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((doc, None)), + owner_id: None, + }, + contract, + document_type, + }, + false, + BlockInfo::default(), + apply, + None, + pv, + None, + ) +} + +fn delete_beat( + drive: &Drive, + contract: &DataContract, + doc: Document, + apply: bool, +) -> Result { + let pv = platform_version(); + let document_type = contract + .document_type_for_name(BEAT_DOCTYPE) + .expect("beat doctype exists"); + drive.delete_index_only_document_for_contract( + doc, + contract, + document_type, + BlockInfo::default(), + apply, + None, + pv, + None, + ) +} + +/// One commitment entry per containing bucket, under the grid-qualified +/// level key, at bucket starts pinned by arithmetic — plus the plain +/// `byHashtag` entry on the same row. +#[test] +fn beat_insert_writes_one_entry_per_containing_bucket() { + let (drive, contract) = setup_likes(); + let beat = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS, 1); + insert_beat(&drive, &contract, &beat, true).expect("insert beat"); + + let document_type = contract + .document_type_for_name(BEAT_DOCTYPE) + .expect("beat doctype exists"); + let expected_commitment = + crate::drive::document::index_only_row_commitment(&beat, document_type, platform_version()) + .expect("commitment computes"); + + for bucket_start in BEAT_BUCKET_STARTS_MS { + let mut bucket_entry = beat_doctype_path(&contract); + bucket_entry.extend([ + BEAT_GRID_LEVEL.to_vec(), + encode_timestamp(bucket_start), + b"hashtag".to_vec(), + b"dash".to_vec(), + vec![0], + ]); + match read_grove_element(&drive, &bucket_entry, &OWNER_1) { + Some(Element::Item(data, _)) => assert_eq!( + data, + expected_commitment.to_vec(), + "bucket {bucket_start}: every bucket's entry carries the row commitment" + ), + other => panic!("expected the entry in bucket {bucket_start}, got {other:?}"), + } + } + + // No entry exists one step below the earliest containing bucket. + let mut outside = beat_doctype_path(&contract); + outside.extend([ + BEAT_GRID_LEVEL.to_vec(), + encode_timestamp(BEAT_BUCKET_STARTS_MS[0] - 900_000), + b"hashtag".to_vec(), + b"dash".to_vec(), + vec![0], + ]); + assert!( + read_grove_element(&drive, &outside, &OWNER_1).is_none(), + "no entry outside the containing buckets" + ); + + // The plain proof index holds the same row's commitment. + let mut by_hashtag = beat_doctype_path(&contract); + by_hashtag.extend([b"hashtag".to_vec(), b"dash".to_vec(), vec![0]]); + match read_grove_element(&drive, &by_hashtag, &OWNER_1) { + Some(Element::Item(data, _)) => assert_eq!(data, expected_commitment.to_vec()), + other => panic!("expected the byHashtag entry, got {other:?}"), + } + + assert_grovedb_is_consistent(&drive); +} + +/// The trending surface: a resolved `IN_TIME_RANGE` count over one bucket +/// counts the bucket's entries per hashtag, unproved and proved agreeing. +#[test] +fn beat_bucket_counts_serve_trending() { + use crate::query::drive_document_count_query::{ + CountMode, DocumentCountRequest, DocumentCountResponse, DriveDocumentCountQuery, + }; + use crate::query::ResolvedTimeRange; + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::document_type::TimeRangeTransform; + + let (drive, contract) = setup_likes(); + // Two dash beats and one btc beat, all at BEAT_T_MS. + for (owner, hashtag, seed) in [ + (OWNER_1, "dash", 1u64), + (OWNER_2, "dash", 2), + (OWNER_3, "btc", 3), + ] { + let beat = build_beat(&contract, hashtag, owner, BEAT_T_MS, seed); + insert_beat(&drive, &contract, &beat, true).expect("insert beat"); + } + + let document_type = contract + .document_type_for_name(BEAT_DOCTYPE) + .expect("beat doctype exists"); + let transform = TimeRangeTransform { + source: "$createdAt".to_string(), + range_seconds: 3600, + step_seconds: 900, + phase_seconds: 0, + }; + let resolved = vec![ResolvedTimeRange { + transform: transform.clone(), + }]; + let bucket = BEAT_BUCKET_STARTS_MS[3]; + let count_in_bucket = |hashtag: &str, prove: bool| { + let drive_config = crate::config::DriveConfig::default(); + let request = DocumentCountRequest { + contract: &contract, + document_type, + where_clauses: vec![ + crate::query::WhereClause { + field: "$createdAt".to_string(), + operator: crate::query::WhereOperator::Equal, + value: Value::U64(bucket), + }, + crate::query::WhereClause { + field: "hashtag".to_string(), + operator: crate::query::WhereOperator::Equal, + value: Value::Text(hashtag.to_string()), + }, + ], + resolved_time_ranges: resolved.clone(), + order_clauses: Vec::new(), + mode: CountMode::Aggregate, + limit: None, + prove, + drive_config: &drive_config, + }; + drive + .execute_document_count_request(request, None, platform_version()) + .expect("the bucketed count must execute") + }; + + match count_in_bucket("dash", false) { + DocumentCountResponse::Aggregate(count) => { + assert_eq!(count, 2, "two dash beats in the bucket") + } + other => panic!("expected an aggregate count, got {other:?}"), + } + match count_in_bucket("btc", false) { + DocumentCountResponse::Aggregate(count) => { + assert_eq!(count, 1, "one btc beat in the bucket") + } + other => panic!("expected an aggregate count, got {other:?}"), + } + + // Proved parity for the dash count. + let proof_bytes = match count_in_bucket("dash", true) { + DocumentCountResponse::Proof(bytes) => bytes, + other => panic!("expected proof bytes, got {other:?}"), + }; + let where_clauses = vec![ + crate::query::WhereClause { + field: "$createdAt".to_string(), + operator: crate::query::WhereOperator::Equal, + value: Value::U64(bucket), + }, + crate::query::WhereClause { + field: "hashtag".to_string(), + operator: crate::query::WhereOperator::Equal, + value: Value::Text("dash".to_string()), + }, + ]; + let index = DriveDocumentCountQuery::find_countable_index_for_where_clauses( + document_type.indexes(), + &where_clauses, + &resolved, + ) + .expect("byHourHashtag covers the resolved bucket count"); + let count_query = DriveDocumentCountQuery { + document_type, + contract_id: contract.id().to_buffer(), + document_type_name: BEAT_DOCTYPE.to_string(), + index, + where_clauses, + }; + let verifier_path_query = count_query + .point_lookup_count_path_query(platform_version()) + .expect("verifier path query builds"); + let (_root_hash, proved) = grovedb::GroveDb::verify_query( + &proof_bytes, + &verifier_path_query, + &platform_version().drive.grove_version, + ) + .expect("the count proof must verify"); + let proved_count: u64 = proved + .into_iter() + .filter_map(|(_path, _key, element)| element) + .map(|element| element.count_value_or_default()) + .sum(); + assert_eq!(proved_count, 2, "proved and unproved counts must agree"); + + assert_grovedb_is_consistent(&drive); +} + +/// Document synthesis over the bucketed index is refused with guidance — +/// the bucket level carries bucket-start granularity, not the document's +/// timestamp. +#[test] +fn beat_synthesis_over_bucketed_index_is_refused() { + use crate::query::{DriveDocumentQuery, InternalClauses, ResolvedTimeRange}; + use dpp::data_contract::document_type::TimeRangeTransform; + + let (drive, contract) = setup_likes(); + let beat = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS, 1); + insert_beat(&drive, &contract, &beat, true).expect("insert beat"); + + let document_type = contract + .document_type_for_name(BEAT_DOCTYPE) + .expect("beat doctype exists"); + let query = DriveDocumentQuery { + contract: &contract, + document_type, + internal_clauses: InternalClauses::extract_from_clauses( + vec![ + crate::query::WhereClause { + field: "$createdAt".to_string(), + operator: crate::query::WhereOperator::Equal, + value: Value::U64(BEAT_BUCKET_STARTS_MS[3]), + }, + crate::query::WhereClause { + field: "hashtag".to_string(), + operator: crate::query::WhereOperator::Equal, + value: Value::Text("dash".to_string()), + }, + ], + platform_version(), + ) + .expect("clauses extract"), + offset: None, + limit: Some(10), + order_by: Default::default(), + start_at: None, + start_at_included: false, + block_time_ms: None, + resolved_time_ranges: vec![ResolvedTimeRange { + transform: TimeRangeTransform { + source: "$createdAt".to_string(), + range_seconds: 3600, + step_seconds: 900, + phase_seconds: 0, + }, + }], + }; + let error = drive + .query_documents(query, None, false, None, None) + .expect_err("document synthesis over a bucketed indexOnly index must be refused"); + assert!( + error + .to_string() + .contains("IN_TIME_RANGE document queries are not supported on an indexOnly type"), + "expected the bucketed-synthesis refusal, got: {error}" + ); +} + +/// Delete-by-values removes every bucket entry (the carried `$createdAt` +/// reproduces the exact bucket set), drained groups prune, and a falsified +/// timestamp addressing the same buckets dies on the commitment probe. +#[test] +fn beat_delete_removes_every_bucket_entry() { + let (drive, contract) = setup_likes(); + let beat = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS, 1); + insert_beat(&drive, &contract, &beat, true).expect("insert beat"); + + // A tuple whose timestamp differs by 1ms lands in the SAME buckets — + // its entries exist — but the commitment binds the exact timestamp. + let falsified = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS + 1, 2); + let error = delete_beat(&drive, &contract, falsified, true) + .expect_err("a falsified timestamp must be refused"); + assert!( + matches!( + error, + crate::error::Error::Drive( + crate::error::drive::DriveError::DeletingDocumentThatDoesNotExist(_) + ) + ), + "expected the row-integrity refusal, got: {error}" + ); + + delete_beat(&drive, &contract, beat, true).expect("delete beat"); + + let mut grid_tree = beat_doctype_path(&contract); + grid_tree.push(BEAT_GRID_LEVEL.to_vec()); + for bucket_start in BEAT_BUCKET_STARTS_MS { + assert!( + read_grove_element(&drive, &grid_tree, &encode_timestamp(bucket_start)).is_none(), + "bucket {bucket_start} must be pruned after the last entry leaves" + ); + } + let mut hashtag_tree = beat_doctype_path(&contract); + hashtag_tree.push(b"hashtag".to_vec()); + assert!( + read_grove_element(&drive, &hashtag_tree, b"dash").is_none(), + "the plain index's group prunes too" + ); + + assert_grovedb_is_consistent(&drive); +} + +/// The estimation twin fans out identically: dry-run fees upper-bound +/// applied fees across the bucket fan-out. +#[test] +fn beat_estimated_fees_upper_bound_actual_fees() { + let (drive, contract) = setup_likes(); + let beat = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS, 1); + + let estimated_insert = + insert_beat(&drive, &contract, &beat, false).expect("estimated insert must work"); + let actual_insert = insert_beat(&drive, &contract, &beat, true).expect("actual insert"); + assert!( + estimated_insert.storage_fee >= actual_insert.storage_fee, + "estimated insert storage fee {} must upper-bound actual {}", + estimated_insert.storage_fee, + actual_insert.storage_fee + ); + + let estimated_delete = + delete_beat(&drive, &contract, beat.clone(), false).expect("estimated delete must work"); + let actual_delete = delete_beat(&drive, &contract, beat, true).expect("actual delete"); + assert!(estimated_delete.processing_fee > 0); + assert!(actual_delete.processing_fee > 0); + + assert_grovedb_is_consistent(&drive); +} + +/// Duplicate detection probes the bucketed index too: re-creating the same +/// beat is refused, and a beat one step-width later shares three of four +/// buckets yet is a distinct row. +#[test] +fn beat_duplicate_and_overlapping_rows() { + let (drive, contract) = setup_likes(); + let beat = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS, 1); + insert_beat(&drive, &contract, &beat, true).expect("first insert"); + + let error = + insert_beat(&drive, &contract, &beat, true).expect_err("the identical beat must refuse"); + assert!( + matches!( + error, + crate::error::Error::Drive(crate::error::drive::DriveError::CorruptedContractIndexes( + _ + )) + ), + "expected the CorruptedContractIndexes backstop, got: {error}" + ); + + // Same owner and hashtag one step later is a different value tuple, + // but it still collides: its bucketed entries share three of four + // bucket positions with the first beat (same member key), and the + // plain byHashtag entry collides outright — ANY colliding entry + // refuses the create, which is the structural-uniqueness rule + // (here: one beat per (hashtag, owner), and per shared bucket). + let overlapping = build_beat(&contract, "dash", OWNER_1, BEAT_T_MS + 900_000, 2); + let error = insert_beat(&drive, &contract, &overlapping, true) + .expect_err("an overlapping beat by the same owner must refuse"); + assert!( + matches!( + error, + crate::error::Error::Drive(crate::error::drive::DriveError::CorruptedContractIndexes( + _ + )) + ), + "expected the duplicate refusal on the shared buckets, got: {error}" + ); + + // A different owner in the same buckets is a different member key. + let other_owner = build_beat(&contract, "dash", OWNER_2, BEAT_T_MS, 3); + insert_beat(&drive, &contract, &other_owner, true) + .expect("another owner may beat in the same buckets"); + + assert_grovedb_is_consistent(&drive); +} diff --git a/packages/rs-drive/src/drive/document/index_level_tree_types.rs b/packages/rs-drive/src/drive/document/index_level_tree_types.rs index 32cb1f6eea..dc01146528 100644 --- a/packages/rs-drive/src/drive/document/index_level_tree_types.rs +++ b/packages/rs-drive/src/drive/document/index_level_tree_types.rs @@ -229,6 +229,24 @@ pub(crate) fn terminal_member_tree_type(index_type: &IndexLevelTypeInfo) -> Tree } } +/// The value-tree type an index's terminal level lives inside, derived +/// from the level info's four terminator flags — the tree the `0` member +/// bucket is inserted INTO. Used by the indexOnly terminal branch's +/// stateless apply type so estimation accounts the parent's aggregate +/// bytes (a `NormalTree` claim under-counts a count-bearing value tree's +/// per-child propagation, and the bucket fan-out multiplies the gap). +/// Continuations only demote provable variants, whose stateless costs +/// match their demoted forms at this call site, so `false` is passed. +pub(crate) fn terminal_value_tree_type(index_type: &IndexLevelTypeInfo) -> TreeType { + derive_value_tree_type( + index_type.countable.is_countable(), + index_type.range_countable, + index_type.summable.is_some(), + index_type.range_summable, + false, + ) +} + /// Pure derivation of the value-tree type over the level's four /// terminator flags plus whether continuations hang beneath it. Split /// out so the full input space is unit-testable without constructing diff --git a/packages/rs-drive/src/drive/document/index_only.rs b/packages/rs-drive/src/drive/document/index_only.rs index cde4626337..dbbafb1691 100644 --- a/packages/rs-drive/src/drive/document/index_only.rs +++ b/packages/rs-drive/src/drive/document/index_only.rs @@ -32,6 +32,11 @@ use dpp::identifier::Identifier; use dpp::version::PlatformVersion; use grovedb::TransactionArg; +/// The entry paths a document's values produce under one index — one per +/// containing bucket for a time-range index, exactly one otherwise — +/// paired with the shared member key (the terminal property's value). +pub type IndexOnlyEntryPathsAndKey = (Vec>>, Vec); + impl Drive { /// Reconstruct the document an indexOnly delete's entries were written /// from: the carried values plus the owner, with `$createdAt` moved @@ -83,16 +88,29 @@ impl Drive { ))) } - /// The grove path and member key of `document`'s entry under `index`: - /// `[DataContractDocuments, contract_id, 1, doctype, (, )*, 0]` with the terminal property's value as the key. - pub fn index_only_entry_path_and_key( + /// The grove paths and member key of `document`'s entries under + /// `index`: each path is `[DataContractDocuments, contract_id, 1, + /// doctype, (, )*, 0]` with the terminal + /// property's value as the member key. + /// + /// A plain index produces exactly one path. A time-range (bucketed) + /// index produces one path per bucket containing the document's + /// timestamp: the first level's segment is the grid-qualified + /// [`level_key`](Index::level_key) and its value keys come from + /// [`TimeRangeTransform::entry_keys_for_raw`] — the same derivation + /// the index walkers write with, so probe and write paths cannot + /// drift (including the edge rules: a pre-origin timestamp produces + /// NO entries, so it produces no probe paths either). + /// + /// [`TimeRangeTransform::entry_keys_for_raw`]: + /// dpp::data_contract::document_type::TimeRangeTransform::entry_keys_for_raw + pub fn index_only_entry_paths_and_key( contract_id: Identifier, document_type: DocumentTypeRef, index: &Index, document: &Document, platform_version: &PlatformVersion, - ) -> Result<(Vec>, Vec), Error> { + ) -> Result { let owner_id = Some(document.owner_id().to_buffer()); let raw_value_for = |property_name: &str| -> Result, Error> { @@ -109,28 +127,52 @@ impl Drive { ))) }; - let mut path: Vec> = Vec::with_capacity(5 + index.properties.len() * 2); - path.push(vec![crate::drive::RootTree::DataContractDocuments as u8]); - path.push(contract_id.to_vec()); - path.push(vec![1]); - path.push(document_type.name().as_bytes().to_vec()); - for property in index.properties.iter() { - path.push(property.name.as_bytes().to_vec()); - path.push(raw_value_for(&property.name)?); + let prefix: Vec> = vec![ + vec![crate::drive::RootTree::DataContractDocuments as u8], + contract_id.to_vec(), + vec![1], + document_type.name().as_bytes().to_vec(), + ]; + let mut paths: Vec>> = vec![prefix]; + for (position, property) in index.properties.iter().enumerate() { + let level_key = index.level_key(position, &property.name); + let raw = raw_value_for(&property.name)?; + // Only a time-range index's first property fans out; every + // other level extends each path with its single value key. + let value_keys: Vec> = match index.time_range.as_ref() { + Some(transform) if position == 0 => transform.entry_keys_for_raw(&raw), + _ => vec![raw], + }; + paths = paths + .into_iter() + .flat_map(|base| { + value_keys + .iter() + .map(|value_key| { + let mut path = base.clone(); + path.push(level_key.as_bytes().to_vec()); + path.push(value_key.clone()); + path + }) + .collect::>() + }) + .collect(); + } + for path in paths.iter_mut() { + path.push(vec![0]); } - path.push(vec![0]); let terminal = index .terminal .as_deref() .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( - "index_only_entry_path_and_key requires an indexOnly index (terminal is \ + "index_only_entry_paths_and_key requires an indexOnly index (terminal is \ always Some there after parse normalization)", )))?; let member_key = raw_value_for(terminal)?; - Ok((path, member_key)) + Ok((paths, member_key)) } /// Whether `document`'s entry under `index` exists AND carries @@ -154,36 +196,54 @@ impl Drive { drive_operations: &mut Vec, platform_version: &PlatformVersion, ) -> Result { - let (path, member_key) = Self::index_only_entry_path_and_key( + let (paths, member_key) = Self::index_only_entry_paths_and_key( contract_id, document_type, index, document, platform_version, )?; - let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); - let element = self.grove_get_raw_optional( - path_refs.as_slice().into(), - member_key.as_slice(), - DirectQueryType::StatefulDirectQuery, - transaction, - drive_operations, - &platform_version.drive, - )?; - Ok(match element { - Some(grovedb::Element::Item(payload, _)) => payload == expected_commitment.as_slice(), - // Summable indexes store `ItemWithSumItem(commitment, amount)`; - // the commitment payload plays the same binding role, and the - // amount needs no separate check — it is one of the document's - // properties, so it is already covered by the commitment. - Some(grovedb::Element::ItemWithSumItem(payload, _, _)) => { - payload == expected_commitment.as_slice() + // ALL of the index's entries must carry the commitment — for a + // bucketed index that is every containing bucket's entry (the + // write path creates them atomically, so anything less means the + // values do not describe an existing row). Zero paths (a bucketed + // index over a pre-origin timestamp) is vacuously consistent: the + // write path wrote nothing there either. + for path in paths { + let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); + let element = self.grove_get_raw_optional( + path_refs.as_slice().into(), + member_key.as_slice(), + DirectQueryType::StatefulDirectQuery, + transaction, + drive_operations, + &platform_version.drive, + )?; + let matches = match element { + Some(grovedb::Element::Item(payload, _)) => { + payload == expected_commitment.as_slice() + } + // Summable indexes store `ItemWithSumItem(commitment, + // amount)`; the commitment payload plays the same binding + // role, and the amount needs no separate check — it is one + // of the document's properties, so it is already covered + // by the commitment. + Some(grovedb::Element::ItemWithSumItem(payload, _, _)) => { + payload == expected_commitment.as_slice() + } + _ => false, + }; + if !matches { + return Ok(false); } - _ => false, - }) + } + Ok(true) } - /// Whether `document`'s entry under `index` exists (stateful read). + /// Whether any of `document`'s entries under `index` exists (stateful + /// read). For a bucketed index the entries are written atomically, so + /// ANY existing bucket entry means the projection exists — the + /// duplicate-detection contract the create-side probes rely on. #[allow(clippy::too_many_arguments)] pub fn has_index_only_document_entry( &self, @@ -195,21 +255,26 @@ impl Drive { drive_operations: &mut Vec, platform_version: &PlatformVersion, ) -> Result { - let (path, member_key) = Self::index_only_entry_path_and_key( + let (paths, member_key) = Self::index_only_entry_paths_and_key( contract_id, document_type, index, document, platform_version, )?; - let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); - self.grove_has_raw( - path_refs.as_slice().into(), - member_key.as_slice(), - DirectQueryType::StatefulDirectQuery, - transaction, - drive_operations, - &platform_version.drive, - ) + for path in paths { + let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); + if self.grove_has_raw( + path_refs.as_slice().into(), + member_key.as_slice(), + DirectQueryType::StatefulDirectQuery, + transaction, + drive_operations, + &platform_version.drive, + )? { + return Ok(true); + } + } + Ok(false) } } diff --git a/packages/rs-drive/src/drive/document/insert/add_reference_for_index_level_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/insert/add_reference_for_index_level_for_contract_operations/v0/mod.rs index becd4f0915..c25bb65ec5 100644 --- a/packages/rs-drive/src/drive/document/insert/add_reference_for_index_level_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_reference_for_index_level_for_contract_operations/v0/mod.rs @@ -401,7 +401,17 @@ impl Drive { BatchInsertTreeApplyType::StatefulBatchInsertTree } else { BatchInsertTreeApplyType::StatelessBatchInsertTree { - in_tree_type: TreeType::NormalTree, + // The `0` tree's parent is the index's value tree, which + // aggregates whenever the index counts or sums — claiming + // `NormalTree` here under-estimates the parent's per-child + // aggregate bytes, and a time-range index's bucket fan-out + // multiplies the gap past the item-size padding (the + // `estimated_fees_upper_bound_actual_fees` e2e tests pin + // the invariant). + in_tree_type: + crate::drive::document::index_level_tree_types::terminal_value_tree_type( + index_type, + ), tree_type: member_tree_type, flags_len: storage_flags .map(|s| s.serialized_size()) @@ -422,16 +432,19 @@ impl Drive { index_path_info.push(Key(vec![0]))?; - // An empty-payload item plus flags is the whole element. // The payload is the 32-byte row commitment; the estimated per-item // value size is padded above it because the estimation layers - // under-count the serialized item envelope (enum tag, length - // prefix, flags option) by a handful of bytes, and estimation must - // UPPER-bound the applied fee — the - // `estimated_fees_upper_bound_actual_fees` e2e test pins the - // invariant. + // under-count each entry's chain (the serialized item envelope — + // enum tag, length prefix, flags option — plus the per-entry share + // of parent-tree aggregate bytes), and estimation must UPPER-bound + // the applied fee. The padding is per entry, so it scales with a + // time-range index's bucket fan-out (one entry chain per bucket), + // where the original 16-byte pad measurably under-ran — the + // `estimated_fees_upper_bound_actual_fees` e2e tests (like / tip / + // beat) pin the invariant across the plain, summable and bucketed + // shapes. const INDEX_ONLY_ITEM_ESTIMATED_VALUE_SIZE: u32 = - crate::drive::document::INDEX_ONLY_ROW_COMMITMENT_SIZE + 16; + crate::drive::document::INDEX_ONLY_ROW_COMMITMENT_SIZE + 32; // Sum-bearing entries additionally carry the i64 sum item in the // element envelope; 10 bytes is the worst case the sum-aware space diff --git a/packages/rs-drive/src/query/index_only_synthesis.rs b/packages/rs-drive/src/query/index_only_synthesis.rs index 35ac428a87..7738c50de6 100644 --- a/packages/rs-drive/src/query/index_only_synthesis.rs +++ b/packages/rs-drive/src/query/index_only_synthesis.rs @@ -108,9 +108,10 @@ impl DriveDocumentQuery<'_> { // Shapes the terminal route can never serve opt out up front, so // their generic-route miss errors propagate untouched: a resolved - // time range needs a bucketed index (which an indexOnly type - // cannot even declare), and the multi-`In` machinery has its own - // error surface. + // time range binds to a bucketed index (whose entries serve only + // the aggregate surfaces — see + // `refuse_bucketed_index_only_synthesis`), and the multi-`In` + // machinery has its own error surface. if !self.resolved_time_ranges.is_empty() || self.internal_clauses.in_clauses.len() > 1 { return Ok(None); } @@ -165,7 +166,11 @@ impl DriveDocumentQuery<'_> { fields.as_slice(), in_field, order_by_keys.as_slice(), - |_| true, + // Bucketed indexes never serve the terminal route: only + // resolved time ranges may bind to bucket keys, and those + // opted out above — a raw query name-matching a bucketed + // index's properties must not walk its grid-keyed levels. + |index| index.time_range.is_none(), platform_version, ) .map_err(|e| Error::Protocol(Box::new(e)))? @@ -474,7 +479,10 @@ impl DriveDocumentQuery<'_> { platform_version: &PlatformVersion, ) -> Result<&Index, Error> { match self.select_best_index(platform_version)? { - crate::query::BestIndexOutcome::Matched(index) => Ok(index), + crate::query::BestIndexOutcome::Matched(index) => { + Self::refuse_bucketed_index_only_synthesis(index)?; + Ok(index) + } crate::query::BestIndexOutcome::NoIndexMatches(no_index_error) => { match self.index_only_terminal_clause_selection(platform_version)? { Some(route) => Ok(route.index), @@ -484,6 +492,32 @@ impl DriveDocumentQuery<'_> { } } + /// Document synthesis over a bucketed indexOnly index is not + /// supported: the bucket level is a derived value — the synthesized + /// `$createdAt` would carry bucket-start granularity, not the + /// document's timestamp — and the type's non-bucketed indexes (the + /// proof-index rule guarantees at least one exists) serve the raw + /// entries. Bucketed indexOnly indexes exist for the aggregate + /// surfaces (`IN_TIME_RANGE` count / range count), which never open + /// value trees. Only a resolved `IN_TIME_RANGE` query can select a + /// bucketed index (raw queries are inadmissible against them), so this + /// fires exactly on "documents in this time bucket" requests. + fn refuse_bucketed_index_only_synthesis(index: &Index) -> Result<(), Error> { + if index.time_range.is_some() { + return Err(Error::Query( + crate::error::query::QuerySyntaxError::Unsupported( + "IN_TIME_RANGE document queries are not supported on an indexOnly type: \ + the bucketed entries carry bucket-start time granularity, so documents \ + cannot be synthesized from them; use the count aggregate surfaces over \ + the bucketed index, or query the raw entries through a non-bucketed \ + index" + .to_string(), + ), + )); + } + Ok(()) + } + /// Route an indexOnly query: `Ok(Some(..))` with the terminal-route /// path query when the generic index matcher cannot serve the query /// but a terminal clause can, `Ok(None)` when the generic route owns @@ -495,7 +529,13 @@ impl DriveDocumentQuery<'_> { platform_version: &PlatformVersion, ) -> Result, Error> { match self.select_best_index(platform_version)? { - crate::query::BestIndexOutcome::Matched(_) => Ok(None), + crate::query::BestIndexOutcome::Matched(index) => { + // Refused here as well as in `index_only_query_index` so + // the prover and the no-proof executor fail before + // building a path query the synthesis side would refuse. + Self::refuse_bucketed_index_only_synthesis(index)?; + Ok(None) + } crate::query::BestIndexOutcome::NoIndexMatches(no_index_error) => { match self.index_only_terminal_clause_selection(platform_version)? { Some(route) => self @@ -691,6 +731,18 @@ pub fn index_only_entry_path_and_key_from_values( .map_err(|e| Error::Protocol(Box::new(e))) }; + // Bare property names are correct here because a bucketed index can + // never reach this builder: it is used with the proof index (which by + // the contract-admission rule involves no $createdAt, so it cannot be + // bucketed) and with terminal-route indexes (which exclude bucketed + // indexes at selection). Guarded rather than assumed. + if index.time_range.is_some() { + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "index_only_entry_path_and_key_from_values cannot address a bucketed index: \ + its levels are keyed by the grid-qualified storage key, not the property name", + ))); + } + let mut path: Vec> = Vec::with_capacity(5 + index.properties.len() * 2); path.push(vec![crate::drive::RootTree::DataContractDocuments as u8]); path.push(contract_id.to_vec()); diff --git a/packages/rs-drive/tests/supporting_files/contract/yappr-likes/yappr-likes-contract.json b/packages/rs-drive/tests/supporting_files/contract/yappr-likes/yappr-likes-contract.json index f9edc64f2e..34c59edd24 100644 --- a/packages/rs-drive/tests/supporting_files/contract/yappr-likes/yappr-likes-contract.json +++ b/packages/rs-drive/tests/supporting_files/contract/yappr-likes/yappr-likes-contract.json @@ -107,6 +107,55 @@ ], "additionalProperties": false }, + "beat": { + "type": "object", + "indexOnly": true, + "documentsMutable": false, + "canBeDeleted": true, + "indices": [ + { + "name": "byHourHashtag", + "properties": [ + { + "$createdAt": "asc" + }, + { + "hashtag": "asc" + } + ], + "terminal": "$ownerId", + "timeRange": { + "on": "$createdAt", + "range": 3600, + "step": 900 + }, + "countable": "countable", + "rangeCountable": true + }, + { + "name": "byHashtag", + "properties": [ + { + "hashtag": "asc" + } + ], + "terminal": "$ownerId" + } + ], + "properties": { + "hashtag": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "position": 0 + } + }, + "required": [ + "hashtag", + "$createdAt" + ], + "additionalProperties": false + }, "tip": { "type": "object", "indexOnly": true,