diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 8b84d7dc1..665613a1d 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -37,7 +37,7 @@ class PAIMON_EXPORT CommitContext { public: CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& memory_pool, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& options); @@ -59,6 +59,10 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } + bool AppendCommitCheckConflict() const { + return append_commit_check_conflict_; + } + std::shared_ptr GetMemoryPool() const { return memory_pool_; } @@ -80,6 +84,7 @@ class PAIMON_EXPORT CommitContext { std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; + bool append_commit_check_conflict_; std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; @@ -125,6 +130,11 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); + /// Sets whether append commits should perform conflict checking (default is false). + /// @param append_commit_check_conflict True to enable append conflict checks. + /// @return Reference to this builder for method chaining. + CommitContextBuilder& AppendCommitCheckConflict(bool append_commit_check_conflict); + /// Sets the memory pool to be used for memory allocation during commit operations. /// @param memory_pool Shared pointer to the memory pool instance. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 1130bd619..fb14b7a3a 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -230,6 +230,16 @@ struct PAIMON_EXPORT Options { /// "commit.max-retries" - Maximum number of retries when commit failed. Default value is 10. static const char COMMIT_MAX_RETRIES[]; + /// "commit.min-retry-wait" - Min retry wait time when commit failed. Default value is 10ms. + static const char COMMIT_MIN_RETRY_WAIT[]; + + /// "commit.max-retry-wait" - Max retry wait time when commit failed. Default value is 10s. + static const char COMMIT_MAX_RETRY_WAIT[]; + + /// "commit.discard-duplicate-files" - Whether to discard duplicate files on APPEND commit. + /// Default value is "false". + static const char COMMIT_DISCARD_DUPLICATE_FILES[]; + /// "compaction.max-size-amplification-percent" - The size amplification is defined as the /// amount (in percentage) of additional storage needed to store a single byte of data in the /// merge tree for changelog mode table. Default value is 200. @@ -265,6 +275,15 @@ struct PAIMON_EXPORT Options { /// level 0 files in candidates. Default value is false. static const char COMPACTION_FORCE_UP_LEVEL_0[]; + /// "overwrite-upgrade" - Whether to try upgrading the data files after overwriting a + /// primary key table. Default value is true. + static const char OVERWRITE_UPGRADE[]; + + /// "dynamic-partition-overwrite" - Whether only overwrite dynamic partition when + /// overwriting a partitioned table with dynamic partition columns. Works only when + /// the table has partition keys. Default value is true. + static const char DYNAMIC_PARTITION_OVERWRITE[]; + /// "lookup-compact.max-interval" - The max interval for a gentle mode lookup compaction to be /// triggered. For every interval, a forced lookup compaction will be performed to flush L0 /// files to higher level. This option is only valid when lookup-compact mode is gentle. No @@ -444,6 +463,12 @@ struct PAIMON_EXPORT Options { /// "write-only" - If set to "true", compactions and snapshot expiration will be skipped. This /// option is used along with dedicated compact jobs. Default value is "false". static const char WRITE_ONLY[]; + /// "bucket-append-ordered" - Whether append writes in fixed bucket mode are ordered. This + /// option is used by commit conflict checks. Default value is "false". + static const char BUCKET_APPEND_ORDERED[]; + /// "write.sequence-number-init-mode" - Specify how to initialize the next sequence number for + /// primary key table writers. Values can be: "scan", "snapshot". Default value is "scan". + static const char WRITE_SEQUENCE_NUMBER_INIT_MODE[]; /// "compaction.min.file-num" - For file set [f_0,...,f_N], the minimum file number to trigger a /// compaction for append-only table. Default value is 5. static const char COMPACTION_MIN_FILE_NUM[]; diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index f52fb3730..2802f8f5c 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -83,7 +83,7 @@ class PAIMON_EXPORT FileStoreCommit { /// Overwrite from manifest committable and partition. /// - /// @param partitions A single partition maps each partition key to a partition value. Depending + /// @param partition A single partition maps each partition key to a partition value. Depending /// on the user-defined statement, the partition might not include all partition keys. Also /// note that this partition does not necessarily equal to the partitions of the newly added /// key-values. This is just the partition to be cleaned up. @@ -92,7 +92,7 @@ class PAIMON_EXPORT FileStoreCommit { /// @param watermark An optional event-time watermark used to indicate the progress of data /// processing. Default is std::nullopt. /// @return Result of the operation. - virtual Status Overwrite(const std::vector>& partitions, + virtual Status Overwrite(const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) = 0; @@ -100,14 +100,14 @@ class PAIMON_EXPORT FileStoreCommit { /// This is a temporary interface for internal use. It will be removed in a future version. /// Please do not rely on it for long-term use. /// - /// @param partitions Description of the partitions. + /// @param partition Description of the partition. /// @param commit_messages Description of the commit messages. /// @param commit_identifier Unique identifier. /// @param watermark An optional event-time watermark used to indicate the progress of data /// processing. Default is std::nullopt. /// @return Result of the operation. virtual Result FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) = 0; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 75ba6cabf..2841d445e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -287,6 +287,15 @@ set(PAIMON_CORE_SRCS core/operation/abstract_split_read.cpp core/operation/append_only_file_store_scan.cpp core/operation/append_only_file_store_write.cpp + core/operation/commit/conflict_detection.cpp + core/operation/commit/commit_scanner.cpp + core/operation/commit/commit_changes_provider.cpp + core/operation/commit/overwrite_changes_provider.cpp + core/operation/commit/row_id_column_conflict_checker.cpp + core/operation/commit/manifest_entry_changes.cpp + core/operation/commit/row_tracking_commit_utils.cpp + core/operation/commit/sequence_snapshot_properties.cpp + core/operation/commit/retry_waiter.cpp core/operation/commit_context.cpp core/operation/expire_snapshots.cpp core/operation/file_store_commit.cpp @@ -695,11 +704,16 @@ if(PAIMON_BUILD_TESTS) core/operation/metrics/compaction_metrics_test.cpp core/operation/data_evolution_file_store_scan_test.cpp core/operation/data_evolution_split_read_test.cpp + core/operation/commit/conflict_detection_test.cpp + core/operation/commit/manifest_entry_changes_test.cpp + core/operation/commit/row_tracking_commit_utils_test.cpp + core/operation/commit/retry_waiter_test.cpp core/operation/key_value_file_store_write_test.cpp core/operation/internal_read_context_test.cpp core/operation/abstract_split_read_test.cpp core/operation/append_only_file_store_write_test.cpp core/operation/commit_metrics_test.cpp + core/operation/commit_context_test.cpp core/operation/expire_snapshots_test.cpp core/operation/file_store_commit_impl_test.cpp core/operation/file_store_commit_test.cpp diff --git a/src/paimon/common/data/binary_row.h b/src/paimon/common/data/binary_row.h index 71359d918..2ab0d9fa9 100644 --- a/src/paimon/common/data/binary_row.h +++ b/src/paimon/common/data/binary_row.h @@ -156,6 +156,19 @@ struct hash> { } }; +/// for std::unordered_map, ...> +template <> +struct hash> { + size_t operator()( + const std::tuple& partition_bucket_level) const { + const auto& [partition, bucket, level] = partition_bucket_level; + size_t hash = paimon::MurmurHashUtils::HashUnsafeBytes( + reinterpret_cast(&bucket), 0, sizeof(bucket), partition.HashCode()); + return paimon::MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&level), 0, + sizeof(level), hash); + } +}; + template <> struct hash { size_t operator()(const paimon::BinaryRow& row) const { diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 499e61973..7582d6284 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -63,6 +63,9 @@ const char Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES[] = "snapshot.clean-empty-d const char Options::COMMIT_FORCE_COMPACT[] = "commit.force-compact"; const char Options::COMMIT_TIMEOUT[] = "commit.timeout"; const char Options::COMMIT_MAX_RETRIES[] = "commit.max-retries"; +const char Options::COMMIT_MIN_RETRY_WAIT[] = "commit.min-retry-wait"; +const char Options::COMMIT_MAX_RETRY_WAIT[] = "commit.max-retry-wait"; +const char Options::COMMIT_DISCARD_DUPLICATE_FILES[] = "commit.discard-duplicate-files"; const char Options::SEQUENCE_FIELD[] = "sequence.field"; const char Options::SEQUENCE_FIELD_SORT_ORDER[] = "sequence.field.sort-order"; const char Options::MERGE_ENGINE[] = "merge-engine"; @@ -111,6 +114,8 @@ const char Options::SCAN_TIMESTAMP_MILLIS[] = "scan.timestamp-millis"; const char Options::SCAN_TIMESTAMP[] = "scan.timestamp"; const char Options::SCAN_TAG_NAME[] = "scan.tag-name"; const char Options::WRITE_ONLY[] = "write-only"; +const char Options::BUCKET_APPEND_ORDERED[] = "bucket-append-ordered"; +const char Options::WRITE_SEQUENCE_NUMBER_INIT_MODE[] = "write.sequence-number-init-mode"; const char Options::COMPACTION_MIN_FILE_NUM[] = "compaction.min.file-num"; const char Options::COMPACTION_FORCE_REWRITE_ALL_FILES[] = "compaction.force-rewrite-all-files"; const char Options::COMPACTION_OPTIMIZATION_INTERVAL[] = "compaction.optimization-interval"; @@ -135,6 +140,8 @@ const char Options::NUM_SORTED_RUNS_COMPACTION_TRIGGER[] = "num-sorted-run.compa const char Options::NUM_SORTED_RUNS_STOP_TRIGGER[] = "num-sorted-run.stop-trigger"; const char Options::NUM_LEVELS[] = "num-levels"; const char Options::COMPACTION_FORCE_UP_LEVEL_0[] = "compaction.force-up-level-0"; +const char Options::OVERWRITE_UPGRADE[] = "overwrite-upgrade"; +const char Options::DYNAMIC_PARTITION_OVERWRITE[] = "dynamic-partition-overwrite"; const char Options::LOOKUP_WAIT[] = "lookup-wait"; const char Options::LOOKUP_COMPACT[] = "lookup-compact"; const char Options::LOOKUP_COMPACT_MAX_INTERVAL[] = "lookup-compact.max-interval"; diff --git a/src/paimon/core/catalog/commit_table_request_test.cpp b/src/paimon/core/catalog/commit_table_request_test.cpp index 3307cf47c..15b6585c2 100644 --- a/src/paimon/core/catalog/commit_table_request_test.cpp +++ b/src/paimon/core/catalog/commit_table_request_test.cpp @@ -37,7 +37,7 @@ TEST(CommitTableRequestTest, TestSimple) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); std::vector partition_statistics = { @@ -61,7 +61,6 @@ TEST(CommitTableRequestTest, TestSimple) { "commitIdentifier": 9223372036854775807, "commitKind": "APPEND", "timeMillis": 1758097357597, - "logOffsets": {}, "totalRecordCount": 5, "deltaRecordCount": 5, "changelogRecordCount": 0, diff --git a/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp b/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp index 62ad368d3..e4cdea52e 100644 --- a/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp +++ b/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp @@ -46,7 +46,7 @@ TEST(RenamingSnapshotCommitTest, TestSimple) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); ASSERT_OK_AND_ASSIGN(bool success, commit->Commit(snapshot, /*statistics=*/{})); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 450c5c179..b55eadfad 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -368,6 +368,8 @@ struct CoreOptions::Impl { int64_t manifest_full_compaction_file_size = 16 * 1024 * 1024; int64_t write_buffer_size = 256 * 1024 * 1024; int64_t commit_timeout = std::numeric_limits::max(); + int64_t commit_min_retry_wait = 10; + int64_t commit_max_retry_wait = 10 * 1000; std::shared_ptr file_format; std::shared_ptr file_system; @@ -427,6 +429,9 @@ struct CoreOptions::Impl { bool ignore_delete = false; bool write_buffer_spillable = true; bool write_only = false; + bool bucket_append_ordered = false; + CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = + CoreOptions::SequenceNumberInitMode::SCAN; bool deletion_vectors_enabled = false; bool deletion_vectors_bitmap64 = false; bool force_lookup = false; @@ -445,6 +450,9 @@ struct CoreOptions::Impl { bool global_index_enabled = true; std::optional global_index_thread_num; bool commit_force_compact = false; + bool commit_discard_duplicate_files = false; + bool dynamic_partition_overwrite = true; + bool overwrite_upgrade = true; bool compaction_force_rewrite_all_files = false; bool compaction_force_up_level_0 = false; std::optional global_index_external_path; @@ -519,6 +527,9 @@ struct CoreOptions::Impl { specified_file_system, &file_system)); // Parse write-only - if true, compactions and snapshot expiration will be skipped PAIMON_RETURN_NOT_OK(parser.Parse(Options::WRITE_ONLY, &write_only)); + // Parse bucket-append-ordered - append writes in fixed-bucket mode are ordered + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BUCKET_APPEND_ORDERED, &bucket_append_ordered)); // Parse partition.legacy-name - use legacy ToString for partition names, default true PAIMON_RETURN_NOT_OK(parser.Parse(Options::PARTITION_GENERATE_LEGACY_NAME, &legacy_partition_name_enabled)); @@ -637,6 +648,22 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseTimeDuration(Options::COMMIT_TIMEOUT, &commit_timeout)); // Parse commit.max-retries - maximum retries when commit failed, default 10 PAIMON_RETURN_NOT_OK(parser.Parse(Options::COMMIT_MAX_RETRIES, &commit_max_retries)); + // Parse commit.min-retry-wait - minimum retry wait when commit failed, default 10ms + PAIMON_RETURN_NOT_OK( + parser.ParseTimeDuration(Options::COMMIT_MIN_RETRY_WAIT, &commit_min_retry_wait)); + // Parse commit.max-retry-wait - maximum retry wait when commit failed, default 10s + PAIMON_RETURN_NOT_OK( + parser.ParseTimeDuration(Options::COMMIT_MAX_RETRY_WAIT, &commit_max_retry_wait)); + // Parse commit.discard-duplicate-files - whether to discard duplicate files on append + PAIMON_RETURN_NOT_OK(parser.Parse(Options::COMMIT_DISCARD_DUPLICATE_FILES, + &commit_discard_duplicate_files)); + // Parse dynamic-partition-overwrite - whether overwrite only dynamic partitions + // for partitioned table overwrite. + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::DYNAMIC_PARTITION_OVERWRITE, &dynamic_partition_overwrite)); + // Parse overwrite-upgrade - whether to try upgrading data files after overwrite on + // primary key table + PAIMON_RETURN_NOT_OK(parser.Parse(Options::OVERWRITE_UPGRADE, &overwrite_upgrade)); return Status::OK(); } @@ -647,6 +674,19 @@ struct CoreOptions::Impl { Options::SEQUENCE_FIELD, Options::FIELDS_SEPARATOR, &sequence_field)); // Parse sequence.field.sort-order - order of sequence field, default "ascending" PAIMON_RETURN_NOT_OK(parser.ParseSortOrder(&sequence_field_sort_order)); + // Parse write-sequence-number-init-mode - sequence init mode for write path + std::string write_sequence_init_mode_str = "scan"; + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, &write_sequence_init_mode_str)); + write_sequence_init_mode_str = StringUtils::ToLowerCase(write_sequence_init_mode_str); + if (write_sequence_init_mode_str == "scan") { + write_sequence_number_init_mode = CoreOptions::SequenceNumberInitMode::SCAN; + } else if (write_sequence_init_mode_str == "snapshot") { + write_sequence_number_init_mode = CoreOptions::SequenceNumberInitMode::SNAPSHOT; + } else { + return Status::Invalid(fmt::format("invalid write sequence number init mode: {}", + write_sequence_init_mode_str)); + } // Parse sort-engine - sort engine for primary key table, default "loser-tree" PAIMON_RETURN_NOT_OK(parser.ParseSortEngine(&sort_engine)); // Parse merge-engine - merge engine for primary key table, default "deduplicate" @@ -1037,6 +1077,26 @@ int32_t CoreOptions::GetCommitMaxRetries() const { return impl_->commit_max_retries; } +int64_t CoreOptions::GetCommitMinRetryWait() const { + return impl_->commit_min_retry_wait; +} + +int64_t CoreOptions::GetCommitMaxRetryWait() const { + return impl_->commit_max_retry_wait; +} + +bool CoreOptions::CommitDiscardDuplicateFiles() const { + return impl_->commit_discard_duplicate_files; +} + +bool CoreOptions::DynamicPartitionOverwrite() const { + return impl_->dynamic_partition_overwrite; +} + +bool CoreOptions::OverwriteUpgrade() const { + return impl_->overwrite_upgrade; +} + int32_t CoreOptions::GetCompactionMinFileNum() const { return impl_->compaction_min_file_num; } @@ -1133,6 +1193,14 @@ bool CoreOptions::WriteOnly() const { return impl_->write_only; } +bool CoreOptions::BucketAppendOrdered() const { + return impl_->bucket_append_ordered; +} + +CoreOptions::SequenceNumberInitMode CoreOptions::WriteSequenceNumberInitMode() const { + return impl_->write_sequence_number_init_mode; +} + std::optional CoreOptions::GetFieldsDefaultFunc() const { return impl_->field_default_func; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 72e57bcd7..e640e92c9 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -48,6 +48,15 @@ class Cache; class PAIMON_EXPORT CoreOptions { public: + // Specifies how to initialize the next sequence number for primary key table writers. + enum class SequenceNumberInitMode { + // initialize by scanning existing file metadata. + SCAN, + // initialize from the maximum sequence number recorded in snapshot properties, + // which can avoid scanning existing file metadata in write-only mode. + SNAPSHOT, + }; + static Result FromMap( const std::map& options_map, const std::shared_ptr& specified_file_system = nullptr, @@ -100,6 +109,11 @@ class PAIMON_EXPORT CoreOptions { bool CompactionForceUpLevel0() const; int64_t GetCommitTimeout() const; int32_t GetCommitMaxRetries() const; + int64_t GetCommitMinRetryWait() const; + int64_t GetCommitMaxRetryWait() const; + bool CommitDiscardDuplicateFiles() const; + bool DynamicPartitionOverwrite() const; + bool OverwriteUpgrade() const; int32_t GetCompactionMinFileNum() const; int32_t GetCompactionMaxSizeAmplificationPercent() const; int32_t GetCompactionSizeRatio() const; @@ -115,6 +129,8 @@ class PAIMON_EXPORT CoreOptions { SortEngine GetSortEngine() const; bool IgnoreDelete() const; bool WriteOnly() const; + bool BucketAppendOrdered() const; + SequenceNumberInitMode WriteSequenceNumberInitMode() const; std::optional GetFieldsDefaultFunc() const; Result> GetFieldAggFunc(const std::string& field_name) const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index aa567c3c7..55b8eb5a5 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -67,8 +67,13 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ("zstd", core_options.GetSpillCompressOptions().compress); ASSERT_EQ(1, core_options.GetSpillCompressOptions().zstd_level); ASSERT_FALSE(core_options.CommitForceCompact()); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + ASSERT_TRUE(core_options.OverwriteUpgrade()); ASSERT_EQ(std::numeric_limits::max(), core_options.GetCommitTimeout()); ASSERT_EQ(10, core_options.GetCommitMaxRetries()); + ASSERT_EQ(10, core_options.GetCommitMinRetryWait()); + ASSERT_EQ(10 * 1000, core_options.GetCommitMaxRetryWait()); + ASSERT_FALSE(core_options.CommitDiscardDuplicateFiles()); ExpireConfig expire_config = core_options.GetExpireConfig(); ASSERT_EQ(10, expire_config.GetSnapshotRetainMin()); ASSERT_EQ(std::numeric_limits::max(), expire_config.GetSnapshotRetainMax()); @@ -81,6 +86,9 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(SortEngine::LOSER_TREE, core_options.GetSortEngine()); ASSERT_FALSE(core_options.IgnoreDelete()); ASSERT_FALSE(core_options.WriteOnly()); + ASSERT_FALSE(core_options.BucketAppendOrdered()); + ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SCAN, + core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(5, core_options.GetCompactionMinFileNum()); ASSERT_FALSE(core_options.CompactionForceRewriteAllFiles()); ASSERT_FALSE(core_options.CompactionForceUpLevel0()); @@ -182,6 +190,11 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::COMMIT_FORCE_COMPACT, "true"}, {Options::COMMIT_TIMEOUT, "120s"}, {Options::COMMIT_MAX_RETRIES, "20"}, + {Options::COMMIT_MIN_RETRY_WAIT, "5ms"}, + {Options::COMMIT_MAX_RETRY_WAIT, "3s"}, + {Options::COMMIT_DISCARD_DUPLICATE_FILES, "true"}, + {Options::DYNAMIC_PARTITION_OVERWRITE, "false"}, + {Options::OVERWRITE_UPGRADE, "false"}, {Options::SCAN_SNAPSHOT_ID, "5"}, {Options::SCAN_MODE, "from-snapshot-full"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "15"}, @@ -232,6 +245,8 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"}, {Options::SCAN_TAG_NAME, "test-tag"}, {Options::WRITE_ONLY, "true"}, + {Options::BUCKET_APPEND_ORDERED, "true"}, + {Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot"}, {Options::COMPACTION_MIN_FILE_NUM, "10"}, {Options::COMPACTION_FORCE_REWRITE_ALL_FILES, "true"}, {Options::COMPACTION_FORCE_UP_LEVEL_0, "true"}, @@ -301,8 +316,13 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ("lz4", core_options.GetSpillCompressOptions().compress); ASSERT_EQ(2, core_options.GetSpillCompressOptions().zstd_level); ASSERT_TRUE(core_options.CommitForceCompact()); + ASSERT_FALSE(core_options.DynamicPartitionOverwrite()); + ASSERT_FALSE(core_options.OverwriteUpgrade()); ASSERT_EQ(120 * 1000, core_options.GetCommitTimeout()); ASSERT_EQ(20, core_options.GetCommitMaxRetries()); + ASSERT_EQ(5, core_options.GetCommitMinRetryWait()); + ASSERT_EQ(3 * 1000, core_options.GetCommitMaxRetryWait()); + ASSERT_TRUE(core_options.CommitDiscardDuplicateFiles()); ASSERT_EQ(5, core_options.GetScanSnapshotId().value_or(-1)); ExpireConfig expire_config = core_options.GetExpireConfig(); ASSERT_EQ(15, expire_config.GetSnapshotRetainMin()); @@ -375,6 +395,9 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_TRUE(core_options.WriteOnly()); + ASSERT_TRUE(core_options.BucketAppendOrdered()); + ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SNAPSHOT, + core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(10, core_options.GetCompactionMinFileNum()); ASSERT_EQ(123, core_options.GetCompactionMaxSizeAmplificationPercent()); ASSERT_EQ(9, core_options.GetCompactionSizeRatio()); @@ -434,6 +457,9 @@ TEST(CoreOptionsTest, TestInvalidCase) { "The high priority pool ratio should in the range [0, 1), while input is 1.1"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET_FUNCTION_TYPE, "invalid"}}), "invalid bucket function type: invalid"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "invalid"}}), + "invalid write sequence number init mode: invalid"); } TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) { @@ -445,6 +471,27 @@ TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) { ASSERT_EQ(13, core_options.GetLookupCompactMaxInterval()); } +TEST(CoreOptionsTest, TestDynamicPartitionOverwriteOption) { + { + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + } + + { + ASSERT_OK_AND_ASSIGN( + CoreOptions core_options, + CoreOptions::FromMap({{Options::DYNAMIC_PARTITION_OVERWRITE, "false"}})); + ASSERT_FALSE(core_options.DynamicPartitionOverwrite()); + } + + { + ASSERT_OK_AND_ASSIGN( + CoreOptions core_options, + CoreOptions::FromMap({{Options::DYNAMIC_PARTITION_OVERWRITE, "true"}})); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + } +} + TEST(CoreOptionsTest, TestNumSortedRunsStopTriggerFloorAndDefault) { { std::map options = { diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp index 35783f1b0..0fe512539 100644 --- a/src/paimon/core/index/index_file_handler_test.cpp +++ b/src/paimon/core/index/index_file_handler_test.cpp @@ -265,9 +265,9 @@ TEST_F(IndexFileHandlerTest, TestScanWithNoIndexManifest) { snapshot.DeltaManifestListSize(), snapshot.ChangelogManifestList(), snapshot.ChangelogManifestListSize(), /*index_manifest=*/std::nullopt, snapshot.CommitUser(), snapshot.CommitIdentifier(), snapshot.GetCommitKind(), - snapshot.TimeMillis(), snapshot.LogOffsets(), snapshot.TotalRecordCount(), - snapshot.DeltaRecordCount(), snapshot.ChangelogRecordCount(), snapshot.Watermark(), - snapshot.Statistics(), snapshot.Properties(), snapshot.NextRowId()); + snapshot.TimeMillis(), snapshot.TotalRecordCount(), snapshot.DeltaRecordCount(), + snapshot.ChangelogRecordCount(), snapshot.Watermark(), snapshot.Statistics(), + snapshot.Properties(), snapshot.NextRowId()); auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get()); std::unordered_set partitions = {partition}; diff --git a/src/paimon/core/manifest/manifest_committable.h b/src/paimon/core/manifest/manifest_committable.h index 777fd8605..44c80db64 100644 --- a/src/paimon/core/manifest/manifest_committable.h +++ b/src/paimon/core/manifest/manifest_committable.h @@ -40,15 +40,13 @@ class ManifestCommittable { : ManifestCommittable(identifier, std::nullopt) {} ManifestCommittable(int64_t identifier, std::optional watermark) - : ManifestCommittable(identifier, watermark, {}, {}, {}) {} + : ManifestCommittable(identifier, watermark, {}, {}) {} ManifestCommittable(int64_t identifier, std::optional watermark, - const std::map& log_offsets, const std::map& properties, const std::vector>& commit_messages) : identifier_(identifier), watermark_(watermark), - log_offsets_(log_offsets), properties_(properties), commit_messages_(commit_messages) {} @@ -60,10 +58,6 @@ class ManifestCommittable { return watermark_; } - const std::map& LogOffsets() const { - return log_offsets_; - } - const std::map& Properties() const { return properties_; } @@ -86,12 +80,6 @@ class ManifestCommittable { std::string watermark_str = watermark_ == std::nullopt ? "null" : std::to_string(watermark_.value()); - std::vector log_offsets_str; - log_offsets_str.reserve(log_offsets_.size()); - for (const auto& [key, value] : log_offsets_) { - log_offsets_str.emplace_back(fmt::format("{}: {}", key, value)); - } - std::vector properties_str; properties_str.reserve(properties_.size()); for (const auto& [key, value] : properties_) { @@ -99,16 +87,15 @@ class ManifestCommittable { } return fmt::format( - "ManifestCommittable {{identifier = {}, watermark = {}, logOffsets = {}, " + "ManifestCommittable {{identifier = {}, watermark = {}, " "commitMessages = {}, properties = {}}}", - identifier_, watermark_str, fmt::join(log_offsets_str, ", "), - fmt::join(commit_messages_str, ", "), fmt::join(properties_str, ", ")); + identifier_, watermark_str, fmt::join(commit_messages_str, ", "), + fmt::join(properties_str, ", ")); } private: int64_t identifier_; std::optional watermark_; - std::map log_offsets_; std::map properties_; std::vector> commit_messages_; }; diff --git a/src/paimon/core/manifest/manifest_committable_test.cpp b/src/paimon/core/manifest/manifest_committable_test.cpp index 9d2fbdbfa..c6761fc8e 100644 --- a/src/paimon/core/manifest/manifest_committable_test.cpp +++ b/src/paimon/core/manifest/manifest_committable_test.cpp @@ -29,28 +29,6 @@ namespace paimon::test { class ManifestCommittableTest : public testing::Test { private: - bool IsEqualMap(const std::map& actual_map, - const std::map& expected_map) { - if (expected_map.size() != actual_map.size()) { - return false; - } - for (const auto& kv : expected_map) { - const auto& key = kv.first; - const auto& value = kv.second; - auto iter = actual_map.find(key); - if (iter != actual_map.end()) { - if (iter->second == value) { - continue; - } else { - return false; - } - } else { - return false; - } - } - return true; - } - std::vector> GetCommitMessages(const std::string& path, int32_t version) const { auto file_system = std::make_shared(); @@ -99,26 +77,22 @@ TEST_F(ManifestCommittableTest, TestSimple) { ASSERT_EQ(committable.Watermark().value(), 456); } { - std::map log_offsets = {{123, 444}, {234, 555}}; std::map properties = {}; std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); - ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, log_offsets, - properties, msgs); - ASSERT_TRUE(IsEqualMap(committable.LogOffsets(), log_offsets)); + ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, properties, msgs); + ASSERT_EQ(committable.Properties(), properties); ASSERT_TRUE(IsEqualMsgs(msgs, committable.FileCommittables())); } { - std::map log_offsets = {}; std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); - ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, - /*log_offsets=*/{}, properties, msgs); + ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, properties, msgs); ASSERT_EQ(committable.Properties(), properties); ASSERT_TRUE(IsEqualMsgs(msgs, committable.FileCommittables())); } diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp index 7c72cb3c9..f1ff3b1ed 100644 --- a/src/paimon/core/operation/abstract_file_store_write.cpp +++ b/src/paimon/core/operation/abstract_file_store_write.cpp @@ -281,7 +281,8 @@ int32_t AbstractFileStoreWrite::GetDefaultBucketNum() const { Result> AbstractFileStoreWrite::ScanExistingFileMetas( const BinaryRow& partition, int32_t bucket) const { - PAIMON_ASSIGN_OR_RAISE(auto part_values, + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, file_store_path_factory_->GeneratePartitionVector(partition)); std::map part_values_map; for (const auto& [key, value] : part_values) { diff --git a/src/paimon/core/operation/commit/commit_changes_provider.cpp b/src/paimon/core/operation/commit/commit_changes_provider.cpp new file mode 100644 index 000000000..9960e0d11 --- /dev/null +++ b/src/paimon/core/operation/commit/commit_changes_provider.cpp @@ -0,0 +1,39 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/commit_changes_provider.h" + +#include + +namespace paimon { + +StaticCommitChangesProvider::StaticCommitChangesProvider( + const std::vector& delta_files, + const std::vector& changelog_files, + const std::vector& index_entries) + : delta_files_(delta_files), changelog_files_(changelog_files), index_entries_(index_entries) {} + +Status StaticCommitChangesProvider::Provide(const std::optional&, + std::vector* delta_files, + std::vector* changelog_files, + std::vector* index_entries) const { + *delta_files = delta_files_; + *changelog_files = changelog_files_; + *index_entries = index_entries_; + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_changes_provider.h b/src/paimon/core/operation/commit/commit_changes_provider.h new file mode 100644 index 000000000..c628143f4 --- /dev/null +++ b/src/paimon/core/operation/commit/commit_changes_provider.h @@ -0,0 +1,58 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/snapshot.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +class CommitChangesProvider { + public: + virtual ~CommitChangesProvider() = default; + + virtual Status Provide(const std::optional& latest_snapshot, + std::vector* delta_files, + std::vector* changelog_files, + std::vector* index_entries) const = 0; +}; + +class StaticCommitChangesProvider final : public CommitChangesProvider { + public: + StaticCommitChangesProvider(const std::vector& delta_files, + const std::vector& changelog_files, + const std::vector& index_entries); + + Status Provide(const std::optional& latest_snapshot, + std::vector* delta_files, + std::vector* changelog_files, + std::vector* index_entries) const override; + + private: + const std::vector& delta_files_; + const std::vector& changelog_files_; + const std::vector& index_entries_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_scanner.cpp b/src/paimon/core/operation/commit/commit_scanner.cpp new file mode 100644 index 000000000..679feec31 --- /dev/null +++ b/src/paimon/core/operation/commit/commit_scanner.cpp @@ -0,0 +1,203 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/commit_scanner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/core/core_options.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/operation/commit/overwrite_changes_provider.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/scan_context.h" + +namespace paimon { + +CommitScanner::CommitScanner(const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& index_manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& schema, + const CoreOptions& core_options, + const std::shared_ptr& executor, + const std::shared_ptr& pool, + const BinaryRowPartitionComputer* partition_computer, + ScanSupplier scan_supplier) + : snapshot_manager_(snapshot_manager), + schema_manager_(schema_manager), + manifest_list_(manifest_list), + manifest_file_(manifest_file), + index_manifest_file_(index_manifest_file), + table_schema_(table_schema), + schema_(schema), + core_options_(core_options), + executor_(executor), + pool_(pool), + partition_computer_(partition_computer), + scan_supplier_(std::move(scan_supplier)) {} + +Result>> CommitScanner::ToPartitionFilters( + const std::vector& changed_partitions) const { + std::vector> partition_filters; + partition_filters.reserve(changed_partitions.size()); + + for (const BinaryRow& changed_partition : changed_partitions) { + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, + partition_computer_->GeneratePartitionVector(changed_partition)); + std::map partition_filter; + for (const auto& [key, value] : part_values) { + partition_filter[key] = value; + } + partition_filters.push_back(std::move(partition_filter)); + } + + return partition_filters; +} + +Result> CommitScanner::ReadAllEntriesFromChangedPartitions( + const Snapshot& snapshot, const std::vector& changed_partitions) const { + if (changed_partitions.empty()) { + return std::vector{}; + } + + std::vector> partition_filters; + PAIMON_ASSIGN_OR_RAISE(partition_filters, ToPartitionFilters(changed_partitions)); + + auto scan_filter = std::make_shared(/*predicate=*/nullptr, partition_filters, + /*bucket_filter=*/std::nullopt); + if (!scan_supplier_) { + return Status::Invalid("CommitScanner requires non-empty scan supplier."); + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, scan_supplier_(scan_filter)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan()); + return plan->Files(); +} + +Result> CommitScanner::ReadAllEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const { + auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions, + /*bucket_filter=*/std::nullopt); + if (!scan_supplier_) { + return Status::Invalid("CommitScanner requires non-empty scan supplier."); + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, scan_supplier_(scan_filter)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan()); + return plan->Files(); +} + +Result> CommitScanner::ReadAllIndexEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const { + std::vector index_entries; + if (!snapshot.IndexManifest()) { + return index_entries; + } + + auto filter = [this, &partitions](const IndexManifestEntry& entry) -> Result { + if (partitions.empty()) { + return true; + } + + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, + partition_computer_->GeneratePartitionVector(entry.partition)); + std::map partition; + for (const auto& [key, value] : part_values) { + partition[key] = value; + } + + for (const auto& partition_spec : partitions) { + bool matched = true; + for (const auto& [key, value] : partition_spec) { + auto iter = partition.find(key); + if (iter == partition.end() || iter->second != value) { + matched = false; + break; + } + } + if (matched) { + return true; + } + } + return false; + }; + + PAIMON_RETURN_NOT_OK( + index_manifest_file_->Read(snapshot.IndexManifest().value(), filter, &index_entries)); + return index_entries; +} + +std::shared_ptr CommitScanner::OverwriteChangesProvider( + const std::vector>& partitions, + const std::vector& changes, + const std::vector& index_entries) const { + return std::make_shared( + changes, index_entries, + [this, partitions](const Snapshot& snapshot) { + return ReadAllEntriesFromPartitions(snapshot, partitions); + }, + [this, partitions](const Snapshot& snapshot) { + return ReadAllIndexEntriesFromPartitions(snapshot, partitions); + }); +} + +Result> CommitScanner::ReadTotalBuckets( + const Snapshot& snapshot, const std::vector& changed_partitions) const { + std::unordered_map total_buckets; + if (changed_partitions.empty()) { + return total_buckets; + } + + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + ReadAllEntriesFromChangedPartitions(snapshot, changed_partitions)); + + std::unordered_set remaining_partitions(changed_partitions.begin(), + changed_partitions.end()); + for (const ManifestEntry& entry : entries) { + if (remaining_partitions.empty()) { + break; + } + if (!(entry.Kind() == FileKind::Add()) || entry.TotalBuckets() <= 0) { + continue; + } + if (remaining_partitions.erase(entry.Partition()) > 0) { + total_buckets.emplace(entry.Partition(), entry.TotalBuckets()); + } + } + + return total_buckets; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_scanner.h b/src/paimon/core/operation/commit/commit_scanner.h new file mode 100644 index 000000000..2a53f8821 --- /dev/null +++ b/src/paimon/core/operation/commit/commit_scanner.h @@ -0,0 +1,108 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/core/core_options.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class BinaryRowPartitionComputer; +class CommitChangesProvider; +class Executor; +class FileStoreScan; +struct IndexManifestEntry; +class IndexManifestFile; +class ManifestEntry; +class ManifestFile; +class ManifestList; +class MemoryPool; +class ScanFilter; +class SchemaManager; +class Snapshot; +class SnapshotManager; +class TableSchema; + +// Manifest entries scanner for commit operations. +class CommitScanner { + public: + using ScanSupplier = + std::function>(const std::shared_ptr&)>; + + CommitScanner(const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& index_manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& schema, const CoreOptions& core_options, + const std::shared_ptr& executor, + const std::shared_ptr& pool, + const BinaryRowPartitionComputer* partition_computer, ScanSupplier scan_supplier); + + Result> ReadAllEntriesFromChangedPartitions( + const Snapshot& snapshot, const std::vector& changed_partitions) const; + + Result> ReadTotalBuckets( + const Snapshot& snapshot, const std::vector& changed_partitions) const; + + Result> ReadAllEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const; + + Result> ReadAllIndexEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const; + + std::shared_ptr OverwriteChangesProvider( + const std::vector>& partitions, + const std::vector& changes, + const std::vector& index_entries) const; + + private: + Result>> ToPartitionFilters( + const std::vector& changed_partitions) const; + + private: + std::shared_ptr snapshot_manager_; + std::shared_ptr schema_manager_; + std::shared_ptr manifest_list_; + std::shared_ptr manifest_file_; + std::shared_ptr index_manifest_file_; + std::shared_ptr table_schema_; + std::shared_ptr schema_; + CoreOptions core_options_; + std::shared_ptr executor_; + std::shared_ptr pool_; + const BinaryRowPartitionComputer* partition_computer_; + ScanSupplier scan_supplier_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/conflict_detection.cpp b/src/paimon/core/operation/commit/conflict_detection.cpp new file mode 100644 index 000000000..e60a05091 --- /dev/null +++ b/src/paimon/core/operation/commit/conflict_detection.cpp @@ -0,0 +1,537 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/conflict_detection.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/manifest/file_entry.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/commit/manifest_entry_changes.h" +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/utils/range.h" + +namespace paimon { + +namespace { + +bool IsVectorStoreFile(const std::string& file_name) { + return file_name.find(".vector.") != std::string::npos; +} + +bool IsDedicatedStorageFile(const std::string& file_name) { + return BlobUtils::IsBlobFile(file_name) || IsVectorStoreFile(file_name); +} + +} // namespace + +ConflictDetection::ConflictDetection(std::shared_ptr table_schema, + const CoreOptions& options, + std::shared_ptr snapshot_manager, + std::shared_ptr manifest_list, + std::shared_ptr manifest_file) + : table_schema_(std::move(table_schema)), + options_(options), + snapshot_manager_(std::move(snapshot_manager)), + manifest_list_(std::move(manifest_list)), + manifest_file_(std::move(manifest_file)) {} + +void ConflictDetection::SetRowIdCheckFromSnapshot( + const std::optional& row_id_check_from_snapshot) { + row_id_check_from_snapshot_ = row_id_check_from_snapshot; +} + +bool ConflictDetection::HasRowIdCheckFromSnapshot() const { + return row_id_check_from_snapshot_.has_value(); +} + +Status ConflictDetection::CheckConflicts( + const Snapshot& latest_snapshot, const std::vector& base_entries, + const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker, + const Snapshot::CommitKind& commit_kind) const { + if (options_.DeletionVectorsEnabled() && + options_.GetBucket() == BucketModeDefine::UNAWARE_BUCKET) { + return Status::NotImplemented( + "check conflicts failed. not yet support dv with BUCKET_UNAWARE mode"); + } + + std::vector all_entries = base_entries; + all_entries.insert(all_entries.end(), delta_entries.begin(), delta_entries.end()); + PAIMON_RETURN_NOT_OK(CheckBucketKeepSame(all_entries, commit_kind)); + + // check the delta, it is important not to delete and add the same file. Since scan + // relies on map for deduplication, this may result in the loss of this file + std::vector merged_delta_entries; + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(delta_entries, &merged_delta_entries)); + + std::vector merged_entries; + // merge manifest entries and also check if the files we want to delete are still there + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(all_entries, &merged_entries)); + PAIMON_RETURN_NOT_OK(CheckDeleteInEntries(merged_entries)); + PAIMON_RETURN_NOT_OK(CheckKeyRange(merged_entries)); + if (commit_kind != Snapshot::CommitKind::Compact()) { + PAIMON_RETURN_NOT_OK( + CheckRowIdExistence(base_entries, delta_entries, latest_snapshot.NextRowId())); + } + PAIMON_RETURN_NOT_OK(CheckRowIdRangeConflicts(commit_kind, merged_entries)); + PAIMON_RETURN_NOT_OK(CheckGlobalIndexRowIdExistence(base_entries, delta_index_entries)); + PAIMON_RETURN_NOT_OK(CheckForRowIdFromSnapshot( + latest_snapshot, delta_entries, delta_index_entries, row_id_column_conflict_checker)); + return Status::OK(); +} + +bool ConflictDetection::ShouldBeOverwriteCommit( + const std::vector& append_table_files, + const std::vector& append_index_files) const { + for (const ManifestEntry& entry : append_table_files) { + if (entry.Kind() == FileKind::Delete()) { + return true; + } + } + + for (const IndexManifestEntry& entry : append_index_files) { + if (entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) { + return true; + } + } + + return false; +} + +Status ConflictDetection::CheckBucketKeepSame(const std::vector& all_entries, + const Snapshot::CommitKind& commit_kind) const { + if (commit_kind == Snapshot::CommitKind::Overwrite()) { + return Status::OK(); + } + + // total buckets within the same partition should remain the same + std::unordered_map total_buckets; + for (const ManifestEntry& entry : all_entries) { + if (entry.TotalBuckets() <= 0) { + continue; + } + if (same_bucket_checked_partitions_.find(entry.Partition()) != + same_bucket_checked_partitions_.end()) { + continue; + } + + auto [iter, inserted] = total_buckets.emplace(entry.Partition(), entry.TotalBuckets()); + if (inserted || iter->second == entry.TotalBuckets()) { + continue; + } + + return BucketNumMismatch(entry.Partition(), entry.TotalBuckets(), iter->second); + } + + MarkBucketCheckedPartitions(total_buckets); + return Status::OK(); +} + +Status ConflictDetection::CollectUncheckedBucketPartitions( + const std::vector& delta_entries, + std::unordered_map* total_buckets) const { + total_buckets->clear(); + for (const ManifestEntry& entry : delta_entries) { + if (!(entry.Kind() == FileKind::Add()) || entry.TotalBuckets() <= 0 || + same_bucket_checked_partitions_.find(entry.Partition()) != + same_bucket_checked_partitions_.end()) { + continue; + } + + auto [iter, inserted] = total_buckets->emplace(entry.Partition(), entry.TotalBuckets()); + if (!inserted && iter->second != entry.TotalBuckets()) { + return BucketNumMismatch(entry.Partition(), entry.TotalBuckets(), iter->second); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckSameBucketByTotalBuckets( + const std::unordered_map& expected_total_buckets, + const std::unordered_map& previous_total_buckets) const { + for (const auto& [partition, total_buckets] : expected_total_buckets) { + auto iter = previous_total_buckets.find(partition); + if (iter != previous_total_buckets.end() && iter->second != total_buckets) { + return BucketNumMismatch(partition, total_buckets, iter->second); + } + } + + MarkBucketCheckedPartitions(expected_total_buckets); + return Status::OK(); +} + +Status ConflictDetection::BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets, + int32_t previous_num_buckets) const { + return Status::Invalid(fmt::format( + "Total buckets of partition {} changed from {} to {} without overwrite. Give up " + "committing.", + partition.ToString(), previous_num_buckets, num_buckets)); +} + +void ConflictDetection::MarkBucketCheckedPartitions( + const std::unordered_map& total_buckets) const { + if (total_buckets.empty()) { + return; + } + + for (const auto& [partition, _] : total_buckets) { + same_bucket_checked_partitions_.insert_or_assign(partition, true); + while (same_bucket_checked_partitions_.size() > kSameBucketCheckCacheMaxSize) { + same_bucket_checked_partitions_.erase(same_bucket_checked_partitions_.begin()->first); + } + } +} + +Status ConflictDetection::CheckDeleteInEntries( + const std::vector& merged_entries) const { + for (const auto& entry : merged_entries) { + if (entry.Kind() == FileKind::Delete()) { + return Status::Invalid(fmt::format( + "Trying to delete file {} which is not previously added.", entry.FileName())); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckKeyRange(const std::vector& merged_entries) const { + if (table_schema_->PrimaryKeys().empty()) { + return Status::OK(); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + table_schema_->TrimmedPrimaryKeyFields()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options_.SequenceFieldSortOrderIsAscending())); + + // group entries by partitions, buckets and levels + std::unordered_map, std::vector> levels; + for (const auto& entry : merged_entries) { + if (!(entry.Kind() == FileKind::Add())) { + continue; + } + int32_t level = entry.Level(); + if (level < 1) { + continue; + } + + levels[std::make_tuple(entry.Partition(), entry.Bucket(), level)].push_back(entry); + } + + // check for all LSM level >= 1, key ranges of files do not intersect + for (auto& [_, entries] : levels) { + std::sort(entries.begin(), entries.end(), + [&key_comparator](const ManifestEntry& a, const ManifestEntry& b) { + return key_comparator->CompareTo(a.MinKey(), b.MinKey()) < 0; + }); + for (size_t i = 0; i + 1 < entries.size(); ++i) { + const ManifestEntry& a = entries[i]; + const ManifestEntry& b = entries[i + 1]; + if (key_comparator->CompareTo(a.MaxKey(), b.MinKey()) >= 0) { + return Status::Invalid(fmt::format( + "LSM conflicts detected! Give up committing. Conflict files are {} and {}.", + a.FileName(), b.FileName())); + } + } + } + return Status::OK(); +} + +Status ConflictDetection::CheckRowIdExistence(const std::vector& base_entries, + const std::vector& delta_entries, + const std::optional& next_row_id) const { + if (!options_.DataEvolutionEnabled()) { + return Status::OK(); + } + + std::vector files_to_check; + files_to_check.reserve(delta_entries.size()); + for (const ManifestEntry& entry : delta_entries) { + if (!(entry.Kind() == FileKind::Add()) || !entry.File()->first_row_id || !next_row_id || + entry.File()->first_row_id.value() >= next_row_id.value()) { + continue; + } + files_to_check.push_back(entry); + } + if (files_to_check.empty()) { + return Status::OK(); + } + + std::vector existing_ranges; + std::set> exact_ranges; + existing_ranges.reserve(base_entries.size()); + for (const ManifestEntry& entry : base_entries) { + if (!entry.File()->first_row_id || IsDedicatedStorageFile(entry.FileName())) { + continue; + } + int64_t range_from = entry.File()->first_row_id.value(); + int64_t range_to = range_from + entry.File()->row_count - 1; + existing_ranges.emplace_back(range_from, range_to); + exact_ranges.emplace(range_from, range_to); + } + std::vector merged_ranges = Range::SortAndMergeOverlap(existing_ranges, + /*adjacent=*/false); + + for (const ManifestEntry& entry : files_to_check) { + int64_t range_from = entry.File()->first_row_id.value(); + int64_t range_to = range_from + entry.File()->row_count - 1; + Range row_range(range_from, range_to); + + bool exists = false; + if (IsDedicatedStorageFile(entry.FileName())) { + for (const Range& existing_range : merged_ranges) { + if (existing_range.from <= row_range.from && existing_range.to >= row_range.to) { + exists = true; + break; + } + } + } else { + exists = exact_ranges.find({row_range.from, row_range.to}) != exact_ranges.end(); + } + + if (!exists) { + return Status::Invalid(fmt::format( + "Row ID existence conflict: file '{}' references firstRowId={}, rowCount={} in " + "bucket {}, but no matching file exists in the current snapshot.", + entry.FileName(), entry.File()->first_row_id.value(), entry.File()->row_count, + entry.Bucket())); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckRowIdRangeConflicts( + const Snapshot::CommitKind& commit_kind, + const std::vector& merged_entries) const { + if (!options_.DataEvolutionEnabled()) { + return Status::OK(); + } + if (!row_id_check_from_snapshot_ && !(commit_kind == Snapshot::CommitKind::Compact())) { + return Status::OK(); + } + + std::vector> entries_with_ranges; + entries_with_ranges.reserve(merged_entries.size()); + for (const ManifestEntry& entry : merged_entries) { + if (!entry.File()->first_row_id) { + continue; + } + int64_t range_from = entry.File()->first_row_id.value(); + int64_t range_to = range_from + entry.File()->row_count - 1; + entries_with_ranges.emplace_back(Range(range_from, range_to), entry); + } + if (entries_with_ranges.empty()) { + return Status::OK(); + } + + std::sort(entries_with_ranges.begin(), entries_with_ranges.end(), + [](const auto& lhs, const auto& rhs) { return lhs.first.from < rhs.first.from; }); + + size_t group_start = 0; + int64_t group_max_to = entries_with_ranges[0].first.to; + for (size_t i = 1; i <= entries_with_ranges.size(); ++i) { + bool overlap_group_end = + (i == entries_with_ranges.size()) || (entries_with_ranges[i].first.from > group_max_to); + if (!overlap_group_end) { + group_max_to = std::max(group_max_to, entries_with_ranges[i].first.to); + continue; + } + + std::optional> expected_range; + for (size_t j = group_start; j < i; ++j) { + const ManifestEntry& entry = entries_with_ranges[j].second; + if (IsDedicatedStorageFile(entry.FileName())) { + continue; + } + std::pair current = {entries_with_ranges[j].first.from, + entries_with_ranges[j].first.to}; + if (!expected_range) { + expected_range = current; + } else if (expected_range.value() != current) { + return Status::Invalid( + "For Data Evolution table, multiple MERGE INTO/COMPACT operations have " + "encountered row-id range conflicts."); + } + } + + if (i < entries_with_ranges.size()) { + group_start = i; + group_max_to = entries_with_ranges[i].first.to; + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckForRowIdFromSnapshot( + const Snapshot& latest_snapshot, const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker) const { + if (!options_.DataEvolutionEnabled() || !row_id_check_from_snapshot_ || !snapshot_manager_ || + !manifest_list_ || !manifest_file_ || !row_id_column_conflict_checker || + !row_id_column_conflict_checker.value() || + row_id_column_conflict_checker.value()->IsEmpty()) { + return Status::OK(); + } + + if (row_id_check_from_snapshot_.value() > latest_snapshot.Id()) { + return Status::OK(); + } + + PAIMON_ASSIGN_OR_RAISE(Snapshot check_snapshot, + snapshot_manager_->LoadSnapshot(row_id_check_from_snapshot_.value())); + if (!check_snapshot.NextRowId()) { + return Status::OK(); + } + int64_t check_next_row_id = check_snapshot.NextRowId().value(); + + int64_t from_snapshot_id = row_id_check_from_snapshot_.value() + 1; + if (from_snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) { + from_snapshot_id = Snapshot::FIRST_SNAPSHOT_ID; + } + if (from_snapshot_id > latest_snapshot.Id()) { + return Status::OK(); + } + + std::vector changed_partitions = + ManifestEntryChanges::ChangedPartitions(delta_entries, delta_index_entries); + if (changed_partitions.empty()) { + return Status::OK(); + } + std::unordered_set changed_partition_set(changed_partitions.begin(), + changed_partitions.end()); + + for (int64_t snapshot_id = from_snapshot_id; snapshot_id <= latest_snapshot.Id(); + ++snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id)); + if (snapshot.GetCommitKind() == Snapshot::CommitKind::Compact()) { + continue; + } + + std::vector delta_manifests; + PAIMON_RETURN_NOT_OK(manifest_list_->ReadDeltaManifests(snapshot, &delta_manifests)); + for (const ManifestFileMeta& manifest_meta : delta_manifests) { + std::vector history_entries; + PAIMON_RETURN_NOT_OK(manifest_file_->Read(manifest_meta.FileName(), /*filter=*/nullptr, + &history_entries)); + for (const ManifestEntry& history_entry : history_entries) { + if (!(history_entry.Kind() == FileKind::Add()) || + !history_entry.File()->first_row_id || + changed_partition_set.find(history_entry.Partition()) == + changed_partition_set.end()) { + continue; + } + int64_t history_first_row_id = history_entry.File()->first_row_id.value(); + if (history_first_row_id >= check_next_row_id) { + continue; + } + if (row_id_column_conflict_checker.value()->ConflictsWith(*history_entry.File())) { + return Status::Invalid( + "For Data Evolution table, multiple MERGE INTO operations have " + "encountered conflicts while checking row-id history from " + "snapshot."); + } + } + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckGlobalIndexRowIdExistence( + const std::vector& base_entries, + const std::vector& delta_index_entries) const { + if (!options_.DataEvolutionEnabled()) { + return Status::OK(); + } + + std::vector indexes_to_check; + for (const IndexManifestEntry& index_entry : delta_index_entries) { + if (!(index_entry.kind == FileKind::Add()) || + !index_entry.index_file->GetGlobalIndexMeta()) { + continue; + } + indexes_to_check.push_back(index_entry); + } + if (indexes_to_check.empty()) { + return Status::OK(); + } + + for (const IndexManifestEntry& index_entry : indexes_to_check) { + std::vector data_ranges; + for (const ManifestEntry& base_entry : base_entries) { + if (!(base_entry.Kind() == FileKind::Add()) || + !(base_entry.Partition() == index_entry.partition) || + base_entry.Bucket() != index_entry.bucket || !base_entry.File()->first_row_id) { + continue; + } + + int64_t first_row_id = base_entry.File()->first_row_id.value(); + data_ranges.emplace_back(first_row_id, first_row_id + base_entry.File()->row_count - 1); + } + + const GlobalIndexMeta& global_index = index_entry.index_file->GetGlobalIndexMeta().value(); + Range index_range(global_index.row_range_start, global_index.row_range_end); + std::vector merged_ranges = Range::SortAndMergeOverlap(data_ranges, + /*adjacent=*/true); + bool covered = false; + for (const Range& data_range : merged_ranges) { + if (data_range.from <= index_range.from && data_range.to >= index_range.to) { + covered = true; + break; + } + } + if (!covered) { + return Status::Invalid(fmt::format( + "Global index row ID existence conflict: index file '{}' references row range {}, " + "but this range is not fully covered by current data files.", + index_entry.index_file->FileName(), index_range.ToString())); + } + } + + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/conflict_detection.h b/src/paimon/core/operation/commit/conflict_detection.h new file mode 100644 index 000000000..b04ad66fd --- /dev/null +++ b/src/paimon/core/operation/commit/conflict_detection.h @@ -0,0 +1,116 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/common/utils/linked_hash_map.h" +#include "paimon/core/core_options.h" +#include "paimon/core/snapshot.h" +#include "paimon/status.h" + +namespace paimon { + +class ManifestEntry; +struct IndexManifestEntry; +class ManifestFile; +class ManifestList; +class RowIdColumnConflictChecker; +class SnapshotManager; +class TableSchema; + +/// Util class for detecting conflicts between base and delta files. +class ConflictDetection { + public: + ConflictDetection(std::shared_ptr table_schema, const CoreOptions& options, + std::shared_ptr snapshot_manager, + std::shared_ptr manifest_list, + std::shared_ptr manifest_file); + + Status CheckConflicts(const Snapshot& latest_snapshot, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker, + const Snapshot::CommitKind& commit_kind) const; + + void SetRowIdCheckFromSnapshot(const std::optional& row_id_check_from_snapshot); + + bool HasRowIdCheckFromSnapshot() const; + + bool ShouldBeOverwriteCommit(const std::vector& append_table_files, + const std::vector& append_index_files) const; + + Status CollectUncheckedBucketPartitions( + const std::vector& delta_entries, + std::unordered_map* total_buckets) const; + + Status CheckSameBucketByTotalBuckets( + const std::unordered_map& expected_total_buckets, + const std::unordered_map& previous_total_buckets) const; + + private: + Status CheckBucketKeepSame(const std::vector& all_entries, + const Snapshot::CommitKind& commit_kind) const; + + Status BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets, + int32_t previous_num_buckets) const; + + void MarkBucketCheckedPartitions( + const std::unordered_map& total_buckets) const; + + Status CheckDeleteInEntries(const std::vector& merged_entries) const; + + Status CheckKeyRange(const std::vector& merged_entries) const; + + Status CheckRowIdExistence(const std::vector& base_entries, + const std::vector& delta_entries, + const std::optional& next_row_id) const; + + Status CheckRowIdRangeConflicts(const Snapshot::CommitKind& commit_kind, + const std::vector& merged_entries) const; + + Status CheckForRowIdFromSnapshot( + const Snapshot& latest_snapshot, const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker) const; + + Status CheckGlobalIndexRowIdExistence( + const std::vector& base_entries, + const std::vector& delta_index_entries) const; + + private: + static constexpr size_t kSameBucketCheckCacheMaxSize = 1000; + + std::shared_ptr table_schema_; + CoreOptions options_; + std::optional row_id_check_from_snapshot_; + std::shared_ptr snapshot_manager_; + std::shared_ptr manifest_list_; + std::shared_ptr manifest_file_; + mutable LinkedHashMap same_bucket_checked_partitions_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp new file mode 100644 index 000000000..d9937c2ec --- /dev/null +++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp @@ -0,0 +1,497 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/conflict_detection.h" + +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/memory/bytes.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +Snapshot MakeSnapshot(const Snapshot::CommitKind& commit_kind) { + return Snapshot( + /*id=*/1, + /*schema_id=*/1, + /*base_manifest_list=*/"base-manifest-list", + /*base_manifest_list_size=*/std::nullopt, + /*delta_manifest_list=*/"delta-manifest-list", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/std::nullopt, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, + /*commit_user=*/"test-user", + /*commit_identifier=*/1, commit_kind, + /*time_millis=*/0, + /*total_record_count=*/0, + /*delta_record_count=*/0, + /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); +} + +Status CheckConflicts(const ConflictDetection& detection, + const std::vector& base_entries, + const std::vector& delta_entries, + const Snapshot::CommitKind& commit_kind) { + return detection.CheckConflicts(MakeSnapshot(commit_kind), base_entries, delta_entries, + /*delta_index_entries=*/{}, + /*row_id_column_conflict_checker=*/std::nullopt, commit_kind); +} + +Status CheckConflicts(const ConflictDetection& detection, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::vector& delta_index_entries, + const Snapshot::CommitKind& commit_kind) { + return detection.CheckConflicts(MakeSnapshot(commit_kind), base_entries, delta_entries, + delta_index_entries, + /*row_id_column_conflict_checker=*/std::nullopt, commit_kind); +} + +} // namespace + +class ConflictDetectionTest : public testing::Test { + public: + void SetUp() override { + fields_ = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + } + + protected: + ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind) const { + int32_t arity = 1; + BinaryRow row(arity); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, 10); + writer.Complete(); + return CreateManifestEntry(file_name, row, kind); + } + + ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, + const FileKind& kind) const { + return CreateManifestEntry(file_name, partition, kind, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/2, /*bucket=*/0); + } + + ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, + const FileKind& kind, const BinaryRow& min_key, + const BinaryRow& max_key, int32_t level, int32_t bucket = 0, + int32_t total_buckets = 2) const { + auto data_file_meta = std::make_shared( + file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(), + SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32, + /*schema_id=*/1, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/3, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta); + } + + ManifestEntry CreateManifestEntryWithFirstRowId(const std::string& file_name, + const BinaryRow& partition, + const FileKind& kind, int32_t bucket, + int64_t first_row_id, int64_t row_count) const { + auto data_file_meta = std::make_shared( + file_name, 1024, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, + /*max_seq_no=*/32, + /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, first_row_id, + /*write_cols=*/std::nullopt); + return ManifestEntry(kind, partition, bucket, /*total_buckets=*/2, data_file_meta); + } + + IndexManifestEntry CreateGlobalIndexEntry(const std::string& file_name, + const BinaryRow& partition, int32_t bucket, + int64_t row_range_start, + int64_t row_range_end) const { + GlobalIndexMeta global_index_meta(row_range_start, row_range_end, /*index_field_id=*/1, + /*extra_field_ids=*/std::nullopt, + std::make_shared("meta", GetDefaultPool().get())); + auto index_file_meta = std::make_shared( + "HASH", file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index_meta); + return IndexManifestEntry(FileKind::Add(), partition, bucket, index_file_meta); + } + + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; + } + + arrow::FieldVector fields_; +}; + +TEST_F(ConflictDetectionTest, TestFileDeletionConflicts) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("f1", FileKind::Add())); + + std::vector changes; + changes.push_back(CreateManifestEntry("f1", FileKind::Delete())); + + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("f2", FileKind::Add())); + + std::vector changes; + changes.push_back(CreateManifestEntry("f1", FileKind::Delete())); + changes.push_back(CreateManifestEntry("f3", FileKind::Add())); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "Trying to delete file f1"); + } + { + std::vector base_entries; + std::vector changes; + changes.push_back(CreateManifestEntry("f1", FileKind::Delete())); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "Trying to delete file f1"); + } +} + +TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceConflicts) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId("base-1", partition, FileKind::Add(), + /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/10)); + base_entries.push_back(CreateManifestEntryWithFirstRowId("base-2", partition, FileKind::Add(), + /*bucket=*/0, /*first_row_id=*/10, + /*row_count=*/10)); + std::vector changes; + + ASSERT_OK( + CheckConflicts(detection, base_entries, changes, + {CreateGlobalIndexEntry("global-index-covered", partition, /*bucket=*/0, + /*row_range_start=*/0, /*row_range_end=*/19)}, + Snapshot::CommitKind::Append())); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, + {CreateGlobalIndexEntry("global-index-missing", partition, /*bucket=*/0, + /*row_range_start=*/0, /*row_range_end=*/20)}, + Snapshot::CommitKind::Append()), + "Global index row ID existence conflict"); +} + +TEST_F(ConflictDetectionTest, TestBucketKeepSame) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + + const BinaryRow partition = CreateIntRow(10); + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "Total buckets of partition"); + } + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", CreateIntRow(20), FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_OK( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Overwrite())); + } +} + +TEST_F(ConflictDetectionTest, TestBucketKeepSameHelpers) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector changes; + changes.push_back(CreateManifestEntry("delta-1", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + changes.push_back(CreateManifestEntry("delta-2", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/1, /*total_buckets=*/4)); + + std::unordered_map expected_total_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &expected_total_buckets)); + ASSERT_EQ(1U, expected_total_buckets.size()); + ASSERT_EQ(4, expected_total_buckets.at(partition)); + + std::unordered_map previous_total_buckets; + previous_total_buckets.emplace(partition, 4); + ASSERT_OK( + detection.CheckSameBucketByTotalBuckets(expected_total_buckets, previous_total_buckets)); + + std::unordered_map cached_total_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &cached_total_buckets)); + ASSERT_TRUE(cached_total_buckets.empty()); + + ConflictDetection mismatch_detection(table_schema, core_options, nullptr, nullptr, nullptr); + ASSERT_NOK_WITH_MSG( + mismatch_detection.CheckSameBucketByTotalBuckets(expected_total_buckets, {{partition, 2}}), + "Total buckets of partition"); +} + +TEST_F(ConflictDetectionTest, TestCollectUncheckedBucketPartitionsMismatch) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector changes; + changes.push_back(CreateManifestEntry("delta-1", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + changes.push_back(CreateManifestEntry("delta-2", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/1, /*total_buckets=*/4)); + + std::unordered_map total_buckets; + ASSERT_NOK_WITH_MSG(detection.CollectUncheckedBucketPartitions(changes, &total_buckets), + "Total buckets of partition"); +} + +TEST_F(ConflictDetectionTest, TestBucketKeepSameCacheEviction) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + const int32_t total_buckets = 4; + for (int32_t value = 0; value <= 1000; ++value) { + std::vector changes; + changes.push_back(CreateManifestEntry("delta", CreateIntRow(value), FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, total_buckets)); + + std::unordered_map expected_total_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &expected_total_buckets)); + ASSERT_EQ(1U, expected_total_buckets.size()); + ASSERT_OK(detection.CheckSameBucketByTotalBuckets(expected_total_buckets, + expected_total_buckets)); + } + + std::vector evicted_partition_changes; + evicted_partition_changes.push_back( + CreateManifestEntry("delta", CreateIntRow(0), FileKind::Add(), DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, /*bucket=*/0, total_buckets)); + std::unordered_map evicted_partition_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(evicted_partition_changes, + &evicted_partition_buckets)); + ASSERT_EQ(1U, evicted_partition_buckets.size()); +} + +TEST_F(ConflictDetectionTest, TestDeletionVectorsNotSupportedWithBucketUnawareMode) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::BUCKET, "0"}, + {Options::DELETION_VECTORS_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, + Snapshot::CommitKind::Append()), + "not yet support dv with BUCKET_UNAWARE mode"); +} + +TEST_F(ConflictDetectionTest, TestCheckLsmKeyRangeConflict) { + auto fields = {arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::int32(), /*nullable=*/false), + arrow::field("f2", arrow::int32())}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{"f1", "f0"}, {{Options::BUCKET, "4"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "4"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(3), + CreateIntRow(5), /*level=*/1)); + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "LSM conflicts detected"); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(4), + CreateIntRow(5), /*level=*/1)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/0)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(2), + CreateIntRow(5), /*level=*/0)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1, /*bucket=*/0)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(2), + CreateIntRow(5), /*level=*/1, + /*bucket=*/1)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", CreateIntRow(20), FileKind::Add(), + CreateIntRow(2), CreateIntRow(5), /*level=*/1)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/manifest_entry_changes.cpp b/src/paimon/core/operation/commit/manifest_entry_changes.cpp new file mode 100644 index 000000000..f14c27481 --- /dev/null +++ b/src/paimon/core/operation/commit/manifest_entry_changes.cpp @@ -0,0 +1,146 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/manifest_entry_changes.h" + +#include + +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_increment.h" + +namespace paimon { + +ManifestEntryChanges::ManifestEntryChanges(int32_t default_num_bucket) + : default_num_bucket_(default_num_bucket) {} + +Status ManifestEntryChanges::Collect(const std::shared_ptr& message) { + auto commit_message = std::dynamic_pointer_cast(message); + if (!commit_message) { + return Status::Invalid("fail to cast commit message to commit message impl"); + } + + DataIncrement new_files_increment = commit_message->GetNewFilesIncrement(); + for (const std::shared_ptr& file : new_files_increment.NewFiles()) { + append_table_files.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : new_files_increment.DeletedFiles()) { + append_table_files.push_back(MakeEntry(FileKind::Delete(), commit_message, file)); + } + for (const std::shared_ptr& file : new_files_increment.ChangelogFiles()) { + append_changelog.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : new_files_increment.DeletedIndexFiles()) { + append_index_files.emplace_back(FileKind::Delete(), commit_message->Partition(), + commit_message->Bucket(), file); + } + for (const std::shared_ptr& file : new_files_increment.NewIndexFiles()) { + append_index_files.emplace_back(FileKind::Add(), commit_message->Partition(), + commit_message->Bucket(), file); + } + + CompactIncrement compact_increment = commit_message->GetCompactIncrement(); + for (const std::shared_ptr& file : compact_increment.CompactBefore()) { + compact_table_files.push_back(MakeEntry(FileKind::Delete(), commit_message, file)); + } + for (const std::shared_ptr& file : compact_increment.CompactAfter()) { + compact_table_files.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : compact_increment.ChangelogFiles()) { + compact_changelog.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : compact_increment.DeletedIndexFiles()) { + compact_index_files.emplace_back(FileKind::Delete(), commit_message->Partition(), + commit_message->Bucket(), file); + } + for (const std::shared_ptr& file : compact_increment.NewIndexFiles()) { + compact_index_files.emplace_back(FileKind::Add(), commit_message->Partition(), + commit_message->Bucket(), file); + } + + return Status::OK(); +} + +bool ManifestEntryChanges::HasAppendChanges() const { + return !append_table_files.empty() || !append_changelog.empty() || !append_index_files.empty(); +} + +bool ManifestEntryChanges::HasGlobalIndexFileAdditions() const { + for (const IndexManifestEntry& index_entry : append_index_files) { + if (index_entry.kind == FileKind::Add() && index_entry.index_file->GetGlobalIndexMeta()) { + return true; + } + } + return false; +} + +bool ManifestEntryChanges::HasCompactChanges() const { + return !compact_table_files.empty() || !compact_changelog.empty() || + !compact_index_files.empty(); +} + +std::string ManifestEntryChanges::ToString() const { + std::vector parts; + if (!append_table_files.empty()) { + parts.push_back(fmt::format("{} append table files", append_table_files.size())); + } + if (!append_changelog.empty()) { + parts.push_back(fmt::format("{} append Changelogs", append_changelog.size())); + } + if (!append_index_files.empty()) { + parts.push_back(fmt::format("{} append index files", append_index_files.size())); + } + if (!compact_table_files.empty()) { + parts.push_back(fmt::format("{} compact table files", compact_table_files.size())); + } + if (!compact_changelog.empty()) { + parts.push_back(fmt::format("{} compact Changelogs", compact_changelog.size())); + } + if (!compact_index_files.empty()) { + parts.push_back(fmt::format("{} compact index files", compact_index_files.size())); + } + return fmt::format("{}", fmt::join(parts, ", ")); +} + +std::vector ManifestEntryChanges::ChangedPartitions( + const std::vector& data_file_changes, + const std::vector& index_file_changes) { + std::unordered_set changed_partitions; + for (const ManifestEntry& file : data_file_changes) { + changed_partitions.insert(file.Partition()); + } + for (const IndexManifestEntry& file : index_file_changes) { + if (file.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX || + file.index_file->GetGlobalIndexMeta()) { + changed_partitions.insert(file.partition); + } + } + return std::vector(changed_partitions.begin(), changed_partitions.end()); +} + +ManifestEntry ManifestEntryChanges::MakeEntry( + const FileKind& kind, const std::shared_ptr& commit_message, + const std::shared_ptr& file) const { + int32_t total_buckets = commit_message->TotalBuckets() == std::nullopt + ? default_num_bucket_ + : commit_message->TotalBuckets().value(); + return ManifestEntry(kind, commit_message->Partition(), commit_message->Bucket(), total_buckets, + file); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/manifest_entry_changes.h b/src/paimon/core/operation/commit/manifest_entry_changes.h new file mode 100644 index 000000000..2ccec1864 --- /dev/null +++ b/src/paimon/core/operation/commit/manifest_entry_changes.h @@ -0,0 +1,72 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/commit_message.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/status.h" + +namespace paimon { + +/// Detailed changes from `CommitMessage`s. +class ManifestEntryChanges { + public: + explicit ManifestEntryChanges(int32_t default_num_bucket); + + Status Collect(const std::shared_ptr& message); + + bool HasAppendChanges() const; + + bool HasGlobalIndexFileAdditions() const; + + bool HasCompactChanges() const; + + std::string ToString() const; + + static std::vector ChangedPartitions( + const std::vector& data_file_changes, + const std::vector& index_file_changes); + + public: + std::vector append_table_files; + std::vector append_changelog; + std::vector append_index_files; + std::vector compact_table_files; + std::vector compact_changelog; + std::vector compact_index_files; + + private: + ManifestEntry MakeEntry(const FileKind& kind, + const std::shared_ptr& commit_message, + const std::shared_ptr& file) const; + + private: + int32_t default_num_bucket_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp new file mode 100644 index 000000000..6fae6cf96 --- /dev/null +++ b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp @@ -0,0 +1,185 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/manifest_entry_changes.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_increment.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/memory/bytes.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class ManifestEntryChangesTest : public testing::Test { + protected: + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; + } + + std::shared_ptr CreateDataFileMeta(const std::string& file_name) const { + return std::make_shared( + file_name, 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, + /*max_seq_no=*/32, + /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + + std::shared_ptr CreateIndexFileMeta( + const std::string& file_name, const std::string& index_type = "bitmap") const { + return std::make_shared( + index_type, file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, + /*global_index_meta=*/std::nullopt); + } + + std::shared_ptr CreateGlobalIndexFileMeta(const std::string& file_name) const { + GlobalIndexMeta global_index(/*row_range_start=*/0, /*row_range_end=*/3, + /*index_field_id=*/1, /*extra_field_ids=*/std::nullopt, + std::make_shared("meta", GetDefaultPool().get())); + return std::make_shared( + "HASH", file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index); + } +}; + +TEST_F(ManifestEntryChangesTest, TestCollectAndSummary) { + const BinaryRow partition = CreateIntRow(10); + + DataIncrement data_increment( + {CreateDataFileMeta("append-add")}, {CreateDataFileMeta("append-del")}, + {CreateDataFileMeta("append-changelog")}, {CreateIndexFileMeta("append-index-add")}, + {CreateIndexFileMeta("append-index-del")}); + CompactIncrement compact_increment( + {CreateDataFileMeta("compact-before")}, {CreateDataFileMeta("compact-after")}, + {CreateDataFileMeta("compact-changelog")}, {CreateIndexFileMeta("compact-index-add")}, + {CreateIndexFileMeta("compact-index-del")}); + + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/4, data_increment, compact_increment); + + ManifestEntryChanges changes(/*default_num_bucket=*/8); + ASSERT_OK(changes.Collect(message)); + + ASSERT_EQ(2u, changes.append_table_files.size()); + ASSERT_EQ(1u, changes.append_changelog.size()); + ASSERT_EQ(2u, changes.append_index_files.size()); + ASSERT_EQ(2u, changes.compact_table_files.size()); + ASSERT_EQ(1u, changes.compact_changelog.size()); + ASSERT_EQ(2u, changes.compact_index_files.size()); + + EXPECT_TRUE(changes.HasAppendChanges()); + EXPECT_FALSE(changes.HasGlobalIndexFileAdditions()); + EXPECT_TRUE(changes.HasCompactChanges()); + + EXPECT_EQ(FileKind::Add(), changes.append_table_files[0].Kind()); + EXPECT_EQ(FileKind::Delete(), changes.append_table_files[1].Kind()); + EXPECT_EQ(4, changes.append_table_files[0].TotalBuckets()); + + std::string summary = changes.ToString(); + EXPECT_NE(std::string::npos, summary.find("2 append table files")); + EXPECT_NE(std::string::npos, summary.find("1 append Changelogs")); + EXPECT_NE(std::string::npos, summary.find("2 compact index files")); +} + +TEST_F(ManifestEntryChangesTest, TestHasGlobalIndexFileAdditions) { + const BinaryRow partition = CreateIntRow(10); + + DataIncrement data_increment( + /*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{}, + /*new_index_files=*/{CreateGlobalIndexFileMeta("append-global-index")}, + /*deleted_index_files=*/{}); + CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{}, + /*changelog_files=*/{}, + /*new_index_files=*/{}, + /*deleted_index_files=*/{}); + + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/4, data_increment, compact_increment); + + ManifestEntryChanges changes(/*default_num_bucket=*/8); + ASSERT_OK(changes.Collect(message)); + + EXPECT_TRUE(changes.HasGlobalIndexFileAdditions()); +} + +TEST_F(ManifestEntryChangesTest, TestCollectInvalidCommitMessageType) { + ManifestEntryChanges changes(/*default_num_bucket=*/8); + std::shared_ptr invalid_message = std::make_shared(); + ASSERT_NOK_WITH_MSG(changes.Collect(invalid_message), + "fail to cast commit message to commit message impl"); +} + +TEST_F(ManifestEntryChangesTest, TestChangedPartitionsIncludesDvAndGlobalIndex) { + const BinaryRow partition_data = CreateIntRow(10); + const BinaryRow partition_dv = CreateIntRow(20); + const BinaryRow partition_global = CreateIntRow(30); + const BinaryRow partition_plain_index = CreateIntRow(40); + + std::vector data_changes; + data_changes.emplace_back(FileKind::Add(), partition_data, /*bucket=*/0, /*total_buckets=*/2, + CreateDataFileMeta("data-file")); + + std::vector index_changes; + index_changes.emplace_back(FileKind::Add(), partition_dv, /*bucket=*/0, + CreateIndexFileMeta("dv-file", "DELETION_VECTORS")); + index_changes.emplace_back(FileKind::Add(), partition_global, /*bucket=*/0, + CreateGlobalIndexFileMeta("global-index")); + index_changes.emplace_back(FileKind::Add(), partition_plain_index, /*bucket=*/0, + CreateIndexFileMeta("plain-index", "bitmap")); + + std::vector changed = + ManifestEntryChanges::ChangedPartitions(data_changes, index_changes); + + auto contains = [&changed](const BinaryRow& target) { + return std::find(changed.begin(), changed.end(), target) != changed.end(); + }; + + EXPECT_TRUE(contains(partition_data)); + EXPECT_TRUE(contains(partition_dv)); + EXPECT_TRUE(contains(partition_global)); + EXPECT_FALSE(contains(partition_plain_index)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider.cpp new file mode 100644 index 000000000..776d302f1 --- /dev/null +++ b/src/paimon/core/operation/commit/overwrite_changes_provider.cpp @@ -0,0 +1,76 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/overwrite_changes_provider.h" + +#include +#include + +#include "paimon/core/manifest/file_entry.h" +#include "paimon/core/manifest/file_kind.h" + +namespace paimon { + +OverwriteChangesProvider::OverwriteChangesProvider( + const std::vector& changes, const std::vector& index_entries, + ManifestScan manifest_scan, IndexScan index_scan) + : changes_(changes), + index_entries_(index_entries), + manifest_scan_(std::move(manifest_scan)), + index_scan_(std::move(index_scan)) {} + +Status OverwriteChangesProvider::Provide(const std::optional& latest_snapshot, + std::vector* delta_files, + std::vector* changelog_files, + std::vector* index_entries) const { + changelog_files->clear(); + *index_entries = index_entries_; + delta_files->clear(); + + if (!latest_snapshot) { + delta_files->insert(delta_files->end(), changes_.begin(), changes_.end()); + return Status::OK(); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + manifest_scan_(latest_snapshot.value())); + std::unordered_set existing_identifiers; + existing_identifiers.reserve(entries.size()); + for (const auto& entry : entries) { + existing_identifiers.insert(entry.CreateIdentifier()); + delta_files->emplace_back(FileKind::Delete(), entry.Partition(), entry.Bucket(), + entry.TotalBuckets(), entry.File()); + } + + for (const auto& change : changes_) { + if (change.Kind() == FileKind::Add() && + existing_identifiers.find(change.CreateIdentifier()) != existing_identifiers.end()) { + continue; + } + delta_files->push_back(change); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector previous_index_entries, + index_scan_(latest_snapshot.value())); + for (const auto& entry : previous_index_entries) { + index_entries->emplace_back(FileKind::Delete(), entry.partition, entry.bucket, + entry.index_file); + } + + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider.h b/src/paimon/core/operation/commit/overwrite_changes_provider.h new file mode 100644 index 000000000..c8ec79ecc --- /dev/null +++ b/src/paimon/core/operation/commit/overwrite_changes_provider.h @@ -0,0 +1,49 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/operation/commit/commit_changes_provider.h" + +namespace paimon { + +class OverwriteChangesProvider final : public CommitChangesProvider { + public: + using ManifestScan = + std::function>(const Snapshot& snapshot)>; + using IndexScan = + std::function>(const Snapshot& snapshot)>; + + OverwriteChangesProvider(const std::vector& changes, + const std::vector& index_entries, + ManifestScan manifest_scan, IndexScan index_scan); + + Status Provide(const std::optional& latest_snapshot, + std::vector* delta_files, + std::vector* changelog_files, + std::vector* index_entries) const override; + + private: + const std::vector& changes_; + const std::vector& index_entries_; + ManifestScan manifest_scan_; + IndexScan index_scan_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/retry_waiter.cpp b/src/paimon/core/operation/commit/retry_waiter.cpp new file mode 100644 index 000000000..3fda445fc --- /dev/null +++ b/src/paimon/core/operation/commit/retry_waiter.cpp @@ -0,0 +1,50 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/retry_waiter.h" + +#include +#include +#include +#include +#include +#include + +namespace paimon { + +RetryWaiter::RetryWaiter(int64_t min_retry_wait_ms, int64_t max_retry_wait_ms) + : min_retry_wait_ms_(std::max(0, min_retry_wait_ms)), + max_retry_wait_ms_(std::max(0, max_retry_wait_ms)) {} + +void RetryWaiter::RetryWait(int32_t retry_count) const { + int32_t non_negative_retry_count = std::max(0, retry_count); + double exponential = std::pow(2.0, static_cast(non_negative_retry_count)); + int64_t retry_wait = static_cast(min_retry_wait_ms_ * exponential); + retry_wait = std::min(retry_wait, max_retry_wait_ms_); + + int64_t jitter_upper = std::max(1, static_cast(retry_wait * 0.2)); + std::mt19937 rng(std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution dist(0, jitter_upper - 1); + retry_wait += dist(rng); + + if (retry_wait <= 0) { + return; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(retry_wait)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/retry_waiter.h b/src/paimon/core/operation/commit/retry_waiter.h new file mode 100644 index 000000000..c7cbd6763 --- /dev/null +++ b/src/paimon/core/operation/commit/retry_waiter.h @@ -0,0 +1,34 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace paimon { + +class RetryWaiter { + public: + RetryWaiter(int64_t min_retry_wait_ms, int64_t max_retry_wait_ms); + + void RetryWait(int32_t retry_count) const; + + private: + int64_t min_retry_wait_ms_; + int64_t max_retry_wait_ms_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/retry_waiter_test.cpp b/src/paimon/core/operation/commit/retry_waiter_test.cpp new file mode 100644 index 000000000..d7b15900a --- /dev/null +++ b/src/paimon/core/operation/commit/retry_waiter_test.cpp @@ -0,0 +1,36 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/retry_waiter.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(RetryWaiterTest, TestRetryWaitWithZeroBoundsReturnsQuickly) { + RetryWaiter waiter(/*min_retry_wait_ms=*/0, /*max_retry_wait_ms=*/0); + + auto begin = std::chrono::steady_clock::now(); + waiter.RetryWait(/*retry_count=*/3); + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - begin); + + ASSERT_LT(elapsed.count(), 20); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp new file mode 100644 index 000000000..d5adc878f --- /dev/null +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp @@ -0,0 +1,224 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" + +#include +#include + +#include "paimon/common/table/special_fields.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/status.h" + +namespace paimon { + +namespace { + +struct DataFileRangeItem { + Range range; + const DataFileMeta* file; +}; + +} // namespace + +Result> RowIdColumnConflictChecker::FromDataFiles( + const std::shared_ptr& schema_manager, + const std::vector>& data_files) { + auto checker = + std::shared_ptr(new RowIdColumnConflictChecker(schema_manager)); + + std::vector row_id_files; + row_id_files.reserve(data_files.size()); + for (const auto& file : data_files) { + if (file && file->first_row_id.has_value()) { + row_id_files.push_back(file.get()); + } + } + + PAIMON_RETURN_NOT_OK(checker->BuildWriteRanges(row_id_files)); + return checker; +} + +Result> RowIdColumnConflictChecker::FromManifestEntries( + const std::shared_ptr& schema_manager, + const std::vector& delta_entries) { + std::vector> data_files; + data_files.reserve(delta_entries.size()); + for (const auto& entry : delta_entries) { + data_files.push_back(entry.File()); + } + return FromDataFiles(schema_manager, data_files); +} + +Status RowIdColumnConflictChecker::BuildWriteRanges( + const std::vector& row_id_files) { + if (row_id_files.empty()) { + write_ranges_.clear(); + return Status::OK(); + } + + std::vector sorted_files; + sorted_files.reserve(row_id_files.size()); + for (const auto* file : row_id_files) { + const int64_t from = file->first_row_id.value(); + sorted_files.push_back(DataFileRangeItem{Range(from, from + file->row_count - 1), file}); + } + + std::sort(sorted_files.begin(), sorted_files.end(), + [](const DataFileRangeItem& a, const DataFileRangeItem& b) { + if (a.range.from != b.range.from) { + return a.range.from < b.range.from; + } + return a.range.to < b.range.to; + }); + + write_ranges_.clear(); + + size_t i = 0; + while (i < sorted_files.size()) { + int64_t merged_from = sorted_files[i].range.from; + int64_t merged_to = sorted_files[i].range.to; + std::unordered_set field_ids; + PAIMON_RETURN_NOT_OK(AddWriteFieldIds(&field_ids, *sorted_files[i].file)); + + size_t j = i + 1; + while (j < sorted_files.size() && sorted_files[j].range.from <= merged_to) { + merged_to = std::max(merged_to, sorted_files[j].range.to); + PAIMON_RETURN_NOT_OK(AddWriteFieldIds(&field_ids, *sorted_files[j].file)); + ++j; + } + + write_ranges_.push_back(WriteRange{Range(merged_from, merged_to), std::move(field_ids)}); + i = j; + } + + std::sort(write_ranges_.begin(), write_ranges_.end(), + [](const WriteRange& a, const WriteRange& b) { + if (a.range.from != b.range.from) { + return a.range.from < b.range.from; + } + return a.range.to < b.range.to; + }); + + return Status::OK(); +} + +Status RowIdColumnConflictChecker::AddWriteFieldIds(std::unordered_set* field_ids, + const DataFileMeta& file) { + if (!file.write_cols.has_value()) { + const std::map* field_id_by_name = nullptr; + PAIMON_RETURN_NOT_OK(FieldIdByName(file.schema_id, &field_id_by_name)); + for (const auto& entry : *field_id_by_name) { + field_ids->insert(entry.second); + } + return Status::OK(); + } + + for (const auto& write_col : file.write_cols.value()) { + PAIMON_ASSIGN_OR_RAISE(std::optional field_id, ResolveFieldId(file, write_col)); + if (field_id.has_value()) { + field_ids->insert(field_id.value()); + } + } + + return Status::OK(); +} + +bool RowIdColumnConflictChecker::ConflictsWith(const DataFileMeta& file) const { + if (!file.first_row_id.has_value()) { + return false; + } + + Range target(file.first_row_id.value(), file.first_row_id.value() + file.row_count - 1); + + int low = 0; + int high = static_cast(write_ranges_.size()); + while (low < high) { + const int mid = (low + high) >> 1; + if (write_ranges_[mid].range.to < target.from) { + low = mid + 1; + } else { + high = mid; + } + } + + for (int index = low; index < static_cast(write_ranges_.size()); ++index) { + const auto& write_range = write_ranges_[index]; + if (write_range.range.from > target.to) { + return false; + } + if (!Range::HasIntersection(write_range.range, target)) { + continue; + } + + if (!file.write_cols.has_value()) { + return true; + } + + for (const auto& write_col : file.write_cols.value()) { + auto field_id_result = ResolveFieldId(file, write_col); + if (!field_id_result.ok()) { + return false; + } + const auto& field_id = field_id_result.value(); + if (field_id.has_value() && write_range.field_ids.count(field_id.value()) > 0) { + return true; + } + } + } + + return false; +} + +Result> RowIdColumnConflictChecker::ResolveFieldId( + const DataFileMeta& file, const std::string& write_col) const { + const std::map* field_id_by_name = nullptr; + PAIMON_RETURN_NOT_OK(FieldIdByName(file.schema_id, &field_id_by_name)); + auto it = field_id_by_name->find(write_col); + if (it != field_id_by_name->end()) { + return std::optional(it->second); + } + + if (SpecialFields::IsSpecialFieldName(write_col)) { + return std::optional(); + } + + return Status::Invalid("Cannot find write column '" + write_col + "' in schema " + + std::to_string(file.schema_id) + "."); +} + +Status RowIdColumnConflictChecker::FieldIdByName( + int64_t schema_id, const std::map** field_id_by_name) const { + auto cache_it = field_id_by_name_cache_.find(schema_id); + if (cache_it != field_id_by_name_cache_.end()) { + *field_id_by_name = &cache_it->second; + return Status::OK(); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + schema_manager_->ReadSchema(schema_id)); + std::map mapping; + for (const auto& field : schema->Fields()) { + mapping[field.Name()] = field.Id(); + } + + auto [it, inserted] = field_id_by_name_cache_.emplace(schema_id, std::move(mapping)); + (void)inserted; + *field_id_by_name = &it->second; + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker.h b/src/paimon/core/operation/commit/row_id_column_conflict_checker.h new file mode 100644 index 000000000..13e7a12a9 --- /dev/null +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker.h @@ -0,0 +1,74 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/utils/range.h" + +namespace paimon { + +class ManifestEntry; + +// Java-aligned row-id conflict checker: overlap in row-id range AND write columns. +class RowIdColumnConflictChecker { + public: + static Result> FromDataFiles( + const std::shared_ptr& schema_manager, + const std::vector>& data_files); + + static Result> FromManifestEntries( + const std::shared_ptr& schema_manager, + const std::vector& delta_entries); + + bool IsEmpty() const { + return write_ranges_.empty(); + } + + bool ConflictsWith(const DataFileMeta& file) const; + + private: + struct WriteRange { + Range range; + std::unordered_set field_ids; + }; + + explicit RowIdColumnConflictChecker(const std::shared_ptr& schema_manager) + : schema_manager_(schema_manager) {} + + Status BuildWriteRanges(const std::vector& row_id_files); + Status AddWriteFieldIds(std::unordered_set* field_ids, const DataFileMeta& file); + Result> ResolveFieldId(const DataFileMeta& file, + const std::string& write_col) const; + Status FieldIdByName(int64_t schema_id, + const std::map** field_id_by_name) const; + + private: + std::shared_ptr schema_manager_; + mutable std::map> field_id_by_name_cache_; + std::vector write_ranges_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp new file mode 100644 index 000000000..bd2e9091d --- /dev/null +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp @@ -0,0 +1,133 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/row_tracking_commit_utils.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/status.h" + +namespace paimon { + +namespace { + +bool IsVectorStoreFile(const std::string& file_name) { + return file_name.find(".vector.") != std::string::npos; +} + +} // namespace + +Result RowTrackingCommitUtils::AssignRowTracking( + int64_t new_snapshot_id, int64_t first_row_id_start, + const std::vector& delta_files) { + std::vector snapshot_assigned; + AssignSnapshotId(new_snapshot_id, delta_files, &snapshot_assigned); + + std::vector row_id_assigned; + PAIMON_ASSIGN_OR_RAISE( + int64_t next_row_id_start, + AssignRowTrackingMeta(first_row_id_start, snapshot_assigned, &row_id_assigned)); + return RowTrackingAssigned{next_row_id_start, std::move(row_id_assigned)}; +} + +void RowTrackingCommitUtils::AssignSnapshotId(int64_t snapshot_id, + const std::vector& delta_files, + std::vector* snapshot_assigned) { + for (const auto& entry : delta_files) { + ManifestEntry assigned_entry = entry; + int64_t min_seq_number = assigned_entry.File()->min_sequence_number; + int64_t max_seq_number = assigned_entry.File()->max_sequence_number; + if (min_seq_number == 0L) { + assigned_entry.AssignSequenceNumber(snapshot_id, snapshot_id); + } else if (max_seq_number == 0L) { + assigned_entry.AssignSequenceNumber(min_seq_number, snapshot_id); + } + snapshot_assigned->emplace_back(std::move(assigned_entry)); + } +} + +Result RowTrackingCommitUtils::AssignRowTrackingMeta( + int64_t first_row_id_start, const std::vector& delta_files, + std::vector* row_id_assigned) { + if (delta_files.empty()) { + return first_row_id_start; + } + + int64_t start = first_row_id_start; + int64_t blob_start_default = first_row_id_start; + std::map blob_starts; + int64_t vector_store_start = first_row_id_start; + + for (const auto& entry : delta_files) { + ManifestEntry assigned_entry = entry; + if (!entry.File()->file_source) { + return Status::Invalid( + "This is a bug, file source field for row-tracking table must present."); + } + + bool contains_row_id = + entry.File()->write_cols.has_value() && + std::find(entry.File()->write_cols->begin(), entry.File()->write_cols->end(), + SpecialFields::RowId().Name()) != entry.File()->write_cols->end(); + + if (entry.File()->file_source.value() == FileSource::Append() && + entry.File()->first_row_id == std::nullopt && !contains_row_id) { + int64_t row_count = entry.File()->row_count; + if (BlobUtils::IsBlobFile(entry.File()->file_name)) { + if (!entry.File()->write_cols || entry.File()->write_cols->empty()) { + return Status::Invalid(fmt::format( + "invalid blob file {}: does not have write_cols", entry.File()->file_name)); + } + std::string blob_field_name = entry.File()->write_cols->at(0); + int64_t blob_start = blob_starts.count(blob_field_name) + ? blob_starts[blob_field_name] + : blob_start_default; + if (blob_start >= start) { + return Status::Invalid( + fmt::format("This is a bug, blobStart {} should be less than start {} when " + "assigning a blob entry file.", + blob_start, start)); + } + assigned_entry.AssignFirstRowId(blob_start); + blob_starts[blob_field_name] = blob_start + row_count; + } else if (IsVectorStoreFile(entry.File()->file_name)) { + if (vector_store_start >= start) { + return Status::Invalid(fmt::format( + "This is a bug, vectorStoreStart {} should be less than start {} " + "when assigning a vector-store entry file.", + vector_store_start, start)); + } + assigned_entry.AssignFirstRowId(vector_store_start); + vector_store_start += row_count; + } else { + assigned_entry.AssignFirstRowId(start); + blob_start_default = start; + blob_starts.clear(); + start += row_count; + } + } + row_id_assigned->emplace_back(std::move(assigned_entry)); + } + return start; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.h b/src/paimon/core/operation/commit/row_tracking_commit_utils.h new file mode 100644 index 000000000..df5869958 --- /dev/null +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.h @@ -0,0 +1,48 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/result.h" + +namespace paimon { + +class RowTrackingCommitUtils { + public: + struct RowTrackingAssigned { + int64_t next_row_id_start; + std::vector assigned_entries; + }; + + // Assign sequence numbers and row ids for row-tracking commit. + static Result AssignRowTracking( + int64_t new_snapshot_id, int64_t first_row_id_start, + const std::vector& delta_files); + + private: + static void AssignSnapshotId(int64_t snapshot_id, const std::vector& delta_files, + std::vector* snapshot_assigned); + + static Result AssignRowTrackingMeta(int64_t first_row_id_start, + const std::vector& delta_files, + std::vector* row_id_assigned); +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp new file mode 100644 index 000000000..5b26b7cdb --- /dev/null +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp @@ -0,0 +1,131 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/row_tracking_commit_utils.h" + +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class RowTrackingCommitUtilsTest : public testing::Test { + protected: + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; + } + + ManifestEntry CreateEntry(const std::string& file_name, int64_t row_count, + int64_t min_seq_number, int64_t max_seq_number, + const std::optional& file_source, + const std::optional>& write_cols) const { + auto file_meta = std::make_shared( + file_name, /*file_size=*/row_count, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + min_seq_number, max_seq_number, + /*schema_id=*/1, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, file_source, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, write_cols); + return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1, + file_meta); + } +}; + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingStampsSequence) { + std::vector input; + input.push_back(CreateEntry("new-file", /*row_count=*/10, /*min_seq=*/0, /*max_seq=*/0, + FileSource::Append(), std::vector{"f0"})); + input.push_back(CreateEntry("partial-modified", /*row_count=*/8, /*min_seq=*/7, + /*max_seq=*/0, FileSource::Append(), + std::vector{"f0"})); + input.push_back(CreateEntry("compact-file", /*row_count=*/6, /*min_seq=*/3, /*max_seq=*/5, + FileSource::Compact(), std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(3u, assigned.assigned_entries.size()); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number); + EXPECT_EQ(7, assigned.assigned_entries[1].File()->min_sequence_number); + EXPECT_EQ(100, assigned.assigned_entries[1].File()->max_sequence_number); + EXPECT_EQ(3, assigned.assigned_entries[2].File()->min_sequence_number); + EXPECT_EQ(5, assigned.assigned_entries[2].File()->max_sequence_number); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTracking) { + std::vector input; + input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq=*/0, /*max_seq=*/0, + FileSource::Append(), std::vector{"f0"})); + input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq=*/0, /*max_seq=*/0, + FileSource::Append(), std::vector{"blob_a"})); + input.push_back(CreateEntry("blob-a-2.blob", /*row_count=*/2, /*min_seq=*/0, /*max_seq=*/0, + FileSource::Append(), std::vector{"blob_a"})); + input.push_back(CreateEntry("vector-1.vector.data", /*row_count=*/4, /*min_seq=*/0, + /*max_seq=*/0, FileSource::Append(), + std::vector{"vec"})); + input.push_back(CreateEntry("normal-file-2", /*row_count=*/5, /*min_seq=*/0, /*max_seq=*/0, + FileSource::Append(), std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/200, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(5u, assigned.assigned_entries.size()); + EXPECT_EQ(0, assigned.assigned_entries[0].File()->first_row_id.value()); + EXPECT_EQ(0, assigned.assigned_entries[1].File()->first_row_id.value()); + EXPECT_EQ(3, assigned.assigned_entries[2].File()->first_row_id.value()); + EXPECT_EQ(0, assigned.assigned_entries[3].File()->first_row_id.value()); + EXPECT_EQ(10, assigned.assigned_entries[4].File()->first_row_id.value()); + EXPECT_EQ(15, assigned.next_row_id_start); + + for (const auto& entry : assigned.assigned_entries) { + EXPECT_EQ(200, entry.File()->min_sequence_number); + EXPECT_EQ(200, entry.File()->max_sequence_number); + } +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingWithoutFileSource) { + std::vector input; + input.push_back(CreateEntry("invalid-no-source", /*row_count=*/1, /*min_seq=*/0, + /*max_seq=*/0, std::nullopt, std::vector{"f0"})); + + ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/1, /*first_row_id_start=*/0, input), + "file source field for row-tracking table must present"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp new file mode 100644 index 000000000..7f6052c6a --- /dev/null +++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp @@ -0,0 +1,94 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/commit/sequence_snapshot_properties.h" + +#include + +#include +#include + +#include "paimon/core/manifest/file_kind.h" + +namespace paimon { + +Result> SequenceSnapshotProperties::MaxSequenceNumber( + const std::optional& snapshot) { + if (!snapshot || !snapshot.value().Properties()) { + return std::optional(); + } + + const auto& properties = snapshot.value().Properties().value(); + auto iter = properties.find(kMaxSequenceNumberKey); + if (iter == properties.end()) { + return std::optional(); + } + + try { + size_t parsed = 0; + int64_t value = std::stoll(iter->second, &parsed); + if (parsed != iter->second.size()) { + return Status::Invalid( + fmt::format("Invalid {} value '{}': trailing characters are not allowed", + kMaxSequenceNumberKey, iter->second)); + } + return std::optional(value); + } catch (const std::exception& e) { + return Status::Invalid(fmt::format("Invalid {} value '{}': {}", kMaxSequenceNumberKey, + iter->second, e.what())); + } +} + +std::optional SequenceSnapshotProperties::MaxSequenceNumberFromFiles( + const std::vector& files) { + int64_t max_sequence_number = std::numeric_limits::min(); + bool found = false; + for (const auto& file : files) { + if (!(file.Kind() == FileKind::Add())) { + continue; + } + max_sequence_number = std::max(max_sequence_number, file.File()->max_sequence_number); + found = true; + } + + if (!found) { + return std::nullopt; + } + return max_sequence_number; +} + +std::map SequenceSnapshotProperties::MergeMaxSequenceNumber( + const std::map& properties, + const std::optional& latest_max_sequence_number, + const std::vector& delta_files) { + std::map snapshot_properties = properties; + + std::optional delta_max_sequence_number = MaxSequenceNumberFromFiles(delta_files); + if (delta_max_sequence_number || latest_max_sequence_number) { + int64_t merged_max_sequence_number = latest_max_sequence_number + ? latest_max_sequence_number.value() + : delta_max_sequence_number.value(); + if (delta_max_sequence_number) { + merged_max_sequence_number = + std::max(merged_max_sequence_number, delta_max_sequence_number.value()); + } + snapshot_properties[kMaxSequenceNumberKey] = std::to_string(merged_max_sequence_number); + } + + return snapshot_properties; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.h b/src/paimon/core/operation/commit/sequence_snapshot_properties.h new file mode 100644 index 000000000..a3c731020 --- /dev/null +++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.h @@ -0,0 +1,53 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/snapshot.h" +#include "paimon/result.h" + +namespace paimon { + +class ManifestFile; +class ManifestFileMeta; + +class SequenceSnapshotProperties { + public: + static constexpr const char* kMaxSequenceNumberKey = "sequence.generation.max-sequence-number"; + + static Result> MaxSequenceNumber( + const std::optional& snapshot); + + static std::optional MaxSequenceNumberFromFiles( + const std::vector& files); + + static std::map MergeMaxSequenceNumber( + const std::map& properties, + const std::optional& latest_max_sequence_number, + const std::vector& delta_files); + + private: + SequenceSnapshotProperties() = delete; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index d9349a21a..dd8c9308f 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -28,6 +28,7 @@ namespace paimon { CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, @@ -36,6 +37,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), + append_commit_check_conflict_(append_commit_check_conflict), memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), @@ -50,6 +52,7 @@ class CommitContextBuilder::Impl { void Reset() { ignore_empty_commit_ = true; use_rest_catalog_commit_ = false; + append_commit_check_conflict_ = false; memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); @@ -61,6 +64,7 @@ class CommitContextBuilder::Impl { std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; + bool append_commit_check_conflict_ = false; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; @@ -98,6 +102,12 @@ CommitContextBuilder& CommitContextBuilder::UseRESTCatalogCommit(bool use_rest_c return *this; } +CommitContextBuilder& CommitContextBuilder::AppendCommitCheckConflict( + bool append_commit_check_conflict) { + impl_->append_commit_check_conflict_ = append_commit_check_conflict; + return *this; +} + CommitContextBuilder& CommitContextBuilder::WithMemoryPool( const std::shared_ptr& memory_pool) { impl_->memory_pool_ = memory_pool; @@ -123,8 +133,8 @@ Result> CommitContextBuilder::Finish() { } auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, - impl_->use_rest_catalog_commit_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_); + impl_->use_rest_catalog_commit_, impl_->append_commit_check_conflict_, impl_->memory_pool_, + impl_->executor_, impl_->specific_file_system_, impl_->options_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp new file mode 100644 index 000000000..c6be4fc74 --- /dev/null +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -0,0 +1,86 @@ +/* + * Copyright 2024-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/commit_context.h" + +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/executor.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(CommitContextTest, TestDefaultValue) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); + + ASSERT_EQ(ctx->GetRootPath(), expected_root_path); + ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); + ASSERT_TRUE(ctx->IgnoreEmptyCommit()); + ASSERT_FALSE(ctx->UseRESTCatalogCommit()); + ASSERT_FALSE(ctx->AppendCommitCheckConflict()); + ASSERT_TRUE(ctx->GetMemoryPool()); + ASSERT_TRUE(ctx->GetExecutor()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(ctx->GetOptions().empty()); +} + +TEST(CommitContextTest, TestSetContent) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + + auto memory_pool = GetDefaultPool(); + std::shared_ptr executor = CreateDefaultExecutor(); + auto fs = std::make_shared(); + + ASSERT_OK_AND_ASSIGN(auto ctx, builder.IgnoreEmptyCommit(false) + .UseRESTCatalogCommit(true) + .AppendCommitCheckConflict(true) + .WithMemoryPool(memory_pool) + .WithExecutor(executor) + .WithFileSystem(fs) + .AddOption("key", "value") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); + ASSERT_EQ(ctx->GetRootPath(), expected_root_path); + ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); + ASSERT_FALSE(ctx->IgnoreEmptyCommit()); + ASSERT_TRUE(ctx->UseRESTCatalogCommit()); + ASSERT_TRUE(ctx->AppendCommitCheckConflict()); + ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); + ASSERT_EQ(ctx->GetExecutor(), executor); + ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); + + std::map expected_options = {{"key", "value"}}; + ASSERT_EQ(ctx->GetOptions(), expected_options); +} + +TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + builder.AddOption("old", "value"); + builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); + + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + + std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; + ASSERT_EQ(ctx->GetOptions(), expected_options); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 82f1b6416..fee41a065 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -27,11 +27,13 @@ #include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/append_only_file_store_scan.h" #include "paimon/core/operation/expire_snapshots.h" #include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -45,6 +47,50 @@ class Schema; namespace paimon { +namespace { + +CommitScanner::ScanSupplier CreateAppendScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +CommitScanner::ScanSupplier CreatePkScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace + Result> FileStoreCommit::Create( std::unique_ptr ctx) { if (ctx == nullptr) { @@ -67,15 +113,6 @@ Result> FileStoreCommit::Create( return Status::Invalid("not found latest schema"); } const auto& schema = table_schema.value(); - if (!schema->PrimaryKeys().empty() && - ctx->GetOptions().find("enable-pk-commit-in-inte-test") == ctx->GetOptions().end()) { - // Postpone bucket mode (bucket=-2) writes all data files to the bucket-postpone/ directory. - // A compaction job will later redistribute files into real buckets. The commit logic - // (manifest and snapshot generation) is the same as append tables, so we allow it. - if (schema->NumBuckets() != BucketModeDefine::POSTPONE_BUCKET) { - return Status::NotImplemented("not support pk table commit yet"); - } - } auto opts = schema->Options(); for (const auto& [key, value] : ctx->GetOptions()) { opts[key] = value; @@ -135,11 +172,23 @@ Result> FileStoreCommit::Create( snapshot_manager, path_factory, manifest_list, manifest_file, options.GetFileSystem(), options.GetExpireConfig(), ctx->GetExecutor()); + CommitScanner::ScanSupplier scan_supplier; + if (table_schema.value()->PrimaryKeys().empty()) { + scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema.value(), arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); + } else { + scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema.value(), arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); + } + return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), options, path_factory, std::move(partition_computer), snapshot_manager, - ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), table_schema.value(), manifest_file, - manifest_list, index_manifest_file, expire_snapshots, schema_manager); + ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), ctx->AppendCommitCheckConflict(), + table_schema.value(), manifest_file, manifest_list, index_manifest_file, expire_snapshots, + schema_manager, std::move(scan_supplier)); } } // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 3f263ab19..6bfc70e85 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -18,10 +18,13 @@ #include #include +#include #include +#include #include #include #include +#include #include #include "fmt/format.h" @@ -32,8 +35,11 @@ #include "paimon/common/executor/future.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/renaming_snapshot_commit.h" @@ -54,14 +60,17 @@ #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/manifest/partition_entry.h" -#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/commit/commit_changes_provider.h" +#include "paimon/core/operation/commit/conflict_detection.h" +#include "paimon/core/operation/commit/row_tracking_commit_utils.h" +#include "paimon/core/operation/commit/sequence_snapshot_properties.h" #include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/manifest_file_merger.h" #include "paimon/core/operation/metrics/commit_metrics.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" @@ -69,12 +78,26 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/metrics.h" -#include "paimon/scan_context.h" namespace paimon { class Executor; class MemoryPool; +namespace { + +bool MatchPartitionSpec(const std::map& partition, + const std::map& partition_spec) { + for (const auto& [key, value] : partition_spec) { + auto iter = partition.find(key); + if (iter == partition.end() || iter->second != value) { + return false; + } + } + return true; +} + +} // namespace + FileStoreCommitImpl::FileStoreCommitImpl( const std::shared_ptr& pool, const std::shared_ptr& executor, const std::shared_ptr& schema, const std::string& root_path, @@ -82,16 +105,18 @@ FileStoreCommitImpl::FileStoreCommitImpl( const std::shared_ptr& path_factory, std::unique_ptr partition_computer, const std::shared_ptr& snapshot_manager, bool ignore_empty_commit, - bool use_rest_catalog_commit, const std::shared_ptr& table_schema, + bool use_rest_catalog_commit, bool append_commit_check_conflict, + const std::shared_ptr& table_schema, const std::shared_ptr& manifest_file, const std::shared_ptr& manifest_list, const std::shared_ptr& index_manifest_file, const std::shared_ptr& expire_snapshots, - const std::shared_ptr& schema_manager) + const std::shared_ptr& schema_manager, CommitScanner::ScanSupplier scan_supplier) : memory_pool_(pool), executor_(executor), schema_(schema), root_path_(root_path), + table_name_(PathUtil::GetName(root_path)), commit_user_(commit_user), options_(options), path_factory_(path_factory), @@ -99,8 +124,15 @@ FileStoreCommitImpl::FileStoreCommitImpl( partition_computer_(std::move(partition_computer)), snapshot_manager_(snapshot_manager), ignore_empty_commit_(ignore_empty_commit), + append_commit_check_conflict_(append_commit_check_conflict), + retry_waiter_(options.GetCommitMinRetryWait(), options.GetCommitMaxRetryWait()), num_bucket_(options.GetBucket()), + bucket_mode_(ResolveBucketMode(options.GetBucket(), table_schema)), table_schema_(table_schema), + commit_scanner_(snapshot_manager, schema_manager, manifest_list, manifest_file, + index_manifest_file, table_schema, schema, options, executor, pool, + partition_computer_.get(), std::move(scan_supplier)), + conflict_detection_(table_schema, options, snapshot_manager_, manifest_list, manifest_file), manifest_file_(manifest_file), manifest_list_(manifest_list), index_manifest_file_(index_manifest_file), @@ -128,7 +160,10 @@ Status FileStoreCommitImpl::DropPartition( } std::string log_msg = fmt::format("Ready to drop partitions {}", partitions); PAIMON_LOG_DEBUG(logger_, "%s", log_msg.c_str()); - return TryOverwrite(partitions, {}, commit_identifier, std::nullopt); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t attempt, + TryOverwrite(partitions, {}, /*index_entries=*/{}, commit_identifier, + std::nullopt, /*properties=*/{})); + return Status::OK(); } Result FileStoreCommitImpl::FilterAndCommit( @@ -238,6 +273,7 @@ Result>> FileStoreCommitImpl::F "unexpected."); } } + // TODO(yonghao.fyh): support commit strict mode last safe snapshot PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, snapshot_manager_->LatestSnapshotOfUser(commit_user_)); if (latest_snapshot) { @@ -258,189 +294,334 @@ Result>> FileStoreCommitImpl::F } Status FileStoreCommitImpl::Overwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t identifier, std::optional watermark) { std::shared_ptr committable = CreateManifestCommittable(identifier, commit_messages, watermark); - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - PAIMON_RETURN_NOT_OK(CollectChanges(committable->FileCommittables(), &append_table_files, - &append_changelog_files, &compact_table_files, - &compact_changelog_files, &append_table_index_files, - &compact_table_index_files)); - if (!append_table_index_files.empty()) { - return Status::NotImplemented("Overwrite not support index for now"); - } - return TryOverwrite(partitions, append_table_files, identifier, watermark); + PAIMON_LOG_INFO(logger_, "Ready to overwrite to table %s, number of commit messages: %zu", + table_name_.c_str(), committable->FileCommittables().size()); + PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString()); + std::string partition_str = fmt::format("{}", partition); + std::string properties_str = fmt::format("{}", committable->Properties()); + PAIMON_LOG_DEBUG(logger_, + "Ready to overwrite partitions %s\nManifestCommittable: %s\nProperties: " + "%s", + partition_str.c_str(), committable_str.c_str(), properties_str.c_str()); + + Duration duration; + int32_t generated_snapshot = 0; + int32_t attempt = 0; + + std::vector> partitions; + if (!partition.empty()) { + partitions.push_back(partition); + } + + PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, + CollectChanges(committable->FileCommittables())); + ManifestEntryChanges report_changes = changes; + report_changes.append_changelog.clear(); + report_changes.compact_changelog.clear(); + ScopeGuard report_guard([&]() { + PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); + ReportCommit(report_changes, duration.Get(), generated_snapshot, attempt); + }); + + PAIMON_RETURN_NOT_OK(ExecuteOverwrite(partitions, &changes, identifier, watermark, + committable->Properties(), &generated_snapshot, + &attempt)); + return Status::OK(); } Result FileStoreCommitImpl::FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t identifier, std::optional watermark) { std::shared_ptr committable = CreateManifestCommittable(identifier, commit_messages, watermark); + PAIMON_LOG_INFO(logger_, "Ready to overwrite to table %s, number of commit messages: %zu", + table_name_.c_str(), committable->FileCommittables().size()); + PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString()); + std::string partition_str = fmt::format("{}", partition); + std::string properties_str = fmt::format("{}", committable->Properties()); + PAIMON_LOG_DEBUG(logger_, + "Ready to overwrite partitions %s\nManifestCommittable: %s\nProperties: " + "%s", + partition_str.c_str(), committable_str.c_str(), properties_str.c_str()); + + Duration duration; + int32_t generated_snapshot = 0; + int32_t attempt = 0; + + std::vector> partitions; + if (!partition.empty()) { + partitions.push_back(partition); + } + std::vector> committables; committables.push_back(committable); PAIMON_ASSIGN_OR_RAISE(std::vector> actual_committables, FilterCommitted(committables)); if (!actual_committables.empty()) { - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - PAIMON_RETURN_NOT_OK(CollectChanges(actual_committables[0]->FileCommittables(), - &append_table_files, &append_changelog_files, - &compact_table_files, &compact_changelog_files, - &append_table_index_files, &compact_table_index_files)); - if (!append_table_index_files.empty()) { - return Status::NotImplemented("FilterAndOverwrite not support index for now"); - } - PAIMON_RETURN_NOT_OK(TryOverwrite(partitions, append_table_files, identifier, watermark)); + PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, + CollectChanges(actual_committables[0]->FileCommittables())); + ManifestEntryChanges report_changes = changes; + report_changes.append_changelog.clear(); + report_changes.compact_changelog.clear(); + ScopeGuard report_guard([&]() { + PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); + ReportCommit(report_changes, duration.Get(), generated_snapshot, attempt); + }); + + PAIMON_RETURN_NOT_OK(ExecuteOverwrite(partitions, &changes, identifier, watermark, + actual_committables[0]->Properties(), + &generated_snapshot, &attempt)); + } else { + PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); } return actual_committables.size(); } +Status FileStoreCommitImpl::ExecuteOverwrite( + const std::vector>& partitions, + ManifestEntryChanges* changes, int64_t identifier, std::optional watermark, + const std::map& properties, int32_t* generated_snapshot, + int32_t* attempt) { + if (!changes->append_changelog.empty() || !changes->compact_changelog.empty()) { + std::string warning = + "Overwrite mode currently does not commit any changelog.\n" + "Please make sure that the partition you're overwriting is not being consumed by a " + "streaming reader.\n" + "Ignored changelog files are:\n"; + for (const auto& entry : changes->append_changelog) { + warning += fmt::format(" * {}\n", entry.ToString()); + } + for (const auto& entry : changes->compact_changelog) { + warning += fmt::format(" * {}\n", entry.ToString()); + } + PAIMON_LOG_WARN(logger_, "%s", warning.c_str()); + } + + bool with_compact = + !changes->compact_table_files.empty() || !changes->compact_index_files.empty(); + if (!with_compact) { + PAIMON_ASSIGN_OR_RAISE(changes->append_table_files, + TryUpgrade(changes->append_table_files)); + } + + bool skip_overwrite = false; + std::vector> overwrite_partitions = partitions; + if (!table_schema_->PartitionKeys().empty() && options_.DynamicPartitionOverwrite()) { + if (changes->append_table_files.empty()) { + skip_overwrite = true; + } else { + std::set> dynamic_partitions; + for (const auto& entry : changes->append_table_files) { + std::map partition_map; + PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition())); + dynamic_partitions.insert(std::move(partition_map)); + } + overwrite_partitions.assign(dynamic_partitions.begin(), dynamic_partitions.end()); + } + } else if (!partitions.empty()) { + for (const auto& entry : changes->append_table_files) { + std::map partition_map; + PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition())); + bool belongs_to_overwrite_partition = false; + for (const auto& partition_spec : partitions) { + if (MatchPartitionSpec(partition_map, partition_spec)) { + belongs_to_overwrite_partition = true; + break; + } + } + if (!belongs_to_overwrite_partition) { + return Status::Invalid(fmt::format( + "Trying to overwrite partitions {}, but the changes in {} does not belong to " + "this partition", + partitions, partition_map)); + } + } + } + + if (!skip_overwrite) { + PAIMON_ASSIGN_OR_RAISE( + int32_t cnt, + TryOverwrite(overwrite_partitions, changes->append_table_files, + changes->append_index_files, identifier, watermark, properties)); + *attempt += cnt; + *generated_snapshot += 1; + } + + if (with_compact) { + PAIMON_ASSIGN_OR_RAISE( + int32_t cnt, TryCommit(changes->compact_table_files, /*changelog_files=*/{}, + changes->compact_index_files, identifier, watermark, properties, + Snapshot::CommitKind::Compact(), + /*detect_conflicts=*/true)); + *attempt += cnt; + *generated_snapshot += 1; + } + + return Status::OK(); +} + Result FileStoreCommitImpl::GetLastCommitTableRequest() { return snapshot_commit_->GetLastCommitTableRequest(); } Result> FileStoreCommitImpl::GetAllFiles( - const Snapshot& snapshot, const std::vector>& partitions) { - auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions, - /*bucket_filter=*/std::nullopt); - PAIMON_ASSIGN_OR_RAISE( - auto scan, AppendOnlyFileStoreScan::Create( - snapshot_manager_, schema_manager_, manifest_list_, manifest_file_, - table_schema_, schema_, scan_filter, options_, executor_, memory_pool_)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, - scan->WithSnapshot(snapshot)->CreatePlan()); - // scan existing file metas - return plan->Files(); + const Snapshot& snapshot, + const std::vector>& partitions) const { + return commit_scanner_.ReadAllEntriesFromPartitions(snapshot, partitions); } -Status FileStoreCommitImpl::TryOverwrite( - const std::vector>& partitions, - const std::vector& changes, int64_t commit_identifier, - std::optional watermark) { - int32_t retry_count = 0; - while (true) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager_->LatestSnapshot()); - std::vector changes_with_overwrite; - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE(std::vector entries, - GetAllFiles(latest_snapshot.value(), partitions)); - for (const auto& entry : entries) { - changes_with_overwrite.emplace_back(FileKind::Delete(), entry.Partition(), - entry.Bucket(), entry.TotalBuckets(), - entry.File()); +Result> FileStoreCommitImpl::PartitionToMap( + const BinaryRow& partition) const { + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, partition_computer_->GeneratePartitionVector(partition)); + std::map partition_map; + for (const auto& [key, value] : part_values) { + partition_map[key] = value; + } + return partition_map; +} + +Result> FileStoreCommitImpl::TryUpgrade( + const std::vector& append_files) const { + if (!options_.OverwriteUpgrade()) { + return append_files; + } + + if (table_schema_->PrimaryKeys().empty()) { + return append_files; + } + + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema_->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + table_schema_->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options_.SequenceFieldSortOrderIsAscending())); + + for (const auto& entry : append_files) { + if (entry.Level() > 0 || entry.Bucket() < 0) { + return append_files; + } + } + + std::unordered_map, std::vector> buckets; + for (const auto& entry : append_files) { + buckets[std::make_pair(entry.Partition(), entry.Bucket())].emplace_back(entry); + } + + std::vector results; + int32_t max_level = options_.GetNumLevels() - 1; + for (auto& [_, entries] : buckets) { + std::vector new_entries = entries; + std::sort(new_entries.begin(), new_entries.end(), + [&key_comparator](const ManifestEntry& a, const ManifestEntry& b) { + return key_comparator->CompareTo(a.MinKey(), b.MinKey()) < 0; + }); + + bool overlap = false; + for (size_t i = 0; i + 1 < new_entries.size(); ++i) { + if (key_comparator->CompareTo(new_entries[i].MaxKey(), new_entries[i + 1].MinKey()) >= + 0) { + overlap = true; + break; } } - changes_with_overwrite.insert(changes_with_overwrite.end(), changes.begin(), changes.end()); - PAIMON_ASSIGN_OR_RAISE(bool commit_success, - TryCommitOnce(changes_with_overwrite, /*index_entries=*/{}, - commit_identifier, watermark, - /*log_offsets=*/{}, /*properties=*/{}, - Snapshot::CommitKind::Overwrite(), latest_snapshot, - /*need_conflict_check=*/true)); - if (commit_success) { - break; + + if (overlap) { + results.insert(results.end(), entries.begin(), entries.end()); + continue; } - if (retry_count >= options_.GetCommitMaxRetries()) { - return Status::Invalid( - fmt::format("Commit failed after {} attempts, there maybe exist commit conflicts " - "between multiple jobs.", - options_.GetCommitMaxRetries())); + + PAIMON_LOG_INFO(logger_, "%s", "Upgraded for overwrite commit."); + for (const auto& entry : new_entries) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr upgraded_file, + entry.File()->Upgrade(max_level)); + results.emplace_back(entry.Kind(), entry.Partition(), entry.Bucket(), + entry.TotalBuckets(), upgraded_file); } - retry_count++; } - return Status::OK(); + + return results; +} + +Result FileStoreCommitImpl::TryOverwrite( + const std::vector>& partitions, + const std::vector& changes, const std::vector& index_entries, + int64_t commit_identifier, std::optional watermark, + const std::map& properties) { + std::shared_ptr changes_provider = + commit_scanner_.OverwriteChangesProvider(partitions, changes, index_entries); + return TryCommit(changes_provider, commit_identifier, watermark, properties, + Snapshot::CommitKind::Overwrite(), /*detect_conflicts=*/true); } Status FileStoreCommitImpl::Commit(const std::shared_ptr& committable, bool check_append_files) { - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - PAIMON_RETURN_NOT_OK(CollectChanges(committable->FileCommittables(), &append_table_files, - &append_changelog_files, &compact_table_files, - &compact_changelog_files, &append_table_index_files, - &compact_table_index_files)); + PAIMON_LOG_INFO(logger_, "Ready to commit to table %s, number of commit messages: %zu", + table_name_.c_str(), committable->FileCommittables().size()); + PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString()); + PAIMON_LOG_DEBUG(logger_, "Ready to commit\n%s", committable_str.c_str()); - int32_t attempt = 0; - int32_t generated_snapshot = 0; Duration duration; - if (!ignore_empty_commit_ || !append_table_files.empty() || !append_table_index_files.empty()) { - PAIMON_ASSIGN_OR_RAISE(int32_t cnt, - TryCommit(append_table_files, append_table_index_files, - committable->Identifier(), committable->Watermark(), - committable->LogOffsets(), committable->Properties(), - Snapshot::CommitKind::Append(), check_append_files)); - attempt += cnt; - ++generated_snapshot; - } + int32_t generated_snapshot = 0; + int32_t attempt = 0; + + PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, + CollectChanges(committable->FileCommittables())); + ScopeGuard report_guard([&]() { + PAIMON_LOG_INFO(logger_, + "Finished (Uncertain of success) commit to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); + ReportCommit(changes, duration.Get(), generated_snapshot, attempt); + }); + + if (!ignore_empty_commit_ || changes.HasAppendChanges()) { + Snapshot::CommitKind commit_kind = Snapshot::CommitKind::Append(); + if (append_commit_check_conflict_) { + check_append_files = true; + } + + if (conflict_detection_.ShouldBeOverwriteCommit(changes.append_table_files, + changes.append_index_files)) { + commit_kind = Snapshot::CommitKind::Overwrite(); + check_append_files = true; + } + if (conflict_detection_.HasRowIdCheckFromSnapshot()) { + check_append_files = true; + } + if (changes.HasGlobalIndexFileAdditions()) { + check_append_files = true; + } - if (!compact_table_files.empty() || !compact_table_index_files.empty()) { PAIMON_ASSIGN_OR_RAISE( - int32_t cnt, TryCommit(compact_table_files, compact_table_index_files, - committable->Identifier(), committable->Watermark(), - committable->LogOffsets(), committable->Properties(), - Snapshot::CommitKind::Compact(), /*check_append_files=*/true)); + int32_t cnt, TryCommit(changes.append_table_files, changes.append_changelog, + changes.append_index_files, committable->Identifier(), + committable->Watermark(), committable->Properties(), commit_kind, + check_append_files)); attempt += cnt; - ++generated_snapshot; + generated_snapshot += 1; } - auto table_files_added = static_cast(append_table_files.size()); - int32_t table_files_deleted = 0; - int64_t compaction_input_file_size = 0; - int64_t compaction_output_file_size = 0; - for (const auto& entry : compact_table_files) { - const auto& kind = entry.Kind(); - if (kind == FileKind::Add()) { - ++table_files_added; - compaction_output_file_size += entry.File()->file_size; - } else if (kind == FileKind::Delete()) { - ++table_files_deleted; - compaction_input_file_size += entry.File()->file_size; - } + + if (changes.HasCompactChanges()) { + PAIMON_ASSIGN_OR_RAISE(int32_t cnt, + TryCommit(changes.compact_table_files, changes.compact_changelog, + changes.compact_index_files, committable->Identifier(), + committable->Watermark(), committable->Properties(), + Snapshot::CommitKind::Compact(), + /*detect_conflicts=*/true)); + attempt += cnt; + generated_snapshot += 1; } - metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, duration.Get()); - metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED, append_table_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED, - compact_table_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED, - append_changelog_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED, - compact_changelog_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS, generated_snapshot); - metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED, RowCounts(append_table_files)); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED, - RowCounts(append_changelog_files)); - metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED, - RowCounts(compact_table_files)); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED, - RowCounts(compact_changelog_files)); - metrics_->SetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN, - NumChangedPartitions({append_table_files, compact_table_files})); - metrics_->SetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN, - NumChangedBuckets({append_table_files, compact_table_files})); - metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE, - compaction_input_file_size); - metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE, - compaction_output_file_size); return Status::OK(); } @@ -453,24 +634,46 @@ Status FileStoreCommitImpl::Commit( } Result FileStoreCommitImpl::TryCommit(const std::vector& delta_files, + const std::vector& changelog_files, const std::vector& index_entries, int64_t identifier, std::optional watermark, - std::map log_offsets, const std::map& properties, Snapshot::CommitKind commit_kind, - bool check_append_files) { + bool detect_conflicts) { + std::shared_ptr changes_provider = + std::make_shared(delta_files, changelog_files, index_entries); + return TryCommit(changes_provider, identifier, watermark, properties, commit_kind, + detect_conflicts); +} + +Result FileStoreCommitImpl::TryCommit( + const std::shared_ptr& changes_provider, int64_t identifier, + std::optional watermark, const std::map& properties, + Snapshot::CommitKind commit_kind, bool detect_conflicts) { + conflict_detection_.SetRowIdCheckFromSnapshot(std::nullopt); int32_t retry_count = 0; int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; + std::optional retry_start_snapshot_id; while (true) { PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, snapshot_manager_->LatestSnapshot()); + std::vector delta_files; + std::vector changelog_files; + std::vector index_entries; + PAIMON_RETURN_NOT_OK(changes_provider->Provide(latest_snapshot, &delta_files, + &changelog_files, &index_entries)); PAIMON_ASSIGN_OR_RAISE( bool commit_success, - TryCommitOnce(delta_files, index_entries, identifier, watermark, log_offsets, - properties, commit_kind, latest_snapshot, check_append_files)); + TryCommitOnce(delta_files, changelog_files, index_entries, identifier, watermark, + properties, commit_kind, latest_snapshot, detect_conflicts, + retry_start_snapshot_id)); if (commit_success) { break; } + retry_start_snapshot_id = latest_snapshot + ? std::optional(latest_snapshot.value().Id() + 1) + : std::optional(Snapshot::FIRST_SNAPSHOT_ID); + conflict_detection_.SetRowIdCheckFromSnapshot(retry_start_snapshot_id); int64_t current_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; if (current_millis - start_millis > options_.GetCommitTimeout() || retry_count >= options_.GetCommitMaxRetries()) { @@ -479,88 +682,129 @@ Result FileStoreCommitImpl::TryCommit(const std::vector& "commit conflicts between multiple jobs.", options_.GetCommitTimeout(), options_.GetCommitMaxRetries())); } + retry_waiter_.RetryWait(retry_count); retry_count++; } return retry_count + 1; } -Result>> FileStoreCommitImpl::ChangedPartitions( - const std::vector& data_files, - const std::vector& index_files) const { - std::set> partitions; - auto add_partition = [&, this](const BinaryRow& partition_row) -> Status { - std::vector> part_values; - PAIMON_ASSIGN_OR_RAISE(part_values, - partition_computer_->GeneratePartitionVector(partition_row)); - if (part_values.empty()) { - return Status::OK(); - } - std::map part_values_map; - for (const auto& [key, value] : part_values) { - part_values_map[key] = value; +Result FileStoreCommitImpl::CheckCommitted(const std::optional& latest_snapshot, + std::optional retry_start_snapshot_id, + int64_t identifier, + const Snapshot::CommitKind& commit_kind) const { + if (!latest_snapshot || !retry_start_snapshot_id || + retry_start_snapshot_id.value() > latest_snapshot.value().Id()) { + return false; + } + + for (int64_t snapshot_id = retry_start_snapshot_id.value(); + snapshot_id <= latest_snapshot.value().Id(); ++snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id)); + if (snapshot.CommitUser() == commit_user_ && snapshot.CommitIdentifier() == identifier && + snapshot.GetCommitKind() == commit_kind) { + return true; } - partitions.insert(part_values_map); + } + return false; +} + +Status FileStoreCommitImpl::CheckSameBucketFromSnapshot( + const std::vector& delta_entries, + const std::optional& latest_snapshot) const { + if (!latest_snapshot) { + return Status::OK(); + } + + std::unordered_map expected_total_buckets; + PAIMON_RETURN_NOT_OK(conflict_detection_.CollectUncheckedBucketPartitions( + delta_entries, &expected_total_buckets)); + if (expected_total_buckets.empty()) { return Status::OK(); - }; + } - for (const ManifestEntry& entry : data_files) { - PAIMON_RETURN_NOT_OK(add_partition(entry.Partition())); + std::vector changed_partitions; + changed_partitions.reserve(expected_total_buckets.size()); + for (const auto& [partition, _] : expected_total_buckets) { + changed_partitions.push_back(partition); } - for (const IndexManifestEntry& entry : index_files) { - if (entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) { - PAIMON_RETURN_NOT_OK(add_partition(entry.partition)); - } + + PAIMON_ASSIGN_OR_RAISE( + auto previous_total_buckets, + commit_scanner_.ReadTotalBuckets(latest_snapshot.value(), changed_partitions)); + + return conflict_detection_.CheckSameBucketByTotalBuckets(expected_total_buckets, + previous_total_buckets); +} + +bool FileStoreCommitImpl::ShouldCheckSameBucket(const Snapshot::CommitKind& commit_kind) const { + return commit_kind == Snapshot::CommitKind::Append() && + bucket_mode_ == BucketMode::HASH_FIXED && + (IsUnorderedWriteOnlyAppend() || IsWriteOnlySnapshotSequenceAppend()); +} + +BucketMode FileStoreCommitImpl::ResolveBucketMode( + int32_t bucket, const std::shared_ptr& table_schema) { + if (bucket == BucketModeDefine::POSTPONE_BUCKET) { + return BucketMode::POSTPONE_MODE; + } + if (bucket == -1) { + return table_schema->PrimaryKeys().empty() ? BucketMode::BUCKET_UNAWARE + : BucketMode::HASH_DYNAMIC; + } + if (bucket == BucketModeDefine::UNAWARE_BUCKET) { + return BucketMode::BUCKET_UNAWARE; } - return partitions; + return BucketMode::HASH_FIXED; } -Result> FileStoreCommitImpl::ReadAllEntriesFromChangedPartitions( - const Snapshot& latest_snapshot, - const std::set>& partitions) const { - std::vector> partition_filters(partitions.begin(), - partitions.end()); - auto scan_filter = std::make_shared(/*predicate=*/nullptr, partition_filters, - /*bucket_filter=*/std::nullopt); - PAIMON_ASSIGN_OR_RAISE( - auto scan, AppendOnlyFileStoreScan::Create( - snapshot_manager_, schema_manager_, manifest_list_, manifest_file_, - table_schema_, schema_, scan_filter, options_, executor_, memory_pool_)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, - scan->WithSnapshot(latest_snapshot)->CreatePlan()); - // scan existing file metas - return plan->Files(); -} - -Status FileStoreCommitImpl::NoConflictsOrFail(const std::string& base_commit_user, - const std::vector& base_entries, - const std::vector& changes) const { - ScopeGuard guard([&]() { - PAIMON_LOG_WARN(logger_, "File deletion conflicts detected! Give up committing. %s", - base_commit_user.c_str()); - }); - std::vector all_entries = base_entries; - all_entries.insert(all_entries.end(), changes.begin(), changes.end()); - std::vector merged_entries; - PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(all_entries, &merged_entries)); - for (const auto& entry : merged_entries) { - if (entry.Kind() == FileKind::Delete()) { - return Status::Invalid(fmt::format( - "Trying to delete file {} which is not previously added.", entry.FileName())); - } - } - // TODO(yonghao.fyh): check for all LSM level >= 1, key ranges of files do not intersect - guard.Release(); - return Status::OK(); +bool FileStoreCommitImpl::IsUnorderedWriteOnlyAppend() const { + return options_.WriteOnly() && !options_.BucketAppendOrdered(); +} + +bool FileStoreCommitImpl::IsWriteOnlySnapshotSequenceAppend() const { + return options_.WriteOnly() && + options_.WriteSequenceNumberInitMode() == CoreOptions::SequenceNumberInitMode::SNAPSHOT; +} + +Result> FileStoreCommitImpl::MaxSequenceNumber( + const std::vector& manifests) const { + int64_t max_from_manifest = std::numeric_limits::min(); + bool found = false; + for (const auto& manifest : manifests) { + std::vector entries; + PAIMON_RETURN_NOT_OK(manifest_file_->Read( + manifest.FileName(), [](const ManifestEntry&) -> Result { return true; }, + &entries)); + std::optional current_max = + SequenceSnapshotProperties::MaxSequenceNumberFromFiles(entries); + if (current_max) { + max_from_manifest = std::max(max_from_manifest, current_max.value()); + found = true; + } + } + + if (!found) { + return std::optional(); + } + return std::optional(max_from_manifest); } Result FileStoreCommitImpl::TryCommitOnce( const std::vector& delta_entries, + const std::vector& changelog_entries, const std::vector& index_entries, int64_t identifier, - std::optional watermark, std::map log_offsets, - const std::map& properties, Snapshot::CommitKind commit_kind, - const std::optional& latest_snapshot, bool need_conflict_check) { + std::optional watermark, const std::map& properties, + Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot, + bool detect_conflicts, std::optional retry_start_snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(bool committed, CheckCommitted(latest_snapshot, retry_start_snapshot_id, + identifier, commit_kind)); + if (committed) { + return true; + } + std::vector delta_files = delta_entries; int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; + int64_t new_snapshot_id = Snapshot::FIRST_SNAPSHOT_ID; int64_t first_row_id_start = 0; if (latest_snapshot) { @@ -571,19 +815,68 @@ Result FileStoreCommitImpl::TryCommitOnce( } } - PAIMON_LOG_DEBUG(logger_, "Ready to commit table files to snapshot #%ld", new_snapshot_id); + // TODO(yonghao.fyh): support strict mode checker + + PAIMON_LOG_DEBUG(logger_, "Ready to commit table files to snapshot %ld", new_snapshot_id); for (const ManifestEntry& entry : delta_files) { PAIMON_LOG_DEBUG(logger_, " * %s", entry.ToString().c_str()); } + PAIMON_LOG_DEBUG(logger_, "Ready to commit changelog files to snapshot %ld", new_snapshot_id); + for (const ManifestEntry& entry : changelog_entries) { + PAIMON_LOG_DEBUG(logger_, " * %s", entry.ToString().c_str()); + } - if (need_conflict_check && latest_snapshot) { - std::set> changed_partitions; - PAIMON_ASSIGN_OR_RAISE(changed_partitions, ChangedPartitions(delta_files, index_entries)); - PAIMON_ASSIGN_OR_RAISE( - std::vector base_data_files, - ReadAllEntriesFromChangedPartitions(latest_snapshot.value(), changed_partitions)); - PAIMON_RETURN_NOT_OK( - NoConflictsOrFail(latest_snapshot.value().CommitUser(), base_data_files, delta_files)); + bool discard_duplicate = + options_.CommitDiscardDuplicateFiles() && commit_kind == Snapshot::CommitKind::Append(); + bool check_conflicts = latest_snapshot.has_value() && (discard_duplicate || detect_conflicts); + // By default, if checkConflicts is required, we do not have to do the extra check bucket + // here. + if (!check_conflicts && ShouldCheckSameBucket(commit_kind)) { + PAIMON_RETURN_NOT_OK(CheckSameBucketFromSnapshot(delta_files, latest_snapshot)); + } + + if (check_conflicts) { + // latestSnapshotId is different from the snapshot id we've checked for conflicts, + // so we have to check again + std::vector changed_partitions = + ManifestEntryChanges::ChangedPartitions(delta_files, index_entries); + PAIMON_ASSIGN_OR_RAISE(std::vector base_data_files, + commit_scanner_.ReadAllEntriesFromChangedPartitions( + latest_snapshot.value(), changed_partitions)); + + if (discard_duplicate) { + std::unordered_set base_identifiers; + base_identifiers.reserve(base_data_files.size()); + for (const auto& entry : base_data_files) { + base_identifiers.insert(entry.CreateIdentifier()); + } + + delta_files.erase( + std::remove_if(delta_files.begin(), delta_files.end(), + [&base_identifiers](const ManifestEntry& entry) { + return base_identifiers.find(entry.CreateIdentifier()) != + base_identifiers.end(); + }), + delta_files.end()); + } + + std::optional> row_id_column_conflict_checker = + std::nullopt; + if (conflict_detection_.HasRowIdCheckFromSnapshot()) { + std::vector> delta_data_files; + delta_data_files.reserve(delta_files.size()); + for (const auto& entry : delta_files) { + delta_data_files.push_back(entry.File()); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr checker, + RowIdColumnConflictChecker::FromDataFiles(schema_manager_, delta_data_files)); + row_id_column_conflict_checker = checker; + } + + PAIMON_RETURN_NOT_OK(conflict_detection_.CheckConflicts( + latest_snapshot.value(), base_data_files, delta_files, index_entries, + row_id_column_conflict_checker, commit_kind)); } std::vector merge_before_manifests; @@ -613,25 +906,13 @@ Result FileStoreCommitImpl::TryCommitOnce( if (latest_snapshot) { old_index_manifest = latest_snapshot.value().IndexManifest(); - // TODO(yonghao.fyh): total record count should call scan when its std::nullopt - previous_total_record_count = latest_snapshot.value().TotalRecordCount() != std::nullopt - ? latest_snapshot.value().TotalRecordCount().value() - : 0; + previous_total_record_count = latest_snapshot.value().TotalRecordCount(); std::vector previous_manifests; // read all previous manifest files PAIMON_RETURN_NOT_OK( manifest_list_->ReadDataManifests(latest_snapshot.value(), &previous_manifests)); merge_before_manifests.insert(merge_before_manifests.end(), previous_manifests.begin(), previous_manifests.end()); - // read the last snapshot to complete the bucket's offsets when logOffsets does not - // contain all buckets - std::optional> latest_log_offsets = - latest_snapshot.value().LogOffsets(); - if (latest_log_offsets) { - for (const auto& [key, value] : latest_log_offsets.value()) { - log_offsets.emplace(key, value); - } - } std::optional latest_watermark = latest_snapshot.value().Watermark(); if (latest_watermark) { if (watermark == std::nullopt) { @@ -665,11 +946,12 @@ Result FileStoreCommitImpl::TryCommitOnce( std::make_move_iterator(entries.end())); } } - // assigned snapshot id to delta files - AssignSnapshotId(new_snapshot_id, &delta_files); - // assign row id for new files - PAIMON_ASSIGN_OR_RAISE(next_row_id_start, - AssignRowTrackingMeta(first_row_id_start, &delta_files)); + + PAIMON_ASSIGN_OR_RAISE(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + new_snapshot_id, first_row_id_start, delta_files)); + next_row_id_start = assigned.next_row_id_start; + delta_files = std::move(assigned.assigned_entries); } // the added records subtract the deleted records from @@ -690,12 +972,20 @@ Result FileStoreCommitImpl::TryCommitOnce( new_changes_manifests.end()); PAIMON_ASSIGN_OR_RAISE(delta_manifest_list, manifest_list_->Write(new_changes_manifests)); + // write changelog into manifest files + std::optional> changelog_manifest_list; + if (!changelog_entries.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::vector changelog_manifests, + manifest_file_->Write(changelog_entries)); + PAIMON_ASSIGN_OR_RAISE(changelog_manifest_list, manifest_list_->Write(changelog_manifests)); + } + PAIMON_ASSIGN_OR_RAISE(index_manifest_name, index_manifest_file_->WriteIndexFiles( old_index_manifest, index_entries)); - std::optional> changelog_manifest_list; - std::optional statistics; - int64_t changelog_record_count = 0; + std::optional statistics = + latest_snapshot ? latest_snapshot.value().Statistics() : std::nullopt; + int64_t changelog_record_count = RowCounts(changelog_entries); int64_t schema_id = 0; PAIMON_ASSIGN_OR_RAISE(std::optional> table_schema, schema_manager_->Latest()); @@ -703,6 +993,24 @@ Result FileStoreCommitImpl::TryCommitOnce( schema_id = table_schema.value()->Id(); } + // Keep Java semantics: inherit previous stats only when schema matches. + if (statistics && latest_snapshot && latest_snapshot.value().SchemaId() != schema_id) { + PAIMON_LOG_WARN(logger_, "%s", "Schema changed, stats will not be inherited"); + statistics = std::nullopt; + } + + std::map snapshot_properties = properties; + if (options_.WriteSequenceNumberInitMode() == CoreOptions::SequenceNumberInitMode::SNAPSHOT) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_max_sequence_number, + SequenceSnapshotProperties::MaxSequenceNumber(latest_snapshot)); + if (!latest_max_sequence_number && latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE(latest_max_sequence_number, + MaxSequenceNumber(merge_before_manifests)); + } + snapshot_properties = SequenceSnapshotProperties::MergeMaxSequenceNumber( + snapshot_properties, latest_max_sequence_number, delta_files); + } + Snapshot new_snapshot( new_snapshot_id, schema_id, base_manifest_list.first, base_manifest_list.second, delta_manifest_list.first, delta_manifest_list.second, @@ -711,28 +1019,20 @@ Result FileStoreCommitImpl::TryCommitOnce( changelog_manifest_list ? std::optional(changelog_manifest_list.value().second) : std::nullopt, index_manifest_name, commit_user_, identifier, commit_kind, - DateTimeUtils::GetCurrentUTCTimeUs() / 1000, log_offsets, total_record_count, - delta_record_count, changelog_record_count, watermark, statistics, - properties.empty() ? std::nullopt - : std::optional>(properties), + DateTimeUtils::GetCurrentUTCTimeUs() / 1000, total_record_count, delta_record_count, + changelog_record_count, watermark, statistics, + snapshot_properties.empty() + ? std::nullopt + : std::optional>(snapshot_properties), next_row_id_start); Result commit_result = CommitSnapshotImpl(new_snapshot, delta_statistics); if (!commit_result.ok()) { - // commit exception, not sure about the situation and should not clean up the files. - PAIMON_LOG_WARN(logger_, "You need call FilterAndCommit to retry commit for exception. %s", + // commit exception is uncertain; retry after checking whether this commit already exists. + PAIMON_LOG_WARN(logger_, "Retry commit for exception. %s", commit_result.status().ToString().c_str()); - - // To prevent the case where an atomic write times out but actually succeeds, - // retrying the commit could lead to the snapshot file being committed multiple times. - // Therefore, retries should be handled by the upper layer, - // which should call FilterAndCommit to avoid duplicate commits. - // Therefore, we should not trigger cleanup here, - // as it may delete meta files from a snapshot that was just written by ourselves, - // leading to an incomplete or corrupted snapshot. guard.Release(); - return Status::Invalid("You need call FilterAndCommit to retry commit for exception. ", - commit_result.status().ToString()); + return false; } bool commit_success = commit_result.value(); if (commit_success) { @@ -750,70 +1050,6 @@ Result FileStoreCommitImpl::TryCommitOnce( } } -void FileStoreCommitImpl::AssignSnapshotId(int64_t snapshot_id, - std::vector* delta_files) const { - for (auto& entry : *delta_files) { - entry.AssignSequenceNumber(/*min_sequence_number=*/snapshot_id, - /*max_sequence_number=*/snapshot_id); - } -} - -Result FileStoreCommitImpl::AssignRowTrackingMeta( - int64_t first_row_id_start, std::vector* delta_files) const { - if (delta_files->empty()) { - return first_row_id_start; - } - // assign row id for new files - int64_t start = first_row_id_start; - int64_t blob_start_default = first_row_id_start; - // Per-blob-field row id tracking: each blob field maintains its own start position, - // keyed by the blob field name (from write_cols[0]). - std::map blob_starts; - // TODO(xinyu.lxy): support vector store file row tracking when vector store is implemented - for (auto& entry : *delta_files) { - if (entry.File()->file_source == std::nullopt) { - return Status::Invalid( - "This is a bug, file source field for row-tracking table must present."); - } - bool contains_row_id = - entry.File()->write_cols.has_value() && - std::find(entry.File()->write_cols->begin(), entry.File()->write_cols->end(), - SpecialFields::RowId().Name()) != entry.File()->write_cols->end(); - if (entry.File()->file_source.value() == FileSource::Append() && - entry.File()->first_row_id == std::nullopt && !contains_row_id) { - int64_t row_count = entry.File()->row_count; - if (BlobUtils::IsBlobFile(entry.File()->file_name)) { - // Use the first write_col as the blob field name to support - // independent row tracking per blob field. - std::string blob_field_name; - if (!entry.File()->write_cols || entry.File()->write_cols->empty()) { - return Status::Invalid(fmt::format( - "invalid blob file {}: does not have write_cols", entry.File()->file_name)); - } - blob_field_name = entry.File()->write_cols->at(0); - int64_t blob_start = blob_starts.count(blob_field_name) - ? blob_starts[blob_field_name] - : blob_start_default; - if (blob_start >= start) { - return Status::Invalid(fmt::format( - "This is a bug, blob start {} should be less than start {} when " - "assigning a blob entry file.", - blob_start, start)); - } - entry.AssignFirstRowId(blob_start); - blob_starts[blob_field_name] = blob_start + row_count; - } else { - entry.AssignFirstRowId(start); - blob_start_default = start; - blob_starts.clear(); - start += row_count; - } - } - // for compact file, do not assign first row id. - } - return start; -} - Result FileStoreCommitImpl::CommitSnapshotImpl( const Snapshot& new_snapshot, const std::vector& delta_statistics) { std::vector statistics; @@ -882,85 +1118,64 @@ std::shared_ptr FileStoreCommitImpl::CreateManifestCommitta return committable; } -Status FileStoreCommitImpl::CollectChanges( - const std::vector>& commit_messages, - std::vector* append_table_files, - std::vector* append_changelog_files, - std::vector* compact_table_files, - std::vector* compact_changelog_files, - std::vector* append_table_index_files, - std::vector* compact_table_index_files) { +Result FileStoreCommitImpl::CollectChanges( + const std::vector>& commit_messages) { + ManifestEntryChanges changes(num_bucket_); for (const auto& message : commit_messages) { - auto commit_message = std::dynamic_pointer_cast(message); - if (commit_message) { - DataIncrement new_files_increment = commit_message->GetNewFilesIncrement(); - for (const std::shared_ptr& new_file : new_files_increment.NewFiles()) { - append_table_files->push_back(MakeEntry(FileKind::Add(), commit_message, new_file)); - } - for (const std::shared_ptr& deleted_file : - new_files_increment.DeletedFiles()) { - append_table_files->push_back( - MakeEntry(FileKind::Delete(), commit_message, deleted_file)); - } - for (const std::shared_ptr& changelog_file : - new_files_increment.ChangelogFiles()) { - append_changelog_files->push_back( - MakeEntry(FileKind::Add(), commit_message, changelog_file)); - } - for (const std::shared_ptr& deleted_index_file : - new_files_increment.DeletedIndexFiles()) { - append_table_index_files->emplace_back( - FileKind::Delete(), commit_message->Partition(), commit_message->Bucket(), - deleted_index_file); - } - for (const std::shared_ptr& new_index_file : - new_files_increment.NewIndexFiles()) { - append_table_index_files->emplace_back(FileKind::Add(), commit_message->Partition(), - commit_message->Bucket(), new_index_file); - } - CompactIncrement compact_increment = commit_message->GetCompactIncrement(); - for (const std::shared_ptr& compact_before : - compact_increment.CompactBefore()) { - compact_table_files->push_back( - MakeEntry(FileKind::Delete(), commit_message, compact_before)); - } - for (const std::shared_ptr& compact_after : - compact_increment.CompactAfter()) { - compact_table_files->push_back( - MakeEntry(FileKind::Add(), commit_message, compact_after)); - } - for (const std::shared_ptr& changelog_file : - compact_increment.ChangelogFiles()) { - compact_changelog_files->push_back( - MakeEntry(FileKind::Add(), commit_message, changelog_file)); - } - for (const std::shared_ptr& deleted_index_file : - compact_increment.DeletedIndexFiles()) { - compact_table_index_files->emplace_back( - FileKind::Delete(), commit_message->Partition(), commit_message->Bucket(), - deleted_index_file); - } - for (const std::shared_ptr& new_index_file : - compact_increment.NewIndexFiles()) { - compact_table_index_files->emplace_back(FileKind::Add(), - commit_message->Partition(), - commit_message->Bucket(), new_index_file); - } - } else { - return Status::Invalid("fail to cast commit message to commit message impl"); - } + PAIMON_RETURN_NOT_OK(changes.Collect(message)); } - return Status::OK(); + PAIMON_LOG_INFO(logger_, "Finished collecting changes, including: %s", + changes.ToString().c_str()); + return changes; } -ManifestEntry FileStoreCommitImpl::MakeEntry( - const FileKind& kind, const std::shared_ptr& commit_message, - const std::shared_ptr& file) const { - int32_t total_buckets = commit_message->TotalBuckets() == std::nullopt - ? num_bucket_ - : commit_message->TotalBuckets().value(); - return ManifestEntry(kind, commit_message->Partition(), commit_message->Bucket(), total_buckets, - file); +void FileStoreCommitImpl::ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration, + int32_t generated_snapshot, int32_t attempt) { + auto table_files_added = static_cast(changes.append_table_files.size()); + int32_t table_files_deleted = 0; + int64_t compaction_input_file_size = 0; + int64_t compaction_output_file_size = 0; + for (const auto& entry : changes.compact_table_files) { + const auto& kind = entry.Kind(); + if (kind == FileKind::Add()) { + ++table_files_added; + compaction_output_file_size += entry.File()->file_size; + } else if (kind == FileKind::Delete()) { + ++table_files_deleted; + compaction_input_file_size += entry.File()->file_size; + } + } + metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, commit_duration); + metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt); + metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added); + metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted); + metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED, + changes.append_table_files.size()); + metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED, + changes.compact_table_files.size()); + metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED, + changes.append_changelog.size()); + metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED, + changes.compact_changelog.size()); + metrics_->SetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS, generated_snapshot); + metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED, + RowCounts(changes.append_table_files)); + metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED, + RowCounts(changes.append_changelog)); + metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED, + RowCounts(changes.compact_table_files)); + metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED, + RowCounts(changes.compact_changelog)); + metrics_->SetCounter( + CommitMetrics::LAST_PARTITIONS_WRITTEN, + NumChangedPartitions({changes.append_table_files, changes.compact_table_files})); + metrics_->SetCounter( + CommitMetrics::LAST_BUCKETS_WRITTEN, + NumChangedBuckets({changes.append_table_files, changes.compact_table_files})); + metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE, + compaction_input_file_size); + metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE, + compaction_output_file_size); } int64_t FileStoreCommitImpl::RowCounts(const std::vector& files) { diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h index f54c85ac8..4017a9d01 100644 --- a/src/paimon/core/operation/file_store_commit_impl.h +++ b/src/paimon/core/operation/file_store_commit_impl.h @@ -28,7 +28,13 @@ #include "paimon/core/catalog/snapshot_commit.h" #include "paimon/core/core_options.h" #include "paimon/core/manifest/partition_entry.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/commit/conflict_detection.h" +#include "paimon/core/operation/commit/manifest_entry_changes.h" +#include "paimon/core/operation/commit/retry_waiter.h" +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" #include "paimon/file_store_commit.h" #include "paimon/logging.h" #include "paimon/memory/memory_pool.h" @@ -60,6 +66,7 @@ class SnapshotManager; class SchemaManager; class TableSchema; class BinaryRowPartitionComputer; +class CommitChangesProvider; class CommitMessage; class Executor; class FileSystem; @@ -80,12 +87,14 @@ class FileStoreCommitImpl : public FileStoreCommit { std::unique_ptr partition_computer, const std::shared_ptr& snapshot_manager, bool ignore_empty_commit, bool use_rest_catalog_commit, + bool append_commit_check_conflict, const std::shared_ptr& table_schema, const std::shared_ptr& manifest_file, const std::shared_ptr& manifest_list, const std::shared_ptr& index_manifest_file, const std::shared_ptr& expire_snapshots, - const std::shared_ptr& schema_manager); + const std::shared_ptr& schema_manager, + CommitScanner::ScanSupplier scan_supplier); ~FileStoreCommitImpl() override; Status Commit(const std::vector>& commit_messages, @@ -97,13 +106,13 @@ class FileStoreCommitImpl : public FileStoreCommit { commit_identifier_and_messages, std::optional watermark = std::nullopt) override; - Status Overwrite(const std::vector>& partitions, + Status Overwrite(const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) override; Result FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) override; @@ -124,13 +133,26 @@ class FileStoreCommitImpl : public FileStoreCommit { Status Commit(const std::shared_ptr& manifest_committable, bool check_append_files); - Status TryOverwrite(const std::vector>& partition, - const std::vector& changes, int64_t commit_identifier, - std::optional watermark); + Result TryOverwrite(const std::vector>& partition, + const std::vector& changes, + const std::vector& index_entries, + int64_t commit_identifier, std::optional watermark, + const std::map& properties); + + Status ExecuteOverwrite(const std::vector>& partitions, + ManifestEntryChanges* changes, int64_t identifier, + std::optional watermark, + const std::map& properties, + int32_t* generated_snapshot, int32_t* attempt); Result> GetAllFiles( const Snapshot& snapshot, - const std::vector>& partitions); + const std::vector>& partitions) const; + + Result> PartitionToMap(const BinaryRow& partition) const; + + Result> TryUpgrade( + const std::vector& append_files) const; Result>> FilterCommitted( const std::vector>& committables); @@ -139,32 +161,33 @@ class FileStoreCommitImpl : public FileStoreCommit { int64_t identifier, const std::vector>& commit_messages, std::optional watermark); - ManifestEntry MakeEntry(const FileKind& kind, - const std::shared_ptr& commit_message, - const std::shared_ptr& file) const; + Result CollectChanges( + const std::vector>& commit_messages); - Status CollectChanges(const std::vector>& commit_messages, - std::vector* append_table_files, - std::vector* append_changelog_files, - std::vector* compact_table_files, - std::vector* compact_changelog_files, - std::vector* append_table_index_files, - std::vector* compact_table_index_files); + void ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration, + int32_t generated_snapshot, int32_t attempt); Result TryCommit(const std::vector& delta_files, + const std::vector& changelog_files, const std::vector& index_entries, int64_t identifier, std::optional watermark, - std::map log_offsets, const std::map& properties, - Snapshot::CommitKind commit_kind, bool check_append_files); + Snapshot::CommitKind commit_kind, bool detect_conflicts); + + Result TryCommit(const std::shared_ptr& changes_provider, + int64_t identifier, std::optional watermark, + const std::map& properties, + Snapshot::CommitKind commit_kind, bool detect_conflicts); + Result TryCommitOnce(const std::vector& delta_files, + const std::vector& changelog_files, const std::vector& index_entries, int64_t commit_identifier, std::optional watermark, - std::map log_offsets, const std::map& properties, Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot, - bool need_conflict_check); + bool detect_conflicts, + std::optional retry_start_snapshot_id); Result CommitSnapshotImpl(const Snapshot& new_snapshot, const std::vector& delta_statistics); @@ -176,25 +199,24 @@ class FileStoreCommitImpl : public FileStoreCommit { const std::optional& old_index_manifest, const std::optional& new_index_manifest); - Result> ReadAllEntriesFromChangedPartitions( - const Snapshot& latest_snapshot, - const std::set>& partitions) const; + Result CheckCommitted(const std::optional& latest_snapshot, + std::optional retry_start_snapshot_id, int64_t identifier, + const Snapshot::CommitKind& commit_kind) const; - Status NoConflictsOrFail(const std::string& base_commit_user, - const std::vector& base_entries, - const std::vector& changes) const; + Status CheckSameBucketFromSnapshot(const std::vector& delta_entries, + const std::optional& latest_snapshot) const; - Status CheckFilesExistence( - const std::vector>& committables) const; + bool ShouldCheckSameBucket(const Snapshot::CommitKind& commit_kind) const; + + bool IsUnorderedWriteOnlyAppend() const; - void AssignSnapshotId(int64_t snapshot_id, std::vector* delta_files) const; + bool IsWriteOnlySnapshotSequenceAppend() const; - Result AssignRowTrackingMeta(int64_t first_row_id_start, - std::vector* delta_files) const; + Result> MaxSequenceNumber( + const std::vector& manifests) const; - Result>> ChangedPartitions( - const std::vector& data_files, - const std::vector& index_files) const; + Status CheckFilesExistence( + const std::vector>& committables) const; static int64_t RowCounts(const std::vector& files); @@ -202,11 +224,15 @@ class FileStoreCommitImpl : public FileStoreCommit { static int64_t NumChangedBuckets(const std::vector>& changes); + static BucketMode ResolveBucketMode(int32_t bucket, + const std::shared_ptr& table_schema); + private: std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr schema_; std::string root_path_; + std::string table_name_; std::string commit_user_; CoreOptions options_; std::shared_ptr path_factory_; @@ -216,8 +242,13 @@ class FileStoreCommitImpl : public FileStoreCommit { std::shared_ptr snapshot_manager_; std::shared_ptr snapshot_commit_; bool ignore_empty_commit_ = true; + bool append_commit_check_conflict_ = false; + RetryWaiter retry_waiter_; int32_t num_bucket_ = 0; + BucketMode bucket_mode_ = BucketMode::BUCKET_UNAWARE; std::shared_ptr table_schema_; + CommitScanner commit_scanner_; + ConflictDetection conflict_detection_; std::shared_ptr manifest_file_; std::shared_ptr manifest_list_; diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 2ba008de3..2115f09c7 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -40,15 +40,19 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/catalog/commit_table_request.h" +#include "paimon/core/index/global_index_meta.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/file_source.h" #include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_committable.h" #include "paimon/core/manifest/manifest_entry.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/operation/metrics/commit_metrics.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/file_utils.h" @@ -59,6 +63,7 @@ #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/fs/local/local_file_system_factory.h" +#include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" #include "paimon/testing/utils/binary_row_generator.h" @@ -67,6 +72,7 @@ #include "paimon/testing/utils/timezone_guard.h" namespace paimon::test { + class GmockFileSystem : public LocalFileSystem { public: MOCK_METHOD(Status, ReadFile, (const std::string& path, std::string* content), (override)); @@ -158,11 +164,18 @@ class FileStoreCommitImplTest : public testing::Test { ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, const FileKind& kind) const { + return CreateManifestEntry(file_name, partition, kind, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/2, /*bucket=*/0); + } + + ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, + const FileKind& kind, const BinaryRow& min_key, + const BinaryRow& max_key, int32_t level, int32_t bucket = 0, + int32_t total_buckets = 2) const { auto data_file_meta = std::make_shared( - file_name, 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), - SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, - /*max_seq_no=*/32, - /*schema_id=*/1, /*level=*/2, + file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(), + SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32, + /*schema_id=*/1, level, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3, @@ -170,7 +183,15 @@ class FileStoreCommitImplTest : public testing::Test { /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); - return ManifestEntry(kind, partition, 0, 2, data_file_meta); + return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta); + } + + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; } ManifestEntry CreateManifestEntryWithNoPartition(const std::string& file_name, @@ -198,6 +219,41 @@ class FileStoreCommitImplTest : public testing::Test { return result; } + std::shared_ptr CreateIndexFileMeta(const std::string& file_name, + const std::string& index_type = "bitmap") { + return std::make_shared(index_type, file_name, /*file_size=*/100, + /*row_count=*/5, /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, + /*global_index_meta=*/std::nullopt); + } + + std::shared_ptr CreateGlobalIndexFileMeta(const std::string& file_name, + int64_t row_range_start, + int64_t row_range_end) { + GlobalIndexMeta global_index(row_range_start, row_range_end, /*index_field_id=*/1, + /*extra_field_ids=*/std::nullopt, + std::make_shared("meta", GetDefaultPool().get())); + return std::make_shared( + "HASH", file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index); + } + + std::shared_ptr CreateAppendDataFileMeta(const std::string& file_name, + int64_t row_count) { + return std::make_shared( + file_name, 1024, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, + /*max_seq_no=*/32, + /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, FileSource::Append(), + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + bool IsStringInSet(const std::set& strSet, const std::string& target) { return strSet.find(target) != strSet.end(); } @@ -338,7 +394,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); std::vector expected_partition_statistics = { @@ -364,6 +420,48 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { ASSERT_FALSE(exist); } +TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs1 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_GT(msgs1.size(), 0); + ASSERT_OK(commit_impl->Commit(msgs1, 1)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_TRUE(snapshot1.Properties()); + auto iter1 = snapshot1.Properties().value().find("sequence.generation.max-sequence-number"); + ASSERT_TRUE(iter1 != snapshot1.Properties().value().end()); + int64_t max_seq_1 = std::stoll(iter1->second); + + std::vector> msgs2 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-02", + /*version=*/3); + ASSERT_GT(msgs2.size(), 0); + ASSERT_OK(commit_impl->Commit(msgs2, 2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot2, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_TRUE(snapshot2.Properties()); + auto iter2 = snapshot2.Properties().value().find("sequence.generation.max-sequence-number"); + ASSERT_TRUE(iter2 != snapshot2.Properties().value().end()); + int64_t max_seq_2 = std::stoll(iter2->second); + + ASSERT_GE(max_seq_2, max_seq_1); +} + TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) { std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; auto dir = UniqueTestDirectory::Create(); @@ -376,6 +474,8 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::COMMIT_MAX_RETRIES, "10") + .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") + .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") .WithFileSystem(fs) .Finish()); @@ -416,25 +516,25 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") + .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") .WithFileSystem(fs) .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); + EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_)) + .Times(testing::AnyNumber()) + .WillRepeatedly(testing::Invoke([&](const std::string& path, std::string* content) { + return mock_fs->FileSystem::ReadFile(path, content); + })); EXPECT_CALL(*mock_fs, ReadFile(testing::StrEq(latest_hint), testing::_)) .WillRepeatedly(testing::Invoke([](const std::string& path, std::string* content) { *content = "-1"; return Status::OK(); })); - EXPECT_CALL( - *mock_fs, - ReadFile(testing::StrEq(PathUtil::JoinPath(table_path, "snapshot/snapshot-5")), testing::_)) - .WillOnce(testing::Invoke([&](const std::string& path, std::string* content) { - return mock_fs->FileSystem::ReadFile(path, content); - })); - EXPECT_CALL(*mock_fs, ListDir(testing::_, testing::_)).Times(testing::AnyNumber()); EXPECT_CALL(*mock_fs, ListDir(testing::StrEq(PathUtil::JoinPath(table_path, "snapshot")), testing::_)) @@ -495,7 +595,12 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); - ASSERT_NOK(commit->Commit(msgs, /*commit_identifier=*/1)); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1)); + std::shared_ptr metrics = commit->GetCommitMetrics(); + ASSERT_TRUE(metrics); + ASSERT_OK_AND_ASSIGN(uint64_t counter, + metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS)); + ASSERT_EQ(2u, counter); ASSERT_OK_AND_ASSIGN( bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); @@ -508,8 +613,6 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua .Finish()); ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); - ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); - ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_7), testing::_)) .WillOnce(testing::Invoke([&](const std::string& path, const std::string& content) { @@ -738,18 +841,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); auto status = commit->Commit(msgs); io_hook->Clear(); + ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath( + table_path_, "snapshot/snapshot-2"))); // termination conditions: - // 1. status does not hit IOHook, already touch all IO operation - // 2. hit IOHook in commit hint, at this point, the snapshot is already committed - if (!HitIOHook(status) || HitIOHookInCommitHint(status)) { + // 1. hit IOHook in commit hint, at this point, the snapshot is already committed + // 2. snapshot file exists, which means atomic store succeeded even if a later IO failed + if (HitIOHookInCommitHint(status) || exist2) { scanned_all_io_hook = true; - ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath( - table_path_, "snapshot/snapshot-2"))); ASSERT_TRUE(exist2); break; } - ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath( - table_path_, "snapshot/snapshot-2"))); + // For some IO-hook positions, retries may be exhausted and return a generic non-IOHook + // status while the snapshot is still not committed. Keep scanning next positions. + if (!HitIOHook(status)) { + continue; + } ASSERT_FALSE(exist2); std::vector actual_snapshots; ASSERT_OK( @@ -868,7 +974,6 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { ASSERT_OK_AND_ASSIGN(exist, file_system_->Exists(PathUtil::JoinPath( table_path_, "manifest/" + index_manifest.value()))); ASSERT_FALSE(exist); - commit_impl->CleanUpTmpManifests( snapshot.value().BaseManifestList(), snapshot.value().DeltaManifestList(), /*old_metas=*/{}, /*new_metas=*/previous_manifests, /*old_index_manifest=*/std::nullopt, @@ -899,133 +1004,6 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { ASSERT_EQ(0u, counter); } -TEST_F(FileStoreCommitImplTest, TestCheckConflict) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); - auto commit_impl = dynamic_cast(commit.get()); - ASSERT_TRUE(commit_impl); - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file2", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file3", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file4", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file5", FileKind::Delete())); - ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file6", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file2", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file3", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file4", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file5", FileKind::Delete())); - ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file2", FileKind::Add())); - changes.push_back(CreateManifestEntry("file3", FileKind::Add())); - changes.push_back(CreateManifestEntry("file4", FileKind::Add())); - changes.push_back(CreateManifestEntry("file5", FileKind::Add())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Add())); - changes.push_back(CreateManifestEntry("file2", FileKind::Add())); - changes.push_back(CreateManifestEntry("file3", FileKind::Add())); - changes.push_back(CreateManifestEntry("file4", FileKind::Add())); - changes.push_back(CreateManifestEntry("file5", FileKind::Add())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Delete())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Add())); - changes.push_back(CreateManifestEntry("file2", FileKind::Add())); - changes.push_back(CreateManifestEntry("file3", FileKind::Add())); - changes.push_back(CreateManifestEntry("file4", FileKind::Add())); - changes.push_back(CreateManifestEntry("file5", FileKind::Add())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - - std::vector changes; - ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Delete())); - - std::vector changes; - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file2", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file3", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file4", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file5", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file6", FileKind::Delete())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } -} - TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1052,8 +1030,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { std::vector changes; changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add())); std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}}; - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, - /*commit_identifier=*/1, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/1, std::nullopt, + /*properties=*/{})); } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { @@ -1071,8 +1050,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { std::vector changes; changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add())); std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}}; - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, - /*commit_identifier=*/0, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/0, std::nullopt, + /*properties=*/{})); ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {})); ASSERT_EQ(1u, entries1.size()); @@ -1080,8 +1060,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { ASSERT_EQ(FileKind::Add(), entries1[0].Kind()); std::vector changes2; changes2.push_back(CreateManifestEntry("new_file_2", FileKind::Add())); - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, - /*commit_identifier=*/1, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, /*index_entries=*/{}, + /*commit_identifier=*/1, std::nullopt, + /*properties=*/{})); ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {})); ASSERT_EQ(1u, entries2.size()); @@ -1089,6 +1070,34 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { ASSERT_EQ(FileKind::Add(), entries2[0].Kind()); } +TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + + std::vector changes; + changes.push_back(CreateManifestEntry("new_file_with_properties", FileKind::Add())); + std::vector> partitions = {{{"f1", "10"}}}; + std::map properties = {{"overwrite-prop", "v1"}}; + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/0, std::nullopt, properties)); + + ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot.has_value()); + ASSERT_TRUE(snapshot->Properties().has_value()); + auto iter = snapshot->Properties()->find("overwrite-prop"); + ASSERT_TRUE(iter != snapshot->Properties()->end()); + ASSERT_EQ("v1", iter->second); +} + TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1104,8 +1113,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { std::vector changes; changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add())); std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}}; - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, - /*commit_identifier=*/0, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/0, std::nullopt, + /*properties=*/{})); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -1130,8 +1140,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { std::vector changes2; changes2.push_back(CreateManifestEntry("new_file_2", FileKind::Add())); - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, - /*commit_identifier=*/2, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, /*index_entries=*/{}, + /*commit_identifier=*/2, std::nullopt, + /*properties=*/{})); ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {})); ASSERT_EQ(1u, entries2.size()); @@ -1272,36 +1283,31 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { /*version=*/3); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - ASSERT_OK(commit_impl->CollectChanges(msgs, &append_table_files, &append_changelog_files, - &compact_table_files, &compact_changelog_files, - &append_table_index_files, &compact_table_index_files)); - ASSERT_EQ(append_table_files.size(), 3u); - ASSERT_EQ(append_changelog_files.size(), 0u); - ASSERT_EQ(compact_table_files.size(), 0u); - ASSERT_EQ(compact_changelog_files.size(), 0u); - ASSERT_EQ(append_table_index_files.size(), 0u); - ASSERT_EQ(compact_table_index_files.size(), 0u); - ASSERT_EQ(append_table_files[0].Kind(), FileKind::Add()); - ASSERT_EQ(append_table_files[0].Bucket(), 0); - ASSERT_EQ(append_table_files[0].TotalBuckets(), 10); - ASSERT_EQ(append_table_files[0].Level(), 0); - ASSERT_EQ(append_table_files[0].FileName(), "data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc"); - ASSERT_EQ(append_table_files[1].Kind(), FileKind::Add()); - ASSERT_EQ(append_table_files[1].Bucket(), 1); - ASSERT_EQ(append_table_files[1].TotalBuckets(), 10); - ASSERT_EQ(append_table_files[1].Level(), 0); - ASSERT_EQ(append_table_files[1].FileName(), "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc"); - ASSERT_EQ(append_table_files[2].Kind(), FileKind::Add()); - ASSERT_EQ(append_table_files[2].Bucket(), 0); - ASSERT_EQ(append_table_files[2].TotalBuckets(), 10); - ASSERT_EQ(append_table_files[2].Level(), 0); - ASSERT_EQ(append_table_files[2].FileName(), "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc"); + ASSERT_OK_AND_ASSIGN(ManifestEntryChanges changes, commit_impl->CollectChanges(msgs)); + ASSERT_EQ(changes.append_table_files.size(), 3u); + ASSERT_EQ(changes.append_changelog.size(), 0u); + ASSERT_EQ(changes.compact_table_files.size(), 0u); + ASSERT_EQ(changes.compact_changelog.size(), 0u); + ASSERT_EQ(changes.append_index_files.size(), 0u); + ASSERT_EQ(changes.compact_index_files.size(), 0u); + ASSERT_EQ(changes.append_table_files[0].Kind(), FileKind::Add()); + ASSERT_EQ(changes.append_table_files[0].Bucket(), 0); + ASSERT_EQ(changes.append_table_files[0].TotalBuckets(), 10); + ASSERT_EQ(changes.append_table_files[0].Level(), 0); + ASSERT_EQ(changes.append_table_files[0].FileName(), + "data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc"); + ASSERT_EQ(changes.append_table_files[1].Kind(), FileKind::Add()); + ASSERT_EQ(changes.append_table_files[1].Bucket(), 1); + ASSERT_EQ(changes.append_table_files[1].TotalBuckets(), 10); + ASSERT_EQ(changes.append_table_files[1].Level(), 0); + ASSERT_EQ(changes.append_table_files[1].FileName(), + "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc"); + ASSERT_EQ(changes.append_table_files[2].Kind(), FileKind::Add()); + ASSERT_EQ(changes.append_table_files[2].Bucket(), 0); + ASSERT_EQ(changes.append_table_files[2].TotalBuckets(), 10); + ASSERT_EQ(changes.append_table_files[2].Level(), 0); + ASSERT_EQ(changes.append_table_files[2].FileName(), + "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc"); } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { @@ -1484,6 +1490,266 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc")); } +TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files; + new_index_files.push_back(CreateIndexFileMeta("bitmap-index-commit-1")); + DataIncrement data_increment({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit_impl->Commit({msg}, 1)); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Append(), snapshot.GetCommitKind()); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-commit-1", index_entries[0].index_file->FileName()); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::ROW_TRACKING_ENABLED, "true") + .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + DataIncrement data_increment_1({CreateAppendDataFileMeta("data-with-row-id", 10)}, {}, {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_1}, 1)); + + std::vector> global_index_files; + global_index_files.push_back(CreateGlobalIndexFileMeta("global-index-out-of-range", + /*row_range_start=*/0, + /*row_range_end=*/10)); + DataIncrement data_increment_2({}, {}, {}, std::move(global_index_files), {}); + std::shared_ptr msg_2 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_2, CompactIncrement({}, {}, {})); + + ASSERT_NOK_WITH_MSG(commit_impl->Commit({msg_2}, 2), "Global index row ID existence conflict"); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> compact_index_files; + compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-commit-compact-1")); + DataIncrement data_increment({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK(commit_impl->Commit({msg}, 1)); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind()); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-commit-compact-1", index_entries[0].index_file->FileName()); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files; + new_index_files.push_back(CreateIndexFileMeta("bitmap-index-delete-1")); + DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_1}, 1)); + + std::vector> deleted_index_files; + deleted_index_files.push_back(CreateIndexFileMeta("bitmap-index-delete-1")); + DataIncrement data_increment_2({}, {}, {}, {}, std::move(deleted_index_files)); + std::shared_ptr msg_2 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_2, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_2}, 2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_TRUE(index_entries.empty()); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files; + new_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-delete-1")); + DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_1}, 1)); + + std::vector> deleted_index_files; + deleted_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-delete-1")); + DataIncrement data_increment_2({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, {}, std::move(deleted_index_files)); + std::shared_ptr msg_2 = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment_2, compact_increment); + ASSERT_OK(commit_impl->Commit({msg_2}, 2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind()); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_TRUE(index_entries.empty()); +} + +TEST_F(FileStoreCommitImplTest, TestOverwriteWithIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files_1; + new_index_files_1.push_back(CreateIndexFileMeta("bitmap-index-1")); + DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files_1), {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit_impl->Overwrite({}, {msg_1}, 1)); + ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot1.value().IndexManifest()); + std::vector index_entries_1; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot1.value().IndexManifest().value(), + /*filter=*/nullptr, &index_entries_1)); + ASSERT_EQ(1u, index_entries_1.size()); + ASSERT_EQ("bitmap-index-1", index_entries_1[0].index_file->FileName()); + + std::vector> new_index_files_2; + new_index_files_2.push_back(CreateIndexFileMeta("bitmap-index-2")); + DataIncrement data_increment_2({}, {}, {}, std::move(new_index_files_2), {}); + std::shared_ptr msg_2 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_2, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit_impl->Overwrite({}, {msg_2}, 2)); + ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot2.value().IndexManifest()); + std::vector index_entries_2; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot2.value().IndexManifest().value(), + /*filter=*/nullptr, &index_entries_2)); + ASSERT_EQ(1u, index_entries_2.size()); + ASSERT_EQ("bitmap-index-2", index_entries_2[0].index_file->FileName()); +} + +TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> compact_index_files; + compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-1")); + DataIncrement data_increment({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK(commit_impl->Overwrite({}, {msg}, 1)); + + ASSERT_OK_AND_ASSIGN(Snapshot overwrite_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), overwrite_snapshot.GetCommitKind()); + ASSERT_FALSE(overwrite_snapshot.IndexManifest()); + + ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind()); + ASSERT_TRUE(compact_snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(compact_snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-compact-1", index_entries[0].index_file->FileName()); +} + TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1540,6 +1806,96 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc")); } +TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files_1; + new_index_files_1.push_back(CreateIndexFileMeta("bitmap-index-filter-1")); + DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files_1), {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + + ASSERT_OK_AND_ASSIGN(int32_t actual_commit, + commit_impl->FilterAndOverwrite({}, {msg_1}, 1, 10)); + ASSERT_EQ(1, actual_commit); + ASSERT_OK_AND_ASSIGN(actual_commit, commit_impl->FilterAndOverwrite({}, {msg_1}, 1, 5)); + ASSERT_EQ(0, actual_commit); + + std::vector> new_index_files_2; + new_index_files_2.push_back(CreateIndexFileMeta("bitmap-index-filter-2")); + DataIncrement data_increment_2({}, {}, {}, std::move(new_index_files_2), {}); + std::shared_ptr msg_2 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_2, CompactIncrement({}, {}, {})); + + ASSERT_OK_AND_ASSIGN(actual_commit, commit_impl->FilterAndOverwrite({}, {msg_2}, 2, 20)); + ASSERT_EQ(1, actual_commit); + ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot.value().IndexManifest()); + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.value().IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-filter-2", index_entries[0].index_file->FileName()); +} + +TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> compact_index_files; + compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-filter-compact-1")); + DataIncrement data_increment({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK_AND_ASSIGN(int32_t actual_commit, commit_impl->FilterAndOverwrite({}, {msg}, 1, 10)); + ASSERT_EQ(1, actual_commit); + ASSERT_OK_AND_ASSIGN(actual_commit, commit_impl->FilterAndOverwrite({}, {msg}, 1, 5)); + ASSERT_EQ(0, actual_commit); + + ASSERT_OK_AND_ASSIGN(Snapshot overwrite_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), overwrite_snapshot.GetCommitKind()); + ASSERT_TRUE(overwrite_snapshot.Watermark()); + ASSERT_EQ(10, overwrite_snapshot.Watermark().value()); + ASSERT_FALSE(overwrite_snapshot.IndexManifest()); + + ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind()); + ASSERT_TRUE(compact_snapshot.Watermark()); + ASSERT_EQ(10, compact_snapshot.Watermark().value()); + ASSERT_TRUE(compact_snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(compact_snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-filter-compact-1", index_entries[0].index_file->FileName()); +} + TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1566,7 +1922,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { std::map partitions; partitions["f1"] = "10"; - ASSERT_OK(commit_impl->Overwrite({partitions}, msgs2, 2)); + ASSERT_OK(commit_impl->Overwrite(partitions, msgs2, 2)); ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {})); ASSERT_EQ(3u, entries1.size()); @@ -1576,6 +1932,68 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { ASSERT_TRUE(IsStringInSet(file_names, "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc")); } +TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartitionIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + const BinaryRow partition_10 = CreateIntRow(10); + const BinaryRow partition_20 = CreateIntRow(20); + + std::vector> index_files_partition_10_v1; + index_files_partition_10_v1.push_back(CreateIndexFileMeta("bitmap-index-partition-10-v1")); + DataIncrement data_increment_partition_10_v1({}, {}, {}, std::move(index_files_partition_10_v1), + {}); + std::shared_ptr msg_partition_10_v1 = std::make_shared( + partition_10, /*bucket=*/0, /*total_buckets=*/2, data_increment_partition_10_v1, + CompactIncrement({}, {}, {})); + + std::vector> index_files_partition_20_v1; + index_files_partition_20_v1.push_back(CreateIndexFileMeta("bitmap-index-partition-20-v1")); + DataIncrement data_increment_partition_20_v1({}, {}, {}, std::move(index_files_partition_20_v1), + {}); + std::shared_ptr msg_partition_20_v1 = std::make_shared( + partition_20, /*bucket=*/0, /*total_buckets=*/2, data_increment_partition_20_v1, + CompactIncrement({}, {}, {})); + + ASSERT_OK(commit_impl->Overwrite({}, {msg_partition_10_v1, msg_partition_20_v1}, 1)); + + std::vector> index_files_partition_10_v2; + index_files_partition_10_v2.push_back(CreateIndexFileMeta("bitmap-index-partition-10-v2")); + DataIncrement data_increment_partition_10_v2({}, {}, {}, std::move(index_files_partition_10_v2), + {}); + std::shared_ptr msg_partition_10_v2 = std::make_shared( + partition_10, /*bucket=*/0, /*total_buckets=*/2, data_increment_partition_10_v2, + CompactIncrement({}, {}, {})); + + std::map partition_spec; + partition_spec["f1"] = "10"; + ASSERT_OK(commit_impl->Overwrite(partition_spec, {msg_partition_10_v2}, 2)); + + ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot.value().IndexManifest()); + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.value().IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(2u, index_entries.size()); + + std::set index_file_names; + for (const auto& entry : index_entries) { + index_file_names.insert(entry.index_file->FileName()); + } + + ASSERT_TRUE(IsStringInSet(index_file_names, "bitmap-index-partition-10-v2")); + ASSERT_TRUE(IsStringInSet(index_file_names, "bitmap-index-partition-20-v1")); + ASSERT_FALSE(IsStringInSet(index_file_names, "bitmap-index-partition-10-v1")); +} + TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1609,6 +2027,39 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { ASSERT_EQ(0u, entries2.size()); } +TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + + ASSERT_OK(commit_impl->Commit(msgs, 1)); + ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {})); + std::set file_names_before = CollectFileNames(entries1); + ASSERT_FALSE(file_names_before.empty()); + + // Committing exactly the same append files should be accepted and filtered out when + // commit.discard-duplicate-files=true. + ASSERT_OK(commit_impl->Commit(msgs, 2)); + ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {})); + std::set file_names_after = CollectFileNames(entries2); + ASSERT_EQ(file_names_before, file_names_after); +} + TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1689,9 +2140,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_FALSE(json.empty()); } -// Verify that FileStoreCommit::Create succeeds for PK tables with postpone bucket mode (bucket=-2) -// without requiring the enable-pk-commit-in-inte-test workaround flag. -TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { +TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { auto pk_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(pk_dir); std::string pk_root = pk_dir->Str(); @@ -1702,14 +2151,13 @@ TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); ::ArrowSchema arrow_schema; ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); - std::map table_options = {{Options::BUCKET, "-2"}}; + std::map table_options = {{Options::BUCKET, "4"}}; ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema, /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - // Create FileStoreCommit WITHOUT the workaround flag — should succeed for postpone bucket CommitContextBuilder builder(pk_table_path, "test_user"); builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); @@ -1717,34 +2165,4 @@ TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { ASSERT_TRUE(committer != nullptr); } -// Verify that FileStoreCommit::Create still rejects PK tables with fixed bucket (bucket > 0) -// when the workaround flag is not set. -TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitRejected) { - auto pk_dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(pk_dir); - std::string pk_root = pk_dir->Str(); - ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {})); - ASSERT_OK(catalog->CreateDatabase("db", {}, false)); - - arrow::Schema pk_schema( - {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); - ::ArrowSchema arrow_schema; - ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); - std::map table_options = {{Options::BUCKET, "4"}}; - ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl_fixed"), &arrow_schema, - /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, - /*ignore_if_exists=*/false)); - - std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl_fixed"); - - CommitContextBuilder builder(pk_table_path, "test_user"); - builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); - ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); - auto result = FileStoreCommit::Create(std::move(commit_context)); - ASSERT_FALSE(result.ok()); - ASSERT_TRUE(result.status().IsNotImplemented()); - ASSERT_TRUE(result.status().ToString().find("not support pk table commit") != - std::string::npos); -} - } // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index 4a6fcd45e..4a20cbe7f 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -26,10 +26,21 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" +#include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/index/deletion_vector_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_increment.h" #include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/result.h" +#include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -72,4 +83,62 @@ TEST(FileStoreCommitTest, TestCreate) { ASSERT_TRUE(commit_impl); } +TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { + auto string_field = arrow::field("f0", arrow::utf8()); + auto int_field = arrow::field("f1", arrow::int32()); + auto int_field1 = arrow::field("f2", arrow::int32()); + auto double_field = arrow::field("f3", arrow::float64()); + auto schema = + arrow::schema(arrow::FieldVector({string_field, int_field, int_field1, double_field})); + + ::ArrowSchema arrow_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok()); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + + std::map options = {{Options::FILE_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}}; + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options)); + ASSERT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &arrow_schema, + /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + CommitContextBuilder context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + LinkedHashMap dv_ranges; + dv_ranges.insert_or_assign("data-file-1", + DeletionVectorMeta( + /*data_file_name=*/"data-file-1", /*offset=*/0, /*length=*/10, + /*cardinality=*/1)); + std::vector> new_index_files; + new_index_files.push_back(std::make_shared( + DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-index-1", 100, 1, + /*dv_ranges=*/dv_ranges, /*external_path=*/std::nullopt)); + DataIncrement data_increment({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg = std::make_shared( + BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()), /*bucket=*/0, + /*total_bucket=*/2, data_increment, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit->Commit({msg}, /*commit_identifier=*/1)); + + auto fs = std::make_shared(); + SnapshotManager snapshot_manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.value().GetCommitKind()); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/orphan_files_cleaner_test.cpp b/src/paimon/core/operation/orphan_files_cleaner_test.cpp index c94220d7c..52ec3bbdb 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_test.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner_test.cpp @@ -230,7 +230,6 @@ TEST(OrphanFilesCleanerTest, TestTableWithChangelog) { "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", "timeMillis" : 1721615035363, - "logOffsets" : { }, "totalRecordCount" : 11, "deltaRecordCount" : 1, "changelogRecordCount" : 0 @@ -263,7 +262,6 @@ TEST(OrphanFilesCleanerTest, TestTableWithIndexManifest) { "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", "timeMillis" : 1721615035363, - "logOffsets" : { }, "totalRecordCount" : 11, "deltaRecordCount" : 1, "changelogRecordCount" : 0 diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index a1f1749d0..c26977d4a 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -79,7 +79,7 @@ bool Snapshot::TEST_Equal(const Snapshot& other) const { return version_ == other.version_ && id_ == other.id_ && schema_id_ == other.schema_id_ && index_manifest_ == other.index_manifest_ && commit_user_ == other.commit_user_ && commit_identifier_ == other.commit_identifier_ && commit_kind_ == other.commit_kind_ && - log_offsets_ == other.log_offsets_ && total_record_count_ == other.total_record_count_ && + total_record_count_ == other.total_record_count_ && delta_record_count_ == other.delta_record_count_ && changelog_record_count_ == other.changelog_record_count_ && watermark_ == other.watermark_ && statistics_ == other.statistics_ && @@ -99,8 +99,7 @@ bool Snapshot::operator==(const Snapshot& other) const { changelog_manifest_list_size_ == other.changelog_manifest_list_size_ && index_manifest_ == other.index_manifest_ && commit_user_ == other.commit_user_ && commit_identifier_ == other.commit_identifier_ && commit_kind_ == other.commit_kind_ && - time_millis_ == other.time_millis_ && log_offsets_ == other.log_offsets_ && - total_record_count_ == other.total_record_count_ && + time_millis_ == other.time_millis_ && total_record_count_ == other.total_record_count_ && delta_record_count_ == other.delta_record_count_ && changelog_record_count_ == other.changelog_record_count_ && watermark_ == other.watermark_ && statistics_ == other.statistics_ && @@ -145,9 +144,7 @@ Snapshot::Snapshot(const std::optional& version, int64_t id, int64_t sc const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, @@ -167,7 +164,6 @@ Snapshot::Snapshot(const std::optional& version, int64_t id, int64_t sc commit_identifier_(commit_identifier), commit_kind_(commit_kind), time_millis_(time_millis), - log_offsets_(log_offsets), total_record_count_(total_record_count), delta_record_count_(delta_record_count), changelog_record_count_(changelog_record_count), @@ -228,17 +224,10 @@ rapidjson::Value Snapshot::ToJson(rapidjson::Document::AllocatorType* allocator) obj.AddMember(rapidjson::StringRef(FIELD_TIME_MILLIS), RapidJsonUtil::SerializeValue(time_millis_, allocator).Move(), *allocator); - if (log_offsets_ != std::nullopt) { - obj.AddMember(rapidjson::StringRef(FIELD_LOG_OFFSETS), - RapidJsonUtil::SerializeValue(log_offsets_.value(), allocator).Move(), - *allocator); - } obj.AddMember(rapidjson::StringRef(FIELD_TOTAL_RECORD_COUNT), - RapidJsonUtil::SerializeValue(total_record_count_.value(), allocator).Move(), - *allocator); + RapidJsonUtil::SerializeValue(total_record_count_, allocator).Move(), *allocator); obj.AddMember(rapidjson::StringRef(FIELD_DELTA_RECORD_COUNT), - RapidJsonUtil::SerializeValue(delta_record_count_.value(), allocator).Move(), - *allocator); + RapidJsonUtil::SerializeValue(delta_record_count_, allocator).Move(), *allocator); if (changelog_record_count_ != std::nullopt) { obj.AddMember( @@ -298,12 +287,10 @@ void Snapshot::FromJson(const rapidjson::Value& obj) noexcept(false) { throw std::invalid_argument("deserialize CommitKind failed"); } time_millis_ = RapidJsonUtil::DeserializeKeyValue(obj, FIELD_TIME_MILLIS); - log_offsets_ = RapidJsonUtil::DeserializeKeyValue>>( - obj, FIELD_LOG_OFFSETS); total_record_count_ = - RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_TOTAL_RECORD_COUNT); + RapidJsonUtil::DeserializeKeyValue(obj, FIELD_TOTAL_RECORD_COUNT); delta_record_count_ = - RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_DELTA_RECORD_COUNT); + RapidJsonUtil::DeserializeKeyValue(obj, FIELD_DELTA_RECORD_COUNT); changelog_record_count_ = RapidJsonUtil::DeserializeKeyValue>( obj, FIELD_CHANGELOG_RECORD_COUNT); watermark_ = RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_WATERMARK); diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h index 012430a61..2ef09e377 100644 --- a/src/paimon/core/snapshot.h +++ b/src/paimon/core/snapshot.h @@ -56,6 +56,11 @@ class Snapshot : public Jsonizable { bool operator==(const CommitKind& other) const { return value_ == other.value_; } + + bool operator!=(const CommitKind& other) const { + return value_ != other.value_; + } + static std::string ToString(const CommitKind& kind); static CommitKind FromString(const std::string& kind); @@ -77,7 +82,6 @@ class Snapshot : public Jsonizable { static constexpr char FIELD_COMMIT_IDENTIFIER[] = "commitIdentifier"; static constexpr char FIELD_COMMIT_KIND[] = "commitKind"; static constexpr char FIELD_TIME_MILLIS[] = "timeMillis"; - static constexpr char FIELD_LOG_OFFSETS[] = "logOffsets"; static constexpr char FIELD_TOTAL_RECORD_COUNT[] = "totalRecordCount"; static constexpr char FIELD_DELTA_RECORD_COUNT[] = "deltaRecordCount"; static constexpr char FIELD_CHANGELOG_RECORD_COUNT[] = "changelogRecordCount"; @@ -96,9 +100,7 @@ class Snapshot : public Jsonizable { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -106,7 +108,7 @@ class Snapshot : public Jsonizable { : Snapshot(CURRENT_VERSION, id, schema_id, base_manifest_list, base_manifest_list_size, delta_manifest_list, delta_manifest_list_size, changelog_manifest_list, changelog_manifest_list_size, index_manifest, commit_user, commit_identifier, - commit_kind, time_millis, log_offsets, total_record_count, delta_record_count, + commit_kind, time_millis, total_record_count, delta_record_count, changelog_record_count, watermark, statistics, properties, next_row_id) {} Snapshot(const std::optional& version, int64_t id, int64_t schema_id, @@ -118,9 +120,7 @@ class Snapshot : public Jsonizable { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -192,15 +192,11 @@ class Snapshot : public Jsonizable { return time_millis_; } - const std::optional>& LogOffsets() const { - return log_offsets_; - } - - const std::optional& TotalRecordCount() const { + int64_t TotalRecordCount() const { return total_record_count_; } - const std::optional& DeltaRecordCount() const { + int64_t DeltaRecordCount() const { return delta_record_count_; } @@ -273,15 +269,11 @@ class Snapshot : public Jsonizable { int64_t time_millis_; - std::optional> log_offsets_; - // record count of all changes occurred in this snapshot - // null for paimon <= 0.3 - std::optional total_record_count_; + int64_t total_record_count_ = 0; // record count of all new changes occurred in this snapshot - // null for paimon <= 0.3 - std::optional delta_record_count_; + int64_t delta_record_count_ = 0; // record count of all changelog produced in this snapshot // null for paimon <= 0.3 diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index f2572dfb8..ed9c7a29e 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -37,7 +37,6 @@ class SnapshotTest : public testing::Test { }; TEST_F(SnapshotTest, TestSimple) { - std::map log_offset = {{25, 30}}; std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; Snapshot snapshot( /*version=*/5, /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"base_manifest_list", 10, @@ -45,7 +44,7 @@ TEST_F(SnapshotTest, TestSimple) { /*changelog_manifest_list=*/"changelog_manifest_list", 30, /*index_manifest=*/"index_manifest", /*commit_user=*/"commit_user_01", /*commit_identifier=*/20, - /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, log_offset, + /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, /*changelog_record_count=*/45, /*watermark=*/50, /*statistics=*/"statistic_test", properties, /*next_row_id=*/0); @@ -63,9 +62,8 @@ TEST_F(SnapshotTest, TestSimple) { ASSERT_EQ(20, snapshot.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind()); ASSERT_EQ(1234, snapshot.TimeMillis()); - ASSERT_EQ(log_offset, snapshot.LogOffsets().value()); - ASSERT_EQ(35, snapshot.TotalRecordCount().value()); - ASSERT_EQ(40, snapshot.DeltaRecordCount().value()); + ASSERT_EQ(35, snapshot.TotalRecordCount()); + ASSERT_EQ(40, snapshot.DeltaRecordCount()); ASSERT_EQ(45, snapshot.ChangelogRecordCount().value()); ASSERT_EQ(50, snapshot.Watermark().value()); ASSERT_EQ("statistic_test", snapshot.Statistics().value()); @@ -92,9 +90,8 @@ TEST_F(SnapshotTest, TestFromPath) { ASSERT_EQ(9223372036854775807ll, snapshot.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Append(), snapshot.GetCommitKind()); ASSERT_EQ(1721614343270ll, snapshot.TimeMillis()); - ASSERT_EQ((std::map()), snapshot.LogOffsets().value()); - ASSERT_EQ(5, snapshot.TotalRecordCount().value()); - ASSERT_EQ(5, snapshot.DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot.TotalRecordCount()); + ASSERT_EQ(5, snapshot.DeltaRecordCount()); ASSERT_EQ(0, snapshot.ChangelogRecordCount().value()); ASSERT_EQ(std::nullopt, snapshot.Watermark()); ASSERT_EQ(std::nullopt, snapshot.Statistics()); @@ -116,7 +113,6 @@ TEST_F(SnapshotTest, TestJsonizable) { "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", "timeMillis" : 1711692199281, - "logOffsets" : { }, "totalRecordCount" : 3, "deltaRecordCount" : 3, "changelogRecordCount" : 0 @@ -133,7 +129,6 @@ TEST_F(SnapshotTest, TestJsonizable) { /*commit_user=*/"0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", /*commit_identifier=*/9223372036854775807ll, /*commit_kind=*/Snapshot::CommitKind::Overwrite(), /*time_millis=*/1711692199281ll, - /*log_offsets=*/std::map(), /*total_record_count=*/3, /*delta_record_count=*/3, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt); @@ -196,10 +191,6 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -226,10 +217,6 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -259,7 +246,6 @@ TEST_F(SnapshotTest, TestCommitKindAnalyze) { /*commit_identifier=*/42, /*commit_kind=*/Snapshot::CommitKind::Analyze(), /*time_millis=*/1700000000000ll, - /*log_offsets=*/std::map(), /*total_record_count=*/0, /*delta_record_count=*/0, /*changelog_record_count=*/0, @@ -286,7 +272,6 @@ TEST_F(SnapshotTest, TestCommitKindAnalyzeSerializeAndDeserialize) { "commitIdentifier" : 42, "commitKind" : "ANALYZE", "timeMillis" : 1700000000000, - "logOffsets" : { }, "totalRecordCount" : 0, "deltaRecordCount" : 0, "changelogRecordCount" : 0, @@ -352,7 +337,6 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { "commitIdentifier" : 100, "commitKind" : "APPEND", "timeMillis" : 1700000000000, - "logOffsets" : { }, "totalRecordCount" : 10, "deltaRecordCount" : 5, "changelogRecordCount" : 3 @@ -383,7 +367,6 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { "commitIdentifier" : 200, "commitKind" : "COMPACT", "timeMillis" : 1700000001000, - "logOffsets" : { }, "totalRecordCount" : 20, "deltaRecordCount" : 10, "changelogRecordCount" : 0 diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index eba5cf856..668844854 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -512,8 +512,8 @@ Result> SnapshotsSystemTable::ArrowSchema() const arrow::field("base_manifest_list", arrow::utf8(), /*nullable=*/false), arrow::field("delta_manifest_list", arrow::utf8(), /*nullable=*/false), arrow::field("changelog_manifest_list", arrow::utf8(), /*nullable=*/true), - arrow::field("total_record_count", arrow::int64(), /*nullable=*/true), - arrow::field("delta_record_count", arrow::int64(), /*nullable=*/true), + arrow::field("total_record_count", arrow::int64(), /*nullable=*/false), + arrow::field("delta_record_count", arrow::int64(), /*nullable=*/false), arrow::field("changelog_record_count", arrow::int64(), /*nullable=*/true), arrow::field("watermark", arrow::int64(), /*nullable=*/true), arrow::field("next_row_id", arrow::int64(), /*nullable=*/true), @@ -542,8 +542,8 @@ Result> SnapshotsSystemTable::BuildRows() const { row.SetField(6, StringValue(snapshot.BaseManifestList())); row.SetField(7, StringValue(snapshot.DeltaManifestList())); row.SetField(8, OptionalStringValue(snapshot.ChangelogManifestList())); - row.SetField(9, OptionalInt64Value(snapshot.TotalRecordCount())); - row.SetField(10, OptionalInt64Value(snapshot.DeltaRecordCount())); + row.SetField(9, snapshot.TotalRecordCount()); + row.SetField(10, snapshot.DeltaRecordCount()); row.SetField(11, OptionalInt64Value(snapshot.ChangelogRecordCount())); row.SetField(12, OptionalInt64Value(snapshot.Watermark())); row.SetField(13, OptionalInt64Value(snapshot.NextRowId())); @@ -625,7 +625,7 @@ Result> TagsSystemTable::ArrowSchema() const { arrow::field("schema_id", arrow::int64(), /*nullable=*/false), arrow::field("commit_time", arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/false), - arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("record_count", arrow::int64(), /*nullable=*/false), arrow::field("create_time", arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/true), arrow::field("time_retained", arrow::utf8(), /*nullable=*/true), @@ -650,7 +650,7 @@ Result> TagsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(VariantType commit_time, LocalTimestampMillisValue(tag.TimeMillis())); row.SetField(3, commit_time); - row.SetField(4, OptionalInt64Value(tag.TotalRecordCount())); + row.SetField(4, tag.TotalRecordCount()); row.SetField(5, OptionalTimestampMillisValue(tag_create_time)); row.SetField(6, OptionalStringValue(OptionalDoubleToString(tag.TagTimeRetained()))); rows.push_back(std::move(row)); diff --git a/src/paimon/core/tag/tag.cpp b/src/paimon/core/tag/tag.cpp index de2d64898..71f18ec0c 100644 --- a/src/paimon/core/tag/tag.cpp +++ b/src/paimon/core/tag/tag.cpp @@ -36,9 +36,7 @@ Tag::Tag(const std::optional& version, const int64_t id, const int64_t const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, const int64_t commit_identifier, const CommitKind commit_kind, const int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + const int64_t total_record_count, const int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -48,7 +46,7 @@ Tag::Tag(const std::optional& version, const int64_t id, const int64_t : Snapshot(version, id, schema_id, base_manifest_list, base_manifest_list_size, delta_manifest_list, delta_manifest_list_size, changelog_manifest_list, changelog_manifest_list_size, index_manifest, commit_user, commit_identifier, - commit_kind, time_millis, log_offsets, total_record_count, delta_record_count, + commit_kind, time_millis, total_record_count, delta_record_count, changelog_record_count, watermark, statistics, properties, next_row_id), tag_create_time_(tag_create_time), tag_time_retained_(tag_time_retained) {} @@ -74,9 +72,8 @@ Result Tag::TrimToSnapshot() const { return Snapshot(Version(), Id(), SchemaId(), BaseManifestList(), BaseManifestListSize(), DeltaManifestList(), DeltaManifestListSize(), ChangelogManifestList(), ChangelogManifestListSize(), IndexManifest(), CommitUser(), CommitIdentifier(), - GetCommitKind(), TimeMillis(), LogOffsets(), TotalRecordCount(), - DeltaRecordCount(), ChangelogRecordCount(), Watermark(), Statistics(), - Properties(), NextRowId()); + GetCommitKind(), TimeMillis(), TotalRecordCount(), DeltaRecordCount(), + ChangelogRecordCount(), Watermark(), Statistics(), Properties(), NextRowId()); } rapidjson::Value Tag::ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) { diff --git a/src/paimon/core/tag/tag.h b/src/paimon/core/tag/tag.h index a90639e5c..3001a8104 100644 --- a/src/paimon/core/tag/tag.h +++ b/src/paimon/core/tag/tag.h @@ -49,9 +49,7 @@ class Tag : public Snapshot { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, diff --git a/src/paimon/core/tag/tag_test.cpp b/src/paimon/core/tag/tag_test.cpp index 8faa6d2d6..f04c81030 100644 --- a/src/paimon/core/tag/tag_test.cpp +++ b/src/paimon/core/tag/tag_test.cpp @@ -39,7 +39,6 @@ class TagTest : public testing::Test { }; TEST_F(TagTest, TestSimple) { - const std::map log_offset = {{25, 30}}; const std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; const auto tag_create_time = std::vector({2026, 1, 2, 3, 4, 5, 6}); const Tag tag( @@ -48,7 +47,7 @@ TEST_F(TagTest, TestSimple) { /*changelog_manifest_list=*/"changelog_manifest_list", 30, /*index_manifest=*/"index_manifest", /*commit_user=*/"commit_user_01", /*commit_identifier=*/20, - /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, log_offset, + /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, /*changelog_record_count=*/45, /*watermark=*/50, /*statistics=*/"statistic_test", properties, /*next_row_id=*/0, @@ -67,9 +66,8 @@ TEST_F(TagTest, TestSimple) { ASSERT_EQ(20, tag.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Compact(), tag.GetCommitKind()); ASSERT_EQ(1234, tag.TimeMillis()); - ASSERT_EQ(log_offset, tag.LogOffsets().value()); - ASSERT_EQ(35, tag.TotalRecordCount().value()); - ASSERT_EQ(40, tag.DeltaRecordCount().value()); + ASSERT_EQ(35, tag.TotalRecordCount()); + ASSERT_EQ(40, tag.DeltaRecordCount()); ASSERT_EQ(45, tag.ChangelogRecordCount().value()); ASSERT_EQ(50, tag.Watermark().value()); ASSERT_EQ("statistic_test", tag.Statistics().value()); @@ -98,9 +96,8 @@ TEST_F(TagTest, TestFromPath) { ASSERT_EQ(9223372036854775807ll, tag.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Append(), tag.GetCommitKind()); ASSERT_EQ(1721614343270ll, tag.TimeMillis()); - ASSERT_EQ((std::map()), tag.LogOffsets().value()); - ASSERT_EQ(5, tag.TotalRecordCount().value()); - ASSERT_EQ(5, tag.DeltaRecordCount().value()); + ASSERT_EQ(5, tag.TotalRecordCount()); + ASSERT_EQ(5, tag.DeltaRecordCount()); ASSERT_EQ(0, tag.ChangelogRecordCount().value()); ASSERT_EQ(std::nullopt, tag.Watermark()); ASSERT_EQ(std::nullopt, tag.Statistics()); @@ -124,7 +121,6 @@ TEST_F(TagTest, TestJsonizable) { "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", "timeMillis" : 1711692199281, - "logOffsets" : { }, "totalRecordCount" : 3, "deltaRecordCount" : 3, "changelogRecordCount" : 0, @@ -144,7 +140,6 @@ TEST_F(TagTest, TestJsonizable) { /*commit_user=*/"0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", /*commit_identifier=*/9223372036854775807ll, /*commit_kind=*/Snapshot::CommitKind::Overwrite(), /*time_millis=*/1711692199281ll, - /*log_offsets=*/std::map(), /*total_record_count=*/3, /*delta_record_count=*/3, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt, @@ -192,10 +187,6 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -223,10 +214,6 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 572e290e6..3ade5c566 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -81,8 +81,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); std::vector datas_2; datas_2.push_back( @@ -101,8 +101,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); std::vector datas_3; datas_3.push_back( @@ -115,8 +115,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot3, helper->LatestSnapshot()); ASSERT_TRUE(snapshot3); ASSERT_EQ(3, snapshot3.value().Id()); - ASSERT_EQ(10, snapshot3.value().TotalRecordCount().value()); - ASSERT_EQ(1, snapshot3.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot3.value().TotalRecordCount()); + ASSERT_EQ(1, snapshot3.value().DeltaRecordCount()); // @note: for append-only tables in Spark, native row-level deletes aren't supported during // writing. Instead, deletions are expressed by committing a Deletion Vector (DV) file @@ -209,8 +209,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -536,8 +536,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) ASSERT_OK(helper2->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper2->LatestSnapshot()); ASSERT_EQ(6, snapshot5.value().Id()); - ASSERT_EQ(8, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(-3, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(-3, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper2->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -618,8 +618,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteBestEffortCompaction) ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -709,8 +709,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteCompactionWithExterna ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index 8d8d4feeb..685645fc5 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -342,8 +342,8 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_TRUE(snapshot_exist); ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, commit_impl->snapshot_manager_->LoadSnapshot(3)); ASSERT_EQ(30, snapshot_3.Watermark().value()); - ASSERT_EQ(-7, snapshot_3.DeltaRecordCount().value()); - ASSERT_EQ(2, snapshot_3.TotalRecordCount().value()); + ASSERT_EQ(-7, snapshot_3.DeltaRecordCount()); + ASSERT_EQ(2, snapshot_3.TotalRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot_3.GetCommitKind()); ASSERT_EQ(2, snapshot_3.CommitIdentifier()); ASSERT_OK_AND_ASSIGN(bool f1_10_bucket_0_exist, @@ -506,8 +506,8 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(snapshot_exist, commit_impl->snapshot_manager_->SnapshotExists(3)); ASSERT_TRUE(snapshot_exist); ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, commit_impl->snapshot_manager_->LoadSnapshot(3)); - ASSERT_EQ(-7, snapshot_3.DeltaRecordCount().value()); - ASSERT_EQ(2, snapshot_3.TotalRecordCount().value()); + ASSERT_EQ(-7, snapshot_3.DeltaRecordCount()); + ASSERT_EQ(2, snapshot_3.TotalRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot_3.GetCommitKind()); ASSERT_EQ(2, snapshot_3.CommitIdentifier()); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index fad7e61dd..15b377c37 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -118,8 +118,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); std::vector datas_2; datas_2.push_back( @@ -138,8 +138,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); std::vector datas_3; datas_3.push_back( @@ -152,8 +152,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot3, helper->LatestSnapshot()); ASSERT_TRUE(snapshot3); ASSERT_EQ(3, snapshot3.value().Id()); - ASSERT_EQ(10, snapshot3.value().TotalRecordCount().value()); - ASSERT_EQ(1, snapshot3.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot3.value().TotalRecordCount()); + ASSERT_EQ(1, snapshot3.value().DeltaRecordCount()); } Result>> WriteArray( @@ -2800,8 +2800,8 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableStreamWriteFullCompaction) { ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(9, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(-2, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(-2, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index a267bfca2..866d4e0ba 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -1052,7 +1052,7 @@ TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) { Timestamp tag_commit_time, DateTimeUtils::ToLocalTimestamp(Timestamp::FromEpochMillis(tag.TimeMillis()))); ASSERT_EQ(tag_commit_time_array->Value(0), tag_commit_time.GetMillisecond()); - ASSERT_EQ(tag_record_count_array->Value(0), tag.TotalRecordCount().value()); + ASSERT_EQ(tag_record_count_array->Value(0), tag.TotalRecordCount()); ASSERT_FALSE(tag_create_time_array->IsNull(0)); ASSERT_EQ(tag_create_time_array->Value(0), 1770185290000); ASSERT_EQ(tag_time_retained_array->GetString(0), "3.000000"); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index bd58eaa96..1c3af431b 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -464,8 +464,8 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -567,8 +567,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -637,8 +637,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(7, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(3, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(7, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(3, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -707,8 +707,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(8, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -772,8 +772,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(16, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(16, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 3); @@ -886,8 +886,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(6, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(6, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(6, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(6, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -949,8 +949,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(10, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -1087,8 +1087,8 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); // round 1 read arrow::FieldVector fields_with_row_kind = fields; @@ -1217,8 +1217,8 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); // round 2 read ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); @@ -1362,8 +1362,8 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_1, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -1506,8 +1506,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); // round1 read arrow::FieldVector fields_with_row_kind = fields; @@ -1610,8 +1610,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); // round2 read ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); @@ -1723,8 +1723,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1797,8 +1797,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -1855,8 +1855,8 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); // read arrow::FieldVector fields_with_row_kind = fields; @@ -1920,8 +1920,8 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); // read arrow::FieldVector fields_with_row_kind = fields; @@ -2806,8 +2806,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -2875,8 +2875,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(7, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(3, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(7, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(3, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); std::string expected_data_2 = @@ -3483,8 +3483,8 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -3854,7 +3854,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { /*key_stats=*/SimpleStats::EmptyStats(), BinaryRowGenerator::GenerateStats({std::string("str_0"), 1}, {std::string("str_3"), 2}, std::vector({0, 2}), pool_.get()), - /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, + /*min_sequence_number=*/4, /*max_sequence_number=*/7, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), @@ -3865,7 +3865,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { "data-xxx.blob", /*file_size=*/764, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), - /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, + /*min_sequence_number=*/4, /*max_sequence_number=*/6, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), @@ -3875,7 +3875,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), - /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, + /*min_sequence_number=*/7, /*max_sequence_number=*/7, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), @@ -3893,8 +3893,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); ASSERT_TRUE(snapshot); ASSERT_EQ(1, snapshot.value().Id()); - ASSERT_EQ(8, snapshot.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot.value().DeltaRecordCount()); ASSERT_EQ(4, snapshot.value().NextRowId().value()); // check data file meta after commit @@ -3965,8 +3965,8 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(2, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(2, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(2, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(2, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -4687,7 +4687,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { /*key_stats=*/SimpleStats::EmptyStats(), BinaryRowGenerator::GenerateStats({std::string("str_0"), 1}, {std::string("str_2"), 2}, std::vector({0, 1}), pool_.get()), - /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, + /*min_sequence_number=*/6, /*max_sequence_number=*/8, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), @@ -4700,7 +4700,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { "data-xxx.blob", /*file_size=*/0, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), - /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, + /*min_sequence_number=*/6, /*max_sequence_number=*/8, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), @@ -4712,7 +4712,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { "data-xxx.blob", /*file_size=*/0, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), - /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, + /*min_sequence_number=*/6, /*max_sequence_number=*/8, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), @@ -4728,8 +4728,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { ASSERT_TRUE(snapshot); ASSERT_EQ(1, snapshot.value().Id()); // 3 rows * 3 files (1 main + 1 blob1 + 1 blob2) = 9 total records - ASSERT_EQ(9, snapshot.value().TotalRecordCount().value()); - ASSERT_EQ(9, snapshot.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot.value().TotalRecordCount()); + ASSERT_EQ(9, snapshot.value().DeltaRecordCount()); ASSERT_EQ(3, snapshot.value().NextRowId().value()); // Check data file meta after commit