From a93d5852dced0b1a5982b4b5abeb5c30ffbf272c Mon Sep 17 00:00:00 2001 From: lc285652 Date: Wed, 8 Jul 2026 15:50:36 +0800 Subject: [PATCH 01/10] build: slim down libzvec_c_api dynamic library (27.15MB -> 17.05MB) Reduce the FAT C API dylib size by ~37% through a 5-layer strategy with zero functional loss; core search code stays at -O3 for performance: - Compiler dead-code sectioning: -ffunction-sections -fdata-sections - Linker pruning: macOS -dead_strip + exported symbol list; Linux --gc-sections + version script, keeping only zvec_* C API symbols - Third-party feature trimming: Arrow disables Dataset & trims Compute kernels; RocksDB disables trace/iostats/perf contexts - Protobuf lite: proto uses LITE_RUNTIME, link libprotobuf-lite - Size-optimized third-party builds: -Os for RocksDB/glog/antlr4, MinSizeRel for Arrow --- CMakeLists.txt | 18 + cmake/bazel.cmake | 2 +- src/binding/c/CMakeLists.txt | 15 + src/binding/c/exported_symbols.lds | 357 ++++++++++++++++++ src/binding/c/exported_symbols.txt | 354 +++++++++++++++++ src/db/CMakeLists.txt | 3 +- src/db/index/CMakeLists.txt | 1 - src/db/index/segment/segment.cc | 12 +- .../index/storage/bufferpool_forward_store.cc | 2 +- src/db/index/storage/memory_forward_store.cc | 2 +- src/db/index/storage/mmap_forward_store.cc | 4 +- src/db/index/storage/mmap_forward_store.h | 1 - src/db/index/storage/store_helper.h | 79 ++-- src/db/proto/zvec.proto | 1 + tests/db/index/CMakeLists.txt | 1 - thirdparty/CMakeLists.txt | 7 + thirdparty/arrow/CMakeLists.txt | 43 +-- thirdparty/arrow/arrow.slim_compute.patch | 130 +++++++ thirdparty/rocksdb/CMakeLists.txt | 3 + 19 files changed, 963 insertions(+), 72 deletions(-) create mode 100644 src/binding/c/exported_symbols.lds create mode 100644 src/binding/c/exported_symbols.txt create mode 100644 thirdparty/arrow/arrow.slim_compute.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 956ee5599..b5442d926 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,9 @@ if(MSVC) else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") + # Fine-grained dead code elimination: place each function/data in its own section + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections -fdata-sections") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT IOS) @@ -49,6 +52,21 @@ if(NOT DEFINED PROJECT_ROOT_DIR OR NOT PROJECT_ROOT_DIR) set(PROJECT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "Root directory of the project" FORCE) endif() +# LTO (Link-Time Optimization) — helps on Linux with --gc-sections but may +# increase size on macOS due to aggressive inlining. Default OFF; enable +# explicitly with -DZVEC_ENABLE_LTO=ON when targeting Linux. +option(ZVEC_ENABLE_LTO "Enable Link-Time Optimization for Release builds" OFF) +if(ZVEC_ENABLE_LTO AND CMAKE_BUILD_TYPE STREQUAL "Release") + include(CheckIPOSupported) + check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_OUTPUT) + if(IPO_SUPPORTED) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) + message(STATUS "LTO enabled for Release build") + else() + message(STATUS "LTO not supported: ${IPO_OUTPUT}") + endif() +endif() + message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}") include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index 01d950b79..c86c01337 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -1502,7 +1502,7 @@ function(_find_protobuf _VERSION) "${protoc_INCLUDE_DIR}" CACHE STRING "Protobuf includes" ) set( - CC_PROTOBUF_LIBS_${_VERSION} libprotobuf CACHE STRING "Protobuf libraries" + CC_PROTOBUF_LIBS_${_VERSION} libprotobuf-lite CACHE STRING "Protobuf libraries" ) endfunction() diff --git a/src/binding/c/CMakeLists.txt b/src/binding/c/CMakeLists.txt index a23a64373..bf605f73f 100644 --- a/src/binding/c/CMakeLists.txt +++ b/src/binding/c/CMakeLists.txt @@ -222,6 +222,21 @@ target_compile_options(zvec_c_api PRIVATE $<$:-Wall -Wextra -Wpedantic> ) +# Linker-level dead code elimination with explicit export list. +# Only symbols in the export list are kept as global; the linker can then +# aggressively strip all code unreachable from those roots. +if(APPLE) + target_link_options(zvec_c_api PRIVATE + -Wl,-dead_strip + -Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols.txt + ) +elseif(UNIX) + target_link_options(zvec_c_api PRIVATE + -Wl,--gc-sections + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols.lds + ) +endif() + # Strip symbols in release builds to reduce library size if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") if(UNIX AND NOT APPLE) diff --git a/src/binding/c/exported_symbols.lds b/src/binding/c/exported_symbols.lds new file mode 100644 index 000000000..db41de1e0 --- /dev/null +++ b/src/binding/c/exported_symbols.lds @@ -0,0 +1,357 @@ +{ global: + zvec_bin_create; + zvec_byte_array_create; + zvec_byte_array_destroy; + zvec_check_version; + zvec_clear_error; + zvec_collection_add_column; + zvec_collection_alter_column; + zvec_collection_close; + zvec_collection_create_and_open; + zvec_collection_create_index; + zvec_collection_delete; + zvec_collection_delete_by_filter; + zvec_collection_delete_with_results; + zvec_collection_destroy; + zvec_collection_drop_column; + zvec_collection_drop_index; + zvec_collection_fetch; + zvec_collection_flush; + zvec_collection_get_options; + zvec_collection_get_schema; + zvec_collection_get_stats; + zvec_collection_insert; + zvec_collection_insert_with_results; + zvec_collection_multi_query; + zvec_collection_open; + zvec_collection_optimize; + zvec_collection_options_create; + zvec_collection_options_destroy; + zvec_collection_options_get_enable_mmap; + zvec_collection_options_get_max_buffer_size; + zvec_collection_options_get_read_only; + zvec_collection_options_set_enable_mmap; + zvec_collection_options_set_max_buffer_size; + zvec_collection_options_set_read_only; + zvec_collection_query; + zvec_collection_schema_add_field; + zvec_collection_schema_add_index; + zvec_collection_schema_alter_field; + zvec_collection_schema_create; + zvec_collection_schema_destroy; + zvec_collection_schema_drop_field; + zvec_collection_schema_drop_index; + zvec_collection_schema_get_all_field_names; + zvec_collection_schema_get_field; + zvec_collection_schema_get_forward_field; + zvec_collection_schema_get_forward_field_names; + zvec_collection_schema_get_forward_field_names_with_index; + zvec_collection_schema_get_forward_fields; + zvec_collection_schema_get_forward_fields_with_index; + zvec_collection_schema_get_max_doc_count_per_segment; + zvec_collection_schema_get_name; + zvec_collection_schema_get_vector_field; + zvec_collection_schema_get_vector_fields; + zvec_collection_schema_has_field; + zvec_collection_schema_has_index; + zvec_collection_schema_set_max_doc_count_per_segment; + zvec_collection_schema_set_name; + zvec_collection_schema_validate; + zvec_collection_stats_destroy; + zvec_collection_stats_get_doc_count; + zvec_collection_stats_get_index_completeness; + zvec_collection_stats_get_index_count; + zvec_collection_stats_get_index_name; + zvec_collection_update; + zvec_collection_update_with_results; + zvec_collection_upsert; + zvec_collection_upsert_with_results; + zvec_config_data_create; + zvec_config_data_destroy; + zvec_config_data_get_brute_force_by_keys_ratio; + zvec_config_data_get_fts_brute_force_by_keys_ratio; + zvec_config_data_get_invert_to_forward_scan_ratio; + zvec_config_data_get_jieba_dict_dir; + zvec_config_data_get_log_type; + zvec_config_data_get_memory_limit; + zvec_config_data_get_optimize_thread_count; + zvec_config_data_get_query_thread_count; + zvec_config_data_set_brute_force_by_keys_ratio; + zvec_config_data_set_fts_brute_force_by_keys_ratio; + zvec_config_data_set_invert_to_forward_scan_ratio; + zvec_config_data_set_jieba_dict_dir; + zvec_config_data_set_log_config; + zvec_config_data_set_memory_limit; + zvec_config_data_set_optimize_thread_count; + zvec_config_data_set_query_thread_count; + zvec_config_log_create_console; + zvec_config_log_create_file; + zvec_config_log_destroy; + zvec_config_log_get_basename; + zvec_config_log_get_dir; + zvec_config_log_get_file_size; + zvec_config_log_get_level; + zvec_config_log_get_overdue_days; + zvec_config_log_is_file_type; + zvec_config_log_set_basename; + zvec_config_log_set_dir; + zvec_config_log_set_file_size; + zvec_config_log_set_level; + zvec_config_log_set_overdue_days; + zvec_data_type_to_string; + zvec_doc_add_field_by_struct; + zvec_doc_add_field_by_value; + zvec_doc_clear; + zvec_doc_create; + zvec_doc_deserialize; + zvec_doc_destroy; + zvec_doc_get_doc_id; + zvec_doc_get_field_count; + zvec_doc_get_field_names; + zvec_doc_get_field_value_basic; + zvec_doc_get_field_value_copy; + zvec_doc_get_field_value_pointer; + zvec_doc_get_operator; + zvec_doc_get_pk_copy; + zvec_doc_get_pk_pointer; + zvec_doc_get_score; + zvec_doc_has_field; + zvec_doc_has_field_value; + zvec_doc_is_empty; + zvec_doc_is_field_null; + zvec_doc_memory_usage; + zvec_doc_merge; + zvec_doc_remove_field; + zvec_doc_serialize; + zvec_doc_set_doc_id; + zvec_doc_set_field_null; + zvec_doc_set_operator; + zvec_doc_set_pk; + zvec_doc_set_score; + zvec_doc_to_detail_string; + zvec_docs_free; + zvec_error_code_to_string; + zvec_field_schema_create; + zvec_field_schema_destroy; + zvec_field_schema_get_data_type; + zvec_field_schema_get_dimension; + zvec_field_schema_get_element_data_size; + zvec_field_schema_get_element_data_type; + zvec_field_schema_get_index_params; + zvec_field_schema_get_index_type; + zvec_field_schema_get_name; + zvec_field_schema_has_index; + zvec_field_schema_has_invert_index; + zvec_field_schema_is_array_type; + zvec_field_schema_is_dense_vector; + zvec_field_schema_is_nullable; + zvec_field_schema_is_sparse_vector; + zvec_field_schema_is_vector_field; + zvec_field_schema_set_data_type; + zvec_field_schema_set_dimension; + zvec_field_schema_set_index_params; + zvec_field_schema_set_name; + zvec_field_schema_set_nullable; + zvec_field_schema_validate; + zvec_float_array_create; + zvec_float_array_destroy; + zvec_free; + zvec_free_field_schema; + zvec_free_str_array; + zvec_free_string; + zvec_free_uint8_array; + zvec_fts_create; + zvec_fts_destroy; + zvec_fts_get_match_string; + zvec_fts_get_query_string; + zvec_fts_set_match_string; + zvec_fts_set_query_string; + zvec_get_default_jieba_dict_dir; + zvec_get_io_backend_description; + zvec_get_io_backend_type; + zvec_get_io_backend_type_name; + zvec_get_last_error; + zvec_get_last_error_details; + zvec_get_version; + zvec_get_version_major; + zvec_get_version_minor; + zvec_get_version_patch; + zvec_group_by_vector_query_create; + zvec_group_by_vector_query_destroy; + zvec_group_by_vector_query_get_field_name; + zvec_group_by_vector_query_get_filter; + zvec_group_by_vector_query_get_group_by_field_name; + zvec_group_by_vector_query_get_group_count; + zvec_group_by_vector_query_get_include_vector; + zvec_group_by_vector_query_get_output_fields; + zvec_group_by_vector_query_get_topk_per_group; + zvec_group_by_vector_query_set_diskann_params; + zvec_group_by_vector_query_set_field_name; + zvec_group_by_vector_query_set_filter; + zvec_group_by_vector_query_set_flat_params; + zvec_group_by_vector_query_set_group_by_field_name; + zvec_group_by_vector_query_set_group_count; + zvec_group_by_vector_query_set_hnsw_params; + zvec_group_by_vector_query_set_include_vector; + zvec_group_by_vector_query_set_ivf_params; + zvec_group_by_vector_query_set_output_fields; + zvec_group_by_vector_query_set_query_params; + zvec_group_by_vector_query_set_query_vector; + zvec_group_by_vector_query_set_topk_per_group; + zvec_group_by_vector_query_set_vamana_params; + zvec_index_params_create; + zvec_index_params_destroy; + zvec_index_params_get_diskann_list_size; + zvec_index_params_get_diskann_max_degree; + zvec_index_params_get_diskann_pq_chunk_num; + zvec_index_params_get_fts_params; + zvec_index_params_get_hnsw_ef_construction; + zvec_index_params_get_hnsw_m; + zvec_index_params_get_invert_params; + zvec_index_params_get_ivf_params; + zvec_index_params_get_metric_type; + zvec_index_params_get_quantize_type; + zvec_index_params_get_quantizer_enable_rotate; + zvec_index_params_get_type; + zvec_index_params_get_vamana_params; + zvec_index_params_set_diskann_params; + zvec_index_params_set_fts_params; + zvec_index_params_set_hnsw_params; + zvec_index_params_set_invert_params; + zvec_index_params_set_ivf_params; + zvec_index_params_set_metric_type; + zvec_index_params_set_quantize_type; + zvec_index_params_set_quantizer_enable_rotate; + zvec_index_params_set_vamana_params; + zvec_index_type_to_string; + zvec_initialize; + zvec_int64_array_create; + zvec_int64_array_destroy; + zvec_is_initialized; + zvec_malloc; + zvec_metric_type_to_string; + zvec_multi_query_add_sub_query; + zvec_multi_query_create; + zvec_multi_query_destroy; + zvec_multi_query_get_filter; + zvec_multi_query_get_include_vector; + zvec_multi_query_get_output_fields; + zvec_multi_query_get_sub_query_count; + zvec_multi_query_get_topk; + zvec_multi_query_set_filter; + zvec_multi_query_set_include_vector; + zvec_multi_query_set_output_fields; + zvec_multi_query_set_rerank_rrf; + zvec_multi_query_set_rerank_weighted; + zvec_multi_query_set_topk; + zvec_query_params_diskann_create; + zvec_query_params_diskann_destroy; + zvec_query_params_diskann_get_is_linear; + zvec_query_params_diskann_get_is_using_refiner; + zvec_query_params_diskann_get_list_size; + zvec_query_params_diskann_get_radius; + zvec_query_params_diskann_set_is_linear; + zvec_query_params_diskann_set_is_using_refiner; + zvec_query_params_diskann_set_list_size; + zvec_query_params_diskann_set_radius; + zvec_query_params_flat_create; + zvec_query_params_flat_destroy; + zvec_query_params_flat_get_is_linear; + zvec_query_params_flat_get_is_using_refiner; + zvec_query_params_flat_get_radius; + zvec_query_params_flat_get_scale_factor; + zvec_query_params_flat_set_is_linear; + zvec_query_params_flat_set_is_using_refiner; + zvec_query_params_flat_set_radius; + zvec_query_params_flat_set_scale_factor; + zvec_query_params_fts_create; + zvec_query_params_fts_destroy; + zvec_query_params_fts_get_default_operator; + zvec_query_params_fts_set_default_operator; + zvec_query_params_hnsw_create; + zvec_query_params_hnsw_destroy; + zvec_query_params_hnsw_get_ef; + zvec_query_params_hnsw_get_is_linear; + zvec_query_params_hnsw_get_is_using_refiner; + zvec_query_params_hnsw_get_radius; + zvec_query_params_hnsw_set_ef; + zvec_query_params_hnsw_set_is_linear; + zvec_query_params_hnsw_set_is_using_refiner; + zvec_query_params_hnsw_set_radius; + zvec_query_params_ivf_create; + zvec_query_params_ivf_destroy; + zvec_query_params_ivf_get_is_linear; + zvec_query_params_ivf_get_is_using_refiner; + zvec_query_params_ivf_get_nprobe; + zvec_query_params_ivf_get_radius; + zvec_query_params_ivf_get_scale_factor; + zvec_query_params_ivf_set_is_linear; + zvec_query_params_ivf_set_is_using_refiner; + zvec_query_params_ivf_set_nprobe; + zvec_query_params_ivf_set_radius; + zvec_query_params_ivf_set_scale_factor; + zvec_query_params_vamana_create; + zvec_query_params_vamana_destroy; + zvec_query_params_vamana_get_ef_search; + zvec_query_params_vamana_get_is_linear; + zvec_query_params_vamana_get_is_using_refiner; + zvec_query_params_vamana_get_radius; + zvec_query_params_vamana_set_ef_search; + zvec_query_params_vamana_set_is_linear; + zvec_query_params_vamana_set_is_using_refiner; + zvec_query_params_vamana_set_radius; + zvec_set_default_jieba_dict_dir; + zvec_shutdown; + zvec_string_array_add; + zvec_string_array_create; + zvec_string_array_destroy; + zvec_string_c_str; + zvec_string_compare; + zvec_string_copy; + zvec_string_create; + zvec_string_create_from_view; + zvec_string_length; + zvec_sub_query_create; + zvec_sub_query_destroy; + zvec_sub_query_get_field_name; + zvec_sub_query_get_num_candidates; + zvec_sub_query_set_diskann_params; + zvec_sub_query_set_field_name; + zvec_sub_query_set_flat_params; + zvec_sub_query_set_fts; + zvec_sub_query_set_fts_params; + zvec_sub_query_set_hnsw_params; + zvec_sub_query_set_ivf_params; + zvec_sub_query_set_num_candidates; + zvec_sub_query_set_query_vector; + zvec_sub_query_set_sparse_indices; + zvec_sub_query_set_sparse_values; + zvec_sub_query_set_sparse_vector; + zvec_sub_query_set_vamana_params; + zvec_vector_query_create; + zvec_vector_query_destroy; + zvec_vector_query_get_field_name; + zvec_vector_query_get_filter; + zvec_vector_query_get_fts; + zvec_vector_query_get_include_doc_id; + zvec_vector_query_get_include_vector; + zvec_vector_query_get_output_fields; + zvec_vector_query_get_topk; + zvec_vector_query_set_diskann_params; + zvec_vector_query_set_field_name; + zvec_vector_query_set_filter; + zvec_vector_query_set_flat_params; + zvec_vector_query_set_fts; + zvec_vector_query_set_fts_params; + zvec_vector_query_set_hnsw_params; + zvec_vector_query_set_include_doc_id; + zvec_vector_query_set_include_vector; + zvec_vector_query_set_ivf_params; + zvec_vector_query_set_output_fields; + zvec_vector_query_set_query_params; + zvec_vector_query_set_query_vector; + zvec_vector_query_set_topk; + zvec_vector_query_set_vamana_params; + zvec_write_results_free; + local: *; +}; diff --git a/src/binding/c/exported_symbols.txt b/src/binding/c/exported_symbols.txt new file mode 100644 index 000000000..dd914c0dc --- /dev/null +++ b/src/binding/c/exported_symbols.txt @@ -0,0 +1,354 @@ +_zvec_bin_create +_zvec_byte_array_create +_zvec_byte_array_destroy +_zvec_check_version +_zvec_clear_error +_zvec_collection_add_column +_zvec_collection_alter_column +_zvec_collection_close +_zvec_collection_create_and_open +_zvec_collection_create_index +_zvec_collection_delete +_zvec_collection_delete_by_filter +_zvec_collection_delete_with_results +_zvec_collection_destroy +_zvec_collection_drop_column +_zvec_collection_drop_index +_zvec_collection_fetch +_zvec_collection_flush +_zvec_collection_get_options +_zvec_collection_get_schema +_zvec_collection_get_stats +_zvec_collection_insert +_zvec_collection_insert_with_results +_zvec_collection_multi_query +_zvec_collection_open +_zvec_collection_optimize +_zvec_collection_options_create +_zvec_collection_options_destroy +_zvec_collection_options_get_enable_mmap +_zvec_collection_options_get_max_buffer_size +_zvec_collection_options_get_read_only +_zvec_collection_options_set_enable_mmap +_zvec_collection_options_set_max_buffer_size +_zvec_collection_options_set_read_only +_zvec_collection_query +_zvec_collection_schema_add_field +_zvec_collection_schema_add_index +_zvec_collection_schema_alter_field +_zvec_collection_schema_create +_zvec_collection_schema_destroy +_zvec_collection_schema_drop_field +_zvec_collection_schema_drop_index +_zvec_collection_schema_get_all_field_names +_zvec_collection_schema_get_field +_zvec_collection_schema_get_forward_field +_zvec_collection_schema_get_forward_field_names +_zvec_collection_schema_get_forward_field_names_with_index +_zvec_collection_schema_get_forward_fields +_zvec_collection_schema_get_forward_fields_with_index +_zvec_collection_schema_get_max_doc_count_per_segment +_zvec_collection_schema_get_name +_zvec_collection_schema_get_vector_field +_zvec_collection_schema_get_vector_fields +_zvec_collection_schema_has_field +_zvec_collection_schema_has_index +_zvec_collection_schema_set_max_doc_count_per_segment +_zvec_collection_schema_set_name +_zvec_collection_schema_validate +_zvec_collection_stats_destroy +_zvec_collection_stats_get_doc_count +_zvec_collection_stats_get_index_completeness +_zvec_collection_stats_get_index_count +_zvec_collection_stats_get_index_name +_zvec_collection_update +_zvec_collection_update_with_results +_zvec_collection_upsert +_zvec_collection_upsert_with_results +_zvec_config_data_create +_zvec_config_data_destroy +_zvec_config_data_get_brute_force_by_keys_ratio +_zvec_config_data_get_fts_brute_force_by_keys_ratio +_zvec_config_data_get_invert_to_forward_scan_ratio +_zvec_config_data_get_jieba_dict_dir +_zvec_config_data_get_log_type +_zvec_config_data_get_memory_limit +_zvec_config_data_get_optimize_thread_count +_zvec_config_data_get_query_thread_count +_zvec_config_data_set_brute_force_by_keys_ratio +_zvec_config_data_set_fts_brute_force_by_keys_ratio +_zvec_config_data_set_invert_to_forward_scan_ratio +_zvec_config_data_set_jieba_dict_dir +_zvec_config_data_set_log_config +_zvec_config_data_set_memory_limit +_zvec_config_data_set_optimize_thread_count +_zvec_config_data_set_query_thread_count +_zvec_config_log_create_console +_zvec_config_log_create_file +_zvec_config_log_destroy +_zvec_config_log_get_basename +_zvec_config_log_get_dir +_zvec_config_log_get_file_size +_zvec_config_log_get_level +_zvec_config_log_get_overdue_days +_zvec_config_log_is_file_type +_zvec_config_log_set_basename +_zvec_config_log_set_dir +_zvec_config_log_set_file_size +_zvec_config_log_set_level +_zvec_config_log_set_overdue_days +_zvec_data_type_to_string +_zvec_doc_add_field_by_struct +_zvec_doc_add_field_by_value +_zvec_doc_clear +_zvec_doc_create +_zvec_doc_deserialize +_zvec_doc_destroy +_zvec_doc_get_doc_id +_zvec_doc_get_field_count +_zvec_doc_get_field_names +_zvec_doc_get_field_value_basic +_zvec_doc_get_field_value_copy +_zvec_doc_get_field_value_pointer +_zvec_doc_get_operator +_zvec_doc_get_pk_copy +_zvec_doc_get_pk_pointer +_zvec_doc_get_score +_zvec_doc_has_field +_zvec_doc_has_field_value +_zvec_doc_is_empty +_zvec_doc_is_field_null +_zvec_doc_memory_usage +_zvec_doc_merge +_zvec_doc_remove_field +_zvec_doc_serialize +_zvec_doc_set_doc_id +_zvec_doc_set_field_null +_zvec_doc_set_operator +_zvec_doc_set_pk +_zvec_doc_set_score +_zvec_doc_to_detail_string +_zvec_docs_free +_zvec_error_code_to_string +_zvec_field_schema_create +_zvec_field_schema_destroy +_zvec_field_schema_get_data_type +_zvec_field_schema_get_dimension +_zvec_field_schema_get_element_data_size +_zvec_field_schema_get_element_data_type +_zvec_field_schema_get_index_params +_zvec_field_schema_get_index_type +_zvec_field_schema_get_name +_zvec_field_schema_has_index +_zvec_field_schema_has_invert_index +_zvec_field_schema_is_array_type +_zvec_field_schema_is_dense_vector +_zvec_field_schema_is_nullable +_zvec_field_schema_is_sparse_vector +_zvec_field_schema_is_vector_field +_zvec_field_schema_set_data_type +_zvec_field_schema_set_dimension +_zvec_field_schema_set_index_params +_zvec_field_schema_set_name +_zvec_field_schema_set_nullable +_zvec_field_schema_validate +_zvec_float_array_create +_zvec_float_array_destroy +_zvec_free +_zvec_free_field_schema +_zvec_free_str_array +_zvec_free_string +_zvec_free_uint8_array +_zvec_fts_create +_zvec_fts_destroy +_zvec_fts_get_match_string +_zvec_fts_get_query_string +_zvec_fts_set_match_string +_zvec_fts_set_query_string +_zvec_get_default_jieba_dict_dir +_zvec_get_io_backend_description +_zvec_get_io_backend_type +_zvec_get_io_backend_type_name +_zvec_get_last_error +_zvec_get_last_error_details +_zvec_get_version +_zvec_get_version_major +_zvec_get_version_minor +_zvec_get_version_patch +_zvec_group_by_vector_query_create +_zvec_group_by_vector_query_destroy +_zvec_group_by_vector_query_get_field_name +_zvec_group_by_vector_query_get_filter +_zvec_group_by_vector_query_get_group_by_field_name +_zvec_group_by_vector_query_get_group_count +_zvec_group_by_vector_query_get_include_vector +_zvec_group_by_vector_query_get_output_fields +_zvec_group_by_vector_query_get_topk_per_group +_zvec_group_by_vector_query_set_diskann_params +_zvec_group_by_vector_query_set_field_name +_zvec_group_by_vector_query_set_filter +_zvec_group_by_vector_query_set_flat_params +_zvec_group_by_vector_query_set_group_by_field_name +_zvec_group_by_vector_query_set_group_count +_zvec_group_by_vector_query_set_hnsw_params +_zvec_group_by_vector_query_set_include_vector +_zvec_group_by_vector_query_set_ivf_params +_zvec_group_by_vector_query_set_output_fields +_zvec_group_by_vector_query_set_query_params +_zvec_group_by_vector_query_set_query_vector +_zvec_group_by_vector_query_set_topk_per_group +_zvec_group_by_vector_query_set_vamana_params +_zvec_index_params_create +_zvec_index_params_destroy +_zvec_index_params_get_diskann_list_size +_zvec_index_params_get_diskann_max_degree +_zvec_index_params_get_diskann_pq_chunk_num +_zvec_index_params_get_fts_params +_zvec_index_params_get_hnsw_ef_construction +_zvec_index_params_get_hnsw_m +_zvec_index_params_get_invert_params +_zvec_index_params_get_ivf_params +_zvec_index_params_get_metric_type +_zvec_index_params_get_quantize_type +_zvec_index_params_get_quantizer_enable_rotate +_zvec_index_params_get_type +_zvec_index_params_get_vamana_params +_zvec_index_params_set_diskann_params +_zvec_index_params_set_fts_params +_zvec_index_params_set_hnsw_params +_zvec_index_params_set_invert_params +_zvec_index_params_set_ivf_params +_zvec_index_params_set_metric_type +_zvec_index_params_set_quantize_type +_zvec_index_params_set_quantizer_enable_rotate +_zvec_index_params_set_vamana_params +_zvec_index_type_to_string +_zvec_initialize +_zvec_int64_array_create +_zvec_int64_array_destroy +_zvec_is_initialized +_zvec_malloc +_zvec_metric_type_to_string +_zvec_multi_query_add_sub_query +_zvec_multi_query_create +_zvec_multi_query_destroy +_zvec_multi_query_get_filter +_zvec_multi_query_get_include_vector +_zvec_multi_query_get_output_fields +_zvec_multi_query_get_sub_query_count +_zvec_multi_query_get_topk +_zvec_multi_query_set_filter +_zvec_multi_query_set_include_vector +_zvec_multi_query_set_output_fields +_zvec_multi_query_set_rerank_rrf +_zvec_multi_query_set_rerank_weighted +_zvec_multi_query_set_topk +_zvec_query_params_diskann_create +_zvec_query_params_diskann_destroy +_zvec_query_params_diskann_get_is_linear +_zvec_query_params_diskann_get_is_using_refiner +_zvec_query_params_diskann_get_list_size +_zvec_query_params_diskann_get_radius +_zvec_query_params_diskann_set_is_linear +_zvec_query_params_diskann_set_is_using_refiner +_zvec_query_params_diskann_set_list_size +_zvec_query_params_diskann_set_radius +_zvec_query_params_flat_create +_zvec_query_params_flat_destroy +_zvec_query_params_flat_get_is_linear +_zvec_query_params_flat_get_is_using_refiner +_zvec_query_params_flat_get_radius +_zvec_query_params_flat_get_scale_factor +_zvec_query_params_flat_set_is_linear +_zvec_query_params_flat_set_is_using_refiner +_zvec_query_params_flat_set_radius +_zvec_query_params_flat_set_scale_factor +_zvec_query_params_fts_create +_zvec_query_params_fts_destroy +_zvec_query_params_fts_get_default_operator +_zvec_query_params_fts_set_default_operator +_zvec_query_params_hnsw_create +_zvec_query_params_hnsw_destroy +_zvec_query_params_hnsw_get_ef +_zvec_query_params_hnsw_get_is_linear +_zvec_query_params_hnsw_get_is_using_refiner +_zvec_query_params_hnsw_get_radius +_zvec_query_params_hnsw_set_ef +_zvec_query_params_hnsw_set_is_linear +_zvec_query_params_hnsw_set_is_using_refiner +_zvec_query_params_hnsw_set_radius +_zvec_query_params_ivf_create +_zvec_query_params_ivf_destroy +_zvec_query_params_ivf_get_is_linear +_zvec_query_params_ivf_get_is_using_refiner +_zvec_query_params_ivf_get_nprobe +_zvec_query_params_ivf_get_radius +_zvec_query_params_ivf_get_scale_factor +_zvec_query_params_ivf_set_is_linear +_zvec_query_params_ivf_set_is_using_refiner +_zvec_query_params_ivf_set_nprobe +_zvec_query_params_ivf_set_radius +_zvec_query_params_ivf_set_scale_factor +_zvec_query_params_vamana_create +_zvec_query_params_vamana_destroy +_zvec_query_params_vamana_get_ef_search +_zvec_query_params_vamana_get_is_linear +_zvec_query_params_vamana_get_is_using_refiner +_zvec_query_params_vamana_get_radius +_zvec_query_params_vamana_set_ef_search +_zvec_query_params_vamana_set_is_linear +_zvec_query_params_vamana_set_is_using_refiner +_zvec_query_params_vamana_set_radius +_zvec_set_default_jieba_dict_dir +_zvec_shutdown +_zvec_string_array_add +_zvec_string_array_create +_zvec_string_array_destroy +_zvec_string_c_str +_zvec_string_compare +_zvec_string_copy +_zvec_string_create +_zvec_string_create_from_view +_zvec_string_length +_zvec_sub_query_create +_zvec_sub_query_destroy +_zvec_sub_query_get_field_name +_zvec_sub_query_get_num_candidates +_zvec_sub_query_set_diskann_params +_zvec_sub_query_set_field_name +_zvec_sub_query_set_flat_params +_zvec_sub_query_set_fts +_zvec_sub_query_set_fts_params +_zvec_sub_query_set_hnsw_params +_zvec_sub_query_set_ivf_params +_zvec_sub_query_set_num_candidates +_zvec_sub_query_set_query_vector +_zvec_sub_query_set_sparse_indices +_zvec_sub_query_set_sparse_values +_zvec_sub_query_set_sparse_vector +_zvec_sub_query_set_vamana_params +_zvec_vector_query_create +_zvec_vector_query_destroy +_zvec_vector_query_get_field_name +_zvec_vector_query_get_filter +_zvec_vector_query_get_fts +_zvec_vector_query_get_include_doc_id +_zvec_vector_query_get_include_vector +_zvec_vector_query_get_output_fields +_zvec_vector_query_get_topk +_zvec_vector_query_set_diskann_params +_zvec_vector_query_set_field_name +_zvec_vector_query_set_filter +_zvec_vector_query_set_flat_params +_zvec_vector_query_set_fts +_zvec_vector_query_set_fts_params +_zvec_vector_query_set_hnsw_params +_zvec_vector_query_set_include_doc_id +_zvec_vector_query_set_include_vector +_zvec_vector_query_set_ivf_params +_zvec_vector_query_set_output_fields +_zvec_vector_query_set_query_params +_zvec_vector_query_set_query_vector +_zvec_vector_query_set_topk +_zvec_vector_query_set_vamana_params +_zvec_write_results_free diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt index b8bb1de8d..a2fb0052c 100644 --- a/src/db/CMakeLists.txt +++ b/src/db/CMakeLists.txt @@ -50,14 +50,13 @@ cc_library( roaring rocksdb antlr4 - libprotobuf + libprotobuf-lite FastPFOR cppjieba snowball Arrow::arrow_static Arrow::parquet_static Arrow::arrow_compute - Arrow::arrow_dataset Arrow::arrow_acero utf8proc DEPS zvec_proto diff --git a/src/db/index/CMakeLists.txt b/src/db/index/CMakeLists.txt index f383bd144..eb3f42386 100644 --- a/src/db/index/CMakeLists.txt +++ b/src/db/index/CMakeLists.txt @@ -27,7 +27,6 @@ cc_library( Arrow::arrow_static Arrow::parquet_static Arrow::arrow_compute - Arrow::arrow_dataset cppjieba snowball FastPFOR diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 36a3b5e9b..251b9c3aa 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -24,8 +24,6 @@ #include #include #include -#include -#include #include #include #include @@ -3161,13 +3159,13 @@ Status SegmentImpl::add_column(FieldSchema::Ptr column_schema, } auto expr = p_result.ValueOrDie(); - auto result = ReadBlocksAsDataset(scalar_blocks, path_, segment_meta_->id(), + auto result = ReadBlocksAsTable(scalar_blocks, path_, segment_meta_->id(), !options_.enable_mmap_); if (!result.ok()) { return Status::InternalError(result.status().message()); } auto dataset = std::move(result).ValueOrDie(); - auto eval_result = EvaluateExpressionWithDataset( + auto eval_result = EvaluateExpressionOnTable( dataset, column_schema->name(), expr, expected_type); if (!eval_result.ok()) { return Status::InternalError("evaluate expression failed:", @@ -3307,15 +3305,15 @@ Status SegmentImpl::alter_column(const std::string &column_name, } } - auto result = ReadBlocksAsDataset( + auto result = ReadBlocksAsTable( filter_column_blocks, path_, segment_meta_->id(), !options_.enable_mmap_); if (!result.ok()) { return Status::InternalError(result.status().message()); } auto dataset = std::move(result).ValueOrDie(); - arrow::Expression expr = arrow::compute::field_ref(old_field_schema->name()); - auto eval_result = EvaluateExpressionWithDataset( + arrow::compute::Expression expr = arrow::compute::field_ref(old_field_schema->name()); + auto eval_result = EvaluateExpressionOnTable( dataset, new_column_name, expr, new_arrow_field->type()); if (!eval_result.ok()) { return Status::InternalError("evaluate expression failed:", diff --git a/src/db/index/storage/bufferpool_forward_store.cc b/src/db/index/storage/bufferpool_forward_store.cc index 2bd0bd66d..9b399601f 100644 --- a/src/db/index/storage/bufferpool_forward_store.cc +++ b/src/db/index/storage/bufferpool_forward_store.cc @@ -355,7 +355,7 @@ ExecBatchPtr BufferPoolForwardStore::fetch( scalars.emplace_back(std::move(scalar_result.ValueOrDie())); } - return std::make_shared(std::move(scalars), 1); + return std::make_shared(std::move(scalars), 1); } RecordBatchReaderPtr BufferPoolForwardStore::scan( diff --git a/src/db/index/storage/memory_forward_store.cc b/src/db/index/storage/memory_forward_store.cc index 268a7c581..deabf8a1f 100644 --- a/src/db/index/storage/memory_forward_store.cc +++ b/src/db/index/storage/memory_forward_store.cc @@ -463,7 +463,7 @@ ExecBatchPtr MemForwardStore::fetch(const std::vector &columns, scalars.emplace_back(std::move(scalar_result.ValueOrDie())); } - return std::make_shared(std::move(scalars), 1); + return std::make_shared(std::move(scalars), 1); } RecordBatchReaderPtr MemForwardStore::scan( diff --git a/src/db/index/storage/mmap_forward_store.cc b/src/db/index/storage/mmap_forward_store.cc index dcaff40b0..d180990d6 100644 --- a/src/db/index/storage/mmap_forward_store.cc +++ b/src/db/index/storage/mmap_forward_store.cc @@ -376,7 +376,7 @@ ExecBatchPtr MmapForwardStore::FetchParquet( scalars.emplace_back(std::move(scalar_result.ValueOrDie())); } - return std::make_shared(std::move(scalars), 1); + return std::make_shared(std::move(scalars), 1); } TablePtr MmapForwardStore::FetchIPC(const std::vector &columns, @@ -458,7 +458,7 @@ ExecBatchPtr MmapForwardStore::FetchIPC(const std::vector &columns, } } - return std::make_shared(std::move(scalars), 1); + return std::make_shared(std::move(scalars), 1); } int MmapForwardStore::FindRowGroupForRow(int64_t row) { diff --git a/src/db/index/storage/mmap_forward_store.h b/src/db/index/storage/mmap_forward_store.h index 8c018a6eb..149bf2906 100644 --- a/src/db/index/storage/mmap_forward_store.h +++ b/src/db/index/storage/mmap_forward_store.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/src/db/index/storage/store_helper.h b/src/db/index/storage/store_helper.h index abb4599e5..03662e64c 100644 --- a/src/db/index/storage/store_helper.h +++ b/src/db/index/storage/store_helper.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -758,10 +757,10 @@ inline arrow::Result> SelectArrayByIndices( return arrow::compute::Take(*arr, *indices_array); } -inline arrow::Result> -ReadBlocksAsDataset(const std::vector &scalar_blocks, - const std::string &base_path, uint32_t collection_id, - bool use_parquet) { +inline arrow::Result> +ReadBlocksAsTable(const std::vector &scalar_blocks, + const std::string &base_path, uint32_t collection_id, + bool use_parquet) { auto fs = std::make_shared(); auto pool = arrow::default_memory_pool(); @@ -858,45 +857,57 @@ ReadBlocksAsDataset(const std::vector &scalar_blocks, ARROW_ASSIGN_OR_RAISE(auto final_table, arrow::ConcatenateTables(segment_tables)); - auto dataset = std::make_shared(final_table); - return dataset; + return final_table; } inline arrow::Result> -EvaluateExpressionWithDataset( - const std::shared_ptr &dataset, - const std::string &new_column_name, const arrow::compute::Expression &expr, +EvaluateExpressionOnTable( + const std::shared_ptr &table, + const std::string &new_column_name, + const arrow::compute::Expression &expr, const std::shared_ptr &expected_type) { - auto new_scan_result = dataset->NewScan(); - if (!new_scan_result.ok()) { - return arrow::Status::Invalid("Failed to create scanner builder"); - } - auto scanner_builder = std::move(new_scan_result.ValueOrDie()); - arrow::compute::CastOptions cast_options; cast_options.to_type = expected_type; cast_options.allow_int_overflow = true; cast_options.allow_float_truncate = true; - arrow::Expression cast_expr = call("cast", {expr}, cast_options); - - auto status = scanner_builder->Project({cast_expr}, {new_column_name}); - if (!status.ok()) { - return arrow::Status::Invalid("Failed to project expression: ", - status.ToString()); - } - auto scanner_result = scanner_builder->Finish(); - if (!scanner_result.ok()) { - return arrow::Status::Invalid("Failed to finish scanner builder: ", - scanner_result.status().ToString()); + arrow::compute::Expression cast_expr = + arrow::compute::call("cast", {expr}, cast_options); + + ARROW_ASSIGN_OR_RAISE(auto bound_expr, + cast_expr.Bind(*table->schema())); + + auto batches = arrow::TableBatchReader(*table); + std::vector> result_arrays; + + std::shared_ptr batch; + while (true) { + ARROW_RETURN_NOT_OK(batches.ReadNext(&batch)); + if (!batch) break; + + arrow::compute::ExecBatch exec_batch(*batch); + ARROW_ASSIGN_OR_RAISE( + auto datum, + arrow::compute::ExecuteScalarExpression(bound_expr, exec_batch)); + + if (datum.is_array()) { + result_arrays.push_back(datum.make_array()); + } else if (datum.is_scalar()) { + ARROW_ASSIGN_OR_RAISE( + auto arr, + arrow::MakeArrayFromScalar(*datum.scalar(), batch->num_rows())); + result_arrays.push_back(arr); + } else { + return arrow::Status::Invalid( + "Expression result is neither array nor scalar"); + } } - auto scanner = std::move(scanner_result.ValueOrDie()); - auto to_table_result = scanner->ToTable(); - if (!to_table_result.ok()) { - return arrow::Status::Invalid("Failed to convert scanner to table: ", - to_table_result.status().ToString()); - } - auto result_table = std::move(to_table_result.ValueOrDie()); + auto result_chunked = + std::make_shared(result_arrays, expected_type); + auto result_field = arrow::field(new_column_name, expected_type); + auto result_schema = arrow::schema({result_field}); + auto result_table = arrow::Table::Make( + result_schema, {result_chunked}, table->num_rows()); return result_table; } diff --git a/src/db/proto/zvec.proto b/src/db/proto/zvec.proto index f2c18f5ad..4fdd5ea84 100644 --- a/src/db/proto/zvec.proto +++ b/src/db/proto/zvec.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package zvec.proto; option cc_enable_arenas = true; +option optimize_for = LITE_RUNTIME; // The Go package name, refers to // https://developers.google.com/protocol-buffers/docs/reference/go-generated#package diff --git a/tests/db/index/CMakeLists.txt b/tests/db/index/CMakeLists.txt index 03da6ffd5..1bd92c8f6 100644 --- a/tests/db/index/CMakeLists.txt +++ b/tests/db/index/CMakeLists.txt @@ -46,7 +46,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) core_knn_hnsw core_knn_hnsw_sparse sparsehash core_knn_flat core_knn_flat_sparse core_knn_ivf core_knn_hnsw_rabitq core_mix_reducer - Arrow::arrow_dataset ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS} SRCS ${CC_SRCS} utils/utils.cc diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index b71904371..c93ff4b03 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -4,6 +4,13 @@ project(thirdparty) include(${PROJECT_ROOT_DIR}/cmake/utils.cmake) +# Optimize third-party libraries for size (-Os) instead of speed (-O3) +# to reduce the final binary size. zvec's own code remains at -O3. +if(NOT MSVC AND CMAKE_BUILD_TYPE STREQUAL "Release") + set(CMAKE_C_FLAGS_RELEASE "-Os -DNDEBUG") + set(CMAKE_CXX_FLAGS_RELEASE "-Os -DNDEBUG") +endif() + set(CMAKE_MACOSX_RPATH ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(EXTERNAL_BINARY_DIR ${CMAKE_BINARY_DIR}/external) diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index d6d926db7..42e084e65 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -13,6 +13,9 @@ if(MSVC) set(ARROW_WIN_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.windows.patch) apply_patch_once("arrow_windows_crt_fix" "${ARROW_SRC_DIR}" "${ARROW_WIN_PATCH}") endif() +# Trim unused compute kernels to reduce binary size (~14MB savings) +set(ARROW_SLIM_COMPUTE_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.slim_compute.patch) +apply_patch_once("arrow_slim_compute" "${ARROW_SRC_DIR}" "${ARROW_SLIM_COMPUTE_PATCH}") include(ExternalProject) include(ProcessorCount) @@ -23,6 +26,17 @@ set(ARROW_CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}") if(NOT ARROW_CMAKE_BUILD_TYPE) set(ARROW_CMAKE_BUILD_TYPE Release) endif() +# Use MinSizeRel (-Os) for Arrow to reduce binary size while keeping +# zvec's own code at -O3 for performance. +if(ARROW_CMAKE_BUILD_TYPE STREQUAL "Release") + set(ARROW_CMAKE_BUILD_TYPE MinSizeRel) +endif() + +# Pass -ffunction-sections -fdata-sections to Arrow so the linker can +# perform fine-grained dead code elimination on Arrow objects. +if(NOT MSVC) + set(_ARROW_SIZE_FLAGS "-DCMAKE_C_FLAGS=-ffunction-sections -fdata-sections" "-DCMAKE_CXX_FLAGS=-ffunction-sections -fdata-sections") +endif() if(MSVC) set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/parquet_static.lib) @@ -30,14 +44,12 @@ if(MSVC) set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/arrow_compute_static.lib) set(LIB_ACERO ${EXTERNAL_LIB_DIR}/arrow_acero_static.lib) set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/arrow_bundled_dependencies.lib) - set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/arrow_dataset_static.lib) else() set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/libparquet.a) set(LIB_ARROW ${EXTERNAL_LIB_DIR}/libarrow.a) set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/libarrow_compute.a) set(LIB_ACERO ${EXTERNAL_LIB_DIR}/libarrow_acero.a) set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/libarrow_bundled_dependencies.a) - set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/libarrow_dataset.a) endif() set(CONFIGURE_ENV_LIST "") @@ -79,10 +91,10 @@ if(ANDROID) SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DANDROID_ABI=${ANDROID_ABI} -DANDROID_NATIVE_API_LEVEL=${ANDROID_NATIVE_API_LEVEL} -DARROW_WITH_MUSL=OFF "/cpp" + CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DANDROID_ABI=${ANDROID_ABI} -DANDROID_NATIVE_API_LEVEL=${ANDROID_NATIVE_API_LEVEL} -DARROW_WITH_MUSL=OFF "/cpp" BUILD_COMMAND "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC} INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local - BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} + BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON @@ -106,10 +118,10 @@ elseif(IOS) SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND env ${_arrow_ios_env} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} -DARROW_CPU_FLAG=${IOS_ARROW_CPU_FLAG} "/cpp" + CONFIGURE_COMMAND env ${_arrow_ios_env} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} -DARROW_CPU_FLAG=${IOS_ARROW_CPU_FLAG} "/cpp" BUILD_COMMAND env "IPHONEOS_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}" "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC} INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local - BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} + BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON @@ -134,10 +146,10 @@ elseif (MSVC) SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND "${CMAKE_COMMAND}" -E env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${ARROW_EXTRA_CMAKE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=$ -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" + CONFIGURE_COMMAND "${CMAKE_COMMAND}" -E env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${ARROW_EXTRA_CMAKE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=$ -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" BUILD_COMMAND "${CMAKE_COMMAND}" --build . -j ${NPROC} --config $ INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local --config $ - BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} + BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON @@ -149,10 +161,10 @@ else () SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" + CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} ${_ARROW_SIZE_FLAGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" BUILD_COMMAND "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC} INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local - BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} + BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON @@ -222,14 +234,3 @@ set_target_properties( INTERFACE_COMPILE_DEFINITIONS "ARROW_ACERO_STATIC" ) add_dependencies(Arrow::arrow_acero ARROW.BUILD) - -add_library(Arrow::arrow_dataset UNKNOWN IMPORTED GLOBAL) -set_target_properties( - Arrow::arrow_dataset PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} - INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} - IMPORTED_LOCATION "${LIB_ARROW_DATASET}" - INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static;Arrow::arrow_compute;Arrow::arrow_acero" - INTERFACE_COMPILE_DEFINITIONS "ARROW_DS_STATIC" -) -add_dependencies(Arrow::arrow_dataset ARROW.BUILD) diff --git a/thirdparty/arrow/arrow.slim_compute.patch b/thirdparty/arrow/arrow.slim_compute.patch new file mode 100644 index 000000000..02a572ce6 --- /dev/null +++ b/thirdparty/arrow/arrow.slim_compute.patch @@ -0,0 +1,130 @@ +diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt +index b33acae5b9..c7c394e96a 100644 +--- a/cpp/src/arrow/CMakeLists.txt ++++ b/cpp/src/arrow/CMakeLists.txt +@@ -747,45 +747,32 @@ if(ARROW_COMPUTE) + string(APPEND ARROW_COMPUTE_PC_CFLAGS "${ARROW_COMPUTE_PC_CFLAGS_PRIVATE}") + set(ARROW_COMPUTE_PC_CFLAGS_PRIVATE "") + endif() +- # Include the remaining kernels ++ # Include only kernels required by zvec: ++ # - scalar_arithmetic: add/subtract/multiply/divide for add_column expressions ++ # - scalar_boolean: AND/OR/NOT for filter expressions ++ # - scalar_compare: comparison operators for filter expressions ++ # - scalar_if_else: conditional expression support ++ # - scalar_set_lookup: is_in / index_in for filter ++ # - scalar_validity: is_null / is_valid ++ # - vector_swizzle: Take (index-based row selection) ++ # - util_internal: shared helpers for kernels above ++ # - ree_util_internal: required by vector_selection ++ # Removed (~14MB of .o): aggregate_*, hash_aggregate_*, scalar_round, ++ # scalar_string_*, scalar_temporal_*, scalar_nested, scalar_random, ++ # vector_sort, vector_select_k, vector_array_sort, vector_cumulative_ops, ++ # vector_nested, vector_pairwise, vector_rank, vector_replace, ++ # vector_run_end_encode, vector_statistics, pivot_internal + list(APPEND + ARROW_COMPUTE_LIB_SRCS + compute/initialize.cc +- compute/kernels/aggregate_basic.cc +- compute/kernels/aggregate_mode.cc +- compute/kernels/aggregate_pivot.cc +- compute/kernels/aggregate_quantile.cc +- compute/kernels/aggregate_tdigest.cc +- compute/kernels/aggregate_var_std.cc +- compute/kernels/hash_aggregate.cc +- compute/kernels/hash_aggregate_numeric.cc +- compute/kernels/hash_aggregate_pivot.cc +- compute/kernels/pivot_internal.cc + compute/kernels/ree_util_internal.cc + compute/kernels/scalar_arithmetic.cc + compute/kernels/scalar_boolean.cc + compute/kernels/scalar_compare.cc + compute/kernels/scalar_if_else.cc +- compute/kernels/scalar_nested.cc +- compute/kernels/scalar_random.cc +- compute/kernels/scalar_round.cc + compute/kernels/scalar_set_lookup.cc +- compute/kernels/scalar_string_ascii.cc +- compute/kernels/scalar_string_utf8.cc +- compute/kernels/scalar_temporal_binary.cc +- compute/kernels/scalar_temporal_unary.cc + compute/kernels/scalar_validity.cc + compute/kernels/util_internal.cc +- compute/kernels/vector_array_sort.cc +- compute/kernels/vector_cumulative_ops.cc +- compute/kernels/vector_nested.cc +- compute/kernels/vector_pairwise.cc +- compute/kernels/vector_rank.cc +- compute/kernels/vector_replace.cc +- compute/kernels/vector_run_end_encode.cc +- compute/kernels/vector_select_k.cc +- compute/kernels/vector_sort.cc +- compute/kernels/vector_statistics.cc + compute/kernels/vector_swizzle.cc + compute/key_hash_internal.cc + compute/key_map_internal.cc +@@ -798,9 +785,6 @@ if(ARROW_COMPUTE) + compute/util.cc + compute/util_internal.cc) + +- append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/kernels/aggregate_basic_avx2.cc) +- append_runtime_avx512_src(ARROW_COMPUTE_LIB_SRCS +- compute/kernels/aggregate_basic_avx512.cc) + append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/key_hash_internal_avx2.cc) + append_runtime_avx2_bmi2_src(ARROW_COMPUTE_LIB_SRCS compute/key_map_internal_avx2.cc) + append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/row/compare_internal_avx2.cc) +diff --git a/cpp/src/arrow/compute/initialize.cc b/cpp/src/arrow/compute/initialize.cc +index d126ac951f..abeb3f4bc6 100644 +--- a/cpp/src/arrow/compute/initialize.cc ++++ b/cpp/src/arrow/compute/initialize.cc +@@ -26,47 +26,18 @@ namespace { + Status RegisterComputeKernels() { + auto registry = GetFunctionRegistry(); + +- // Register additional kernels on libarrow_compute +- // Scalar functions ++ // Only register kernels required by zvec: ++ // Scalar functions needed for expressions and filters + internal::RegisterScalarArithmetic(registry); + internal::RegisterScalarBoolean(registry); + internal::RegisterScalarComparison(registry); + internal::RegisterScalarIfElse(registry); +- internal::RegisterScalarNested(registry); +- internal::RegisterScalarRandom(registry); // Nullary +- internal::RegisterScalarRoundArithmetic(registry); + internal::RegisterScalarSetLookup(registry); +- internal::RegisterScalarStringAscii(registry); +- internal::RegisterScalarStringUtf8(registry); +- internal::RegisterScalarTemporalBinary(registry); +- internal::RegisterScalarTemporalUnary(registry); + internal::RegisterScalarValidity(registry); + +- // Vector functions +- internal::RegisterVectorArraySort(registry); +- internal::RegisterVectorCumulativeSum(registry); +- internal::RegisterVectorNested(registry); +- internal::RegisterVectorRank(registry); +- internal::RegisterVectorReplace(registry); +- internal::RegisterVectorSelectK(registry); +- internal::RegisterVectorSort(registry); +- internal::RegisterVectorRunEndEncode(registry); +- internal::RegisterVectorRunEndDecode(registry); +- internal::RegisterVectorPairwise(registry); +- internal::RegisterVectorStatistics(registry); ++ // Vector functions needed for row selection + internal::RegisterVectorSwizzle(registry); + +- // Aggregate functions +- internal::RegisterHashAggregateBasic(registry); +- internal::RegisterHashAggregateNumeric(registry); +- internal::RegisterHashAggregatePivot(registry); +- internal::RegisterScalarAggregateBasic(registry); +- internal::RegisterScalarAggregateMode(registry); +- internal::RegisterScalarAggregatePivot(registry); +- internal::RegisterScalarAggregateQuantile(registry); +- internal::RegisterScalarAggregateTDigest(registry); +- internal::RegisterScalarAggregateVariance(registry); +- + return Status::OK(); + } + diff --git a/thirdparty/rocksdb/CMakeLists.txt b/thirdparty/rocksdb/CMakeLists.txt index 8d95e882f..110e380e5 100644 --- a/thirdparty/rocksdb/CMakeLists.txt +++ b/thirdparty/rocksdb/CMakeLists.txt @@ -42,6 +42,9 @@ set(WITH_CORE_TOOLS OFF CACHE BOOL "build with ldb and sst_dump" FORCE) set(WITH_TOOLS OFF CACHE BOOL "build with tools" FORCE) set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE) set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE) +set(WITH_TRACE_TOOLS OFF CACHE BOOL "Disable trace tools to reduce binary size" FORCE) +set(WITH_IOSTATS_CONTEXT OFF CACHE BOOL "Disable IO stats context to reduce binary size" FORCE) +set(WITH_PERF_CONTEXT OFF CACHE BOOL "Disable perf context to reduce binary size" FORCE) set(WITH_WINDOWS_UTF8_FILENAMES ON CACHE BOOL "use UTF8 as characterset for opening files, regardles of the system code page" FORCE) # TODO(windows): verify set(ROCKSDB_SKIP_THIRDPARTY ON CACHE BOOL "skip thirdparty.inc" FORCE) From 3c3b2cf1bdd57bbdec0b05b51d71a7ebddd766ed Mon Sep 17 00:00:00 2001 From: lc285652 Date: Fri, 24 Jul 2026 19:10:46 +0800 Subject: [PATCH 02/10] build: slim C++ shared lib exports and add MSVC size optimization Extend the size-reduction work to the C++ all-in-one shared library and the Windows build: - libzvec C++ SDK (macOS/Linux): restrict exported symbols to zvec's own public API (namespace zvec, incl. nested zvec::ailego/core/turbo) via -exported_symbols_list / --version-script + -dead_strip / --gc-sections. Hides ~14.5k third-party symbols; libzvec.dylib 24.41MB -> 19.05MB (-21.9%), total -34.1% vs main baseline. Public C++ headers never expose third-party types, so external C++ consumers are unaffected. - Third-party libs (MSVC): favor small code (/O1 /Ob1) over speed (/O2) to mirror the -Os / MinSizeRel size optimization used on GCC/Clang, including Arrow's Release config. --- src/CMakeLists.txt | 24 ++++++++++++++++++++++++ src/exported_symbols_cpp.lds | 10 ++++++++++ src/exported_symbols_cpp.txt | 11 +++++++++++ thirdparty/CMakeLists.txt | 13 +++++++++++-- thirdparty/arrow/CMakeLists.txt | 4 ++++ 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 src/exported_symbols_cpp.lds create mode 100644 src/exported_symbols_cpp.txt diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2520d9074..fef294639 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -146,6 +146,30 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) endif() endif() + # Restrict exported symbols to zvec's own public C++ API (namespace zvec, + # which includes nested zvec::ailego / zvec::core / zvec::turbo / + # zvec::reranker). All third-party symbols (arrow, rocksdb, parquet, + # protobuf, antlr4) are hidden, and unreferenced code is dead-stripped to + # reduce the shared library size. The public C++ headers never expose + # third-party types, so hiding them is safe for external C++ consumers. + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + if(APPLE) + set(_ZVEC_ALLIN_EXPORTS ${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols_cpp.txt) + target_link_options(${TARGET_NAME} PRIVATE + "-Wl,-exported_symbols_list,${_ZVEC_ALLIN_EXPORTS}" + "-Wl,-dead_strip") + set_property(TARGET ${TARGET_NAME} APPEND PROPERTY + LINK_DEPENDS ${_ZVEC_ALLIN_EXPORTS}) + elseif(UNIX) + set(_ZVEC_ALLIN_EXPORTS ${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols_cpp.lds) + target_link_options(${TARGET_NAME} PRIVATE + "-Wl,--version-script,${_ZVEC_ALLIN_EXPORTS}" + "-Wl,--gc-sections") + set_property(TARGET ${TARGET_NAME} APPEND PROPERTY + LINK_DEPENDS ${_ZVEC_ALLIN_EXPORTS}) + endif() + endif() + target_include_directories(${TARGET_NAME} PUBLIC $ diff --git a/src/exported_symbols_cpp.lds b/src/exported_symbols_cpp.lds new file mode 100644 index 000000000..421bc3c9e --- /dev/null +++ b/src/exported_symbols_cpp.lds @@ -0,0 +1,10 @@ +{ + global: + extern "C++" { + "zvec::*"; + zvec::*; + }; + zvec_*; + local: + *; +}; diff --git a/src/exported_symbols_cpp.txt b/src/exported_symbols_cpp.txt new file mode 100644 index 000000000..0b09f2b7b --- /dev/null +++ b/src/exported_symbols_cpp.txt @@ -0,0 +1,11 @@ +__ZN4zvec* +__ZNK4zvec* +__ZZN4zvec* +__ZZNK4zvec* +__ZTVN4zvec* +__ZTIN4zvec* +__ZTSN4zvec* +__ZGVN4zvec* +__ZGVZN4zvec* +__ZGVZNK4zvec* +_zvec_* diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index c93ff4b03..a66850c32 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -4,11 +4,20 @@ project(thirdparty) include(${PROJECT_ROOT_DIR}/cmake/utils.cmake) -# Optimize third-party libraries for size (-Os) instead of speed (-O3) -# to reduce the final binary size. zvec's own code remains at -O3. +# Optimize third-party libraries for size instead of speed to reduce the final +# binary size. zvec's own code (built under src/) keeps its default speed +# optimization since these flags are directory-scoped to thirdparty/. +# - GCC/Clang: use -Os in place of -O3 +# - MSVC: favor small code (/O1, /Ob1) in place of /O2, /Ob2, while preserving +# the runtime library and other default Release flags if(NOT MSVC AND CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_C_FLAGS_RELEASE "-Os -DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE "-Os -DNDEBUG") +elseif(MSVC) + string(REPLACE "/O2" "/O1" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") + string(REPLACE "/O2" "/O1" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + string(REPLACE "/Ob2" "/Ob1" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") + string(REPLACE "/Ob2" "/Ob1" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") endif() set(CMAKE_MACOSX_RPATH ON) diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index 42e084e65..781db2415 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -140,6 +140,10 @@ elseif (MSVC) "-DCMAKE_CXX_FLAGS=${_ARROW_CRT_FLAG}" -DARROW_USE_STATIC_CRT=${ZVEC_USE_STATIC_CRT} "-DCMAKE_MSVC_RUNTIME_LIBRARY=${_ARROW_MSVC_RUNTIME}" + # Favor small code (/O1, /Ob1) over speed (/O2) for the Release config + # to match the MinSizeRel size optimization used on other platforms. + "-DCMAKE_C_FLAGS_RELEASE=/O1 /Ob1 /DNDEBUG" + "-DCMAKE_CXX_FLAGS_RELEASE=/O1 /Ob1 /DNDEBUG" ) ExternalProject_Add( ARROW.BUILD PREFIX arrow From 8874f17e2f43a980a659aaf0f768685e7234fade Mon Sep 17 00:00:00 2001 From: lc285652 Date: Tue, 28 Jul 2026 19:47:20 +0800 Subject: [PATCH 03/10] style: apply clang-format to segment.cc and store_helper.h --- src/db/index/segment/segment.cc | 17 ++++++++-------- src/db/index/storage/store_helper.h | 30 ++++++++++++----------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 251b9c3aa..ee17fee27 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -3160,13 +3160,13 @@ Status SegmentImpl::add_column(FieldSchema::Ptr column_schema, auto expr = p_result.ValueOrDie(); auto result = ReadBlocksAsTable(scalar_blocks, path_, segment_meta_->id(), - !options_.enable_mmap_); + !options_.enable_mmap_); if (!result.ok()) { return Status::InternalError(result.status().message()); } auto dataset = std::move(result).ValueOrDie(); - auto eval_result = EvaluateExpressionOnTable( - dataset, column_schema->name(), expr, expected_type); + auto eval_result = EvaluateExpressionOnTable(dataset, column_schema->name(), + expr, expected_type); if (!eval_result.ok()) { return Status::InternalError("evaluate expression failed:", eval_result.status().message()); @@ -3305,16 +3305,17 @@ Status SegmentImpl::alter_column(const std::string &column_name, } } - auto result = ReadBlocksAsTable( - filter_column_blocks, path_, segment_meta_->id(), !options_.enable_mmap_); + auto result = ReadBlocksAsTable(filter_column_blocks, path_, + segment_meta_->id(), !options_.enable_mmap_); if (!result.ok()) { return Status::InternalError(result.status().message()); } auto dataset = std::move(result).ValueOrDie(); - arrow::compute::Expression expr = arrow::compute::field_ref(old_field_schema->name()); - auto eval_result = EvaluateExpressionOnTable( - dataset, new_column_name, expr, new_arrow_field->type()); + arrow::compute::Expression expr = + arrow::compute::field_ref(old_field_schema->name()); + auto eval_result = EvaluateExpressionOnTable(dataset, new_column_name, expr, + new_arrow_field->type()); if (!eval_result.ok()) { return Status::InternalError("evaluate expression failed:", eval_result.status().message()); diff --git a/src/db/index/storage/store_helper.h b/src/db/index/storage/store_helper.h index 03662e64c..a58d6f484 100644 --- a/src/db/index/storage/store_helper.h +++ b/src/db/index/storage/store_helper.h @@ -757,10 +757,9 @@ inline arrow::Result> SelectArrayByIndices( return arrow::compute::Take(*arr, *indices_array); } -inline arrow::Result> -ReadBlocksAsTable(const std::vector &scalar_blocks, - const std::string &base_path, uint32_t collection_id, - bool use_parquet) { +inline arrow::Result> ReadBlocksAsTable( + const std::vector &scalar_blocks, const std::string &base_path, + uint32_t collection_id, bool use_parquet) { auto fs = std::make_shared(); auto pool = arrow::default_memory_pool(); @@ -860,11 +859,9 @@ ReadBlocksAsTable(const std::vector &scalar_blocks, return final_table; } -inline arrow::Result> -EvaluateExpressionOnTable( +inline arrow::Result> EvaluateExpressionOnTable( const std::shared_ptr &table, - const std::string &new_column_name, - const arrow::compute::Expression &expr, + const std::string &new_column_name, const arrow::compute::Expression &expr, const std::shared_ptr &expected_type) { arrow::compute::CastOptions cast_options; cast_options.to_type = expected_type; @@ -873,8 +870,7 @@ EvaluateExpressionOnTable( arrow::compute::Expression cast_expr = arrow::compute::call("cast", {expr}, cast_options); - ARROW_ASSIGN_OR_RAISE(auto bound_expr, - cast_expr.Bind(*table->schema())); + ARROW_ASSIGN_OR_RAISE(auto bound_expr, cast_expr.Bind(*table->schema())); auto batches = arrow::TableBatchReader(*table); std::vector> result_arrays; @@ -885,16 +881,14 @@ EvaluateExpressionOnTable( if (!batch) break; arrow::compute::ExecBatch exec_batch(*batch); - ARROW_ASSIGN_OR_RAISE( - auto datum, - arrow::compute::ExecuteScalarExpression(bound_expr, exec_batch)); + ARROW_ASSIGN_OR_RAISE(auto datum, arrow::compute::ExecuteScalarExpression( + bound_expr, exec_batch)); if (datum.is_array()) { result_arrays.push_back(datum.make_array()); } else if (datum.is_scalar()) { - ARROW_ASSIGN_OR_RAISE( - auto arr, - arrow::MakeArrayFromScalar(*datum.scalar(), batch->num_rows())); + ARROW_ASSIGN_OR_RAISE(auto arr, arrow::MakeArrayFromScalar( + *datum.scalar(), batch->num_rows())); result_arrays.push_back(arr); } else { return arrow::Status::Invalid( @@ -906,8 +900,8 @@ EvaluateExpressionOnTable( std::make_shared(result_arrays, expected_type); auto result_field = arrow::field(new_column_name, expected_type); auto result_schema = arrow::schema({result_field}); - auto result_table = arrow::Table::Make( - result_schema, {result_chunked}, table->num_rows()); + auto result_table = + arrow::Table::Make(result_schema, {result_chunked}, table->num_rows()); return result_table; } From 479dc01d2f61ee5f36c096240c59ffb3e1b5fd03 Mon Sep 17 00:00:00 2001 From: lc285652 Date: Wed, 29 Jul 2026 15:32:21 +0800 Subject: [PATCH 04/10] fix: restore production compute kernels and drop dataset dependency from tests The slim compute patch removed kernels that production code paths rely on: - match_like (LIKE queries) from scalar_string_ascii/utf8 - sort_indices (ORDER BY) from vector_sort/vector_array_sort - make_struct / list_value_length from scalar_nested Restore these kernels in arrow.slim_compute.patch while keeping the rest (aggregates, temporal, round, random, etc.) removed. Arrow Dataset stays disabled: sql_expr_validator_test's dataset member was dead code and is removed; sql_expr_parser_test's scan test is rewritten with arrow::compute::ExecuteScalarExpression, preserving the original intent (verify a parsed expression evaluates correctly on real data) without depending on arrow::dataset. All 153 C++ tests pass locally (write_recovery_test flake passes in isolation). --- .../db/index/segment/sql_expr_parser_test.cc | 52 ++++++++----------- .../index/segment/sql_expr_validator_test.cc | 16 ------ thirdparty/arrow/arrow.slim_compute.patch | 46 ++++++++-------- 3 files changed, 45 insertions(+), 69 deletions(-) diff --git a/tests/db/index/segment/sql_expr_parser_test.cc b/tests/db/index/segment/sql_expr_parser_test.cc index b59308cc6..1d563e92b 100644 --- a/tests/db/index/segment/sql_expr_parser_test.cc +++ b/tests/db/index/segment/sql_expr_parser_test.cc @@ -16,9 +16,8 @@ #include "db/index/segment/sql_expr_parser.h" #include #include -#include -#include #include +#include #include #include #include @@ -26,7 +25,6 @@ #include "utils/utils.h" using namespace arrow; -using namespace arrow::dataset; using namespace zvec; class SqlExprParserTest : public ::testing::Test { @@ -237,42 +235,37 @@ std::shared_ptr MakeTestTable() { } -// Convert Table to Dataset (for testing) -arrow::Result> MakeTestDataset( - const std::shared_ptr &table) { - return std::make_shared(table); -} - -TEST_F(SqlExprParserTest, ParseAndScanDataSet) { - auto status = arrow::compute::Initialize(); +// Execute the parsed expression against in-memory data. This used to go +// through arrow::dataset::Scanner, but Arrow Dataset is disabled in the +// slim build; ExecuteScalarExpression preserves the same intent: verify +// that a parsed expression evaluates correctly on real data. +TEST_F(SqlExprParserTest, ParseAndExecuteExpression) { + ASSERT_OK(arrow::compute::Initialize()); auto schema = arrow::schema({arrow::field("int_col", arrow::int32()), arrow::field("double_col", arrow::float64()), arrow::field("string_col", arrow::utf8()), arrow::field("bool_col", arrow::boolean())}); - // Step 1: Create test table + // Step 1: Create test table and turn it into a single record batch auto table = MakeTestTable(); + ASSERT_OK_AND_ASSIGN(auto batch, table->CombineChunksToBatch()); - // Step 2: Convert to Dataset - auto dataset = MakeTestDataset(table).ValueOrDie(); - - // Step 3: Create scanner and project expression A + B - auto scanner_builder = dataset->NewScan().ValueOrDie(); - + // Step 2: Parse and bind the projection expression A + B auto expr = ParseToExpression("int_col + double_col", schema).ValueOrDie(); - status = scanner_builder->Project({expr}, {"sum"}); + ASSERT_OK_AND_ASSIGN(auto bound, expr.Bind(*schema)); - auto scanner = scanner_builder->Finish().ValueOrDie(); + // Step 3: Execute the expression and get results + ASSERT_OK_AND_ASSIGN(auto datum, arrow::compute::ExecuteScalarExpression( + bound, *schema, arrow::Datum(batch))); - // Step 4: Execute and get results - auto result_table = scanner->ToTable().ValueOrDie(); - ASSERT_TRUE(result_table != nullptr); - ASSERT_EQ(result_table->num_rows(), 5); + auto sum_array = datum.make_array(); + ASSERT_TRUE(sum_array != nullptr); + ASSERT_EQ(sum_array->length(), 5); - auto int_col = table->column(0); // int_col - auto double_col = table->column(1); // double_col - auto sum_col = result_table->column(0); // sum column + auto int_col = table->column(0); // int_col + auto double_col = table->column(1); // double_col + auto sum_col = std::static_pointer_cast(sum_array); for (int64_t i = 0; i < table->num_rows(); ++i) { auto int_value = @@ -281,10 +274,7 @@ TEST_F(SqlExprParserTest, ParseAndScanDataSet) { auto double_value = std::static_pointer_cast(double_col->chunk(0)) ->Value(i); - auto sum_value = - std::static_pointer_cast(sum_col->chunk(0)) - ->Value(i); - ASSERT_NEAR(int_value + double_value, sum_value, 1e-10); + ASSERT_NEAR(int_value + double_value, sum_col->Value(i), 1e-10); } } \ No newline at end of file diff --git a/tests/db/index/segment/sql_expr_validator_test.cc b/tests/db/index/segment/sql_expr_validator_test.cc index 93c80ee93..d9178d7a1 100644 --- a/tests/db/index/segment/sql_expr_validator_test.cc +++ b/tests/db/index/segment/sql_expr_validator_test.cc @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -38,24 +37,9 @@ class ExprValidatorTest : public ::testing::Test { schema_ = arrow::schema({arrow::field("int32_col", arrow::int32()), arrow::field("double_col", arrow::float64()), arrow::field("str_col", arrow::utf8())}); - - std::vector> arrays; - for (const auto &field : schema_->fields()) { - std::unique_ptr builder; - ASSERT_TRUE(arrow::MakeBuilder(arrow::default_memory_pool(), - field->type(), &builder) - .ok()); - std::shared_ptr array; - ASSERT_TRUE(builder->Finish(&array).ok()); - arrays.push_back(array); - } - - auto table = arrow::Table::Make(schema_, arrays); - dataset_ = std::make_shared(table); } std::shared_ptr schema_; - std::shared_ptr dataset_; }; TEST_F(ExprValidatorTest, SingleNumericColumn_Valid) { diff --git a/thirdparty/arrow/arrow.slim_compute.patch b/thirdparty/arrow/arrow.slim_compute.patch index 02a572ce6..841278a4d 100644 --- a/thirdparty/arrow/arrow.slim_compute.patch +++ b/thirdparty/arrow/arrow.slim_compute.patch @@ -1,8 +1,8 @@ diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt -index b33acae5b9..c7c394e96a 100644 +index b33acae5b9..6771f4a98d 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt -@@ -747,45 +747,32 @@ if(ARROW_COMPUTE) +@@ -747,45 +747,39 @@ if(ARROW_COMPUTE) string(APPEND ARROW_COMPUTE_PC_CFLAGS "${ARROW_COMPUTE_PC_CFLAGS_PRIVATE}") set(ARROW_COMPUTE_PC_CFLAGS_PRIVATE "") endif() @@ -12,16 +12,18 @@ index b33acae5b9..c7c394e96a 100644 + # - scalar_boolean: AND/OR/NOT for filter expressions + # - scalar_compare: comparison operators for filter expressions + # - scalar_if_else: conditional expression support ++ # - scalar_nested: make_struct (dataset scan), list_value_length + # - scalar_set_lookup: is_in / index_in for filter ++ # - scalar_string_ascii/utf8: match_like for LIKE queries + # - scalar_validity: is_null / is_valid ++ # - vector_array_sort / vector_sort: sort_indices for ORDER BY + # - vector_swizzle: Take (index-based row selection) + # - util_internal: shared helpers for kernels above + # - ree_util_internal: required by vector_selection -+ # Removed (~14MB of .o): aggregate_*, hash_aggregate_*, scalar_round, -+ # scalar_string_*, scalar_temporal_*, scalar_nested, scalar_random, -+ # vector_sort, vector_select_k, vector_array_sort, vector_cumulative_ops, -+ # vector_nested, vector_pairwise, vector_rank, vector_replace, -+ # vector_run_end_encode, vector_statistics, pivot_internal ++ # Removed: aggregate_*, hash_aggregate_*, pivot_internal, scalar_random, ++ # scalar_round, scalar_temporal_*, vector_cumulative_ops, vector_nested, ++ # vector_pairwise, vector_rank, vector_replace, vector_run_end_encode, ++ # vector_select_k, vector_statistics list(APPEND ARROW_COMPUTE_LIB_SRCS compute/initialize.cc @@ -40,17 +42,17 @@ index b33acae5b9..c7c394e96a 100644 compute/kernels/scalar_boolean.cc compute/kernels/scalar_compare.cc compute/kernels/scalar_if_else.cc -- compute/kernels/scalar_nested.cc + compute/kernels/scalar_nested.cc - compute/kernels/scalar_random.cc - compute/kernels/scalar_round.cc compute/kernels/scalar_set_lookup.cc -- compute/kernels/scalar_string_ascii.cc -- compute/kernels/scalar_string_utf8.cc + compute/kernels/scalar_string_ascii.cc + compute/kernels/scalar_string_utf8.cc - compute/kernels/scalar_temporal_binary.cc - compute/kernels/scalar_temporal_unary.cc compute/kernels/scalar_validity.cc compute/kernels/util_internal.cc -- compute/kernels/vector_array_sort.cc + compute/kernels/vector_array_sort.cc - compute/kernels/vector_cumulative_ops.cc - compute/kernels/vector_nested.cc - compute/kernels/vector_pairwise.cc @@ -58,12 +60,12 @@ index b33acae5b9..c7c394e96a 100644 - compute/kernels/vector_replace.cc - compute/kernels/vector_run_end_encode.cc - compute/kernels/vector_select_k.cc -- compute/kernels/vector_sort.cc + compute/kernels/vector_sort.cc - compute/kernels/vector_statistics.cc compute/kernels/vector_swizzle.cc compute/key_hash_internal.cc compute/key_map_internal.cc -@@ -798,9 +785,6 @@ if(ARROW_COMPUTE) +@@ -798,9 +792,6 @@ if(ARROW_COMPUTE) compute/util.cc compute/util_internal.cc) @@ -74,44 +76,44 @@ index b33acae5b9..c7c394e96a 100644 append_runtime_avx2_bmi2_src(ARROW_COMPUTE_LIB_SRCS compute/key_map_internal_avx2.cc) append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/row/compare_internal_avx2.cc) diff --git a/cpp/src/arrow/compute/initialize.cc b/cpp/src/arrow/compute/initialize.cc -index d126ac951f..abeb3f4bc6 100644 +index d126ac951f..b53967d03d 100644 --- a/cpp/src/arrow/compute/initialize.cc +++ b/cpp/src/arrow/compute/initialize.cc -@@ -26,47 +26,18 @@ namespace { +@@ -26,47 +26,23 @@ namespace { Status RegisterComputeKernels() { auto registry = GetFunctionRegistry(); - // Register additional kernels on libarrow_compute - // Scalar functions + // Only register kernels required by zvec: -+ // Scalar functions needed for expressions and filters ++ // Scalar functions needed for expressions, filters, LIKE and dataset scan internal::RegisterScalarArithmetic(registry); internal::RegisterScalarBoolean(registry); internal::RegisterScalarComparison(registry); internal::RegisterScalarIfElse(registry); -- internal::RegisterScalarNested(registry); + internal::RegisterScalarNested(registry); - internal::RegisterScalarRandom(registry); // Nullary - internal::RegisterScalarRoundArithmetic(registry); internal::RegisterScalarSetLookup(registry); -- internal::RegisterScalarStringAscii(registry); -- internal::RegisterScalarStringUtf8(registry); + internal::RegisterScalarStringAscii(registry); + internal::RegisterScalarStringUtf8(registry); - internal::RegisterScalarTemporalBinary(registry); - internal::RegisterScalarTemporalUnary(registry); internal::RegisterScalarValidity(registry); - // Vector functions -- internal::RegisterVectorArraySort(registry); ++ // Vector functions needed for row selection and sorting (sort_indices) + internal::RegisterVectorArraySort(registry); - internal::RegisterVectorCumulativeSum(registry); - internal::RegisterVectorNested(registry); - internal::RegisterVectorRank(registry); - internal::RegisterVectorReplace(registry); - internal::RegisterVectorSelectK(registry); -- internal::RegisterVectorSort(registry); + internal::RegisterVectorSort(registry); - internal::RegisterVectorRunEndEncode(registry); - internal::RegisterVectorRunEndDecode(registry); - internal::RegisterVectorPairwise(registry); - internal::RegisterVectorStatistics(registry); -+ // Vector functions needed for row selection internal::RegisterVectorSwizzle(registry); - // Aggregate functions From 3462df8afa5ed53bd3b0ed1522e6e3d5ead267e2 Mon Sep 17 00:00:00 2001 From: lc285652 Date: Wed, 29 Jul 2026 17:38:57 +0800 Subject: [PATCH 05/10] fix: export vtable/typeinfo symbols of zvec classes in Linux version script The version script only exported zvec::* function/data symbols. vtable, typeinfo, VTT and guard variable symbols demangle to 'vtable for zvec::...' etc., which the zvec::* pattern does not match, so they were localized by 'local: *'. Consumers linking libzvec.so then failed with 'undefined reference to vtable for zvec::core_interface::HNSWIndexParam'. Add the mangled-name patterns (_ZTVN4zvec*, _ZTIN4zvec*, _ZTSN4zvec*, _ZTTN4zvec*, _ZGVN4zvec*), mirroring what exported_symbols_cpp.txt already does on macOS. --- src/exported_symbols_cpp.lds | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/exported_symbols_cpp.lds b/src/exported_symbols_cpp.lds index 421bc3c9e..5835e0ab4 100644 --- a/src/exported_symbols_cpp.lds +++ b/src/exported_symbols_cpp.lds @@ -5,6 +5,16 @@ zvec::*; }; zvec_*; + /* vtable / typeinfo / VTT / guard variables of zvec:: classes. + Their demangled names start with "vtable for" etc., so the + zvec::* patterns above do not match them; without these the + consumers fail with "undefined reference to vtable". Mirrors + __ZTVN4zvec* etc. in exported_symbols_cpp.txt on macOS. */ + _ZTVN4zvec*; + _ZTIN4zvec*; + _ZTSN4zvec*; + _ZTTN4zvec*; + _ZGVN4zvec*; local: *; }; From 7b5b764f6aa53dfe07e05b45f582b6fdf0cc1268 Mon Sep 17 00:00:00 2001 From: lc285652 Date: Thu, 30 Jul 2026 10:47:22 +0800 Subject: [PATCH 06/10] fix: remove quoted literal pattern from version script for lld compatibility In linker version scripts, a quoted "zvec::*" entry is a literal name match (not a glob). GNU ld silently tolerates the never-matching entry, but lld (used by the Android NDK) enforces --no-undefined-version and fails with: ld.lld: error: version script assignment of 'global' to symbol 'zvec::*' failed: symbol not defined Keep only the unquoted glob pattern, which matches all demangled zvec:: symbols on both GNU ld and lld. --- src/exported_symbols_cpp.lds | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/exported_symbols_cpp.lds b/src/exported_symbols_cpp.lds index 5835e0ab4..4e6c9e934 100644 --- a/src/exported_symbols_cpp.lds +++ b/src/exported_symbols_cpp.lds @@ -1,7 +1,10 @@ { global: extern "C++" { - "zvec::*"; + /* Unquoted = glob pattern matching all demangled zvec:: symbols. + Do NOT add a quoted "zvec::*" entry: quotes mean literal match, + and lld (Android NDK) fails with --no-undefined-version when a + literal entry matches no symbol. */ zvec::*; }; zvec_*; From 9a278771899f904e3b6ab02b7aaf3bf86d235bf4 Mon Sep 17 00:00:00 2001 From: lc285652 Date: Thu, 30 Jul 2026 16:29:55 +0800 Subject: [PATCH 07/10] feat(db): add protobuf-free manifest codec with byte-level cross-checks Introduces a self-contained implementation of the manifest on-disk format, in preparation for dropping the libprotobuf/protoc dependency: - pb_wire.h: minimal protobuf wire-format reader/writer (varint, fixed32/64, length-delimited). Unknown fields are consumed and ignored, preserving proto3 forward compatibility; malformed input (bad varint, out-of-range length, groups, field number 0) is reported instead of crashing. - manifest_enum.h: the on-disk enums, mirroring src/db/proto/zvec.proto with identical numeric values. The CodeBooks in type_helper.h now speak these instead of the generated protobuf enums. - manifest_codec.{h,cc}: encodes/decodes all 16 messages directly between zvec's C++ types and bytes, without an intermediate message object. Format compatibility is established by two test suites: - manifest_codec_test.cc cross-checks against libprotobuf while it is still available: every message encoded by both implementations must produce identical bytes, and each side must parse the other's output. This covers the subtle cases - always-present base/quantizer_param sub-messages, the absent quantizer_param of HNSW_RABITQ, Vamana's fixed32 alpha, empty repeated string elements and oneof "last branch wins". - manifest_codec_golden_test.cc pins protobuf's real output as embedded byte arrays and does not depend on libprotobuf, so it keeps guarding the format after the dependency is removed. src/db/proto/zvec.proto is retained as the authoritative documentation of the format. No behaviour change yet: Version::Save/Load still use protobuf. --- src/db/index/common/manifest_codec.cc | 844 ++++++++++++++++++ src/db/index/common/manifest_codec.h | 77 ++ src/db/index/common/manifest_enum.h | 120 +++ src/db/index/common/pb_wire.h | 302 +++++++ src/db/index/common/proto_converter.cc | 122 ++- src/db/index/common/proto_converter.h | 1 + src/db/index/common/type_helper.h | 232 ++--- tests/db/index/common/db_type_helper_test.cc | 283 +++--- .../common/manifest_codec_golden_test.cc | 266 ++++++ tests/db/index/common/manifest_codec_test.cc | 496 ++++++++++ 10 files changed, 2465 insertions(+), 278 deletions(-) create mode 100644 src/db/index/common/manifest_codec.cc create mode 100644 src/db/index/common/manifest_codec.h create mode 100644 src/db/index/common/manifest_enum.h create mode 100644 src/db/index/common/pb_wire.h create mode 100644 tests/db/index/common/manifest_codec_golden_test.cc create mode 100644 tests/db/index/common/manifest_codec_test.cc diff --git a/src/db/index/common/manifest_codec.cc b/src/db/index/common/manifest_codec.cc new file mode 100644 index 000000000..fca36d759 --- /dev/null +++ b/src/db/index/common/manifest_codec.cc @@ -0,0 +1,844 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "db/index/common/manifest_codec.h" +#include +#include "db/index/common/manifest_enum.h" +#include "db/index/common/pb_wire.h" +#include "db/index/common/type_helper.h" + +namespace zvec { +namespace { + +using pbwire::Reader; +using pbwire::Writer; + +// Field numbers, mirroring src/db/proto/zvec.proto. Changing any of these +// breaks compatibility with existing manifest files. +namespace f_quantizer { +constexpr uint32_t kEnableRotate = 1; +} +namespace f_base { +constexpr uint32_t kMetricType = 1; +constexpr uint32_t kQuantizeType = 2; +constexpr uint32_t kQuantizerParam = 4; +} // namespace f_base +namespace f_invert { +constexpr uint32_t kEnableRangeOptimization = 1; +} +namespace f_hnsw { +constexpr uint32_t kBase = 1; +constexpr uint32_t kM = 2; +constexpr uint32_t kEfConstruction = 3; +constexpr uint32_t kUseContiguousMemory = 4; +} // namespace f_hnsw +namespace f_hnsw_rabitq { +constexpr uint32_t kBase = 1; +constexpr uint32_t kM = 2; +constexpr uint32_t kEfConstruction = 3; +constexpr uint32_t kTotalBits = 4; +constexpr uint32_t kNumClusters = 5; +constexpr uint32_t kSampleCount = 6; +} // namespace f_hnsw_rabitq +namespace f_flat { +constexpr uint32_t kBase = 1; +} +namespace f_ivf { +constexpr uint32_t kBase = 1; +constexpr uint32_t kNList = 2; +constexpr uint32_t kNIters = 3; +constexpr uint32_t kUseSoar = 4; +} // namespace f_ivf +namespace f_diskann { +constexpr uint32_t kBase = 1; +constexpr uint32_t kMaxDegree = 2; +constexpr uint32_t kListSize = 3; +constexpr uint32_t kPqChunkNum = 4; +} // namespace f_diskann +namespace f_vamana { +constexpr uint32_t kBase = 1; +constexpr uint32_t kMaxDegree = 2; +constexpr uint32_t kSearchListSize = 3; +constexpr uint32_t kAlpha = 4; +constexpr uint32_t kSaturateGraph = 5; +constexpr uint32_t kUseContiguousMemory = 6; +constexpr uint32_t kUseIdMap = 7; +} // namespace f_vamana +namespace f_fts { +constexpr uint32_t kTokenizerName = 1; +constexpr uint32_t kFilters = 2; +constexpr uint32_t kExtraParams = 3; +} // namespace f_fts +namespace f_index_params { +constexpr uint32_t kInvert = 1; +constexpr uint32_t kHnsw = 2; +constexpr uint32_t kFlat = 3; +constexpr uint32_t kIvf = 4; +constexpr uint32_t kHnswRabitq = 5; +constexpr uint32_t kVamana = 6; +constexpr uint32_t kFts = 7; +constexpr uint32_t kDiskann = 8; +} // namespace f_index_params +namespace f_field { +constexpr uint32_t kName = 1; +constexpr uint32_t kDataType = 2; +constexpr uint32_t kDimension = 3; +constexpr uint32_t kNullable = 4; +constexpr uint32_t kIndexParams = 5; +} // namespace f_field +namespace f_collection { +constexpr uint32_t kName = 1; +constexpr uint32_t kFields = 2; +constexpr uint32_t kMaxDocCountPerSegment = 3; +} // namespace f_collection +namespace f_block { +constexpr uint32_t kBlockId = 1; +constexpr uint32_t kBlockType = 2; +constexpr uint32_t kMinDocId = 3; +constexpr uint32_t kMaxDocId = 4; +constexpr uint32_t kDocCount = 5; +constexpr uint32_t kColumns = 6; +} // namespace f_block +namespace f_segment { +constexpr uint32_t kSegmentId = 1; +constexpr uint32_t kPersistedBlocks = 2; +constexpr uint32_t kWritingForwardBlock = 3; +constexpr uint32_t kIndexedVectorFields = 4; +} // namespace f_segment +namespace f_manifest { +constexpr uint32_t kVersion = 1; +constexpr uint32_t kSchema = 2; +constexpr uint32_t kEnableMmap = 3; +constexpr uint32_t kPersistedSegmentMetas = 4; +constexpr uint32_t kWritingSegmentMeta = 5; +constexpr uint32_t kIdMapPathSuffix = 6; +constexpr uint32_t kDeleteSnapshotPathSuffix = 7; +constexpr uint32_t kNextSegmentId = 8; +} // namespace f_manifest + +//! Mirror of proto BaseIndexParams, shared by all vector index params. +struct BaseParams { + MetricType metric_type{MetricType::UNDEFINED}; + QuantizeType quantize_type{QuantizeType::UNDEFINED}; + // Whether the quantizer_param sub-message is present. The protobuf-based + // converter called mutable_quantizer_param() for every index type except + // HNSW_RABITQ, so presence must be reproduced exactly to keep the encoded + // bytes identical. + bool has_quantizer_param{false}; + bool enable_rotate{false}; +}; + +void EncodeBase(const BaseParams &base, std::string *out) { + Writer w(out); + w.PutVarint(f_base::kMetricType, + static_cast( + wire::ToNumber(MetricTypeCodeBook::Get(base.metric_type)))); + w.PutVarint(f_base::kQuantizeType, + static_cast(wire::ToNumber( + QuantizeTypeCodeBook::Get(base.quantize_type)))); + if (base.has_quantizer_param) { + std::string quantizer; + Writer qw(&quantizer); + qw.PutBool(f_quantizer::kEnableRotate, base.enable_rotate); + w.PutMessage(f_base::kQuantizerParam, quantizer); + } +} + +BaseParams DecodeBase(std::string_view buf) { + BaseParams base; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_base::kMetricType: + base.metric_type = MetricTypeCodeBook::Get( + wire::FromNumber(r.int32_value())); + break; + case f_base::kQuantizeType: + base.quantize_type = QuantizeTypeCodeBook::Get( + wire::FromNumber(r.int32_value())); + break; + case f_base::kQuantizerParam: { + base.has_quantizer_param = true; + Reader qr(r.bytes()); + while (qr.Next()) { + if (qr.field() == f_quantizer::kEnableRotate) { + base.enable_rotate = qr.bool_value(); + } + } + break; + } + default: + break; // unknown field: already consumed by Next() + } + } + return base; +} + +//! Builds the base params of an index that carries a quantizer_param. +template +BaseParams MakeBase(const Params *params) { + BaseParams base; + base.metric_type = params->metric_type(); + base.quantize_type = params->quantize_type(); + base.has_quantizer_param = true; + base.enable_rotate = params->quantizer_param().enable_rotate(); + return base; +} + +void EncodeHnsw(const HnswIndexParams *params, std::string *out) { + std::string base; + EncodeBase(MakeBase(params), &base); + Writer w(out); + w.PutMessage(f_hnsw::kBase, base); + w.PutVarint(f_hnsw::kM, static_cast(params->m())); + w.PutVarint(f_hnsw::kEfConstruction, + static_cast(params->ef_construction())); + w.PutBool(f_hnsw::kUseContiguousMemory, params->use_contiguous_memory()); +} + +HnswIndexParams::OPtr DecodeHnsw(std::string_view buf) { + BaseParams base; + int32_t m = 0; + int32_t ef_construction = 0; + bool use_contiguous_memory = false; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_hnsw::kBase: + base = DecodeBase(r.bytes()); + break; + case f_hnsw::kM: + m = r.int32_value(); + break; + case f_hnsw::kEfConstruction: + ef_construction = r.int32_value(); + break; + case f_hnsw::kUseContiguousMemory: + use_contiguous_memory = r.bool_value(); + break; + default: + break; + } + } + return std::make_shared( + base.metric_type, m, ef_construction, base.quantize_type, + use_contiguous_memory, QuantizerParam(base.enable_rotate)); +} + +void EncodeHnswRabitq(const HnswRabitqIndexParams *params, std::string *out) { + // NOTE: unlike the other vector indexes, the protobuf converter never + // touched quantizer_param here, so the sub-message must stay absent. + BaseParams base; + base.metric_type = params->metric_type(); + base.quantize_type = params->quantize_type(); + base.has_quantizer_param = false; + + std::string base_buf; + EncodeBase(base, &base_buf); + Writer w(out); + w.PutMessage(f_hnsw_rabitq::kBase, base_buf); + w.PutVarint(f_hnsw_rabitq::kM, static_cast(params->m())); + w.PutVarint(f_hnsw_rabitq::kEfConstruction, + static_cast(params->ef_construction())); + w.PutVarint(f_hnsw_rabitq::kTotalBits, + static_cast(params->total_bits())); + w.PutVarint(f_hnsw_rabitq::kNumClusters, + static_cast(params->num_clusters())); + w.PutVarint(f_hnsw_rabitq::kSampleCount, + static_cast(params->sample_count())); +} + +HnswRabitqIndexParams::OPtr DecodeHnswRabitq(std::string_view buf) { + BaseParams base; + int32_t m = 0; + int32_t ef_construction = 0; + int32_t total_bits = 0; + int32_t num_clusters = 0; + int32_t sample_count = 0; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_hnsw_rabitq::kBase: + base = DecodeBase(r.bytes()); + break; + case f_hnsw_rabitq::kM: + m = r.int32_value(); + break; + case f_hnsw_rabitq::kEfConstruction: + ef_construction = r.int32_value(); + break; + case f_hnsw_rabitq::kTotalBits: + total_bits = r.int32_value(); + break; + case f_hnsw_rabitq::kNumClusters: + num_clusters = r.int32_value(); + break; + case f_hnsw_rabitq::kSampleCount: + sample_count = r.int32_value(); + break; + default: + break; + } + } + return std::make_shared(base.metric_type, total_bits, + num_clusters, m, + ef_construction, sample_count); +} + +void EncodeFlat(const FlatIndexParams *params, std::string *out) { + std::string base; + EncodeBase(MakeBase(params), &base); + Writer w(out); + w.PutMessage(f_flat::kBase, base); +} + +FlatIndexParams::OPtr DecodeFlat(std::string_view buf) { + BaseParams base; + Reader r(buf); + while (r.Next()) { + if (r.field() == f_flat::kBase) { + base = DecodeBase(r.bytes()); + } + } + return std::make_shared(base.metric_type, base.quantize_type, + QuantizerParam(base.enable_rotate)); +} + +void EncodeIvf(const IVFIndexParams *params, std::string *out) { + std::string base; + EncodeBase(MakeBase(params), &base); + Writer w(out); + w.PutMessage(f_ivf::kBase, base); + w.PutVarint(f_ivf::kNList, static_cast(params->n_list())); + w.PutVarint(f_ivf::kNIters, static_cast(params->n_iters())); + w.PutBool(f_ivf::kUseSoar, params->use_soar()); +} + +IVFIndexParams::OPtr DecodeIvf(std::string_view buf) { + BaseParams base; + int32_t n_list = 0; + int32_t n_iters = 0; + bool use_soar = false; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_ivf::kBase: + base = DecodeBase(r.bytes()); + break; + case f_ivf::kNList: + n_list = r.int32_value(); + break; + case f_ivf::kNIters: + n_iters = r.int32_value(); + break; + case f_ivf::kUseSoar: + use_soar = r.bool_value(); + break; + default: + break; + } + } + return std::make_shared(base.metric_type, n_list, n_iters, + use_soar, base.quantize_type, + QuantizerParam(base.enable_rotate)); +} + +void EncodeDiskAnn(const DiskAnnIndexParams *params, std::string *out) { + std::string base; + EncodeBase(MakeBase(params), &base); + Writer w(out); + w.PutMessage(f_diskann::kBase, base); + w.PutVarint(f_diskann::kMaxDegree, + static_cast(params->max_degree())); + w.PutVarint(f_diskann::kListSize, static_cast(params->list_size())); + w.PutVarint(f_diskann::kPqChunkNum, + static_cast(params->pq_chunk_num())); +} + +DiskAnnIndexParams::OPtr DecodeDiskAnn(std::string_view buf) { + BaseParams base; + int32_t max_degree = 0; + int32_t list_size = 0; + int32_t pq_chunk_num = 0; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_diskann::kBase: + base = DecodeBase(r.bytes()); + break; + case f_diskann::kMaxDegree: + max_degree = r.int32_value(); + break; + case f_diskann::kListSize: + list_size = r.int32_value(); + break; + case f_diskann::kPqChunkNum: + pq_chunk_num = r.int32_value(); + break; + default: + break; + } + } + return std::make_shared( + base.metric_type, max_degree, list_size, pq_chunk_num, base.quantize_type, + QuantizerParam(base.enable_rotate)); +} + +void EncodeVamana(const VamanaIndexParams *params, std::string *out) { + std::string base; + EncodeBase(MakeBase(params), &base); + Writer w(out); + w.PutMessage(f_vamana::kBase, base); + w.PutVarint(f_vamana::kMaxDegree, + static_cast(params->max_degree())); + w.PutVarint(f_vamana::kSearchListSize, + static_cast(params->search_list_size())); + w.PutFloat(f_vamana::kAlpha, params->alpha()); + w.PutBool(f_vamana::kSaturateGraph, params->saturate_graph()); + w.PutBool(f_vamana::kUseContiguousMemory, params->use_contiguous_memory()); + w.PutBool(f_vamana::kUseIdMap, params->use_id_map()); +} + +VamanaIndexParams::OPtr DecodeVamana(std::string_view buf) { + BaseParams base; + int32_t max_degree = 0; + int32_t search_list_size = 0; + float alpha = 0.0f; + bool saturate_graph = false; + bool use_contiguous_memory = false; + bool use_id_map = false; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_vamana::kBase: + base = DecodeBase(r.bytes()); + break; + case f_vamana::kMaxDegree: + max_degree = r.int32_value(); + break; + case f_vamana::kSearchListSize: + search_list_size = r.int32_value(); + break; + case f_vamana::kAlpha: + alpha = r.float_value(); + break; + case f_vamana::kSaturateGraph: + saturate_graph = r.bool_value(); + break; + case f_vamana::kUseContiguousMemory: + use_contiguous_memory = r.bool_value(); + break; + case f_vamana::kUseIdMap: + use_id_map = r.bool_value(); + break; + default: + break; + } + } + return std::make_shared( + base.metric_type, max_degree, search_list_size, alpha, saturate_graph, + use_contiguous_memory, use_id_map, base.quantize_type, + QuantizerParam(base.enable_rotate)); +} + +void EncodeInvert(const InvertIndexParams *params, std::string *out) { + Writer w(out); + w.PutBool(f_invert::kEnableRangeOptimization, + params->enable_range_optimization()); +} + +InvertIndexParams::OPtr DecodeInvert(std::string_view buf) { + bool enable_range_optimization = false; + Reader r(buf); + while (r.Next()) { + if (r.field() == f_invert::kEnableRangeOptimization) { + enable_range_optimization = r.bool_value(); + } + } + return std::make_shared(enable_range_optimization); +} + +void EncodeFts(const FtsIndexParams *params, std::string *out) { + Writer w(out); + w.PutString(f_fts::kTokenizerName, params->tokenizer_name()); + for (const auto &filter : params->filters()) { + w.AddString(f_fts::kFilters, filter); + } + w.PutString(f_fts::kExtraParams, params->extra_params()); +} + +FtsIndexParams::Ptr DecodeFts(std::string_view buf) { + std::string tokenizer_name; + std::vector filters; + std::string extra_params; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_fts::kTokenizerName: + tokenizer_name = r.string_value(); + break; + case f_fts::kFilters: + filters.push_back(r.string_value()); + break; + case f_fts::kExtraParams: + extra_params = r.string_value(); + break; + default: + break; + } + } + return std::make_shared(tokenizer_name, std::move(filters), + extra_params); +} + +} // namespace + +void ManifestCodec::EncodeIndexParams(const IndexParams *params, + std::string *out) { + if (params == nullptr) { + return; + } + Writer w(out); + std::string payload; + switch (params->type()) { + case IndexType::INVERT: + if (auto *p = dynamic_cast(params)) { + EncodeInvert(p, &payload); + w.PutMessage(f_index_params::kInvert, payload); + } + break; + case IndexType::HNSW: + if (auto *p = dynamic_cast(params)) { + EncodeHnsw(p, &payload); + w.PutMessage(f_index_params::kHnsw, payload); + } + break; + case IndexType::FLAT: + if (auto *p = dynamic_cast(params)) { + EncodeFlat(p, &payload); + w.PutMessage(f_index_params::kFlat, payload); + } + break; + case IndexType::IVF: + if (auto *p = dynamic_cast(params)) { + EncodeIvf(p, &payload); + w.PutMessage(f_index_params::kIvf, payload); + } + break; + case IndexType::HNSW_RABITQ: + if (auto *p = dynamic_cast(params)) { + EncodeHnswRabitq(p, &payload); + w.PutMessage(f_index_params::kHnswRabitq, payload); + } + break; + case IndexType::VAMANA: + if (auto *p = dynamic_cast(params)) { + EncodeVamana(p, &payload); + w.PutMessage(f_index_params::kVamana, payload); + } + break; + case IndexType::FTS: + if (auto *p = dynamic_cast(params)) { + EncodeFts(p, &payload); + w.PutMessage(f_index_params::kFts, payload); + } + break; + case IndexType::DISKANN: + if (auto *p = dynamic_cast(params)) { + EncodeDiskAnn(p, &payload); + w.PutMessage(f_index_params::kDiskann, payload); + } + break; + default: + break; + } +} + +IndexParams::Ptr ManifestCodec::DecodeIndexParams(std::string_view buf) { + // oneof semantics: the last branch present on the wire wins. + IndexParams::Ptr params; + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_index_params::kInvert: + params = DecodeInvert(r.bytes()); + break; + case f_index_params::kHnsw: + params = DecodeHnsw(r.bytes()); + break; + case f_index_params::kFlat: + params = DecodeFlat(r.bytes()); + break; + case f_index_params::kIvf: + params = DecodeIvf(r.bytes()); + break; + case f_index_params::kHnswRabitq: + params = DecodeHnswRabitq(r.bytes()); + break; + case f_index_params::kVamana: + params = DecodeVamana(r.bytes()); + break; + case f_index_params::kFts: + params = DecodeFts(r.bytes()); + break; + case f_index_params::kDiskann: + params = DecodeDiskAnn(r.bytes()); + break; + default: + break; + } + } + return params; +} + +void ManifestCodec::EncodeFieldSchema(const FieldSchema &field, + std::string *out) { + Writer w(out); + w.PutString(f_field::kName, field.name()); + w.PutVarint(f_field::kDataType, + static_cast( + wire::ToNumber(DataTypeCodeBook::Get(field.data_type())))); + w.PutVarint(f_field::kDimension, field.dimension()); + w.PutBool(f_field::kNullable, field.nullable()); + auto index_params = field.index_params(); + if (index_params) { + std::string payload; + EncodeIndexParams(index_params.get(), &payload); + w.PutMessage(f_field::kIndexParams, payload); + } +} + +FieldSchema::Ptr ManifestCodec::DecodeFieldSchema(std::string_view buf) { + auto field = std::make_shared(); + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_field::kName: + field->set_name(r.string_value()); + break; + case f_field::kDataType: + field->set_data_type(DataTypeCodeBook::Get( + wire::FromNumber(r.int32_value()))); + break; + case f_field::kDimension: + field->set_dimension(r.uint32_value()); + break; + case f_field::kNullable: + field->set_nullable(r.bool_value()); + break; + case f_field::kIndexParams: + field->set_index_params(DecodeIndexParams(r.bytes())); + break; + default: + break; + } + } + return field; +} + +void ManifestCodec::EncodeCollectionSchema(const CollectionSchema &schema, + std::string *out) { + Writer w(out); + w.PutString(f_collection::kName, schema.name()); + for (const auto &field : schema.fields()) { + std::string payload; + EncodeFieldSchema(*field, &payload); + w.PutMessage(f_collection::kFields, payload); + } + w.PutVarint(f_collection::kMaxDocCountPerSegment, + schema.max_doc_count_per_segment()); +} + +CollectionSchema::Ptr ManifestCodec::DecodeCollectionSchema( + std::string_view buf) { + auto schema = std::make_shared(); + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_collection::kName: + schema->set_name(r.string_value()); + break; + case f_collection::kFields: + schema->add_field(DecodeFieldSchema(r.bytes())); + break; + case f_collection::kMaxDocCountPerSegment: + schema->set_max_doc_count_per_segment(r.varint()); + break; + default: + break; + } + } + return schema; +} + +void ManifestCodec::EncodeBlockMeta(const BlockMeta &meta, std::string *out) { + Writer w(out); + w.PutVarint(f_block::kBlockId, meta.id()); + w.PutVarint(f_block::kBlockType, static_cast(wire::ToNumber( + BlockTypeCodeBook::Get(meta.type())))); + w.PutVarint(f_block::kMinDocId, meta.min_doc_id()); + w.PutVarint(f_block::kMaxDocId, meta.max_doc_id()); + w.PutVarint(f_block::kDocCount, meta.doc_count()); + for (const auto &column : meta.columns()) { + w.AddString(f_block::kColumns, column); + } +} + +BlockMeta::Ptr ManifestCodec::DecodeBlockMeta(std::string_view buf) { + auto meta = std::make_shared(); + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_block::kBlockId: + meta->set_id(r.uint32_value()); + break; + case f_block::kBlockType: + meta->set_type(BlockTypeCodeBook::Get( + wire::FromNumber(r.int32_value()))); + break; + case f_block::kMinDocId: + meta->set_min_doc_id(r.varint()); + break; + case f_block::kMaxDocId: + meta->set_max_doc_id(r.varint()); + break; + case f_block::kDocCount: + meta->set_doc_count(r.uint32_value()); + break; + case f_block::kColumns: + meta->add_column(r.string_value()); + break; + default: + break; + } + } + return meta; +} + +void ManifestCodec::EncodeSegmentMeta(const SegmentMeta &meta, + std::string *out) { + Writer w(out); + w.PutVarint(f_segment::kSegmentId, meta.id()); + for (const auto &block : meta.persisted_blocks()) { + std::string payload; + EncodeBlockMeta(block, &payload); + w.PutMessage(f_segment::kPersistedBlocks, payload); + } + if (meta.has_writing_forward_block()) { + std::string payload; + EncodeBlockMeta(meta.writing_forward_block().value(), &payload); + w.PutMessage(f_segment::kWritingForwardBlock, payload); + } + for (const auto &field : meta.indexed_vector_fields()) { + w.AddString(f_segment::kIndexedVectorFields, field); + } +} + +SegmentMeta::Ptr ManifestCodec::DecodeSegmentMeta(std::string_view buf) { + auto meta = std::make_shared(0); + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_segment::kSegmentId: + meta->set_id(r.uint32_value()); + break; + case f_segment::kPersistedBlocks: + meta->add_persisted_block(*DecodeBlockMeta(r.bytes())); + break; + case f_segment::kWritingForwardBlock: + meta->set_writing_forward_block(*DecodeBlockMeta(r.bytes())); + break; + case f_segment::kIndexedVectorFields: + meta->add_indexed_vector_field(r.string_value()); + break; + default: + break; + } + } + return meta; +} + +Status ManifestCodec::Encode(const ManifestData &data, std::string *out) { + Writer w(out); + w.PutVarint(f_manifest::kVersion, data.version); + if (data.schema) { + std::string payload; + EncodeCollectionSchema(*data.schema, &payload); + w.PutMessage(f_manifest::kSchema, payload); + } + w.PutBool(f_manifest::kEnableMmap, data.enable_mmap); + for (const auto &meta : data.persisted_segment_metas) { + if (!meta) { + continue; + } + std::string payload; + EncodeSegmentMeta(*meta, &payload); + w.PutMessage(f_manifest::kPersistedSegmentMetas, payload); + } + if (data.writing_segment_meta) { + std::string payload; + EncodeSegmentMeta(*data.writing_segment_meta, &payload); + w.PutMessage(f_manifest::kWritingSegmentMeta, payload); + } + w.PutVarint(f_manifest::kIdMapPathSuffix, data.id_map_path_suffix); + w.PutVarint(f_manifest::kDeleteSnapshotPathSuffix, + data.delete_snapshot_path_suffix); + w.PutVarint(f_manifest::kNextSegmentId, data.next_segment_id); + return Status::OK(); +} + +Status ManifestCodec::Decode(std::string_view buf, ManifestData *data) { + Reader r(buf); + while (r.Next()) { + switch (r.field()) { + case f_manifest::kVersion: + data->version = r.uint32_value(); + break; + case f_manifest::kSchema: + data->schema = DecodeCollectionSchema(r.bytes()); + break; + case f_manifest::kEnableMmap: + data->enable_mmap = r.bool_value(); + break; + case f_manifest::kPersistedSegmentMetas: + data->persisted_segment_metas.push_back(DecodeSegmentMeta(r.bytes())); + break; + case f_manifest::kWritingSegmentMeta: + data->writing_segment_meta = DecodeSegmentMeta(r.bytes()); + break; + case f_manifest::kIdMapPathSuffix: + data->id_map_path_suffix = r.uint32_value(); + break; + case f_manifest::kDeleteSnapshotPathSuffix: + data->delete_snapshot_path_suffix = r.uint32_value(); + break; + case f_manifest::kNextSegmentId: + data->next_segment_id = r.uint32_value(); + break; + default: + break; + } + } + if (!r.ok()) { + return Status::InternalError("Malformed manifest data"); + } + if (!data->schema) { + // An absent schema means the file is not a valid manifest; the protobuf + // parser accepted it but downstream code always expects a schema. + data->schema = std::make_shared(); + } + return Status::OK(); +} + +} // namespace zvec diff --git a/src/db/index/common/manifest_codec.h b/src/db/index/common/manifest_codec.h new file mode 100644 index 000000000..82e7b8810 --- /dev/null +++ b/src/db/index/common/manifest_codec.h @@ -0,0 +1,77 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Encoder/decoder for the collection manifest. +//! +//! The on-disk layout is the protobuf wire format described by +//! src/db/proto/zvec.proto, which remains the authoritative documentation of +//! the format. Field numbers here must stay in sync with that file. +//! +//! Compared with the generated protobuf code this converts directly between +//! zvec's own C++ types and bytes, without an intermediate message object. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "db/index/common/meta.h" + +namespace zvec { + +//! In-memory representation of the manifest file contents. +struct ManifestData { + uint32_t version{0}; + CollectionSchema::Ptr schema; + bool enable_mmap{false}; + std::vector persisted_segment_metas; + SegmentMeta::Ptr writing_segment_meta; // null when absent + uint32_t id_map_path_suffix{0}; + uint32_t delete_snapshot_path_suffix{0}; + uint32_t next_segment_id{0}; +}; + +//! Converts between ManifestData and its on-disk byte representation. +struct ManifestCodec { + //! Serializes a manifest. Appends to `out`. + static Status Encode(const ManifestData &data, std::string *out); + + //! Parses a manifest. Returns an error for malformed input. + static Status Decode(std::string_view buf, ManifestData *data); + + // The helpers below are exposed for unit testing so that each message can + // be checked against the format independently. + + static void EncodeIndexParams(const IndexParams *params, std::string *out); + static IndexParams::Ptr DecodeIndexParams(std::string_view buf); + + static void EncodeFieldSchema(const FieldSchema &field, std::string *out); + static FieldSchema::Ptr DecodeFieldSchema(std::string_view buf); + + static void EncodeCollectionSchema(const CollectionSchema &schema, + std::string *out); + static CollectionSchema::Ptr DecodeCollectionSchema(std::string_view buf); + + static void EncodeBlockMeta(const BlockMeta &meta, std::string *out); + static BlockMeta::Ptr DecodeBlockMeta(std::string_view buf); + + static void EncodeSegmentMeta(const SegmentMeta &meta, std::string *out); + static SegmentMeta::Ptr DecodeSegmentMeta(std::string_view buf); +}; + +} // namespace zvec diff --git a/src/db/index/common/manifest_enum.h b/src/db/index/common/manifest_enum.h new file mode 100644 index 000000000..89beca152 --- /dev/null +++ b/src/db/index/common/manifest_enum.h @@ -0,0 +1,120 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! On-disk enum values of the manifest format. +//! +//! These mirror the enums declared in src/db/proto/zvec.proto one-to-one. +//! The numeric values are part of the persisted manifest format and must +//! never change; only append new entries. + +#pragma once + +#include + +namespace zvec { +namespace wire { + +//! Mirrors proto enum DataType. +enum class DataType : int32_t { + DT_UNDEFINED = 0, + + DT_BINARY = 1, + DT_STRING = 2, + DT_BOOL = 3, + DT_INT32 = 4, + DT_INT64 = 5, + DT_UINT32 = 6, + DT_UINT64 = 7, + DT_FLOAT = 8, + DT_DOUBLE = 9, + + DT_VECTOR_BINARY32 = 20, + DT_VECTOR_BINARY64 = 21, + DT_VECTOR_FP16 = 22, + DT_VECTOR_FP32 = 23, + DT_VECTOR_FP64 = 24, + DT_VECTOR_INT4 = 25, + DT_VECTOR_INT8 = 26, + DT_VECTOR_INT16 = 27, + + DT_SPARSE_VECTOR_FP16 = 30, + DT_SPARSE_VECTOR_FP32 = 31, + + DT_ARRAY_BINARY = 40, + DT_ARRAY_STRING = 41, + DT_ARRAY_BOOL = 42, + DT_ARRAY_INT32 = 43, + DT_ARRAY_INT64 = 44, + DT_ARRAY_UINT32 = 45, + DT_ARRAY_UINT64 = 46, + DT_ARRAY_FLOAT = 47, + DT_ARRAY_DOUBLE = 48, +}; + +//! Mirrors proto enum IndexType. +enum class IndexType : int32_t { + IT_UNDEFINED = 0, + IT_HNSW = 1, + IT_IVF = 2, + IT_FLAT = 3, + IT_HNSW_RABITQ = 4, + IT_VAMANA = 5, + IT_DISKANN = 6, + IT_INVERT = 10, + IT_FTS = 11, +}; + +//! Mirrors proto enum QuantizeType. +enum class QuantizeType : int32_t { + QT_UNDEFINED = 0, + QT_FP16 = 1, + QT_INT8 = 2, + QT_INT4 = 3, + QT_RABITQ = 4, +}; + +//! Mirrors proto enum MetricType. +enum class MetricType : int32_t { + MT_UNDEFINED = 0, + MT_L2 = 1, + MT_IP = 2, + MT_COSINE = 3, +}; + +//! Mirrors proto enum BlockType. +enum class BlockType : int32_t { + BT_UNDEFINED = 0, + BT_SCALAR = 1, + BT_SCALAR_INDEX = 2, + BT_VECTOR_INDEX = 3, + BT_VECTOR_INDEX_QUANTIZE = 4, + BT_FTS_INDEX = 5, +}; + +//! Converts a wire enum to its underlying numeric value for encoding. +template +constexpr int32_t ToNumber(E value) { + return static_cast(value); +} + +//! Converts a numeric value read off the wire back to a wire enum. Unknown +//! values are preserved as-is; the CodeBook mapping turns them into +//! UNDEFINED, matching the previous protobuf-based behaviour. +template +constexpr E FromNumber(int32_t value) { + return static_cast(value); +} + +} // namespace wire +} // namespace zvec diff --git a/src/db/index/common/pb_wire.h b/src/db/index/common/pb_wire.h new file mode 100644 index 000000000..5183560e0 --- /dev/null +++ b/src/db/index/common/pb_wire.h @@ -0,0 +1,302 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Minimal protobuf wire-format reader/writer. +//! +//! zvec persists its manifest in protobuf wire format (see +//! src/db/proto/zvec.proto, kept as the authoritative format documentation). +//! This header implements just enough of the encoding to read and write that +//! format so that the project does not need to depend on libprotobuf/protoc. +//! +//! Wire format reference: +//! https://protobuf.dev/programming-guides/encoding/ +//! +//! Only the wire types actually used by zvec.proto are supported for reading +//! values (varint, 64-bit, length-delimited, 32-bit); deprecated groups +//! (wire types 3 and 4) are rejected as corrupt input. + +#pragma once + +#include +#include +#include +#include + +namespace zvec { +namespace pbwire { + +//! Protobuf wire types. +enum WireType : uint32_t { + kVarint = 0, + kFixed64 = 1, + kLenDelim = 2, + kStartGroup = 3, // deprecated, unsupported + kEndGroup = 4, // deprecated, unsupported + kFixed32 = 5, +}; + +//! Maximum nesting depth accepted while decoding, as a guard against +//! maliciously crafted or corrupted input. +constexpr int kMaxNestingDepth = 16; + +//! Appends protobuf-encoded fields to a string buffer. +//! +//! Fields must be written in ascending field-number order to match the byte +//! layout produced by the protobuf library. +class Writer { + public: + explicit Writer(std::string *out) : out_(out) {} + + //! Writes a varint field. Skipped when the value is zero, matching proto3 + //! semantics where default-valued singular fields are not serialized. + void PutVarint(uint32_t field, uint64_t value) { + if (value == 0) { + return; + } + PutVarintAlways(field, value); + } + + //! Writes a bool field. Skipped when false (proto3 default). + void PutBool(uint32_t field, bool value) { + if (!value) { + return; + } + PutVarintAlways(field, 1); + } + + //! Writes a float field as fixed32. Skipped when the value is +0.0f + //! (proto3 default). Note -0.0f compares equal to 0.0f and is therefore + //! also skipped, which matches the protobuf library behaviour. + void PutFloat(uint32_t field, float value) { + if (value == 0.0f) { + return; + } + PutTag(field, kFixed32); + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + PutLittleEndian32(bits); + } + + //! Writes a singular string field. Skipped when empty (proto3 default). + void PutString(uint32_t field, const std::string &value) { + if (value.empty()) { + return; + } + PutLenDelim(field, value); + } + + //! Writes one element of a repeated string field. Unlike singular fields, + //! repeated elements are always written, including empty strings. + void AddString(uint32_t field, const std::string &value) { + PutLenDelim(field, value); + } + + //! Writes a length-delimited field holding an already encoded sub-message. + //! Always written, including when the payload is empty: an empty but + //! present sub-message is encoded as a zero-length field by protobuf. + void PutMessage(uint32_t field, std::string_view encoded) { + PutLenDelim(field, encoded); + } + + private: + void PutTag(uint32_t field, WireType type) { + PutVarintRaw((static_cast(field) << 3) | + static_cast(type)); + } + + void PutVarintAlways(uint32_t field, uint64_t value) { + PutTag(field, kVarint); + PutVarintRaw(value); + } + + void PutLenDelim(uint32_t field, std::string_view value) { + PutTag(field, kLenDelim); + PutVarintRaw(value.size()); + out_->append(value.data(), value.size()); + } + + void PutVarintRaw(uint64_t value) { + while (value >= 0x80) { + out_->push_back(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + out_->push_back(static_cast(value)); + } + + void PutLittleEndian32(uint32_t value) { + for (int i = 0; i < 4; ++i) { + out_->push_back(static_cast(value & 0xFF)); + value >>= 8; + } + } + + std::string *out_; +}; + +//! Iterates over the fields of a protobuf-encoded buffer. +//! +//! Each successful Next() fully consumes one field, so unknown fields are +//! skipped simply by not reading their value. This preserves protobuf's +//! forward compatibility: manifests written by newer versions carrying extra +//! fields remain readable. +class Reader { + public: + Reader(const char *data, size_t size) : data_(data), size_(size) {} + + explicit Reader(std::string_view buf) : Reader(buf.data(), buf.size()) {} + + //! Advances to the next field. Returns false at end of buffer or on error; + //! use ok() to distinguish the two. + bool Next() { + if (!ok_ || pos_ >= size_) { + return false; + } + uint64_t tag = 0; + if (!ReadVarintRaw(&tag)) { + return Fail(); + } + field_ = static_cast(tag >> 3); + type_ = static_cast(tag & 0x7); + if (field_ == 0) { + return Fail(); // field number 0 is illegal + } + + switch (type_) { + case kVarint: + return ReadVarintRaw(&varint_) ? true : Fail(); + case kFixed64: { + if (size_ - pos_ < 8) { + return Fail(); + } + uint64_t bits = 0; + for (int i = 0; i < 8; ++i) { + bits |= static_cast(static_cast(data_[pos_ + i])) + << (8 * i); + } + pos_ += 8; + varint_ = bits; + return true; + } + case kFixed32: { + if (size_ - pos_ < 4) { + return Fail(); + } + uint32_t bits = 0; + for (int i = 0; i < 4; ++i) { + bits |= static_cast(static_cast(data_[pos_ + i])) + << (8 * i); + } + pos_ += 4; + fixed32_ = bits; + return true; + } + case kLenDelim: { + uint64_t len = 0; + if (!ReadVarintRaw(&len)) { + return Fail(); + } + if (len > size_ - pos_) { + return Fail(); + } + bytes_ = std::string_view(data_ + pos_, static_cast(len)); + pos_ += static_cast(len); + return true; + } + default: + return Fail(); // groups and unknown wire types + } + } + + uint32_t field() const { + return field_; + } + + WireType wire_type() const { + return type_; + } + + //! Value of the current field decoded as a varint. + uint64_t varint() const { + return varint_; + } + + uint32_t uint32_value() const { + return static_cast(varint_); + } + + int32_t int32_value() const { + return static_cast(static_cast(varint_)); + } + + bool bool_value() const { + return varint_ != 0; + } + + //! Value of the current field decoded as a fixed32 float. + float float_value() const { + float value; + std::memcpy(&value, &fixed32_, sizeof(value)); + return value; + } + + //! Payload of the current length-delimited field. The view points into the + //! buffer passed to the constructor and stays valid as long as it does. + std::string_view bytes() const { + return bytes_; + } + + std::string string_value() const { + return std::string(bytes_); + } + + //! False once malformed input has been encountered. + bool ok() const { + return ok_; + } + + private: + bool Fail() { + ok_ = false; + return false; + } + + bool ReadVarintRaw(uint64_t *out) { + uint64_t result = 0; + for (int shift = 0; shift < 64; shift += 7) { + if (pos_ >= size_) { + return false; + } + uint8_t byte = static_cast(data_[pos_++]); + result |= static_cast(byte & 0x7F) << shift; + if ((byte & 0x80) == 0) { + *out = result; + return true; + } + } + return false; // varint longer than 10 bytes + } + + const char *data_; + size_t size_; + size_t pos_{0}; + uint32_t field_{0}; + WireType type_{kVarint}; + uint64_t varint_{0}; + uint32_t fixed32_{0}; + std::string_view bytes_{}; + bool ok_{true}; +}; + +} // namespace pbwire +} // namespace zvec diff --git a/src/db/index/common/proto_converter.cc b/src/db/index/common/proto_converter.cc index 80b4c61ca..f0014479d 100644 --- a/src/db/index/common/proto_converter.cc +++ b/src/db/index/common/proto_converter.cc @@ -13,16 +13,73 @@ // limitations under the License. #include "proto_converter.h" +#include "db/index/common/manifest_enum.h" namespace zvec { +namespace { + +// The CodeBooks in type_helper.h speak the wire enums of the manifest format +// (see manifest_enum.h). These adapters bridge them to the protobuf generated +// enums, whose numeric values are identical by construction. +// +// NOTE: this whole file goes away once Version::Save/Load switch to +// ManifestCodec; it is kept only so that the cross-check tests can compare +// both implementations. +template +inline ProtoEnum ToProtoEnum(WireEnum value) { + return static_cast(wire::ToNumber(value)); +} + +template +inline WireEnum ToWireEnum(ProtoEnum value) { + return wire::FromNumber(static_cast(value)); +} + +struct PbMetric { + static proto::MetricType Get(MetricType type) { + return ToProtoEnum(MetricTypeCodeBook::Get(type)); + } + static MetricType Get(proto::MetricType type) { + return MetricTypeCodeBook::Get(ToWireEnum(type)); + } +}; + +struct PbQuantize { + static proto::QuantizeType Get(QuantizeType type) { + return ToProtoEnum(QuantizeTypeCodeBook::Get(type)); + } + static QuantizeType Get(proto::QuantizeType type) { + return QuantizeTypeCodeBook::Get(ToWireEnum(type)); + } +}; + +struct PbDataType { + static proto::DataType Get(DataType type) { + return ToProtoEnum(DataTypeCodeBook::Get(type)); + } + static DataType Get(proto::DataType type) { + return DataTypeCodeBook::Get(ToWireEnum(type)); + } +}; + +struct PbBlockType { + static proto::BlockType Get(BlockType type) { + return ToProtoEnum(BlockTypeCodeBook::Get(type)); + } + static BlockType Get(proto::BlockType type) { + return BlockTypeCodeBook::Get(ToWireEnum(type)); + } +}; + +} // namespace HnswIndexParams::OPtr ProtoConverter::FromPb( const proto::HnswIndexParams ¶ms_pb) { bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); auto params = std::make_shared( - MetricTypeCodeBook::Get(params_pb.base().metric_type()), params_pb.m(), + PbMetric::Get(params_pb.base().metric_type()), params_pb.m(), params_pb.ef_construction(), - QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()), + PbQuantize::Get(params_pb.base().quantize_type()), params_pb.use_contiguous_memory(), QuantizerParam(enable_rotate)); return params; @@ -31,9 +88,9 @@ HnswIndexParams::OPtr ProtoConverter::FromPb( proto::HnswIndexParams ProtoConverter::ToPb(const HnswIndexParams *params) { proto::HnswIndexParams params_pb; params_pb.mutable_base()->set_metric_type( - MetricTypeCodeBook::Get(params->metric_type())); + PbMetric::Get(params->metric_type())); params_pb.mutable_base()->set_quantize_type( - QuantizeTypeCodeBook::Get(params->quantize_type())); + PbQuantize::Get(params->quantize_type())); params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( params->quantizer_param().enable_rotate()); params_pb.set_ef_construction(params->ef_construction()); @@ -46,9 +103,9 @@ proto::HnswIndexParams ProtoConverter::ToPb(const HnswIndexParams *params) { HnswRabitqIndexParams::OPtr ProtoConverter::FromPb( const proto::HnswRabitqIndexParams ¶ms_pb) { auto params = std::make_shared( - MetricTypeCodeBook::Get(params_pb.base().metric_type()), - params_pb.total_bits(), params_pb.num_clusters(), params_pb.m(), - params_pb.ef_construction(), params_pb.sample_count()); + PbMetric::Get(params_pb.base().metric_type()), params_pb.total_bits(), + params_pb.num_clusters(), params_pb.m(), params_pb.ef_construction(), + params_pb.sample_count()); return params; } @@ -57,9 +114,9 @@ proto::HnswRabitqIndexParams ProtoConverter::ToPb( const HnswRabitqIndexParams *params) { proto::HnswRabitqIndexParams params_pb; params_pb.mutable_base()->set_metric_type( - MetricTypeCodeBook::Get(params->metric_type())); + PbMetric::Get(params->metric_type())); params_pb.mutable_base()->set_quantize_type( - QuantizeTypeCodeBook::Get(params->quantize_type())); + PbQuantize::Get(params->quantize_type())); params_pb.set_m(params->m()); params_pb.set_ef_construction(params->ef_construction()); params_pb.set_total_bits(params->total_bits()); @@ -73,17 +130,17 @@ FlatIndexParams::OPtr ProtoConverter::FromPb( const proto::FlatIndexParams ¶ms_pb) { bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); return std::make_shared( - MetricTypeCodeBook::Get(params_pb.base().metric_type()), - QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()), + PbMetric::Get(params_pb.base().metric_type()), + PbQuantize::Get(params_pb.base().quantize_type()), QuantizerParam(enable_rotate)); } proto::FlatIndexParams ProtoConverter::ToPb(const FlatIndexParams *params) { proto::FlatIndexParams params_pb; params_pb.mutable_base()->set_metric_type( - MetricTypeCodeBook::Get(params->metric_type())); + PbMetric::Get(params->metric_type())); params_pb.mutable_base()->set_quantize_type( - QuantizeTypeCodeBook::Get(params->quantize_type())); + PbQuantize::Get(params->quantize_type())); params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( params->quantizer_param().enable_rotate()); return params_pb; @@ -94,18 +151,18 @@ IVFIndexParams::OPtr ProtoConverter::FromPb( const proto::IVFIndexParams ¶ms_pb) { bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); return std::make_shared( - MetricTypeCodeBook::Get(params_pb.base().metric_type()), - params_pb.n_list(), params_pb.n_iters(), params_pb.use_soar(), - QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()), + PbMetric::Get(params_pb.base().metric_type()), params_pb.n_list(), + params_pb.n_iters(), params_pb.use_soar(), + PbQuantize::Get(params_pb.base().quantize_type()), QuantizerParam(enable_rotate)); } proto::IVFIndexParams ProtoConverter::ToPb(const IVFIndexParams *params) { proto::IVFIndexParams params_pb; params_pb.mutable_base()->set_metric_type( - MetricTypeCodeBook::Get(params->metric_type())); + PbMetric::Get(params->metric_type())); params_pb.mutable_base()->set_quantize_type( - QuantizeTypeCodeBook::Get(params->quantize_type())); + PbQuantize::Get(params->quantize_type())); params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( params->quantizer_param().enable_rotate()); params_pb.set_n_list(params->n_list()); @@ -119,20 +176,19 @@ VamanaIndexParams::OPtr ProtoConverter::FromPb( const proto::VamanaIndexParams ¶ms_pb) { bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); return std::make_shared( - MetricTypeCodeBook::Get(params_pb.base().metric_type()), - params_pb.max_degree(), params_pb.search_list_size(), params_pb.alpha(), + PbMetric::Get(params_pb.base().metric_type()), params_pb.max_degree(), + params_pb.search_list_size(), params_pb.alpha(), params_pb.saturate_graph(), params_pb.use_contiguous_memory(), - params_pb.use_id_map(), - QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()), + params_pb.use_id_map(), PbQuantize::Get(params_pb.base().quantize_type()), QuantizerParam(enable_rotate)); } proto::VamanaIndexParams ProtoConverter::ToPb(const VamanaIndexParams *params) { proto::VamanaIndexParams params_pb; params_pb.mutable_base()->set_metric_type( - MetricTypeCodeBook::Get(params->metric_type())); + PbMetric::Get(params->metric_type())); params_pb.mutable_base()->set_quantize_type( - QuantizeTypeCodeBook::Get(params->quantize_type())); + PbQuantize::Get(params->quantize_type())); params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( params->quantizer_param().enable_rotate()); params_pb.set_max_degree(params->max_degree()); @@ -164,9 +220,9 @@ DiskAnnIndexParams::OPtr ProtoConverter::FromPb( const proto::DiskAnnIndexParams ¶ms_pb) { bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); return std::make_shared( - MetricTypeCodeBook::Get(params_pb.base().metric_type()), - params_pb.max_degree(), params_pb.list_size(), params_pb.pq_chunk_num(), - QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()), + PbMetric::Get(params_pb.base().metric_type()), params_pb.max_degree(), + params_pb.list_size(), params_pb.pq_chunk_num(), + PbQuantize::Get(params_pb.base().quantize_type()), QuantizerParam(enable_rotate)); } @@ -174,9 +230,9 @@ proto::DiskAnnIndexParams ProtoConverter::ToPb( const DiskAnnIndexParams *params) { proto::DiskAnnIndexParams params_pb; params_pb.mutable_base()->set_metric_type( - MetricTypeCodeBook::Get(params->metric_type())); + PbMetric::Get(params->metric_type())); params_pb.mutable_base()->set_quantize_type( - QuantizeTypeCodeBook::Get(params->quantize_type())); + PbQuantize::Get(params->quantize_type())); params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( params->quantizer_param().enable_rotate()); params_pb.set_max_degree(params->max_degree()); @@ -212,7 +268,7 @@ FieldSchema::Ptr ProtoConverter::FromPb(const proto::FieldSchema &schema_pb) { auto schema = std::make_shared(); schema->set_name(schema_pb.name()); - schema->set_data_type(DataTypeCodeBook::Get(schema_pb.data_type())); + schema->set_data_type(PbDataType::Get(schema_pb.data_type())); schema->set_dimension(schema_pb.dimension()); schema->set_nullable(schema_pb.nullable()); if (schema_pb.has_index_params()) { @@ -224,7 +280,7 @@ proto::FieldSchema ProtoConverter::ToPb(const FieldSchema &schema) { proto::FieldSchema schema_pb; schema_pb.set_name(schema.name()); - schema_pb.set_data_type(DataTypeCodeBook::Get(schema.data_type())); + schema_pb.set_data_type(PbDataType::Get(schema.data_type())); schema_pb.set_dimension(schema.dimension()); schema_pb.set_nullable(schema.nullable()); auto index_params = schema.index_params(); @@ -292,7 +348,7 @@ BlockMeta::Ptr ProtoConverter::FromPb(const proto::BlockMeta &meta_pb) { auto block_meta = std::make_shared(); block_meta->set_id(meta_pb.block_id()); - block_meta->set_type(BlockTypeCodeBook::Get(meta_pb.block_type())); + block_meta->set_type(PbBlockType::Get(meta_pb.block_type())); block_meta->set_min_doc_id(meta_pb.min_doc_id()); block_meta->set_max_doc_id(meta_pb.max_doc_id()); block_meta->set_doc_count(meta_pb.doc_count()); @@ -378,7 +434,7 @@ proto::IndexParams ProtoConverter::ToPb(const IndexParams *params) { proto::BlockMeta ProtoConverter::ToPb(const BlockMeta &meta) { proto::BlockMeta meta_pb; meta_pb.set_block_id(meta.id()); - meta_pb.set_block_type(BlockTypeCodeBook::Get(meta.type())); + meta_pb.set_block_type(PbBlockType::Get(meta.type())); meta_pb.set_min_doc_id(meta.min_doc_id()); meta_pb.set_max_doc_id(meta.max_doc_id()); meta_pb.set_doc_count(meta.doc_count()); diff --git a/src/db/index/common/proto_converter.h b/src/db/index/common/proto_converter.h index e0f015cb1..1d9ee0d10 100644 --- a/src/db/index/common/proto_converter.h +++ b/src/db/index/common/proto_converter.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once +#include #include #include #include "db/index/common/meta.h" diff --git a/src/db/index/common/type_helper.h b/src/db/index/common/type_helper.h index d24f1ee52..8d35c17e1 100644 --- a/src/db/index/common/type_helper.h +++ b/src/db/index/common/type_helper.h @@ -16,9 +16,11 @@ #include #include +#include +#include #include #include -#include "proto/zvec.pb.h" +#include "db/index/common/manifest_enum.h" namespace zvec { @@ -40,23 +42,23 @@ bool sort_and_find_duplicates(uint32_t *indices, char *values, size_t n, //! Index Type Codebook struct IndexTypeCodeBook { //! convert protobuf IndexType to C++ IndexType - static IndexType Get(proto::IndexType type) { + static IndexType Get(wire::IndexType type) { switch (type) { - case proto::IT_HNSW: + case wire::IndexType::IT_HNSW: return IndexType::HNSW; - case proto::IT_HNSW_RABITQ: + case wire::IndexType::IT_HNSW_RABITQ: return IndexType::HNSW_RABITQ; - case proto::IT_FLAT: + case wire::IndexType::IT_FLAT: return IndexType::FLAT; - case proto::IT_IVF: + case wire::IndexType::IT_IVF: return IndexType::IVF; - case proto::IT_VAMANA: + case wire::IndexType::IT_VAMANA: return IndexType::VAMANA; - case proto::IT_INVERT: + case wire::IndexType::IT_INVERT: return IndexType::INVERT; - case proto::IT_DISKANN: + case wire::IndexType::IT_DISKANN: return IndexType::DISKANN; - case proto::IT_FTS: + case wire::IndexType::IT_FTS: return IndexType::FTS; default: break; @@ -65,28 +67,28 @@ struct IndexTypeCodeBook { } //! Convert C++ IndexType to protobuf IndexType - static proto::IndexType Get(IndexType type) { + static wire::IndexType Get(IndexType type) { switch (type) { case IndexType::HNSW: - return proto::IT_HNSW; + return wire::IndexType::IT_HNSW; case IndexType::HNSW_RABITQ: - return proto::IT_HNSW_RABITQ; + return wire::IndexType::IT_HNSW_RABITQ; case IndexType::FLAT: - return proto::IT_FLAT; + return wire::IndexType::IT_FLAT; case IndexType::IVF: - return proto::IT_IVF; + return wire::IndexType::IT_IVF; case IndexType::VAMANA: - return proto::IT_VAMANA; + return wire::IndexType::IT_VAMANA; case IndexType::INVERT: - return proto::IT_INVERT; + return wire::IndexType::IT_INVERT; case IndexType::DISKANN: - return proto::IT_DISKANN; + return wire::IndexType::IT_DISKANN; case IndexType::FTS: - return proto::IT_FTS; + return wire::IndexType::IT_FTS; default: break; } - return proto::IT_UNDEFINED; + return wire::IndexType::IT_UNDEFINED; } //! Convert C++ IndexType to C++ String @@ -116,96 +118,96 @@ struct IndexTypeCodeBook { }; struct DataTypeCodeBook { - static bool IsArrayType(proto::DataType type) { - return proto::DataType::DT_ARRAY_BINARY <= type && - type <= proto::DataType::DT_ARRAY_DOUBLE; + static bool IsArrayType(wire::DataType type) { + return wire::DataType::DT_ARRAY_BINARY <= type && + type <= wire::DataType::DT_ARRAY_DOUBLE; } - static DataType Get(proto::DataType type) { + static DataType Get(wire::DataType type) { DataType data_types = DataType::UNDEFINED; switch (type) { - case proto::DataType::DT_BINARY: + case wire::DataType::DT_BINARY: data_types = DataType::BINARY; break; - case proto::DataType::DT_STRING: + case wire::DataType::DT_STRING: data_types = DataType::STRING; break; - case proto::DataType::DT_BOOL: + case wire::DataType::DT_BOOL: data_types = DataType::BOOL; break; - case proto::DataType::DT_INT32: + case wire::DataType::DT_INT32: data_types = DataType::INT32; break; - case proto::DataType::DT_INT64: + case wire::DataType::DT_INT64: data_types = DataType::INT64; break; - case proto::DataType::DT_UINT32: + case wire::DataType::DT_UINT32: data_types = DataType::UINT32; break; - case proto::DataType::DT_UINT64: + case wire::DataType::DT_UINT64: data_types = DataType::UINT64; break; - case proto::DataType::DT_FLOAT: + case wire::DataType::DT_FLOAT: data_types = DataType::FLOAT; break; - case proto::DataType::DT_DOUBLE: + case wire::DataType::DT_DOUBLE: data_types = DataType::DOUBLE; break; - case proto::DataType::DT_VECTOR_BINARY32: + case wire::DataType::DT_VECTOR_BINARY32: data_types = DataType::VECTOR_BINARY32; break; - case proto::DataType::DT_VECTOR_BINARY64: + case wire::DataType::DT_VECTOR_BINARY64: data_types = DataType::VECTOR_BINARY64; break; - case proto::DataType::DT_VECTOR_FP16: + case wire::DataType::DT_VECTOR_FP16: data_types = DataType::VECTOR_FP16; break; - case proto::DataType::DT_VECTOR_FP32: + case wire::DataType::DT_VECTOR_FP32: data_types = DataType::VECTOR_FP32; break; - case proto::DataType::DT_VECTOR_FP64: + case wire::DataType::DT_VECTOR_FP64: data_types = DataType::VECTOR_FP64; break; - case proto::DataType::DT_VECTOR_INT4: + case wire::DataType::DT_VECTOR_INT4: data_types = DataType::VECTOR_INT4; break; - case proto::DataType::DT_VECTOR_INT8: + case wire::DataType::DT_VECTOR_INT8: data_types = DataType::VECTOR_INT8; break; - case proto::DataType::DT_VECTOR_INT16: + case wire::DataType::DT_VECTOR_INT16: data_types = DataType::VECTOR_INT16; break; - case proto::DataType::DT_SPARSE_VECTOR_FP16: + case wire::DataType::DT_SPARSE_VECTOR_FP16: data_types = DataType::SPARSE_VECTOR_FP16; break; - case proto::DataType::DT_SPARSE_VECTOR_FP32: + case wire::DataType::DT_SPARSE_VECTOR_FP32: data_types = DataType::SPARSE_VECTOR_FP32; break; - case proto::DataType::DT_ARRAY_BINARY: + case wire::DataType::DT_ARRAY_BINARY: data_types = DataType::ARRAY_BINARY; break; - case proto::DataType::DT_ARRAY_STRING: + case wire::DataType::DT_ARRAY_STRING: data_types = DataType::ARRAY_STRING; break; - case proto::DataType::DT_ARRAY_BOOL: + case wire::DataType::DT_ARRAY_BOOL: data_types = DataType::ARRAY_BOOL; break; - case proto::DataType::DT_ARRAY_INT32: + case wire::DataType::DT_ARRAY_INT32: data_types = DataType::ARRAY_INT32; break; - case proto::DataType::DT_ARRAY_INT64: + case wire::DataType::DT_ARRAY_INT64: data_types = DataType::ARRAY_INT64; break; - case proto::DataType::DT_ARRAY_UINT32: + case wire::DataType::DT_ARRAY_UINT32: data_types = DataType::ARRAY_UINT32; break; - case proto::DataType::DT_ARRAY_UINT64: + case wire::DataType::DT_ARRAY_UINT64: data_types = DataType::ARRAY_UINT64; break; - case proto::DataType::DT_ARRAY_FLOAT: + case wire::DataType::DT_ARRAY_FLOAT: data_types = DataType::ARRAY_FLOAT; break; - case proto::DataType::DT_ARRAY_DOUBLE: + case wire::DataType::DT_ARRAY_DOUBLE: data_types = DataType::ARRAY_DOUBLE; break; @@ -215,92 +217,92 @@ struct DataTypeCodeBook { return data_types; } - static proto::DataType Get(const DataType type) { - proto::DataType data_type = proto::DataType::DT_UNDEFINED; + static wire::DataType Get(const DataType type) { + wire::DataType data_type = wire::DataType::DT_UNDEFINED; switch (type) { case DataType::BINARY: - data_type = proto::DataType::DT_BINARY; + data_type = wire::DataType::DT_BINARY; break; case DataType::STRING: - data_type = proto::DataType::DT_STRING; + data_type = wire::DataType::DT_STRING; break; case DataType::BOOL: - data_type = proto::DataType::DT_BOOL; + data_type = wire::DataType::DT_BOOL; break; case DataType::INT32: - data_type = proto::DataType::DT_INT32; + data_type = wire::DataType::DT_INT32; break; case DataType::INT64: - data_type = proto::DataType::DT_INT64; + data_type = wire::DataType::DT_INT64; break; case DataType::UINT32: - data_type = proto::DataType::DT_UINT32; + data_type = wire::DataType::DT_UINT32; break; case DataType::UINT64: - data_type = proto::DataType::DT_UINT64; + data_type = wire::DataType::DT_UINT64; break; case DataType::FLOAT: - data_type = proto::DataType::DT_FLOAT; + data_type = wire::DataType::DT_FLOAT; break; case DataType::DOUBLE: - data_type = proto::DataType::DT_DOUBLE; + data_type = wire::DataType::DT_DOUBLE; break; case DataType::VECTOR_BINARY32: - data_type = proto::DataType::DT_VECTOR_BINARY32; + data_type = wire::DataType::DT_VECTOR_BINARY32; break; case DataType::VECTOR_BINARY64: - data_type = proto::DataType::DT_VECTOR_BINARY64; + data_type = wire::DataType::DT_VECTOR_BINARY64; break; case DataType::VECTOR_FP16: - data_type = proto::DataType::DT_VECTOR_FP16; + data_type = wire::DataType::DT_VECTOR_FP16; break; case DataType::VECTOR_FP32: - data_type = proto::DataType::DT_VECTOR_FP32; + data_type = wire::DataType::DT_VECTOR_FP32; break; case DataType::VECTOR_FP64: - data_type = proto::DataType::DT_VECTOR_FP64; + data_type = wire::DataType::DT_VECTOR_FP64; break; case DataType::VECTOR_INT4: - data_type = proto::DataType::DT_VECTOR_INT4; + data_type = wire::DataType::DT_VECTOR_INT4; break; case DataType::VECTOR_INT8: - data_type = proto::DataType::DT_VECTOR_INT8; + data_type = wire::DataType::DT_VECTOR_INT8; break; case DataType::VECTOR_INT16: - data_type = proto::DataType::DT_VECTOR_INT16; + data_type = wire::DataType::DT_VECTOR_INT16; break; case DataType::SPARSE_VECTOR_FP16: - data_type = proto::DataType::DT_SPARSE_VECTOR_FP16; + data_type = wire::DataType::DT_SPARSE_VECTOR_FP16; break; case DataType::SPARSE_VECTOR_FP32: - data_type = proto::DataType::DT_SPARSE_VECTOR_FP32; + data_type = wire::DataType::DT_SPARSE_VECTOR_FP32; break; case DataType::ARRAY_BINARY: - data_type = proto::DataType::DT_ARRAY_BINARY; + data_type = wire::DataType::DT_ARRAY_BINARY; break; case DataType::ARRAY_BOOL: - data_type = proto::DataType::DT_ARRAY_BOOL; + data_type = wire::DataType::DT_ARRAY_BOOL; break; case DataType::ARRAY_DOUBLE: - data_type = proto::DataType::DT_ARRAY_DOUBLE; + data_type = wire::DataType::DT_ARRAY_DOUBLE; break; case DataType::ARRAY_FLOAT: - data_type = proto::DataType::DT_ARRAY_FLOAT; + data_type = wire::DataType::DT_ARRAY_FLOAT; break; case DataType::ARRAY_INT32: - data_type = proto::DataType::DT_ARRAY_INT32; + data_type = wire::DataType::DT_ARRAY_INT32; break; case DataType::ARRAY_INT64: - data_type = proto::DataType::DT_ARRAY_INT64; + data_type = wire::DataType::DT_ARRAY_INT64; break; case DataType::ARRAY_STRING: - data_type = proto::DataType::DT_ARRAY_STRING; + data_type = wire::DataType::DT_ARRAY_STRING; break; case DataType::ARRAY_UINT32: - data_type = proto::DataType::DT_ARRAY_UINT32; + data_type = wire::DataType::DT_ARRAY_UINT32; break; case DataType::ARRAY_UINT64: - data_type = proto::DataType::DT_ARRAY_UINT64; + data_type = wire::DataType::DT_ARRAY_UINT64; break; default: break; @@ -408,29 +410,29 @@ struct DataTypeCodeBook { }; struct MetricTypeCodeBook { - static MetricType Get(proto::MetricType type) { + static MetricType Get(wire::MetricType type) { switch (type) { - case proto::MetricType::MT_IP: + case wire::MetricType::MT_IP: return MetricType::IP; - case proto::MetricType::MT_L2: + case wire::MetricType::MT_L2: return MetricType::L2; - case proto::MetricType::MT_COSINE: + case wire::MetricType::MT_COSINE: return MetricType::COSINE; default: return MetricType::UNDEFINED; } } - static proto::MetricType Get(MetricType type) { + static wire::MetricType Get(MetricType type) { switch (type) { case MetricType::IP: - return proto::MetricType::MT_IP; + return wire::MetricType::MT_IP; case MetricType::L2: - return proto::MetricType::MT_L2; + return wire::MetricType::MT_L2; case MetricType::COSINE: - return proto::MetricType::MT_COSINE; + return wire::MetricType::MT_COSINE; default: - return proto::MetricType::MT_UNDEFINED; + return wire::MetricType::MT_UNDEFINED; } } @@ -449,33 +451,33 @@ struct MetricTypeCodeBook { }; struct QuantizeTypeCodeBook { - static QuantizeType Get(proto::QuantizeType type) { + static QuantizeType Get(wire::QuantizeType type) { switch (type) { - case proto::QuantizeType::QT_FP16: + case wire::QuantizeType::QT_FP16: return QuantizeType::FP16; - case proto::QuantizeType::QT_INT4: + case wire::QuantizeType::QT_INT4: return QuantizeType::INT4; - case proto::QuantizeType::QT_INT8: + case wire::QuantizeType::QT_INT8: return QuantizeType::INT8; - case proto::QuantizeType::QT_RABITQ: + case wire::QuantizeType::QT_RABITQ: return QuantizeType::RABITQ; default: return QuantizeType::UNDEFINED; } } - static proto::QuantizeType Get(QuantizeType type) { + static wire::QuantizeType Get(QuantizeType type) { switch (type) { case QuantizeType::FP16: - return proto::QuantizeType::QT_FP16; + return wire::QuantizeType::QT_FP16; case QuantizeType::INT4: - return proto::QuantizeType::QT_INT4; + return wire::QuantizeType::QT_INT4; case QuantizeType::INT8: - return proto::QuantizeType::QT_INT8; + return wire::QuantizeType::QT_INT8; case QuantizeType::RABITQ: - return proto::QuantizeType::QT_RABITQ; + return wire::QuantizeType::QT_RABITQ; default: - return proto::QuantizeType::QT_UNDEFINED; + return wire::QuantizeType::QT_UNDEFINED; } } @@ -504,22 +506,22 @@ struct QuantizeTypeCodeBook { }; struct BlockTypeCodeBook { - static BlockType Get(proto::BlockType type) { + static BlockType Get(wire::BlockType type) { BlockType block_types = BlockType::UNDEFINED; switch (type) { - case proto::BlockType::BT_SCALAR: + case wire::BlockType::BT_SCALAR: block_types = BlockType::SCALAR; break; - case proto::BlockType::BT_SCALAR_INDEX: + case wire::BlockType::BT_SCALAR_INDEX: block_types = BlockType::SCALAR_INDEX; break; - case proto::BlockType::BT_VECTOR_INDEX: + case wire::BlockType::BT_VECTOR_INDEX: block_types = BlockType::VECTOR_INDEX; break; - case proto::BlockType::BT_VECTOR_INDEX_QUANTIZE: + case wire::BlockType::BT_VECTOR_INDEX_QUANTIZE: block_types = BlockType::VECTOR_INDEX_QUANTIZE; break; - case proto::BlockType::BT_FTS_INDEX: + case wire::BlockType::BT_FTS_INDEX: block_types = BlockType::FTS_INDEX; break; default: @@ -528,23 +530,23 @@ struct BlockTypeCodeBook { return block_types; } - static proto::BlockType Get(BlockType type) { - proto::BlockType block_types = proto::BlockType::BT_UNDEFINED; + static wire::BlockType Get(BlockType type) { + wire::BlockType block_types = wire::BlockType::BT_UNDEFINED; switch (type) { case BlockType::SCALAR: - block_types = proto::BlockType::BT_SCALAR; + block_types = wire::BlockType::BT_SCALAR; break; case BlockType::SCALAR_INDEX: - block_types = proto::BlockType::BT_SCALAR_INDEX; + block_types = wire::BlockType::BT_SCALAR_INDEX; break; case BlockType::VECTOR_INDEX: - block_types = proto::BlockType::BT_VECTOR_INDEX; + block_types = wire::BlockType::BT_VECTOR_INDEX; break; case BlockType::VECTOR_INDEX_QUANTIZE: - block_types = proto::BlockType::BT_VECTOR_INDEX_QUANTIZE; + block_types = wire::BlockType::BT_VECTOR_INDEX_QUANTIZE; break; case BlockType::FTS_INDEX: - block_types = proto::BlockType::BT_FTS_INDEX; + block_types = wire::BlockType::BT_FTS_INDEX; break; default: break; diff --git a/tests/db/index/common/db_type_helper_test.cc b/tests/db/index/common/db_type_helper_test.cc index b3b7cc3d7..a299d9ead 100644 --- a/tests/db/index/common/db_type_helper_test.cc +++ b/tests/db/index/common/db_type_helper_test.cc @@ -19,24 +19,28 @@ using namespace zvec; TEST(IndexTypeCodeBookTest, ProtoToCppConversion) { // Test conversion from protobuf to C++ IndexType - EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_HNSW), IndexType::HNSW); - EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_FLAT), IndexType::FLAT); - EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_IVF), IndexType::IVF); - EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_INVERT), IndexType::INVERT); - EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_UNDEFINED), IndexType::UNDEFINED); - EXPECT_EQ(IndexTypeCodeBook::Get(static_cast(999)), + EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_HNSW), IndexType::HNSW); + EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_FLAT), IndexType::FLAT); + EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_IVF), IndexType::IVF); + EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_INVERT), + IndexType::INVERT); + EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_UNDEFINED), + IndexType::UNDEFINED); + EXPECT_EQ(IndexTypeCodeBook::Get(static_cast(999)), IndexType::UNDEFINED); } TEST(IndexTypeCodeBookTest, CppToProtoConversion) { // Test conversion from C++ IndexType to protobuf IndexType - EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::HNSW), proto::IT_HNSW); - EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::FLAT), proto::IT_FLAT); - EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::IVF), proto::IT_IVF); - EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::INVERT), proto::IT_INVERT); - EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::UNDEFINED), proto::IT_UNDEFINED); + EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::HNSW), wire::IndexType::IT_HNSW); + EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::FLAT), wire::IndexType::IT_FLAT); + EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::IVF), wire::IndexType::IT_IVF); + EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::INVERT), + wire::IndexType::IT_INVERT); + EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::UNDEFINED), + wire::IndexType::IT_UNDEFINED); EXPECT_EQ(IndexTypeCodeBook::Get(static_cast(999)), - proto::IT_UNDEFINED); + wire::IndexType::IT_UNDEFINED); } TEST(IndexTypeCodeBookTest, CppToStringConversion) { @@ -50,138 +54,145 @@ TEST(IndexTypeCodeBookTest, CppToStringConversion) { TEST(DataTypeCodeBookTest, IsArrayType) { // Test array type detection - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_BINARY)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_STRING)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_BOOL)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_INT32)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_INT64)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_UINT32)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_UINT64)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_FLOAT)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_DOUBLE)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_BINARY32)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_BINARY64)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_FP16)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_FP32)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_FP64)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_INT4)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_INT8)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_INT16)); - EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_SPARSE_VECTOR_FP32)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_BINARY)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_STRING)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_BOOL)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_INT32)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_INT64)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_UINT32)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_UINT64)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_FLOAT)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_DOUBLE)); + EXPECT_FALSE( + DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_BINARY32)); + EXPECT_FALSE( + DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_BINARY64)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_FP16)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_FP32)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_FP64)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_INT4)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_INT8)); + EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_INT16)); + EXPECT_FALSE( + DataTypeCodeBook::IsArrayType(wire::DataType::DT_SPARSE_VECTOR_FP32)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_BINARY)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_STRING)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_BOOL)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_INT32)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_INT64)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_UINT32)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_UINT64)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_FLOAT)); - EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_DOUBLE)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_BINARY)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_STRING)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_BOOL)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_INT32)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_INT64)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_UINT32)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_UINT64)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_FLOAT)); + EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_DOUBLE)); } TEST(DataTypeCodeBookTest, ProtoToCppConversion) { // Test conversion from protobuf to C++ DataType - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_BINARY), DataType::BINARY); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_STRING), DataType::STRING); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_BOOL), DataType::BOOL); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_INT32), DataType::INT32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_INT64), DataType::INT64); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_UINT32), DataType::UINT32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_UINT64), DataType::UINT64); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_FLOAT), DataType::FLOAT); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_DOUBLE), DataType::DOUBLE); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_BINARY32), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_BINARY), DataType::BINARY); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_STRING), DataType::STRING); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_BOOL), DataType::BOOL); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_INT32), DataType::INT32); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_INT64), DataType::INT64); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_UINT32), DataType::UINT32); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_UINT64), DataType::UINT64); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_FLOAT), DataType::FLOAT); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_DOUBLE), DataType::DOUBLE); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_BINARY32), DataType::VECTOR_BINARY32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_BINARY64), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_BINARY64), DataType::VECTOR_BINARY64); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_FP16), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_FP16), DataType::VECTOR_FP16); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_FP32), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_FP32), DataType::VECTOR_FP32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_FP64), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_FP64), DataType::VECTOR_FP64); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_INT4), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_INT4), DataType::VECTOR_INT4); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_INT8), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_INT8), DataType::VECTOR_INT8); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_INT16), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_INT16), DataType::VECTOR_INT16); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_SPARSE_VECTOR_FP32), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_SPARSE_VECTOR_FP32), DataType::SPARSE_VECTOR_FP32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_BINARY), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_BINARY), DataType::ARRAY_BINARY); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_STRING), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_STRING), DataType::ARRAY_STRING); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_BOOL), DataType::ARRAY_BOOL); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_INT32), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_BOOL), + DataType::ARRAY_BOOL); + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_INT32), DataType::ARRAY_INT32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_INT64), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_INT64), DataType::ARRAY_INT64); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_UINT32), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_UINT32), DataType::ARRAY_UINT32); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_UINT64), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_UINT64), DataType::ARRAY_UINT64); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_FLOAT), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_FLOAT), DataType::ARRAY_FLOAT); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_DOUBLE), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_DOUBLE), DataType::ARRAY_DOUBLE); - EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_UNDEFINED), DataType::UNDEFINED); - EXPECT_EQ(DataTypeCodeBook::Get(static_cast(999)), + EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_UNDEFINED), + DataType::UNDEFINED); + EXPECT_EQ(DataTypeCodeBook::Get(static_cast(999)), DataType::UNDEFINED); } TEST(DataTypeCodeBookTest, CppToProtoConversion) { // Test conversion from C++ DataType to protobuf DataType - EXPECT_EQ(DataTypeCodeBook::Get(DataType::BINARY), proto::DT_BINARY); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::STRING), proto::DT_STRING); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::BOOL), proto::DT_BOOL); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT32), proto::DT_INT32); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT64), proto::DT_INT64); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT32), proto::DT_UINT32); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT64), proto::DT_UINT64); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::FLOAT), proto::DT_FLOAT); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::DOUBLE), proto::DT_DOUBLE); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::BINARY), wire::DataType::DT_BINARY); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::STRING), wire::DataType::DT_STRING); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::BOOL), wire::DataType::DT_BOOL); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT32), wire::DataType::DT_INT32); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT64), wire::DataType::DT_INT64); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT32), wire::DataType::DT_UINT32); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT64), wire::DataType::DT_UINT64); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::FLOAT), wire::DataType::DT_FLOAT); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::DOUBLE), wire::DataType::DT_DOUBLE); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_BINARY32), - proto::DT_VECTOR_BINARY32); + wire::DataType::DT_VECTOR_BINARY32); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_BINARY64), - proto::DT_VECTOR_BINARY64); + wire::DataType::DT_VECTOR_BINARY64); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_FP16), - proto::DT_VECTOR_FP16); + wire::DataType::DT_VECTOR_FP16); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_FP32), - proto::DT_VECTOR_FP32); + wire::DataType::DT_VECTOR_FP32); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_FP64), - proto::DT_VECTOR_FP64); + wire::DataType::DT_VECTOR_FP64); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_INT4), - proto::DT_VECTOR_INT4); + wire::DataType::DT_VECTOR_INT4); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_INT8), - proto::DT_VECTOR_INT8); + wire::DataType::DT_VECTOR_INT8); EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_INT16), - proto::DT_VECTOR_INT16); + wire::DataType::DT_VECTOR_INT16); EXPECT_EQ(DataTypeCodeBook::Get(DataType::SPARSE_VECTOR_FP16), - proto::DT_SPARSE_VECTOR_FP16); + wire::DataType::DT_SPARSE_VECTOR_FP16); EXPECT_EQ(DataTypeCodeBook::Get(DataType::SPARSE_VECTOR_FP32), - proto::DT_SPARSE_VECTOR_FP32); + wire::DataType::DT_SPARSE_VECTOR_FP32); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_BINARY), - proto::DT_ARRAY_BINARY); + wire::DataType::DT_ARRAY_BINARY); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_STRING), - proto::DT_ARRAY_STRING); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_BOOL), proto::DT_ARRAY_BOOL); + wire::DataType::DT_ARRAY_STRING); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_BOOL), + wire::DataType::DT_ARRAY_BOOL); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_INT32), - proto::DT_ARRAY_INT32); + wire::DataType::DT_ARRAY_INT32); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_INT64), - proto::DT_ARRAY_INT64); + wire::DataType::DT_ARRAY_INT64); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_UINT32), - proto::DT_ARRAY_UINT32); + wire::DataType::DT_ARRAY_UINT32); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_UINT64), - proto::DT_ARRAY_UINT64); + wire::DataType::DT_ARRAY_UINT64); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_FLOAT), - proto::DT_ARRAY_FLOAT); + wire::DataType::DT_ARRAY_FLOAT); EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_DOUBLE), - proto::DT_ARRAY_DOUBLE); - EXPECT_EQ(DataTypeCodeBook::Get(DataType::UNDEFINED), proto::DT_UNDEFINED); + wire::DataType::DT_ARRAY_DOUBLE); + EXPECT_EQ(DataTypeCodeBook::Get(DataType::UNDEFINED), + wire::DataType::DT_UNDEFINED); EXPECT_EQ(DataTypeCodeBook::Get(static_cast(999)), - proto::DT_UNDEFINED); + wire::DataType::DT_UNDEFINED); } TEST(DataTypeCodeBookTest, CppToStringConversion) { @@ -220,72 +231,84 @@ TEST(DataTypeCodeBookTest, CppToStringConversion) { TEST(MetricTypeCodeBookTest, ProtoToCppConversion) { // Test conversion from protobuf to C++ MetricType - EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_IP), MetricType::IP); - EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_L2), MetricType::L2); - EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_COSINE), MetricType::COSINE); - EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_UNDEFINED), + EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_IP), MetricType::IP); + EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_L2), MetricType::L2); + EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_COSINE), + MetricType::COSINE); + EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_UNDEFINED), MetricType::UNDEFINED); - EXPECT_EQ(MetricTypeCodeBook::Get(static_cast(999)), + EXPECT_EQ(MetricTypeCodeBook::Get(static_cast(999)), MetricType::UNDEFINED); } TEST(MetricTypeCodeBookTest, CppToProtoConversion) { // Test conversion from C++ MetricType to protobuf MetricType - EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::IP), proto::MT_IP); - EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::L2), proto::MT_L2); - EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::COSINE), proto::MT_COSINE); + EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::IP), wire::MetricType::MT_IP); + EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::L2), wire::MetricType::MT_L2); + EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::COSINE), + wire::MetricType::MT_COSINE); EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::UNDEFINED), - proto::MT_UNDEFINED); + wire::MetricType::MT_UNDEFINED); EXPECT_EQ(MetricTypeCodeBook::Get(static_cast(999)), - proto::MT_UNDEFINED); + wire::MetricType::MT_UNDEFINED); } TEST(QuantizeTypeCodeBookTest, ProtoToCppConversion) { // Test conversion from protobuf to C++ QuantizeType - EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_FP16), QuantizeType::FP16); - EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_INT4), QuantizeType::INT4); - EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_INT8), QuantizeType::INT8); - EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_UNDEFINED), + EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_FP16), + QuantizeType::FP16); + EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_INT4), + QuantizeType::INT4); + EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_INT8), + QuantizeType::INT8); + EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_UNDEFINED), QuantizeType::UNDEFINED); - EXPECT_EQ(QuantizeTypeCodeBook::Get(static_cast(999)), + EXPECT_EQ(QuantizeTypeCodeBook::Get(static_cast(999)), QuantizeType::UNDEFINED); } TEST(QuantizeTypeCodeBookTest, CppToProtoConversion) { // Test conversion from C++ QuantizeType to protobuf QuantizeType - EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::FP16), proto::QT_FP16); - EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT4), proto::QT_INT4); - EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT8), proto::QT_INT8); + EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::FP16), + wire::QuantizeType::QT_FP16); + EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT4), + wire::QuantizeType::QT_INT4); + EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT8), + wire::QuantizeType::QT_INT8); EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::UNDEFINED), - proto::QT_UNDEFINED); + wire::QuantizeType::QT_UNDEFINED); EXPECT_EQ(QuantizeTypeCodeBook::Get(static_cast(999)), - proto::QT_UNDEFINED); + wire::QuantizeType::QT_UNDEFINED); } TEST(BlockTypeCodeBookTest, ProtoToCppConversion) { // Test conversion from protobuf to C++ BlockType - EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_SCALAR), BlockType::SCALAR); - EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_SCALAR_INDEX), + EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_SCALAR), + BlockType::SCALAR); + EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_SCALAR_INDEX), BlockType::SCALAR_INDEX); - EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_VECTOR_INDEX), + EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_VECTOR_INDEX), BlockType::VECTOR_INDEX); - EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_VECTOR_INDEX_QUANTIZE), + EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_VECTOR_INDEX_QUANTIZE), BlockType::VECTOR_INDEX_QUANTIZE); - EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_UNDEFINED), BlockType::UNDEFINED); - EXPECT_EQ(BlockTypeCodeBook::Get(static_cast(999)), + EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_UNDEFINED), + BlockType::UNDEFINED); + EXPECT_EQ(BlockTypeCodeBook::Get(static_cast(999)), BlockType::UNDEFINED); } TEST(BlockTypeCodeBookTest, CppToProtoConversion) { // Test conversion from C++ BlockType to protobuf BlockType - EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::SCALAR), proto::BT_SCALAR); + EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::SCALAR), + wire::BlockType::BT_SCALAR); EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::SCALAR_INDEX), - proto::BT_SCALAR_INDEX); + wire::BlockType::BT_SCALAR_INDEX); EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::VECTOR_INDEX), - proto::BT_VECTOR_INDEX); + wire::BlockType::BT_VECTOR_INDEX); EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::VECTOR_INDEX_QUANTIZE), - proto::BT_VECTOR_INDEX_QUANTIZE); - EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::UNDEFINED), proto::BT_UNDEFINED); + wire::BlockType::BT_VECTOR_INDEX_QUANTIZE); + EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::UNDEFINED), + wire::BlockType::BT_UNDEFINED); EXPECT_EQ(BlockTypeCodeBook::Get(static_cast(999)), - proto::BT_UNDEFINED); + wire::BlockType::BT_UNDEFINED); } \ No newline at end of file diff --git a/tests/db/index/common/manifest_codec_golden_test.cc b/tests/db/index/common/manifest_codec_golden_test.cc new file mode 100644 index 000000000..5e4803eb1 --- /dev/null +++ b/tests/db/index/common/manifest_codec_golden_test.cc @@ -0,0 +1,266 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Golden-byte and robustness tests for ManifestCodec. +//! +//! The byte arrays below were produced by the protobuf implementation that +//! zvec used before ManifestCodec existed (see the cross-check tests in +//! manifest_codec_test.cc, which verified both implementations agree +//! byte-for-byte). They pin the on-disk manifest format: decoding them must +//! yield the expected values and re-encoding must reproduce the exact same +//! bytes. +//! +//! These tests do not depend on libprotobuf, so they keep guarding format +//! compatibility after that dependency is dropped. +//! +//! Never regenerate these arrays to make a failing test pass: a mismatch means +//! the persisted format changed and existing collections would fail to open. + +#include +#include +#include +#include +#include +#include "db/index/common/manifest_codec.h" +#include "db/index/common/pb_wire.h" + +using namespace zvec; + +namespace { + +//! Manifest of a default-constructed schema with no fields. +const uint8_t kGoldenEmpty[] = { + 0x12, 0x05, 0x18, 0x80, 0xad, 0xe2, 0x04, +}; + +//! Manifest with a scalar field, an HNSW vector field and one segment. +const uint8_t kGoldenSimple[] = { + 0x12, 0x31, 0x0a, 0x06, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x06, + 0x0a, 0x02, 0x70, 0x6b, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x05, 0x64, 0x65, + 0x6e, 0x73, 0x65, 0x10, 0x17, 0x18, 0x80, 0x01, 0x2a, 0x0d, 0x12, 0x0b, + 0x0a, 0x04, 0x08, 0x02, 0x22, 0x00, 0x10, 0x10, 0x18, 0xc8, 0x01, 0x18, + 0xa0, 0x8d, 0x06, 0x18, 0x01, 0x22, 0x0a, 0x12, 0x08, 0x10, 0x01, 0x28, + 0x0a, 0x32, 0x02, 0x70, 0x6b, 0x40, 0x01, +}; + +//! Manifest exercising every IndexParams oneof branch, both segment slots and +//! all scalar manifest fields. +const uint8_t kGoldenAllIndexTypes[] = { + 0x12, 0xc1, 0x02, 0x0a, 0x04, 0x72, 0x69, 0x63, 0x68, 0x12, 0x14, 0x0a, + 0x08, 0x66, 0x5f, 0x69, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x10, 0x05, 0x20, + 0x01, 0x2a, 0x04, 0x0a, 0x02, 0x08, 0x01, 0x12, 0x24, 0x0a, 0x06, 0x66, + 0x5f, 0x68, 0x6e, 0x73, 0x77, 0x10, 0x17, 0x18, 0x80, 0x01, 0x20, 0x01, + 0x2a, 0x13, 0x12, 0x11, 0x0a, 0x08, 0x08, 0x01, 0x10, 0x02, 0x22, 0x02, + 0x08, 0x01, 0x10, 0x10, 0x18, 0xc8, 0x01, 0x20, 0x01, 0x12, 0x1a, 0x0a, + 0x06, 0x66, 0x5f, 0x66, 0x6c, 0x61, 0x74, 0x10, 0x16, 0x18, 0x40, 0x20, + 0x01, 0x2a, 0x0a, 0x1a, 0x08, 0x0a, 0x06, 0x08, 0x02, 0x10, 0x01, 0x22, + 0x00, 0x12, 0x22, 0x0a, 0x05, 0x66, 0x5f, 0x69, 0x76, 0x66, 0x10, 0x1a, + 0x18, 0x20, 0x20, 0x01, 0x2a, 0x13, 0x22, 0x11, 0x0a, 0x08, 0x08, 0x03, + 0x10, 0x03, 0x22, 0x02, 0x08, 0x01, 0x10, 0x80, 0x04, 0x18, 0x0c, 0x20, + 0x01, 0x12, 0x29, 0x0a, 0x08, 0x66, 0x5f, 0x72, 0x61, 0x62, 0x69, 0x74, + 0x71, 0x10, 0x17, 0x18, 0x80, 0x02, 0x20, 0x01, 0x2a, 0x16, 0x2a, 0x14, + 0x0a, 0x04, 0x08, 0x01, 0x10, 0x04, 0x10, 0x18, 0x18, 0x96, 0x01, 0x20, + 0x05, 0x28, 0x80, 0x08, 0x30, 0xd0, 0x86, 0x03, 0x12, 0x2b, 0x0a, 0x08, + 0x66, 0x5f, 0x76, 0x61, 0x6d, 0x61, 0x6e, 0x61, 0x10, 0x17, 0x18, 0x60, + 0x20, 0x01, 0x2a, 0x19, 0x32, 0x17, 0x0a, 0x08, 0x08, 0x01, 0x10, 0x04, + 0x22, 0x02, 0x08, 0x01, 0x10, 0x30, 0x18, 0x60, 0x25, 0xcd, 0xcc, 0xac, + 0x3f, 0x28, 0x01, 0x38, 0x01, 0x12, 0x2b, 0x0a, 0x05, 0x66, 0x5f, 0x66, + 0x74, 0x73, 0x10, 0x02, 0x20, 0x01, 0x2a, 0x1e, 0x3a, 0x1c, 0x0a, 0x05, + 0x6a, 0x69, 0x65, 0x62, 0x61, 0x12, 0x09, 0x6c, 0x6f, 0x77, 0x65, 0x72, + 0x63, 0x61, 0x73, 0x65, 0x12, 0x04, 0x73, 0x74, 0x6f, 0x70, 0x1a, 0x02, + 0x7b, 0x7d, 0x12, 0x25, 0x0a, 0x09, 0x66, 0x5f, 0x64, 0x69, 0x73, 0x6b, + 0x61, 0x6e, 0x6e, 0x10, 0x17, 0x18, 0x80, 0x04, 0x20, 0x01, 0x2a, 0x11, + 0x42, 0x0f, 0x0a, 0x06, 0x08, 0x01, 0x10, 0x02, 0x22, 0x00, 0x10, 0x40, + 0x18, 0x80, 0x01, 0x20, 0x10, 0x12, 0x0d, 0x0a, 0x07, 0x66, 0x5f, 0x70, + 0x6c, 0x61, 0x69, 0x6e, 0x10, 0x30, 0x20, 0x01, 0x18, 0x80, 0x80, 0x40, + 0x18, 0x01, 0x22, 0x16, 0x12, 0x0c, 0x10, 0x01, 0x20, 0xe7, 0x07, 0x28, + 0xe8, 0x07, 0x32, 0x02, 0x70, 0x6b, 0x22, 0x06, 0x66, 0x5f, 0x68, 0x6e, + 0x73, 0x77, 0x22, 0x1d, 0x08, 0x01, 0x12, 0x11, 0x08, 0x01, 0x10, 0x01, + 0x18, 0xe8, 0x07, 0x20, 0xcf, 0x0f, 0x28, 0xe8, 0x07, 0x32, 0x02, 0x70, + 0x6b, 0x22, 0x06, 0x66, 0x5f, 0x68, 0x6e, 0x73, 0x77, 0x2a, 0x06, 0x08, + 0x02, 0x1a, 0x02, 0x10, 0x01, 0x30, 0x05, 0x38, 0x09, 0x40, 0x0c, +}; + +template +std::string Golden(const uint8_t (&bytes)[N]) { + return std::string(reinterpret_cast(bytes), N); +} + +//! Decodes a golden buffer and re-encodes it, expecting identical bytes. +ManifestData ExpectStableRoundTrip(const std::string &golden, + const char *name) { + ManifestData data; + EXPECT_TRUE(ManifestCodec::Decode(golden, &data).ok()) << name; + + std::string reencoded; + EXPECT_TRUE(ManifestCodec::Encode(data, &reencoded).ok()) << name; + EXPECT_EQ(reencoded, golden) + << "re-encoding " << name + << " changed the bytes; the on-disk manifest format must stay stable"; + return data; +} + +} // namespace + +TEST(ManifestCodecGolden, EmptyManifest) { + auto data = ExpectStableRoundTrip(Golden(kGoldenEmpty), "empty"); + ASSERT_NE(data.schema, nullptr); + EXPECT_TRUE(data.schema->name().empty()); + EXPECT_TRUE(data.schema->fields().empty()); + EXPECT_FALSE(data.enable_mmap); + EXPECT_EQ(data.next_segment_id, 0u); +} + +TEST(ManifestCodecGolden, SimpleManifest) { + auto data = ExpectStableRoundTrip(Golden(kGoldenSimple), "simple"); + ASSERT_NE(data.schema, nullptr); + EXPECT_EQ(data.schema->name(), "simple"); + EXPECT_EQ(data.schema->max_doc_count_per_segment(), 100000u); + + const auto fields = data.schema->fields(); + ASSERT_EQ(fields.size(), 2u); + + EXPECT_EQ(fields[0]->name(), "pk"); + EXPECT_EQ(fields[0]->data_type(), DataType::STRING); + EXPECT_EQ(fields[0]->index_params(), nullptr); + + EXPECT_EQ(fields[1]->name(), "dense"); + EXPECT_EQ(fields[1]->data_type(), DataType::VECTOR_FP32); + EXPECT_EQ(fields[1]->dimension(), 128u); + const auto dense_params = fields[1]->index_params(); + ASSERT_NE(dense_params, nullptr); + ASSERT_EQ(dense_params->type(), IndexType::HNSW); + auto *hnsw = dynamic_cast(dense_params.get()); + ASSERT_NE(hnsw, nullptr); + EXPECT_EQ(hnsw->metric_type(), MetricType::IP); + EXPECT_EQ(hnsw->m(), 16); + EXPECT_EQ(hnsw->ef_construction(), 200); + + EXPECT_TRUE(data.enable_mmap); + ASSERT_EQ(data.persisted_segment_metas.size(), 1u); + EXPECT_EQ(data.persisted_segment_metas[0]->id(), 0u); + EXPECT_EQ(data.persisted_segment_metas[0]->persisted_blocks().size(), 1u); + EXPECT_EQ(data.next_segment_id, 1u); +} + +TEST(ManifestCodecGolden, AllIndexTypes) { + auto data = + ExpectStableRoundTrip(Golden(kGoldenAllIndexTypes), "all_index_types"); + ASSERT_NE(data.schema, nullptr); + + const auto fields = data.schema->fields(); + ASSERT_EQ(fields.size(), 9u); + + // Every oneof branch of IndexParams must survive the round trip. + const std::vector expected = { + IndexType::INVERT, IndexType::HNSW, IndexType::FLAT, + IndexType::IVF, IndexType::HNSW_RABITQ, IndexType::VAMANA, + IndexType::FTS, IndexType::DISKANN}; + for (size_t i = 0; i < expected.size(); ++i) { + const auto params = fields[i]->index_params(); + ASSERT_NE(params, nullptr) << "field " << fields[i]->name(); + EXPECT_EQ(params->type(), expected[i]) << "field " << fields[i]->name(); + } + // The last field intentionally carries no index params. + EXPECT_EQ(fields[8]->index_params(), nullptr); + + // Spot-check the only fixed32 field in the whole format. + const auto vamana_params = fields[5]->index_params(); + auto *vamana = dynamic_cast(vamana_params.get()); + ASSERT_NE(vamana, nullptr); + EXPECT_FLOAT_EQ(vamana->alpha(), 1.35f); + EXPECT_TRUE(vamana->saturate_graph()); + EXPECT_TRUE(vamana->use_id_map()); + EXPECT_FALSE(vamana->use_contiguous_memory()); + + // Repeated strings of the FTS branch. + const auto fts_params = fields[6]->index_params(); + auto *fts = dynamic_cast(fts_params.get()); + ASSERT_NE(fts, nullptr); + EXPECT_EQ(fts->tokenizer_name(), "jieba"); + ASSERT_EQ(fts->filters().size(), 2u); + EXPECT_EQ(fts->filters()[0], "lowercase"); + EXPECT_EQ(fts->filters()[1], "stop"); + + EXPECT_EQ(data.persisted_segment_metas.size(), 2u); + ASSERT_NE(data.writing_segment_meta, nullptr); + EXPECT_EQ(data.writing_segment_meta->id(), 2u); + EXPECT_TRUE(data.writing_segment_meta->has_writing_forward_block()); + EXPECT_EQ(data.id_map_path_suffix, 5u); + EXPECT_EQ(data.delete_snapshot_path_suffix, 9u); + EXPECT_EQ(data.next_segment_id, 12u); +} + +TEST(ManifestCodecGolden, UnknownFieldsAreIgnored) { + // Forward compatibility: a manifest written by a newer zvec may carry fields + // this build does not know about. They must be skipped silently. + std::string with_unknown = Golden(kGoldenSimple); + { + pbwire::Writer w(&with_unknown); + w.PutVarint(99, 12345); // unknown varint field + w.PutMessage(100, std::string("x")); // unknown length-delimited field + w.PutFloat(101, 2.5f); // unknown fixed32 field + } + + ManifestData data; + ASSERT_TRUE(ManifestCodec::Decode(with_unknown, &data).ok()); + ASSERT_NE(data.schema, nullptr); + EXPECT_EQ(data.schema->name(), "simple"); + EXPECT_EQ(data.schema->fields().size(), 2u); + EXPECT_EQ(data.next_segment_id, 1u); +} + +TEST(ManifestCodecGolden, TruncatedInputDoesNotCrash) { + const std::string golden = Golden(kGoldenAllIndexTypes); + // Truncation at every offset must either be reported as an error or produce + // a partial but well-formed result; it must never crash or hang. + for (size_t len = 1; len < golden.size(); ++len) { + ManifestData data; + ManifestCodec::Decode(std::string_view(golden.data(), len), &data); + } +} + +TEST(ManifestCodecGolden, CorruptInputIsRejected) { + // A length-delimited field claiming more bytes than are available. + std::string bad_len; + { + pbwire::Writer w(&bad_len); + w.PutMessage(2, std::string("payload")); + } + bad_len[1] = static_cast(0x7F); // overlong length + ManifestData data; + EXPECT_FALSE(ManifestCodec::Decode(bad_len, &data).ok()); + + // A varint that never terminates. + const std::string bad_varint(12, static_cast(0xFF)); + ManifestData data2; + EXPECT_FALSE(ManifestCodec::Decode(bad_varint, &data2).ok()); + + // Field number zero is illegal. + const std::string zero_field(1, static_cast(0x00)); + ManifestData data3; + EXPECT_FALSE(ManifestCodec::Decode(zero_field, &data3).ok()); +} + +TEST(ManifestCodecGolden, WireReaderRejectsGroups) { + // Wire types 3 and 4 (deprecated groups) are never produced by zvec.proto + // and must be treated as corrupt input rather than silently skipped. + for (uint8_t type : {3, 4}) { + std::string buf; + buf.push_back(static_cast((1 << 3) | type)); + pbwire::Reader r(buf); + EXPECT_FALSE(r.Next()); + EXPECT_FALSE(r.ok()); + } +} diff --git a/tests/db/index/common/manifest_codec_test.cc b/tests/db/index/common/manifest_codec_test.cc new file mode 100644 index 000000000..3e917bbd6 --- /dev/null +++ b/tests/db/index/common/manifest_codec_test.cc @@ -0,0 +1,496 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Cross-checks ManifestCodec against the protobuf library. +//! +//! While libprotobuf is still available, every message is encoded with both +//! implementations and the resulting bytes must be identical. This is what +//! guarantees that dropping the protobuf dependency does not change the +//! on-disk manifest format. +//! +//! These tests are removed together with the protobuf dependency; the +//! golden-file tests in manifest_codec_golden_test.cc take over from there. + +#include "db/index/common/manifest_codec.h" +#include +#include +#include +#include +#include "db/index/common/proto_converter.h" +#include "db/index/common/type_helper.h" + +using namespace zvec; + +namespace { + +//! Encodes index params with the new codec. +std::string EncodeNew(const IndexParams *params) { + std::string out; + ManifestCodec::EncodeIndexParams(params, &out); + return out; +} + +//! Encodes index params through the protobuf library. +std::string EncodePb(const IndexParams *params) { + return ProtoConverter::ToPb(params).SerializeAsString(); +} + +//! Asserts both implementations produce the same bytes, and that each side can +//! read what the other produced. +void ExpectIndexParamsRoundTrip(const IndexParams *params) { + const std::string bytes_new = EncodeNew(params); + const std::string bytes_pb = EncodePb(params); + ASSERT_EQ(bytes_new, bytes_pb) << "encoded bytes differ for index type " + << IndexTypeCodeBook::AsString(params->type()); + + // New decoder reads protobuf output. + auto from_pb_bytes = ManifestCodec::DecodeIndexParams(bytes_pb); + ASSERT_NE(from_pb_bytes, nullptr); + EXPECT_EQ(from_pb_bytes->type(), params->type()); + + // protobuf reads new encoder output. + proto::IndexParams pb; + ASSERT_TRUE(pb.ParseFromString(bytes_new)); + auto from_new_bytes = ProtoConverter::FromPb(pb); + ASSERT_NE(from_new_bytes, nullptr); + EXPECT_EQ(from_new_bytes->type(), params->type()); +} + +} // namespace + +TEST(ManifestCodecCrossCheck, InvertIndexParams) { + InvertIndexParams enabled(true); + ExpectIndexParamsRoundTrip(&enabled); + InvertIndexParams disabled(false); + ExpectIndexParamsRoundTrip(&disabled); +} + +TEST(ManifestCodecCrossCheck, HnswIndexParams) { + // All-default values exercise proto3 "skip defaults" plus the always + // present base/quantizer_param sub-messages. + HnswIndexParams defaults(MetricType::UNDEFINED, 0, 0, QuantizeType::UNDEFINED, + false, QuantizerParam()); + ExpectIndexParamsRoundTrip(&defaults); + + HnswIndexParams full(MetricType::COSINE, 32, 200, QuantizeType::INT8, true, + QuantizerParam(true)); + ExpectIndexParamsRoundTrip(&full); +} + +TEST(ManifestCodecCrossCheck, HnswRabitqIndexParams) { + // NOTE: this is the one index type whose quantizer_param sub-message stays + // absent on the wire. + HnswRabitqIndexParams defaults(MetricType::UNDEFINED, 0, 0, 0, 0, 0); + ExpectIndexParamsRoundTrip(&defaults); + + HnswRabitqIndexParams full(MetricType::L2, 7, 4096, 48, 300, 100000); + ExpectIndexParamsRoundTrip(&full); +} + +TEST(ManifestCodecCrossCheck, FlatIndexParams) { + FlatIndexParams defaults(MetricType::UNDEFINED, QuantizeType::UNDEFINED, + QuantizerParam()); + ExpectIndexParamsRoundTrip(&defaults); + + FlatIndexParams full(MetricType::IP, QuantizeType::FP16, + QuantizerParam(true)); + ExpectIndexParamsRoundTrip(&full); +} + +TEST(ManifestCodecCrossCheck, IVFIndexParams) { + IVFIndexParams defaults(MetricType::UNDEFINED, 0, 0, false, + QuantizeType::UNDEFINED, QuantizerParam()); + ExpectIndexParamsRoundTrip(&defaults); + + IVFIndexParams full(MetricType::L2, 2048, 25, true, QuantizeType::INT4, + QuantizerParam(true)); + ExpectIndexParamsRoundTrip(&full); +} + +TEST(ManifestCodecCrossCheck, DiskAnnIndexParams) { + DiskAnnIndexParams defaults(MetricType::UNDEFINED, 0, 0, 0, + QuantizeType::UNDEFINED, QuantizerParam()); + ExpectIndexParamsRoundTrip(&defaults); + + DiskAnnIndexParams full(MetricType::COSINE, 96, 128, 32, QuantizeType::INT8, + QuantizerParam(true)); + ExpectIndexParamsRoundTrip(&full); +} + +TEST(ManifestCodecCrossCheck, VamanaIndexParams) { + // alpha is the only fixed32 field in the whole format. + VamanaIndexParams defaults(MetricType::UNDEFINED, 0, 0, 0.0f, false, false, + false, QuantizeType::UNDEFINED, QuantizerParam()); + ExpectIndexParamsRoundTrip(&defaults); + + VamanaIndexParams full(MetricType::L2, 64, 128, 1.2f, true, true, true, + QuantizeType::RABITQ, QuantizerParam(true)); + ExpectIndexParamsRoundTrip(&full); + + VamanaIndexParams negative_alpha(MetricType::IP, 8, 16, -2.5f, false, false, + false, QuantizeType::UNDEFINED, + QuantizerParam()); + ExpectIndexParamsRoundTrip(&negative_alpha); +} + +TEST(ManifestCodecCrossCheck, FtsIndexParams) { + FtsIndexParams defaults("", {}, ""); + ExpectIndexParamsRoundTrip(&defaults); + + // Repeated strings, including an empty element which must still be written. + FtsIndexParams full("jieba", {"lowercase", "", "stop"}, R"({"k1":1.2})"); + ExpectIndexParamsRoundTrip(&full); + + auto decoded = ManifestCodec::DecodeIndexParams(EncodeNew(&full)); + ASSERT_NE(decoded, nullptr); + auto *fts = dynamic_cast(decoded.get()); + ASSERT_NE(fts, nullptr); + EXPECT_EQ(fts->tokenizer_name(), "jieba"); + ASSERT_EQ(fts->filters().size(), 3u); + EXPECT_EQ(fts->filters()[0], "lowercase"); + EXPECT_EQ(fts->filters()[1], ""); + EXPECT_EQ(fts->filters()[2], "stop"); + EXPECT_EQ(fts->extra_params(), R"({"k1":1.2})"); +} + +TEST(ManifestCodecCrossCheck, FieldSchema) { + // Without index params. + FieldSchema plain; + plain.set_name("id"); + plain.set_data_type(DataType::UINT64); + plain.set_nullable(false); + { + std::string bytes_new; + ManifestCodec::EncodeFieldSchema(plain, &bytes_new); + EXPECT_EQ(bytes_new, ProtoConverter::ToPb(plain).SerializeAsString()); + } + + // With index params and a non-zero dimension. + FieldSchema vector_field; + vector_field.set_name("dense"); + vector_field.set_data_type(DataType::VECTOR_FP32); + vector_field.set_dimension(128); + vector_field.set_nullable(true); + vector_field.set_index_params(std::make_shared( + MetricType::IP, 16, 100, QuantizeType::UNDEFINED, false, + QuantizerParam())); + { + std::string bytes_new; + ManifestCodec::EncodeFieldSchema(vector_field, &bytes_new); + const std::string bytes_pb = + ProtoConverter::ToPb(vector_field).SerializeAsString(); + ASSERT_EQ(bytes_new, bytes_pb); + + auto decoded = ManifestCodec::DecodeFieldSchema(bytes_pb); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->name(), "dense"); + EXPECT_EQ(decoded->data_type(), DataType::VECTOR_FP32); + EXPECT_EQ(decoded->dimension(), 128u); + EXPECT_TRUE(decoded->nullable()); + ASSERT_NE(decoded->index_params(), nullptr); + EXPECT_EQ(decoded->index_params()->type(), IndexType::HNSW); + } +} + +TEST(ManifestCodecCrossCheck, CollectionSchema) { + CollectionSchema empty; + { + std::string bytes_new; + ManifestCodec::EncodeCollectionSchema(empty, &bytes_new); + EXPECT_EQ(bytes_new, ProtoConverter::ToPb(empty).SerializeAsString()); + } + + CollectionSchema schema; + schema.set_name("test_collection"); + schema.set_max_doc_count_per_segment(100000); + + auto pk = std::make_shared(); + pk->set_name("pk"); + pk->set_data_type(DataType::STRING); + schema.add_field(pk); + + auto dense = std::make_shared(); + dense->set_name("dense"); + dense->set_data_type(DataType::VECTOR_FP32); + dense->set_dimension(64); + dense->set_index_params(std::make_shared( + MetricType::IP, QuantizeType::UNDEFINED, QuantizerParam())); + schema.add_field(dense); + + auto text = std::make_shared(); + text->set_name("text"); + text->set_data_type(DataType::STRING); + text->set_index_params(std::make_shared( + "standard", std::vector{"lowercase"}, "")); + schema.add_field(text); + + std::string bytes_new; + ManifestCodec::EncodeCollectionSchema(schema, &bytes_new); + const std::string bytes_pb = ProtoConverter::ToPb(schema).SerializeAsString(); + ASSERT_EQ(bytes_new, bytes_pb); + + auto decoded = ManifestCodec::DecodeCollectionSchema(bytes_pb); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->name(), "test_collection"); + EXPECT_EQ(decoded->max_doc_count_per_segment(), 100000u); + ASSERT_EQ(decoded->fields().size(), 3u); + EXPECT_EQ(decoded->fields()[1]->index_params()->type(), IndexType::FLAT); + EXPECT_EQ(decoded->fields()[2]->index_params()->type(), IndexType::FTS); +} + +TEST(ManifestCodecCrossCheck, BlockMeta) { + BlockMeta empty; + { + std::string bytes_new; + ManifestCodec::EncodeBlockMeta(empty, &bytes_new); + EXPECT_EQ(bytes_new, ProtoConverter::ToPb(empty).SerializeAsString()); + } + + BlockMeta meta; + meta.set_id(7); + meta.set_type(BlockType::VECTOR_INDEX_QUANTIZE); + meta.set_min_doc_id(1); + meta.set_max_doc_id(1ULL << 40); + meta.set_doc_count(4096); + meta.add_column("dense"); + meta.add_column("sparse"); + + std::string bytes_new; + ManifestCodec::EncodeBlockMeta(meta, &bytes_new); + const std::string bytes_pb = ProtoConverter::ToPb(meta).SerializeAsString(); + ASSERT_EQ(bytes_new, bytes_pb); + + auto decoded = ManifestCodec::DecodeBlockMeta(bytes_pb); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->id(), 7u); + EXPECT_EQ(decoded->type(), BlockType::VECTOR_INDEX_QUANTIZE); + EXPECT_EQ(decoded->min_doc_id(), 1u); + EXPECT_EQ(decoded->max_doc_id(), 1ULL << 40); + EXPECT_EQ(decoded->doc_count(), 4096u); + ASSERT_EQ(decoded->columns().size(), 2u); + EXPECT_EQ(decoded->columns()[0], "dense"); +} + +TEST(ManifestCodecCrossCheck, SegmentMeta) { + SegmentMeta empty(0); + { + std::string bytes_new; + ManifestCodec::EncodeSegmentMeta(empty, &bytes_new); + EXPECT_EQ(bytes_new, ProtoConverter::ToPb(empty).SerializeAsString()); + } + + SegmentMeta meta(3); + BlockMeta scalar; + scalar.set_id(0); + scalar.set_type(BlockType::SCALAR); + scalar.set_doc_count(10); + scalar.add_column("pk"); + meta.add_persisted_block(scalar); + + BlockMeta index_block; + index_block.set_id(1); + index_block.set_type(BlockType::VECTOR_INDEX); + index_block.add_column("dense"); + meta.add_persisted_block(index_block); + + BlockMeta writing; + writing.set_id(2); + writing.set_type(BlockType::SCALAR); + meta.set_writing_forward_block(writing); + + meta.add_indexed_vector_field("dense"); + meta.add_indexed_vector_field("sparse"); + + std::string bytes_new; + ManifestCodec::EncodeSegmentMeta(meta, &bytes_new); + const std::string bytes_pb = ProtoConverter::ToPb(meta).SerializeAsString(); + ASSERT_EQ(bytes_new, bytes_pb); + + auto decoded = ManifestCodec::DecodeSegmentMeta(bytes_pb); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->id(), 3u); + EXPECT_EQ(decoded->persisted_blocks().size(), 2u); + ASSERT_TRUE(decoded->has_writing_forward_block()); + EXPECT_EQ(decoded->writing_forward_block()->id(), 2u); + EXPECT_EQ(decoded->indexed_vector_fields().size(), 2u); +} + +namespace { + +//! Builds a manifest covering every oneof branch and both segment slots. +ManifestData MakeRichManifest() { + ManifestData data; + data.enable_mmap = true; + data.id_map_path_suffix = 5; + data.delete_snapshot_path_suffix = 9; + data.next_segment_id = 12; + + auto schema = std::make_shared(); + schema->set_name("rich"); + schema->set_max_doc_count_per_segment(1u << 20); + + struct FieldSpec { + const char *name; + DataType type; + uint32_t dimension; + IndexParams::Ptr params; + }; + const std::vector specs = { + {"f_invert", DataType::INT64, 0, + std::make_shared(true)}, + {"f_hnsw", DataType::VECTOR_FP32, 128, + std::make_shared(MetricType::L2, 16, 200, + QuantizeType::INT8, true, + QuantizerParam(true))}, + {"f_flat", DataType::VECTOR_FP16, 64, + std::make_shared(MetricType::IP, QuantizeType::FP16, + QuantizerParam())}, + {"f_ivf", DataType::VECTOR_INT8, 32, + std::make_shared(MetricType::COSINE, 512, 12, true, + QuantizeType::INT4, + QuantizerParam(true))}, + {"f_rabitq", DataType::VECTOR_FP32, 256, + std::make_shared(MetricType::L2, 5, 1024, 24, 150, + 50000)}, + {"f_vamana", DataType::VECTOR_FP32, 96, + std::make_shared(MetricType::L2, 48, 96, 1.35f, true, + false, true, QuantizeType::RABITQ, + QuantizerParam(true))}, + {"f_fts", DataType::STRING, 0, + std::make_shared( + "jieba", std::vector{"lowercase", "stop"}, "{}")}, + {"f_diskann", DataType::VECTOR_FP32, 512, + std::make_shared( + MetricType::L2, 64, 128, 16, QuantizeType::INT8, QuantizerParam())}, + {"f_plain", DataType::ARRAY_DOUBLE, 0, nullptr}, + }; + + for (const auto &spec : specs) { + auto field = std::make_shared(); + field->set_name(spec.name); + field->set_data_type(spec.type); + field->set_dimension(spec.dimension); + field->set_nullable(true); + if (spec.params) { + field->set_index_params(spec.params); + } + schema->add_field(field); + } + data.schema = schema; + + for (uint32_t id = 0; id < 2; ++id) { + auto segment = std::make_shared(id); + BlockMeta block; + block.set_id(id); + block.set_type(BlockType::SCALAR); + block.set_min_doc_id(id * 1000); + block.set_max_doc_id(id * 1000 + 999); + block.set_doc_count(1000); + block.add_column("pk"); + segment->add_persisted_block(block); + segment->add_indexed_vector_field("f_hnsw"); + data.persisted_segment_metas.push_back(segment); + } + + auto writing = std::make_shared(2); + BlockMeta writing_block; + writing_block.set_id(0); + writing_block.set_type(BlockType::SCALAR); + writing->set_writing_forward_block(writing_block); + data.writing_segment_meta = writing; + + return data; +} + +//! Serializes a manifest through the protobuf library, mirroring what +//! Version::Save used to do. +std::string EncodeManifestPb(const ManifestData &data) { + proto::Manifest manifest; + auto schema_pb = ProtoConverter::ToPb(*data.schema); + manifest.mutable_schema()->Swap(&schema_pb); + manifest.set_enable_mmap(data.enable_mmap); + for (const auto &meta : data.persisted_segment_metas) { + auto meta_pb = ProtoConverter::ToPb(*meta); + manifest.add_persisted_segment_metas()->Swap(&meta_pb); + } + if (data.writing_segment_meta) { + auto meta_pb = ProtoConverter::ToPb(*data.writing_segment_meta); + manifest.mutable_writing_segment_meta()->Swap(&meta_pb); + } + manifest.set_id_map_path_suffix(data.id_map_path_suffix); + manifest.set_delete_snapshot_path_suffix(data.delete_snapshot_path_suffix); + manifest.set_next_segment_id(data.next_segment_id); + return manifest.SerializeAsString(); +} + +} // namespace + +TEST(ManifestCodecCrossCheck, FullManifestBytesMatch) { + const ManifestData data = MakeRichManifest(); + + std::string bytes_new; + ASSERT_TRUE(ManifestCodec::Encode(data, &bytes_new).ok()); + const std::string bytes_pb = EncodeManifestPb(data); + ASSERT_EQ(bytes_new, bytes_pb); + + // protobuf must accept our output. + proto::Manifest parsed; + ASSERT_TRUE(parsed.ParseFromString(bytes_new)); + EXPECT_EQ(parsed.schema().fields_size(), 9); + EXPECT_TRUE(parsed.enable_mmap()); + EXPECT_EQ(parsed.next_segment_id(), 12u); + + // And we must accept protobuf's output, preserving every field. + ManifestData decoded; + ASSERT_TRUE(ManifestCodec::Decode(bytes_pb, &decoded).ok()); + EXPECT_TRUE(decoded.enable_mmap); + EXPECT_EQ(decoded.id_map_path_suffix, 5u); + EXPECT_EQ(decoded.delete_snapshot_path_suffix, 9u); + EXPECT_EQ(decoded.next_segment_id, 12u); + ASSERT_NE(decoded.schema, nullptr); + EXPECT_EQ(decoded.schema->name(), "rich"); + ASSERT_EQ(decoded.schema->fields().size(), 9u); + EXPECT_EQ(decoded.persisted_segment_metas.size(), 2u); + ASSERT_NE(decoded.writing_segment_meta, nullptr); + EXPECT_EQ(decoded.writing_segment_meta->id(), 2u); +} + +TEST(ManifestCodecCrossCheck, EmptyManifest) { + ManifestData data; + data.schema = std::make_shared(); + + std::string bytes_new; + ASSERT_TRUE(ManifestCodec::Encode(data, &bytes_new).ok()); + EXPECT_EQ(bytes_new, EncodeManifestPb(data)); +} + +TEST(ManifestCodecCrossCheck, IndexParamsOneofLastWins) { + // protobuf oneof semantics: when several branches appear on the wire the + // last one wins. Craft such a buffer by concatenating two encodings. + FlatIndexParams flat(MetricType::IP, QuantizeType::UNDEFINED, + QuantizerParam()); + HnswIndexParams hnsw(MetricType::L2, 16, 100, QuantizeType::UNDEFINED, false, + QuantizerParam()); + const std::string combined = EncodeNew(&flat) + EncodeNew(&hnsw); + + auto ours = ManifestCodec::DecodeIndexParams(combined); + ASSERT_NE(ours, nullptr); + + proto::IndexParams pb; + ASSERT_TRUE(pb.ParseFromString(combined)); + auto theirs = ProtoConverter::FromPb(pb); + ASSERT_NE(theirs, nullptr); + + EXPECT_EQ(ours->type(), theirs->type()); +} From b6ec367815a6365ab7b0c927e680ea6f58aee7be Mon Sep 17 00:00:00 2001 From: lc285652 Date: Thu, 30 Jul 2026 16:36:54 +0800 Subject: [PATCH 08/10] refactor(db): persist the manifest through ManifestCodec Version::Load/Save no longer build an intermediate proto::Manifest; they read and write the manifest bytes with ManifestCodec instead. The on-disk format is unchanged - manifest_codec_test.cc verifies byte-for-byte equality with the protobuf implementation, and manifest_codec_golden_test.cc pins the format independently. Manifests are a few kilobytes, so the file is read/written in one go rather than streamed. ProtoConverter is intentionally kept for now: it is what the cross-check tests compare against. It is removed together with the protobuf dependency. --- src/db/index/common/version_manager.cc | 74 ++++++++++++-------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/src/db/index/common/version_manager.cc b/src/db/index/common/version_manager.cc index 1ca095d85..de854bd52 100644 --- a/src/db/index/common/version_manager.cc +++ b/src/db/index/common/version_manager.cc @@ -20,15 +20,15 @@ #include #include #include +#include #include -#include #include #include #include #include #include "db/common/file_helper.h" #include "db/common/typedef.h" -#include "db/index/common/proto_converter.h" +#include "db/index/common/manifest_codec.h" #include "db/index/common/type_helper.h" namespace zvec { @@ -41,35 +41,36 @@ Status Version::Load(const std::string &path, Version *version) { return Status::InternalError("Failed to open file"); } - proto::Manifest manifest; + // Manifests are small (kilobytes), so reading the whole file at once keeps + // the decoder simple. + std::ostringstream buffer; + buffer << ifs.rdbuf(); + const std::string encoded = buffer.str(); - if (!manifest.ParseFromIstream(&ifs)) { + ManifestData manifest; + auto status = ManifestCodec::Decode(encoded, &manifest); + if (!status.ok()) { LOG_ERROR("Failed to parse manifest from file: %s", path.c_str()); return Status::InternalError("Failed to parse manifest"); } - CollectionSchema::Ptr schema = ProtoConverter::FromPb(manifest.schema()); - version->set_schema(*schema); + version->set_schema(*manifest.schema); - version->set_enable_mmap(manifest.enable_mmap()); + version->set_enable_mmap(manifest.enable_mmap); - for (int i = 0; i < manifest.persisted_segment_metas_size(); ++i) { - SegmentMeta::Ptr meta = - ProtoConverter::FromPb(manifest.persisted_segment_metas(i)); + for (auto &meta : manifest.persisted_segment_metas) { version->add_persisted_segment_meta(meta); } - if (manifest.has_writing_segment_meta()) { - SegmentMeta::Ptr meta = - ProtoConverter::FromPb(manifest.writing_segment_meta()); - version->reset_writing_segment_meta(meta); + if (manifest.writing_segment_meta) { + version->reset_writing_segment_meta(manifest.writing_segment_meta); } - version->set_id_map_path_suffix(manifest.id_map_path_suffix()); + version->set_id_map_path_suffix(manifest.id_map_path_suffix); version->set_delete_snapshot_path_suffix( - manifest.delete_snapshot_path_suffix()); + manifest.delete_snapshot_path_suffix); - version->set_next_segment_id(manifest.next_segment_id()); + version->set_next_segment_id(manifest.next_segment_id); return Status::OK(); } @@ -83,31 +84,24 @@ Status Version::Save(const std::string &path, const Version &version) { return Status::InternalError("Failed to open file: %s", path.c_str()); } - proto::Manifest manifest; - - // set schema - auto schema = ProtoConverter::ToPb(version.schema()); - manifest.mutable_schema()->Swap(&schema); - - manifest.set_enable_mmap(version.enable_mmap()); - - // set segments meta - for (auto &meta : version.persisted_segment_metas()) { - auto meta_pb = ProtoConverter::ToPb(*meta); - manifest.add_persisted_segment_metas()->Swap(&meta_pb); - } - - if (version.writing_segment_meta()) { - auto meta_pb = ProtoConverter::ToPb(*version.writing_segment_meta()); - manifest.mutable_writing_segment_meta()->Swap(&meta_pb); + ManifestData manifest; + manifest.schema = std::make_shared(version.schema()); + manifest.enable_mmap = version.enable_mmap(); + manifest.persisted_segment_metas = version.persisted_segment_metas(); + manifest.writing_segment_meta = version.writing_segment_meta(); + manifest.id_map_path_suffix = version.id_map_path_suffix(); + manifest.delete_snapshot_path_suffix = version.delete_snapshot_path_suffix(); + manifest.next_segment_id = version.next_segment_id(); + + std::string encoded; + auto status = ManifestCodec::Encode(manifest, &encoded); + if (!status.ok()) { + LOG_ERROR("Failed to serialize manifest to file: %s", path.c_str()); + return Status::InternalError("Failed to serialize manifest to file"); } - manifest.set_id_map_path_suffix(version.id_map_path_suffix()); - manifest.set_delete_snapshot_path_suffix( - version.delete_snapshot_path_suffix()); - manifest.set_next_segment_id(version.next_segment_id()); - - if (!manifest.SerializeToOstream(&ofs)) { + ofs.write(encoded.data(), static_cast(encoded.size())); + if (!ofs.good()) { LOG_ERROR("Failed to serialize manifest to file: %s", path.c_str()); return Status::InternalError("Failed to serialize manifest to file"); } From d8cd772785820000d37bde02c524f60751ad972f Mon Sep 17 00:00:00 2001 From: lc285652 Date: Thu, 30 Jul 2026 17:32:10 +0800 Subject: [PATCH 09/10] build: drop the protobuf/protoc dependency The manifest is now read and written entirely by ManifestCodec (added and cross-checked against protobuf in the previous commits), so libprotobuf and protoc are no longer needed. Removed: - thirdparty/protobuf submodule and its build integration - src/db/proto compilation (cc_proto_library, zvec.pb.cc, libprotobuf-lite link and zvec_proto target/deps across src and tests) - ProtoConverter and the cross-check test that depended on libprotobuf - the "build host protoc" stage from the Android and iOS CI workflows and the build_android.sh / build_ios.sh scripts, along with the now-unused GLOBAL_CC_PROTOBUF_PROTOC option. Cross-compiling no longer needs a host protoc, simplifying those pipelines noticeably. Kept: - src/db/proto/zvec.proto as the authoritative documentation of the on-disk manifest format - cc_proto_library in cmake/bazel.cmake as a generic helper (no longer used by zvec itself) The manifest on-disk format is unchanged; manifest_codec_golden_test.cc keeps guarding it. Full C++ test suite (156 tests) passes locally. --- .github/workflows/04-android-build.yml | 46 +- .github/workflows/06-ios-build.yml | 20 - .gitmodules | 3 - CMakeLists.txt | 2 +- scripts/build_android.sh | 44 +- scripts/build_ios.sh | 29 +- src/db/CMakeLists.txt | 12 +- src/db/index/CMakeLists.txt | 1 - src/db/index/common/proto_converter.cc | 495 ---------------- src/db/index/common/proto_converter.h | 82 --- tests/db/CMakeLists.txt | 1 - tests/db/crash_recovery/CMakeLists.txt | 1 - tests/db/index/CMakeLists.txt | 1 - .../index/common/db_proto_converter_test.cc | 550 ------------------ tests/db/index/common/manifest_codec_test.cc | 496 ---------------- tests/db/index/common/version_manager_test.cc | 25 +- tests/db/sqlengine/CMakeLists.txt | 1 - thirdparty/CMakeLists.txt | 1 - thirdparty/protobuf/CMakeLists.txt | 76 --- thirdparty/protobuf/protobuf-3.21.12 | 1 - thirdparty/protobuf/protobuf.windows.patch | 15 - 21 files changed, 37 insertions(+), 1865 deletions(-) delete mode 100644 src/db/index/common/proto_converter.cc delete mode 100644 src/db/index/common/proto_converter.h delete mode 100644 tests/db/index/common/db_proto_converter_test.cc delete mode 100644 tests/db/index/common/manifest_codec_test.cc delete mode 100644 thirdparty/protobuf/CMakeLists.txt delete mode 160000 thirdparty/protobuf/protobuf-3.21.12 delete mode 100644 thirdparty/protobuf/protobuf.windows.patch diff --git a/.github/workflows/04-android-build.yml b/.github/workflows/04-android-build.yml index aa44084c9..7d6f6413d 100644 --- a/.github/workflows/04-android-build.yml +++ b/.github/workflows/04-android-build.yml @@ -71,33 +71,8 @@ jobs: sdkmanager --install "system-images;android-${{ matrix.api }};google_apis_playstore;x86_64" 2>/dev/null || \ sdkmanager --install "system-images;android-${{ matrix.api }};default;x86_64" - # ── Step 1: build host protoc (using HOST compiler, NOT NDK) ─────── - - name: Cache host protoc - uses: actions/cache@v6 - with: - path: build_host - key: ${{ runner.os }}-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }} - restore-keys: | - ${{ runner.os }}-host-protoc- - - - name: 'Step 1: Build host protoc' - shell: bash - run: | - if [ ! -f "build_host/bin/protoc" ]; then - git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true - cmake -S . -B build_host \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_TOOLCHAIN_FILE="" \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -G Ninja - cmake --build build_host --target protoc --parallel - else - echo "Using cached host protoc" - fi - - # ── Step 2: cross-compile zvec + tests for Android ───────────────── - - name: 'Step 2: Cross-compile zvec and tests' + # ── Step 1: cross-compile zvec + tests for Android ───────────────────────── + - name: 'Step 1: Cross-compile zvec and tests' shell: bash env: BUILD_DIR: build_android_${{ matrix.abi }} @@ -123,7 +98,6 @@ jobs: -DAUTO_DETECT_ARCH=OFF \ -DENABLE_WERROR=ON \ -DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \ - -DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -150,8 +124,8 @@ jobs: echo "Building ${#TEST_NAMES[@]} test executables..." ninja -C "$BUILD_DIR" -j$(nproc) "${TEST_NAMES[@]}" - # ── Step 3: start emulator ───────────────────────────────────────── - - name: 'Step 3: Start Android emulator' + # ── Step 2: start emulator ───────────────────────────────────────── + - name: 'Step 2: Start Android emulator' shell: bash run: | AVD_NAME="zvec_test_avd" @@ -244,8 +218,8 @@ jobs: echo "Device ABI: $(adb shell getprop ro.product.cpu.abi | tr -d '\r')" echo "ABI list : $(adb shell getprop ro.product.cpu.abilist | tr -d '\r')" - # ── Step 4: run unit tests on emulator ───────────────────────────── - - name: 'Step 4: Run unit tests on emulator' + # ── Step 3: run unit tests on emulator ───────────────────────────── + - name: 'Step 3: Run unit tests on emulator' shell: bash env: BUILD_DIR: build_android_${{ matrix.abi }} @@ -385,8 +359,8 @@ jobs: fi echo "All tests passed!" - # ── Step 5: build and run examples ───────────────────────────────── - - name: 'Step 5: Build and run examples' + # ── Step 4: build and run examples ───────────────────────────────── + - name: 'Step 4: Build and run examples' shell: bash env: BUILD_DIR: build_android_${{ matrix.abi }} @@ -405,8 +379,8 @@ jobs: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build "$EXAMPLES_BUILD" --parallel - # Reuse the shared-library directory from Step 4; push again in - # case Step 4 was skipped or the directory was cleaned. + # Reuse the shared-library directory from Step 3; push again in + # case Step 3 was skipped or the directory was cleaned. DEVICE_LIB_DIR="/data/local/tmp/zvec_tests/lib" adb shell "mkdir -p $DEVICE_LIB_DIR" 2>/dev/null || true SO_COUNT=0 diff --git a/.github/workflows/06-ios-build.yml b/.github/workflows/06-ios-build.yml index 98a444f62..f5b95974b 100644 --- a/.github/workflows/06-ios-build.yml +++ b/.github/workflows/06-ios-build.yml @@ -37,25 +37,6 @@ jobs: key: ios-${{ matrix.platform }} max-size: 150M - - name: Cache host protoc build - uses: actions/cache@v6 - with: - path: build_host - key: macos-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }} - restore-keys: | - macos-host-protoc- - - - name: Build host protoc - run: | - if [ ! -f "build_host/bin/protoc" ]; then - cmake -S . -B build_host -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - cmake --build build_host --target protoc --parallel $(sysctl -n hw.ncpu) - else - echo "Using cached host protoc" - fi - - name: Configure and Build run: | git submodule foreach --recursive 'git stash --include-untracked' || true @@ -73,7 +54,6 @@ jobs: -DBUILD_TOOLS=OFF \ -DENABLE_WERROR=ON \ -DCMAKE_INSTALL_PREFIX="./install" \ - -DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \ -DIOS=ON \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ diff --git a/.gitmodules b/.gitmodules index f919c733b..2af5885e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -24,9 +24,6 @@ path = thirdparty/glog/glog-0.5.0 url = https://github.com/google/glog.git ignore = dirty -[submodule "thirdparty/protobuf/protobuf-3.21.12"] - path = thirdparty/protobuf/protobuf-3.21.12 - url = https://github.com/protocolbuffers/protobuf.git [submodule "thirdparty/lz4/lz4-1.9.4"] path = thirdparty/lz4/lz4-1.9.4 url = https://github.com/lz4/lz4.git diff --git a/CMakeLists.txt b/CMakeLists.txt index b5442d926..da9ad07b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,7 +162,7 @@ cc_directories(src) cc_directories(tests) -add_custom_target(clang_tidy_deps DEPENDS zvec_proto ARROW.BUILD glog gflags Lz4.BUILD) +add_custom_target(clang_tidy_deps DEPENDS ARROW.BUILD glog gflags Lz4.BUILD) if(BUILD_TOOLS) cc_directories(tools) diff --git a/scripts/build_android.sh b/scripts/build_android.sh index a1785a4d1..54337f92f 100755 --- a/scripts/build_android.sh +++ b/scripts/build_android.sh @@ -19,7 +19,7 @@ API_LEVEL=${1:-35} BUILD_TYPE=${2:-"Release"} CORE_COUNT=$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4) -# ── Android SDK paths (set later, after host protoc build) ─────────── +# ── Android SDK paths ──────────────────────────────────────────────── ANDROID_SDK_ROOT=${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk} ANDROID_NDK_HOME=${ANDROID_NDK_HOME:-$(ls -d "$ANDROID_SDK_ROOT/ndk/"* 2>/dev/null | sort -V | tail -1)} @@ -39,40 +39,9 @@ if [ ! -d "$ANDROID_NDK_HOME" ]; then exit 1 fi -# ── Step 1: build host protoc (using HOST compiler, NOT NDK) ───────── +# ── Step 1: cross-compile zvec + tests for Android ─────────────────── echo "" -echo ">>> Step 1: Building protoc for host..." -HOST_BUILD_DIR="build_host" - -git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true - -if [ ! -f "$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc" ]; then - # Explicitly avoid NDK toolchain for host build - cmake -S . -B "$HOST_BUILD_DIR" \ - -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ - -DCMAKE_TOOLCHAIN_FILE="" \ - -G Ninja - cmake --build "$HOST_BUILD_DIR" --target protoc -j"$CORE_COUNT" -else - echo " (cached — skipping)" -fi -PROTOC_EXECUTABLE=$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc -echo ">>> Step 1: Done (protoc=$PROTOC_EXECUTABLE)" - -# ── Now export Android env vars for cross-compilation ──────────────── -export ANDROID_SDK_ROOT -export ANDROID_HOME=$ANDROID_SDK_ROOT -export ANDROID_NDK_HOME -export CMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake - -export PATH=$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin -export PATH=$PATH:$ANDROID_SDK_ROOT/platform-tools -export PATH=$PATH:$ANDROID_SDK_ROOT/emulator -export PATH=$PATH:$ANDROID_NDK_HOME - -# ── Step 2: cross-compile zvec + tests for Android ─────────────────── -echo "" -echo ">>> Step 2: Cross-compiling zvec for Android ($ABI, API $API_LEVEL)..." +echo ">>> Step 1: Cross-compiling zvec for Android ($ABI, API $API_LEVEL)..." # reset thirdparty so the cross toolchain can patch cleanly git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true @@ -94,7 +63,6 @@ cmake -S . -B "$BUILD_DIR" -G Ninja \ -DENABLE_NATIVE=OFF \ -DAUTO_DETECT_ARCH=OFF \ -DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \ - -DGLOBAL_CC_PROTOBUF_PROTOC="$PROTOC_EXECUTABLE" echo " Building library..." cmake --build "$BUILD_DIR" -j"$CORE_COUNT" @@ -130,11 +98,11 @@ fi echo " Building ${#TEST_NAMES[@]} test executables..." ninja -C "$BUILD_DIR" -j"$CORE_COUNT" "${TEST_NAMES[@]}" -echo ">>> Step 2: Done" +echo ">>> Step 1: Done" -# ── Step 3: collect test binaries ──────────────────────────────────── +# ── Step 2: collect test binaries ──────────────────────────────────── echo "" -echo ">>> Step 3: Collecting test binaries..." +echo ">>> Step 2: Collecting test binaries..." TEST_BINS=() for name in "${TEST_NAMES[@]}"; do diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh index 6a6340554..43cef564a 100755 --- a/scripts/build_ios.sh +++ b/scripts/build_ios.sh @@ -34,24 +34,8 @@ echo " Architecture: $ARCH" echo " Build Type: $BUILD_TYPE" echo " iOS Deployment Target: $IOS_DEPLOYMENT_TARGET" -# step1: use host env to compile protoc -echo "step1: building protoc for host..." - -git submodule foreach --recursive 'git stash --include-untracked' - -HOST_BUILD_DIR="build_host" -mkdir -p $HOST_BUILD_DIR -cd $HOST_BUILD_DIR - -cmake -DCMAKE_BUILD_TYPE="$BUILD_TYPE" .. -make -j protoc -PROTOC_EXECUTABLE=$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc -cd $CURRENT_DIR - -echo "step1: Done!!!" - -# step2: cross build zvec for iOS -echo "step2: building zvec for iOS..." +# step1: cross build zvec for iOS +echo "step1: building zvec for iOS..." # reset thirdparty directory git submodule foreach --recursive 'git stash --include-untracked' @@ -79,7 +63,6 @@ cmake \ -DBUILD_PYTHON_BINDINGS=OFF \ -DBUILD_TOOLS=OFF \ -DCMAKE_INSTALL_PREFIX="./install" \ - -DGLOBAL_CC_PROTOBUF_PROTOC=$PROTOC_EXECUTABLE \ -DIOS=ON \ ../ @@ -87,11 +70,11 @@ echo "building..." CORE_COUNT=$(sysctl -n hw.ncpu) make -j$CORE_COUNT -echo "step2: Done!!!" +echo "step1: Done!!!" -# step3: build and run all unit tests on simulator +# step2: build and run all unit tests on simulator if [ "$PLATFORM" != "OS" ]; then - echo "step3: building and running unit tests on simulator..." + echo "step2: building and running unit tests on simulator..." make -j$CORE_COUNT unittest @@ -163,7 +146,7 @@ sys.exit(1) exit 1 fi - echo "step3: Done!!!" + echo "step2: Done!!!" else echo "Skipping tests (device build cannot run on simulator)" fi diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt index a2fb0052c..c6603bab2 100644 --- a/src/db/CMakeLists.txt +++ b/src/db/CMakeLists.txt @@ -1,11 +1,9 @@ include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) include(${PROJECT_ROOT_DIR}/cmake/option.cmake) -cc_proto_library( - NAME zvec_proto STATIC - SRCS proto/*.proto - PROTOROOT ./ -) +# NOTE: proto/zvec.proto is no longer compiled. It is kept as the +# authoritative documentation of the manifest on-disk format, which is now +# read and written by src/db/index/common/manifest_codec.{h,cc}. cc_directory(common) cc_directory(index) @@ -40,7 +38,7 @@ endif() cc_library( NAME zvec STATIC STRICT SRCS_NO_GLOB PACKED ${ZVEC_DB_LIBRARY_OPTIONS} - SRCS ${ALL_DB_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/proto/zvec.pb.cc + SRCS ${ALL_DB_SRCS} INCS . ${CMAKE_CURRENT_BINARY_DIR} PUBINCS ${PROJECT_ROOT_DIR}/src/include LIBS @@ -50,7 +48,6 @@ cc_library( roaring rocksdb antlr4 - libprotobuf-lite FastPFOR cppjieba snowball @@ -59,6 +56,5 @@ cc_library( Arrow::arrow_compute Arrow::arrow_acero utf8proc - DEPS zvec_proto VERSION "${PROXIMA_ZVEC_VERSION}" ) diff --git a/src/db/index/CMakeLists.txt b/src/db/index/CMakeLists.txt index eb3f42386..a5816646e 100644 --- a/src/db/index/CMakeLists.txt +++ b/src/db/index/CMakeLists.txt @@ -21,7 +21,6 @@ cc_library( NAME zvec_index STATIC STRICT SRCS *.cc segment/*.cc column/vector_column/*.cc column/inverted_column/*.cc column/fts_column/*.cc column/fts_column/tokenizer/*.cc column/fts_column/posting/*.cc column/fts_column/iterator/*.cc storage/*.cc storage/wal/*.cc common/*.cc LIBS zvec_common - zvec_proto rocksdb core_interface Arrow::arrow_static diff --git a/src/db/index/common/proto_converter.cc b/src/db/index/common/proto_converter.cc deleted file mode 100644 index f0014479d..000000000 --- a/src/db/index/common/proto_converter.cc +++ /dev/null @@ -1,495 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "proto_converter.h" -#include "db/index/common/manifest_enum.h" - -namespace zvec { -namespace { - -// The CodeBooks in type_helper.h speak the wire enums of the manifest format -// (see manifest_enum.h). These adapters bridge them to the protobuf generated -// enums, whose numeric values are identical by construction. -// -// NOTE: this whole file goes away once Version::Save/Load switch to -// ManifestCodec; it is kept only so that the cross-check tests can compare -// both implementations. -template -inline ProtoEnum ToProtoEnum(WireEnum value) { - return static_cast(wire::ToNumber(value)); -} - -template -inline WireEnum ToWireEnum(ProtoEnum value) { - return wire::FromNumber(static_cast(value)); -} - -struct PbMetric { - static proto::MetricType Get(MetricType type) { - return ToProtoEnum(MetricTypeCodeBook::Get(type)); - } - static MetricType Get(proto::MetricType type) { - return MetricTypeCodeBook::Get(ToWireEnum(type)); - } -}; - -struct PbQuantize { - static proto::QuantizeType Get(QuantizeType type) { - return ToProtoEnum(QuantizeTypeCodeBook::Get(type)); - } - static QuantizeType Get(proto::QuantizeType type) { - return QuantizeTypeCodeBook::Get(ToWireEnum(type)); - } -}; - -struct PbDataType { - static proto::DataType Get(DataType type) { - return ToProtoEnum(DataTypeCodeBook::Get(type)); - } - static DataType Get(proto::DataType type) { - return DataTypeCodeBook::Get(ToWireEnum(type)); - } -}; - -struct PbBlockType { - static proto::BlockType Get(BlockType type) { - return ToProtoEnum(BlockTypeCodeBook::Get(type)); - } - static BlockType Get(proto::BlockType type) { - return BlockTypeCodeBook::Get(ToWireEnum(type)); - } -}; - -} // namespace - -HnswIndexParams::OPtr ProtoConverter::FromPb( - const proto::HnswIndexParams ¶ms_pb) { - bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); - auto params = std::make_shared( - PbMetric::Get(params_pb.base().metric_type()), params_pb.m(), - params_pb.ef_construction(), - PbQuantize::Get(params_pb.base().quantize_type()), - params_pb.use_contiguous_memory(), QuantizerParam(enable_rotate)); - - return params; -} - -proto::HnswIndexParams ProtoConverter::ToPb(const HnswIndexParams *params) { - proto::HnswIndexParams params_pb; - params_pb.mutable_base()->set_metric_type( - PbMetric::Get(params->metric_type())); - params_pb.mutable_base()->set_quantize_type( - PbQuantize::Get(params->quantize_type())); - params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( - params->quantizer_param().enable_rotate()); - params_pb.set_ef_construction(params->ef_construction()); - params_pb.set_m(params->m()); - params_pb.set_use_contiguous_memory(params->use_contiguous_memory()); - return params_pb; -} - -// HnswRabitqIndexParams -HnswRabitqIndexParams::OPtr ProtoConverter::FromPb( - const proto::HnswRabitqIndexParams ¶ms_pb) { - auto params = std::make_shared( - PbMetric::Get(params_pb.base().metric_type()), params_pb.total_bits(), - params_pb.num_clusters(), params_pb.m(), params_pb.ef_construction(), - params_pb.sample_count()); - - return params; -} - -proto::HnswRabitqIndexParams ProtoConverter::ToPb( - const HnswRabitqIndexParams *params) { - proto::HnswRabitqIndexParams params_pb; - params_pb.mutable_base()->set_metric_type( - PbMetric::Get(params->metric_type())); - params_pb.mutable_base()->set_quantize_type( - PbQuantize::Get(params->quantize_type())); - params_pb.set_m(params->m()); - params_pb.set_ef_construction(params->ef_construction()); - params_pb.set_total_bits(params->total_bits()); - params_pb.set_num_clusters(params->num_clusters()); - params_pb.set_sample_count(params->sample_count()); - return params_pb; -} - -// FlatIndexParams -FlatIndexParams::OPtr ProtoConverter::FromPb( - const proto::FlatIndexParams ¶ms_pb) { - bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); - return std::make_shared( - PbMetric::Get(params_pb.base().metric_type()), - PbQuantize::Get(params_pb.base().quantize_type()), - QuantizerParam(enable_rotate)); -} - -proto::FlatIndexParams ProtoConverter::ToPb(const FlatIndexParams *params) { - proto::FlatIndexParams params_pb; - params_pb.mutable_base()->set_metric_type( - PbMetric::Get(params->metric_type())); - params_pb.mutable_base()->set_quantize_type( - PbQuantize::Get(params->quantize_type())); - params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( - params->quantizer_param().enable_rotate()); - return params_pb; -} - -// IVFIndexParams -IVFIndexParams::OPtr ProtoConverter::FromPb( - const proto::IVFIndexParams ¶ms_pb) { - bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); - return std::make_shared( - PbMetric::Get(params_pb.base().metric_type()), params_pb.n_list(), - params_pb.n_iters(), params_pb.use_soar(), - PbQuantize::Get(params_pb.base().quantize_type()), - QuantizerParam(enable_rotate)); -} - -proto::IVFIndexParams ProtoConverter::ToPb(const IVFIndexParams *params) { - proto::IVFIndexParams params_pb; - params_pb.mutable_base()->set_metric_type( - PbMetric::Get(params->metric_type())); - params_pb.mutable_base()->set_quantize_type( - PbQuantize::Get(params->quantize_type())); - params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( - params->quantizer_param().enable_rotate()); - params_pb.set_n_list(params->n_list()); - params_pb.set_n_iters(params->n_iters()); - params_pb.set_use_soar(params->use_soar()); - return params_pb; -} - -// VamanaIndexParams -VamanaIndexParams::OPtr ProtoConverter::FromPb( - const proto::VamanaIndexParams ¶ms_pb) { - bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); - return std::make_shared( - PbMetric::Get(params_pb.base().metric_type()), params_pb.max_degree(), - params_pb.search_list_size(), params_pb.alpha(), - params_pb.saturate_graph(), params_pb.use_contiguous_memory(), - params_pb.use_id_map(), PbQuantize::Get(params_pb.base().quantize_type()), - QuantizerParam(enable_rotate)); -} - -proto::VamanaIndexParams ProtoConverter::ToPb(const VamanaIndexParams *params) { - proto::VamanaIndexParams params_pb; - params_pb.mutable_base()->set_metric_type( - PbMetric::Get(params->metric_type())); - params_pb.mutable_base()->set_quantize_type( - PbQuantize::Get(params->quantize_type())); - params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( - params->quantizer_param().enable_rotate()); - params_pb.set_max_degree(params->max_degree()); - params_pb.set_search_list_size(params->search_list_size()); - params_pb.set_alpha(params->alpha()); - params_pb.set_saturate_graph(params->saturate_graph()); - params_pb.set_use_contiguous_memory(params->use_contiguous_memory()); - params_pb.set_use_id_map(params->use_id_map()); - return params_pb; -} - -// InvertIndexParams -InvertIndexParams::OPtr ProtoConverter::FromPb( - const proto::InvertIndexParams ¶ms_pb) { - auto params = std::make_shared( - params_pb.enable_range_optimization()); - - return params; -} - -proto::InvertIndexParams ProtoConverter::ToPb(const InvertIndexParams *params) { - proto::InvertIndexParams params_pb; - params_pb.set_enable_range_optimization(params->enable_range_optimization()); - return params_pb; -} - -// DiskAnnIndexParams -DiskAnnIndexParams::OPtr ProtoConverter::FromPb( - const proto::DiskAnnIndexParams ¶ms_pb) { - bool enable_rotate = params_pb.base().quantizer_param().enable_rotate(); - return std::make_shared( - PbMetric::Get(params_pb.base().metric_type()), params_pb.max_degree(), - params_pb.list_size(), params_pb.pq_chunk_num(), - PbQuantize::Get(params_pb.base().quantize_type()), - QuantizerParam(enable_rotate)); -} - -proto::DiskAnnIndexParams ProtoConverter::ToPb( - const DiskAnnIndexParams *params) { - proto::DiskAnnIndexParams params_pb; - params_pb.mutable_base()->set_metric_type( - PbMetric::Get(params->metric_type())); - params_pb.mutable_base()->set_quantize_type( - PbQuantize::Get(params->quantize_type())); - params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate( - params->quantizer_param().enable_rotate()); - params_pb.set_max_degree(params->max_degree()); - params_pb.set_list_size(params->list_size()); - params_pb.set_pq_chunk_num(params->pq_chunk_num()); - return params_pb; -} - -// FtsIndexParams -FtsIndexParams::Ptr ProtoConverter::FromPb( - const proto::FtsIndexParams ¶ms_pb) { - std::vector filters; - filters.reserve(params_pb.filters_size()); - for (const auto &filter : params_pb.filters()) { - filters.push_back(filter); - } - return std::make_shared( - params_pb.tokenizer_name(), std::move(filters), params_pb.extra_params()); -} - -proto::FtsIndexParams ProtoConverter::ToPb(const FtsIndexParams *params) { - proto::FtsIndexParams params_pb; - params_pb.set_tokenizer_name(params->tokenizer_name()); - for (const auto &filter : params->filters()) { - params_pb.add_filters(filter); - } - params_pb.set_extra_params(params->extra_params()); - return params_pb; -} - -// FieldSchema -FieldSchema::Ptr ProtoConverter::FromPb(const proto::FieldSchema &schema_pb) { - auto schema = std::make_shared(); - - schema->set_name(schema_pb.name()); - schema->set_data_type(PbDataType::Get(schema_pb.data_type())); - schema->set_dimension(schema_pb.dimension()); - schema->set_nullable(schema_pb.nullable()); - if (schema_pb.has_index_params()) { - schema->set_index_params(ProtoConverter::FromPb(schema_pb.index_params())); - } - return schema; -} -proto::FieldSchema ProtoConverter::ToPb(const FieldSchema &schema) { - proto::FieldSchema schema_pb; - - schema_pb.set_name(schema.name()); - schema_pb.set_data_type(PbDataType::Get(schema.data_type())); - schema_pb.set_dimension(schema.dimension()); - schema_pb.set_nullable(schema.nullable()); - auto index_params = schema.index_params(); - if (index_params) { - auto index_params_pb = schema_pb.mutable_index_params(); - index_params_pb->MergeFrom(ProtoConverter::ToPb(index_params.get())); - } - return schema_pb; -} - -// CollectionSchema -CollectionSchema::Ptr ProtoConverter::FromPb( - const proto::CollectionSchema &schema_pb) { - CollectionSchema::Ptr schema = std::make_shared(); - - schema->set_name(schema_pb.name()); - - for (auto &column_schema_pb : schema_pb.fields()) { - FieldSchema::Ptr column_schema = ProtoConverter::FromPb(column_schema_pb); - schema->add_field(column_schema); - } - - schema->set_max_doc_count_per_segment(schema_pb.max_doc_count_per_segment()); - - return schema; -} - -proto::CollectionSchema ProtoConverter::ToPb(const CollectionSchema &schema) { - proto::CollectionSchema schema_pb; - schema_pb.set_name(schema.name()); - for (auto &column_schema : schema.fields()) { - proto::FieldSchema *column_schema_pb = schema_pb.add_fields(); - column_schema_pb->MergeFrom(ProtoConverter::ToPb(*column_schema)); - } - - schema_pb.set_max_doc_count_per_segment(schema.max_doc_count_per_segment()); - - return schema_pb; -} - -IndexParams::Ptr ProtoConverter::FromPb(const proto::IndexParams ¶ms_pb) { - if (params_pb.has_hnsw()) { - return ProtoConverter::FromPb(params_pb.hnsw()); - } else if (params_pb.has_invert()) { - return ProtoConverter::FromPb(params_pb.invert()); - } else if (params_pb.has_ivf()) { - return ProtoConverter::FromPb(params_pb.ivf()); - } else if (params_pb.has_flat()) { - return ProtoConverter::FromPb(params_pb.flat()); - } else if (params_pb.has_hnsw_rabitq()) { - return ProtoConverter::FromPb(params_pb.hnsw_rabitq()); - } else if (params_pb.has_diskann()) { - return ProtoConverter::FromPb(params_pb.diskann()); - } else if (params_pb.has_vamana()) { - return ProtoConverter::FromPb(params_pb.vamana()); - } else if (params_pb.has_fts()) { - return ProtoConverter::FromPb(params_pb.fts()); - } - - return nullptr; -} - -// BlockMeta -BlockMeta::Ptr ProtoConverter::FromPb(const proto::BlockMeta &meta_pb) { - auto block_meta = std::make_shared(); - - block_meta->set_id(meta_pb.block_id()); - block_meta->set_type(PbBlockType::Get(meta_pb.block_type())); - block_meta->set_min_doc_id(meta_pb.min_doc_id()); - block_meta->set_max_doc_id(meta_pb.max_doc_id()); - block_meta->set_doc_count(meta_pb.doc_count()); - for (auto &column : meta_pb.columns()) { - block_meta->add_column(column); - } - - return block_meta; -} - -proto::IndexParams ProtoConverter::ToPb(const IndexParams *params) { - proto::IndexParams params_pb; - - switch (params->type()) { - case IndexType::INVERT: { - auto invert_params = dynamic_cast(params); - if (invert_params) { - params_pb.mutable_invert()->CopyFrom( - ProtoConverter::ToPb(invert_params)); - } - break; - } - case IndexType::HNSW: { - auto hnsw_params = dynamic_cast(params); - if (hnsw_params) { - params_pb.mutable_hnsw()->CopyFrom(ProtoConverter::ToPb(hnsw_params)); - } - break; - } - case IndexType::IVF: { - auto ivf_params = dynamic_cast(params); - if (ivf_params) { - params_pb.mutable_ivf()->CopyFrom(ProtoConverter::ToPb(ivf_params)); - } - break; - } - case IndexType::FLAT: { - auto flat_params = dynamic_cast(params); - if (flat_params) { - params_pb.mutable_flat()->CopyFrom(ProtoConverter::ToPb(flat_params)); - } - break; - } - case IndexType::HNSW_RABITQ: { - auto hnsw_rabitq_params = - dynamic_cast(params); - if (hnsw_rabitq_params) { - params_pb.mutable_hnsw_rabitq()->CopyFrom( - ProtoConverter::ToPb(hnsw_rabitq_params)); - } - break; - } - case IndexType::DISKANN: { - auto diskann_params = dynamic_cast(params); - if (diskann_params) { - params_pb.mutable_diskann()->CopyFrom( - ProtoConverter::ToPb(diskann_params)); - } - break; - } - case IndexType::VAMANA: { - auto vamana_params = dynamic_cast(params); - if (vamana_params) { - params_pb.mutable_vamana()->CopyFrom( - ProtoConverter::ToPb(vamana_params)); - } - break; - } - case IndexType::FTS: { - auto fts_params = dynamic_cast(params); - if (fts_params) { - params_pb.mutable_fts()->CopyFrom(ProtoConverter::ToPb(fts_params)); - } - break; - } - default: - break; - } - - return params_pb; -} - -proto::BlockMeta ProtoConverter::ToPb(const BlockMeta &meta) { - proto::BlockMeta meta_pb; - meta_pb.set_block_id(meta.id()); - meta_pb.set_block_type(PbBlockType::Get(meta.type())); - meta_pb.set_min_doc_id(meta.min_doc_id()); - meta_pb.set_max_doc_id(meta.max_doc_id()); - meta_pb.set_doc_count(meta.doc_count()); - for (auto &column : meta.columns()) { - meta_pb.add_columns(column); - } - - return meta_pb; -} - -// SegmentMeta -SegmentMeta::Ptr ProtoConverter::FromPb(const proto::SegmentMeta &meta_pb) { - auto meta = std::make_shared(meta_pb.segment_id()); - - auto persisted_blocks = meta_pb.persisted_blocks(); - - for (auto &persisted_block_pb : persisted_blocks) { - BlockMeta::Ptr persisted_block = ProtoConverter::FromPb(persisted_block_pb); - meta->add_persisted_block(*persisted_block); - } - - if (meta_pb.has_writing_forward_block()) { - meta->set_writing_forward_block( - *ProtoConverter::FromPb(meta_pb.writing_forward_block())); - } - - auto indexed_vector_fields = meta_pb.indexed_vector_fields(); - for (auto &indexed_vector_field : indexed_vector_fields) { - meta->add_indexed_vector_field(indexed_vector_field); - } - - return meta; -} - -proto::SegmentMeta ProtoConverter::ToPb(const SegmentMeta &meta) { - proto::SegmentMeta meta_pb; - meta_pb.set_segment_id(meta.id()); - - auto persisted_blocks = meta.persisted_blocks(); - for (auto &persisted_block : persisted_blocks) { - auto persisted_block_pb = ProtoConverter::ToPb(persisted_block); - meta_pb.add_persisted_blocks()->MergeFrom(persisted_block_pb); - } - - if (meta.has_writing_forward_block()) { - meta_pb.mutable_writing_forward_block()->MergeFrom( - ProtoConverter::ToPb(meta.writing_forward_block().value())); - } - - auto indexed_vector_fields = meta.indexed_vector_fields(); - for (auto &field : indexed_vector_fields) { - meta_pb.add_indexed_vector_fields(field); - } - - return meta_pb; -} - -} // namespace zvec \ No newline at end of file diff --git a/src/db/index/common/proto_converter.h b/src/db/index/common/proto_converter.h deleted file mode 100644 index 1d9ee0d10..000000000 --- a/src/db/index/common/proto_converter.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#pragma once - -#include -#include -#include -#include "db/index/common/meta.h" - -namespace zvec { - -struct ProtoConverter { - // HnswIndexParams - static HnswIndexParams::OPtr FromPb(const proto::HnswIndexParams ¶ms_pb); - - static proto::HnswIndexParams ToPb(const HnswIndexParams *params); - - // HnswRabitqIndexParams - static HnswRabitqIndexParams::OPtr FromPb( - const proto::HnswRabitqIndexParams ¶ms_pb); - static proto::HnswRabitqIndexParams ToPb(const HnswRabitqIndexParams *params); - - // FlatIndexParams - static FlatIndexParams::OPtr FromPb(const proto::FlatIndexParams ¶ms_pb); - static proto::FlatIndexParams ToPb(const FlatIndexParams *params); - - // IVFIndexParams - static IVFIndexParams::OPtr FromPb(const proto::IVFIndexParams ¶ms_pb); - static proto::IVFIndexParams ToPb(const IVFIndexParams *params); - - // VamanaIndexParams - static VamanaIndexParams::OPtr FromPb( - const proto::VamanaIndexParams ¶ms_pb); - static proto::VamanaIndexParams ToPb(const VamanaIndexParams *params); - - // InvertIndexParams - static InvertIndexParams::OPtr FromPb( - const proto::InvertIndexParams ¶ms_pb); - static proto::InvertIndexParams ToPb(const InvertIndexParams *params); - - // DiskAnnIndexParams - static DiskAnnIndexParams::OPtr FromPb( - const proto::DiskAnnIndexParams ¶ms_pb); - static proto::DiskAnnIndexParams ToPb(const DiskAnnIndexParams *params); - - // FtsIndexParams - static FtsIndexParams::Ptr FromPb(const proto::FtsIndexParams ¶ms_pb); - static proto::FtsIndexParams ToPb(const FtsIndexParams *params); - - // IndexParams - static IndexParams::Ptr FromPb(const proto::IndexParams ¶ms_pb); - static proto::IndexParams ToPb(const IndexParams *params); - - // FieldSchema - static FieldSchema::Ptr FromPb(const proto::FieldSchema &field_pb); - static proto::FieldSchema ToPb(const FieldSchema &field); - - // CollectionSchema - static CollectionSchema::Ptr FromPb(const proto::CollectionSchema &schema_pb); - static proto::CollectionSchema ToPb(const CollectionSchema &schema); - - // BlockMeta - static BlockMeta::Ptr FromPb(const proto::BlockMeta &meta_pb); - static proto::BlockMeta ToPb(const BlockMeta &meta); - - // SegmentMeta - static SegmentMeta::Ptr FromPb(const proto::SegmentMeta &meta_pb); - static proto::SegmentMeta ToPb(const SegmentMeta &meta); -}; - -} // namespace zvec \ No newline at end of file diff --git a/tests/db/CMakeLists.txt b/tests/db/CMakeLists.txt index bfad2bc57..a93041518 100644 --- a/tests/db/CMakeLists.txt +++ b/tests/db/CMakeLists.txt @@ -27,7 +27,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) cc_gmock( NAME ${CC_TARGET} STRICT LIBS zvec - zvec_proto core_knn_flat core_knn_flat_sparse core_knn_hnsw diff --git a/tests/db/crash_recovery/CMakeLists.txt b/tests/db/crash_recovery/CMakeLists.txt index 98a8a3bc9..4e28c277c 100644 --- a/tests/db/crash_recovery/CMakeLists.txt +++ b/tests/db/crash_recovery/CMakeLists.txt @@ -27,7 +27,6 @@ endif() # Common libraries set(CRASH_RECOVERY_COMMON_LIBS zvec - zvec_proto core_knn_flat core_knn_flat_sparse core_knn_hnsw diff --git a/tests/db/index/CMakeLists.txt b/tests/db/index/CMakeLists.txt index 1bd92c8f6..70c160c27 100644 --- a/tests/db/index/CMakeLists.txt +++ b/tests/db/index/CMakeLists.txt @@ -38,7 +38,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) NAME ${CC_TARGET} STRICT LIBS zvec zvec_ailego - zvec_proto core_knn_diskann core_metric_static core_utility_static diff --git a/tests/db/index/common/db_proto_converter_test.cc b/tests/db/index/common/db_proto_converter_test.cc deleted file mode 100644 index 9c71c3c89..000000000 --- a/tests/db/index/common/db_proto_converter_test.cc +++ /dev/null @@ -1,550 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "db/index/common/proto_converter.h" -#include "db/index/common/type_helper.h" - -using namespace zvec; - -TEST(ConverterTest, InvertIndexParamsConversion) { - // Test conversion from protobuf to C++ InvertIndexParams - proto::InvertIndexParams invert_pb; - invert_pb.set_enable_range_optimization(true); - - auto invert_params = ProtoConverter::FromPb(invert_pb); - ASSERT_NE(invert_params, nullptr); - EXPECT_TRUE(invert_params->enable_range_optimization()); - EXPECT_EQ(invert_params->type(), IndexType::INVERT); - - // Test with false value - proto::InvertIndexParams invert_pb2; - invert_pb2.set_enable_range_optimization(false); - - auto invert_params2 = ProtoConverter::FromPb(invert_pb2); - ASSERT_NE(invert_params2, nullptr); - EXPECT_FALSE(invert_params2->enable_range_optimization()); - - // Test conversion from C++ to protobuf - InvertIndexParams original_params(true); - auto pb_result = ProtoConverter::ToPb(&original_params); - EXPECT_TRUE(pb_result.enable_range_optimization()); -} - -TEST(ConverterTest, HnswIndexParamsConversion) { - // Test conversion from protobuf to C++ HnswIndexParams - proto::HnswIndexParams hnsw_pb; - auto *base_params = hnsw_pb.mutable_base(); - base_params->set_metric_type(proto::MT_L2); - base_params->set_quantize_type(proto::QT_FP16); - hnsw_pb.set_m(16); - hnsw_pb.set_ef_construction(100); - - auto hnsw_params = ProtoConverter::FromPb(hnsw_pb); - ASSERT_NE(hnsw_params, nullptr); - EXPECT_EQ(hnsw_params->metric_type(), MetricType::L2); - EXPECT_EQ(hnsw_params->m(), 16); - EXPECT_EQ(hnsw_params->ef_construction(), 100); - EXPECT_EQ(hnsw_params->quantize_type(), QuantizeType::FP16); - EXPECT_EQ(hnsw_params->type(), IndexType::HNSW); - - // Test conversion from C++ to protobuf - HnswIndexParams original_params(MetricType::IP, 32, 200, QuantizeType::INT8); - auto pb_result = ProtoConverter::ToPb(&original_params); - EXPECT_EQ(pb_result.base().metric_type(), proto::MT_IP); - EXPECT_EQ(pb_result.m(), 32); - EXPECT_EQ(pb_result.ef_construction(), 200); - EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_INT8); -} - -TEST(ConverterTest, FlatIndexParamsConversion) { - // Test conversion from protobuf to C++ FlatIndexParams - proto::FlatIndexParams flat_pb; - auto *base_params = flat_pb.mutable_base(); - base_params->set_metric_type(proto::MT_COSINE); - base_params->set_quantize_type(proto::QT_INT4); - - auto flat_params = ProtoConverter::FromPb(flat_pb); - ASSERT_NE(flat_params, nullptr); - EXPECT_EQ(flat_params->metric_type(), MetricType::COSINE); - EXPECT_EQ(flat_params->quantize_type(), QuantizeType::INT4); - EXPECT_EQ(flat_params->type(), IndexType::FLAT); - - // Test conversion from C++ to protobuf - FlatIndexParams original_params(MetricType::L2, QuantizeType::FP16); - auto pb_result = ProtoConverter::ToPb(&original_params); - EXPECT_EQ(pb_result.base().metric_type(), proto::MT_L2); - EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_FP16); -} - -TEST(ConverterTest, IVFIndexParamsConversion) { - // Test conversion from protobuf to C++ IVFIndexParams - proto::IVFIndexParams ivf_pb; - auto *base_params = ivf_pb.mutable_base(); - base_params->set_metric_type(proto::MT_IP); - base_params->set_quantize_type(proto::QT_INT8); - ivf_pb.set_n_list(128); - - auto ivf_params = ProtoConverter::FromPb(ivf_pb); - ASSERT_NE(ivf_params, nullptr); - EXPECT_EQ(ivf_params->metric_type(), MetricType::IP); - EXPECT_EQ(ivf_params->n_list(), 128); - EXPECT_EQ(ivf_params->quantize_type(), QuantizeType::INT8); - EXPECT_EQ(ivf_params->type(), IndexType::IVF); - - // Test conversion from C++ to protobuf - IVFIndexParams original_params(MetricType::COSINE, 256, 10, false, - QuantizeType::INT4); - auto pb_result = ProtoConverter::ToPb(&original_params); - EXPECT_EQ(pb_result.base().metric_type(), proto::MT_COSINE); - EXPECT_EQ(pb_result.n_list(), 256); - EXPECT_EQ(pb_result.n_iters(), 10); - EXPECT_FALSE(pb_result.use_soar()); - EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_INT4); -} - -TEST(ConverterTest, IndexParamsConversion) { - // Test conversion from protobuf to C++ IndexParams for HNSW - proto::IndexParams index_pb; - auto *hnsw_pb = index_pb.mutable_hnsw(); - auto *base_params = hnsw_pb->mutable_base(); - base_params->set_metric_type(proto::MT_L2); - base_params->set_quantize_type(proto::QT_FP16); - hnsw_pb->set_m(16); - hnsw_pb->set_ef_construction(100); - - auto index_params = ProtoConverter::FromPb(index_pb); - ASSERT_NE(index_params, nullptr); - EXPECT_EQ(index_params->type(), IndexType::HNSW); - auto hnsw_cast = std::dynamic_pointer_cast(index_params); - ASSERT_NE(hnsw_cast, nullptr); - EXPECT_EQ(hnsw_cast->metric_type(), MetricType::L2); - EXPECT_EQ(hnsw_cast->m(), 16); - EXPECT_EQ(hnsw_cast->ef_construction(), 100); - EXPECT_EQ(hnsw_cast->quantize_type(), QuantizeType::FP16); - - // Test conversion from C++ HnswIndexParams to protobuf IndexParams - HnswIndexParams hnsw_original(MetricType::IP, 32, 200); - auto pb_result = ProtoConverter::ToPb(&hnsw_original); - EXPECT_EQ(pb_result.base().metric_type(), proto::MT_IP); - EXPECT_EQ(pb_result.m(), 32); - EXPECT_EQ(pb_result.ef_construction(), 200); - - // Test conversion from protobuf to C++ IndexParams for FLAT - proto::IndexParams index_pb2; - auto *flat_pb = index_pb2.mutable_flat(); - auto *base_params2 = flat_pb->mutable_base(); - base_params2->set_metric_type(proto::MT_COSINE); - base_params2->set_quantize_type(proto::QT_INT8); - - auto index_params2 = ProtoConverter::FromPb(index_pb2); - ASSERT_NE(index_params2, nullptr); - EXPECT_EQ(index_params2->type(), IndexType::FLAT); - auto flat_cast = std::dynamic_pointer_cast(index_params2); - ASSERT_NE(flat_cast, nullptr); - EXPECT_EQ(flat_cast->metric_type(), MetricType::COSINE); - EXPECT_EQ(flat_cast->quantize_type(), QuantizeType::INT8); - - // Test conversion from C++ FlatIndexParams to protobuf IndexParams - FlatIndexParams flat_original(MetricType::L2); - auto pb_result2 = ProtoConverter::ToPb(&flat_original); - EXPECT_EQ(pb_result2.base().metric_type(), proto::MT_L2); - - // Test conversion from protobuf to C++ IndexParams for IVF - proto::IndexParams index_pb3; - auto *ivf_pb = index_pb3.mutable_ivf(); - auto *base_params3 = ivf_pb->mutable_base(); - base_params3->set_metric_type(proto::MT_IP); - base_params3->set_quantize_type(proto::QT_INT4); - ivf_pb->set_n_list(128); - - auto index_params3 = ProtoConverter::FromPb(index_pb3); - ASSERT_NE(index_params3, nullptr); - EXPECT_EQ(index_params3->type(), IndexType::IVF); - auto ivf_cast = std::dynamic_pointer_cast(index_params3); - ASSERT_NE(ivf_cast, nullptr); - EXPECT_EQ(ivf_cast->metric_type(), MetricType::IP); - EXPECT_EQ(ivf_cast->n_list(), 128); - EXPECT_EQ(ivf_cast->quantize_type(), QuantizeType::INT4); - - // Test conversion from C++ IVFIndexParams to protobuf IndexParams - IVFIndexParams ivf_original(MetricType::COSINE, 256); - auto pb_result3 = ProtoConverter::ToPb(&ivf_original); - EXPECT_EQ(pb_result3.base().metric_type(), proto::MT_COSINE); - EXPECT_EQ(pb_result3.n_list(), 256); - - // Test conversion from protobuf to C++ IndexParams for INVERT - proto::IndexParams index_pb4; - auto *invert_pb = index_pb4.mutable_invert(); - invert_pb->set_enable_range_optimization(true); - - auto index_params4 = ProtoConverter::FromPb(index_pb4); - ASSERT_NE(index_params4, nullptr); - EXPECT_EQ(index_params4->type(), IndexType::INVERT); - auto invert_cast = - std::dynamic_pointer_cast(index_params4); - ASSERT_NE(invert_cast, nullptr); - EXPECT_TRUE(invert_cast->enable_range_optimization()); - - // Test conversion from C++ InvertIndexParams to protobuf IndexParams - InvertIndexParams invert_original(false); - auto pb_result4 = ProtoConverter::ToPb(&invert_original); - EXPECT_FALSE(pb_result4.enable_range_optimization()); -} - -TEST(ConverterTest, FieldSchemaConversion) { - // Test conversion from protobuf to C++ FieldSchema - proto::FieldSchema field_pb; - field_pb.set_name("test_field"); - field_pb.set_data_type(proto::DT_VECTOR_FP32); - field_pb.set_dimension(128); - field_pb.set_nullable(true); - - // Add index params - auto *index_params_pb = field_pb.mutable_index_params(); - auto *hnsw_pb = index_params_pb->mutable_hnsw(); - auto *base_params = hnsw_pb->mutable_base(); - base_params->set_metric_type(proto::MT_L2); - base_params->set_quantize_type(proto::QT_FP16); - hnsw_pb->set_m(16); - hnsw_pb->set_ef_construction(100); - - auto field_schema = ProtoConverter::FromPb(field_pb); - ASSERT_NE(field_schema, nullptr); - EXPECT_EQ(field_schema->name(), "test_field"); - EXPECT_EQ(field_schema->data_type(), DataType::VECTOR_FP32); - EXPECT_TRUE(field_schema->nullable()); - EXPECT_EQ(field_schema->dimension(), 128u); - ASSERT_NE(field_schema->index_params(), nullptr); - EXPECT_EQ(field_schema->index_params()->type(), IndexType::HNSW); - - // Test conversion from C++ to protobuf - FieldSchema original_field("another_field", DataType::ARRAY_INT32, 64, false, - nullptr); - auto pb_result = ProtoConverter::ToPb(original_field); - EXPECT_EQ(pb_result.name(), "another_field"); - EXPECT_EQ(pb_result.data_type(), proto::DT_ARRAY_INT32); - EXPECT_FALSE(pb_result.nullable()); - EXPECT_EQ(pb_result.dimension(), 64u); -} - -TEST(ConverterTest, CollectionSchemaConversion) { - // Test conversion from protobuf to C++ CollectionSchema - proto::CollectionSchema schema_pb; - schema_pb.set_name("test_collection"); - schema_pb.set_max_doc_count_per_segment(1000000); - - auto *field1_pb = schema_pb.add_fields(); - field1_pb->set_name("field1"); - field1_pb->set_data_type(proto::DT_STRING); - - auto *field2_pb = schema_pb.add_fields(); - field2_pb->set_name("field2"); - field2_pb->set_data_type(proto::DT_VECTOR_FP32); - field2_pb->set_dimension(128); - - auto collection_schema = ProtoConverter::FromPb(schema_pb); - ASSERT_NE(collection_schema, nullptr); - EXPECT_EQ(collection_schema->name(), "test_collection"); - EXPECT_EQ(collection_schema->fields().size(), 2); - EXPECT_EQ(collection_schema->max_doc_count_per_segment(), 1000000u); - - // Test conversion from C++ to protobuf - CollectionSchema original_schema; - original_schema.set_name("original_collection"); - - auto pb_result = ProtoConverter::ToPb(original_schema); - EXPECT_EQ(pb_result.name(), "original_collection"); -} - -TEST(ConverterTest, BlockMetaConversion) { - // Test conversion from protobuf to C++ BlockMeta - proto::BlockMeta meta_pb; - meta_pb.set_block_id(1); - meta_pb.set_block_type(proto::BT_SCALAR); - meta_pb.set_min_doc_id(100); - meta_pb.set_max_doc_id(200); - meta_pb.set_doc_count(50); - meta_pb.add_columns("col1"); - meta_pb.add_columns("col2"); - - auto block_meta = ProtoConverter::FromPb(meta_pb); - ASSERT_NE(block_meta, nullptr); - EXPECT_EQ(block_meta->id(), 1u); - EXPECT_EQ(block_meta->type(), BlockType::SCALAR); - EXPECT_EQ(block_meta->min_doc_id(), 100u); - EXPECT_EQ(block_meta->max_doc_id(), 200u); - EXPECT_EQ(block_meta->doc_count(), 50u); - EXPECT_EQ(block_meta->columns().size(), 2); - EXPECT_EQ(block_meta->columns()[0], "col1"); - EXPECT_EQ(block_meta->columns()[1], "col2"); - - // Test conversion from C++ to protobuf - BlockMeta original_meta(2, BlockType::VECTOR_INDEX, 300, 400); - original_meta.set_doc_count(75); - original_meta.add_column("col3"); - original_meta.add_column("col4"); - - auto pb_result = ProtoConverter::ToPb(original_meta); - EXPECT_EQ(pb_result.block_id(), 2u); - EXPECT_EQ(pb_result.block_type(), proto::BT_VECTOR_INDEX); - EXPECT_EQ(pb_result.min_doc_id(), 300u); - EXPECT_EQ(pb_result.max_doc_id(), 400u); - EXPECT_EQ(pb_result.doc_count(), 75u); - EXPECT_EQ(pb_result.columns_size(), 2); - EXPECT_EQ(pb_result.columns(0), "col3"); - EXPECT_EQ(pb_result.columns(1), "col4"); -} - -TEST(ConverterTest, SegmentMetaConversion) { - // Test conversion from protobuf to C++ SegmentMeta - proto::SegmentMeta segment_pb; - segment_pb.set_segment_id(10); - - // Add persisted blocks - auto *block1_pb = segment_pb.add_persisted_blocks(); - block1_pb->set_block_id(1); - block1_pb->set_block_type(proto::BT_SCALAR); - block1_pb->set_min_doc_id(0); - block1_pb->set_max_doc_id(100); - block1_pb->set_doc_count(50); - block1_pb->add_columns("col1"); - block1_pb->add_columns("col2"); - - auto *block2_pb = segment_pb.add_persisted_blocks(); - block2_pb->set_block_id(2); - block2_pb->set_block_type(proto::BT_VECTOR_INDEX); - block2_pb->set_min_doc_id(101); - block2_pb->set_max_doc_id(200); - block2_pb->set_doc_count(75); - block2_pb->add_columns("vec_col"); - - // Add writing forward block - auto *writing_block_pb = segment_pb.mutable_writing_forward_block(); - writing_block_pb->set_block_id(3); - writing_block_pb->set_block_type(proto::BT_SCALAR); - writing_block_pb->set_min_doc_id(201); - writing_block_pb->set_max_doc_id(300); - writing_block_pb->set_doc_count(25); - writing_block_pb->add_columns("col3"); - - // Add indexed vector fields - segment_pb.add_indexed_vector_fields("vec_col1"); - segment_pb.add_indexed_vector_fields("vec_col2"); - - auto segment_meta = ProtoConverter::FromPb(segment_pb); - ASSERT_NE(segment_meta, nullptr); - EXPECT_EQ(segment_meta->id(), 10u); - EXPECT_EQ(segment_meta->persisted_blocks().size(), 2); - EXPECT_TRUE(segment_meta->has_writing_forward_block()); - - // Check first persisted block - const auto &block1 = segment_meta->persisted_blocks()[0]; - EXPECT_EQ(block1.id(), 1u); - EXPECT_EQ(block1.type(), BlockType::SCALAR); - EXPECT_EQ(block1.min_doc_id(), 0u); - EXPECT_EQ(block1.max_doc_id(), 100u); - EXPECT_EQ(block1.doc_count(), 50u); - EXPECT_EQ(block1.columns().size(), 2); - EXPECT_EQ(block1.columns()[0], "col1"); - EXPECT_EQ(block1.columns()[1], "col2"); - - // Check second persisted block - const auto &block2 = segment_meta->persisted_blocks()[1]; - EXPECT_EQ(block2.id(), 2u); - EXPECT_EQ(block2.type(), BlockType::VECTOR_INDEX); - EXPECT_EQ(block2.min_doc_id(), 101u); - EXPECT_EQ(block2.max_doc_id(), 200u); - EXPECT_EQ(block2.doc_count(), 75u); - EXPECT_EQ(block2.columns().size(), 1); - EXPECT_EQ(block2.columns()[0], "vec_col"); - - // Check writing forward block - const auto &writing_block = segment_meta->writing_forward_block(); - EXPECT_EQ(writing_block.value().id(), 3u); - EXPECT_EQ(writing_block.value().type(), BlockType::SCALAR); - EXPECT_EQ(writing_block.value().min_doc_id(), 201u); - EXPECT_EQ(writing_block.value().max_doc_id(), 300u); - EXPECT_EQ(writing_block.value().doc_count(), 25u); - EXPECT_EQ(writing_block.value().columns().size(), 1); - EXPECT_EQ(writing_block.value().columns()[0], "col3"); - - // Check indexed vector fields - EXPECT_TRUE(segment_meta->vector_indexed("vec_col1")); - EXPECT_TRUE(segment_meta->vector_indexed("vec_col2")); - EXPECT_FALSE(segment_meta->vector_indexed("non_existent_field")); - - // Test conversion from C++ to protobuf - SegmentMeta original_meta(20); - - // Add persisted blocks - BlockMeta block1_meta(1, BlockType::SCALAR_INDEX, 0, 50); - block1_meta.set_doc_count(25); - block1_meta.add_column("col3"); - block1_meta.add_column("col4"); - original_meta.add_persisted_block(block1_meta); - - BlockMeta block2_meta(2, BlockType::VECTOR_INDEX_QUANTIZE, 51, 100); - block2_meta.set_doc_count(30); - block2_meta.add_column("vec_col2"); - original_meta.add_persisted_block(block2_meta); - - // Set writing forward block - BlockMeta writing_block_meta(3, BlockType::SCALAR, 101, 150); - writing_block_meta.set_doc_count(40); - writing_block_meta.add_column("col5"); - original_meta.set_writing_forward_block(writing_block_meta); - - // Add indexed vector fields - original_meta.add_indexed_vector_field("vec_field1"); - original_meta.add_indexed_vector_field("vec_field2"); - - auto pb_result = ProtoConverter::ToPb(original_meta); - EXPECT_EQ(pb_result.segment_id(), 20u); - EXPECT_EQ(pb_result.persisted_blocks_size(), 2); - - // Check first persisted block - const auto &pb_block1 = pb_result.persisted_blocks(0); - EXPECT_EQ(pb_block1.block_id(), 1u); - EXPECT_EQ(pb_block1.block_type(), proto::BT_SCALAR_INDEX); - EXPECT_EQ(pb_block1.min_doc_id(), 0u); - EXPECT_EQ(pb_block1.max_doc_id(), 50u); - EXPECT_EQ(pb_block1.doc_count(), 25u); - EXPECT_EQ(pb_block1.columns_size(), 2); - EXPECT_EQ(pb_block1.columns(0), "col3"); - EXPECT_EQ(pb_block1.columns(1), "col4"); - - // Check second persisted block - const auto &pb_block2 = pb_result.persisted_blocks(1); - EXPECT_EQ(pb_block2.block_id(), 2u); - EXPECT_EQ(pb_block2.block_type(), proto::BT_VECTOR_INDEX_QUANTIZE); - EXPECT_EQ(pb_block2.min_doc_id(), 51u); - EXPECT_EQ(pb_block2.max_doc_id(), 100u); - EXPECT_EQ(pb_block2.doc_count(), 30u); - EXPECT_EQ(pb_block2.columns_size(), 1); - EXPECT_EQ(pb_block2.columns(0), "vec_col2"); - - // Check writing forward block - const auto &pb_writing_block = pb_result.writing_forward_block(); - EXPECT_EQ(pb_writing_block.block_id(), 3u); - EXPECT_EQ(pb_writing_block.block_type(), proto::BT_SCALAR); - EXPECT_EQ(pb_writing_block.min_doc_id(), 101u); - EXPECT_EQ(pb_writing_block.max_doc_id(), 150u); - EXPECT_EQ(pb_writing_block.doc_count(), 40u); - EXPECT_EQ(pb_writing_block.columns_size(), 1); - EXPECT_EQ(pb_writing_block.columns(0), "col5"); - - // Check indexed vector fields - EXPECT_EQ(pb_result.indexed_vector_fields_size(), 2); - EXPECT_EQ(pb_result.indexed_vector_fields(0), "vec_field1"); - EXPECT_EQ(pb_result.indexed_vector_fields(1), "vec_field2"); -} - -TEST(ConverterTest, SegmentMetaWithEmptyFields) { - // Test conversion with minimal data - proto::SegmentMeta segment_pb; - segment_pb.set_segment_id(1); - - auto segment_meta = ProtoConverter::FromPb(segment_pb); - ASSERT_NE(segment_meta, nullptr); - EXPECT_EQ(segment_meta->id(), 1u); - EXPECT_EQ(segment_meta->persisted_blocks().size(), 0); - EXPECT_FALSE(segment_meta->has_writing_forward_block()); - EXPECT_EQ(segment_meta->indexed_vector_fields().size(), 0); - - // Test conversion from C++ to protobuf with minimal data - SegmentMeta original_meta(5); - auto pb_result = ProtoConverter::ToPb(original_meta); - EXPECT_EQ(pb_result.segment_id(), 5u); - EXPECT_EQ(pb_result.persisted_blocks_size(), 0); - EXPECT_FALSE(pb_result.has_writing_forward_block()); - EXPECT_EQ(pb_result.indexed_vector_fields_size(), 0); -} - -// ==================== enable_rotate roundtrip tests ==================== - -TEST(ConverterTest, HnswIndexParamsWithEnableRotate) { - // C++ -> PB -> C++ roundtrip with enable_rotate = true - HnswIndexParams original(MetricType::COSINE, 16, 200, QuantizeType::INT8, - false, QuantizerParam(true)); - EXPECT_TRUE(original.quantizer_param().enable_rotate()); - - auto pb = ProtoConverter::ToPb(&original); - EXPECT_TRUE(pb.base().quantizer_param().enable_rotate()); - - auto restored = ProtoConverter::FromPb(pb); - ASSERT_NE(restored, nullptr); - EXPECT_TRUE(restored->quantizer_param().enable_rotate()); - EXPECT_TRUE(restored->enable_rotate()); // convenience getter - EXPECT_EQ(restored->metric_type(), MetricType::COSINE); - EXPECT_EQ(restored->m(), 16); - EXPECT_EQ(restored->ef_construction(), 200); - EXPECT_EQ(restored->quantize_type(), QuantizeType::INT8); - - // C++ -> PB -> C++ roundtrip with enable_rotate = false - HnswIndexParams original_no_rot(MetricType::L2, 32, 100, QuantizeType::FP16); - auto pb2 = ProtoConverter::ToPb(&original_no_rot); - EXPECT_FALSE(pb2.base().quantizer_param().enable_rotate()); - auto restored2 = ProtoConverter::FromPb(pb2); - ASSERT_NE(restored2, nullptr); - EXPECT_FALSE(restored2->quantizer_param().enable_rotate()); -} - -TEST(ConverterTest, FlatIndexParamsWithEnableRotate) { - FlatIndexParams original(MetricType::IP, QuantizeType::INT8, - QuantizerParam(true)); - EXPECT_TRUE(original.quantizer_param().enable_rotate()); - - auto pb = ProtoConverter::ToPb(&original); - EXPECT_TRUE(pb.base().quantizer_param().enable_rotate()); - - auto restored = ProtoConverter::FromPb(pb); - ASSERT_NE(restored, nullptr); - EXPECT_TRUE(restored->quantizer_param().enable_rotate()); - EXPECT_EQ(restored->metric_type(), MetricType::IP); - EXPECT_EQ(restored->quantize_type(), QuantizeType::INT8); - - // enable_rotate = false - FlatIndexParams original_no_rot(MetricType::L2, QuantizeType::FP16); - auto pb2 = ProtoConverter::ToPb(&original_no_rot); - EXPECT_FALSE(pb2.base().quantizer_param().enable_rotate()); - auto restored2 = ProtoConverter::FromPb(pb2); - EXPECT_FALSE(restored2->quantizer_param().enable_rotate()); -} - -TEST(ConverterTest, IVFIndexParamsWithEnableRotate) { - IVFIndexParams original(MetricType::COSINE, 256, 20, true, QuantizeType::INT8, - QuantizerParam(true)); - EXPECT_TRUE(original.quantizer_param().enable_rotate()); - - auto pb = ProtoConverter::ToPb(&original); - EXPECT_TRUE(pb.base().quantizer_param().enable_rotate()); - - auto restored = ProtoConverter::FromPb(pb); - ASSERT_NE(restored, nullptr); - EXPECT_TRUE(restored->quantizer_param().enable_rotate()); - EXPECT_EQ(restored->metric_type(), MetricType::COSINE); - EXPECT_EQ(restored->n_list(), 256); - EXPECT_EQ(restored->n_iters(), 20); - EXPECT_TRUE(restored->use_soar()); - EXPECT_EQ(restored->quantize_type(), QuantizeType::INT8); - - // enable_rotate = false - IVFIndexParams original_no_rot(MetricType::L2, 128, 10, false, - QuantizeType::FP16); - auto pb2 = ProtoConverter::ToPb(&original_no_rot); - EXPECT_FALSE(pb2.base().quantizer_param().enable_rotate()); - auto restored2 = ProtoConverter::FromPb(pb2); - EXPECT_FALSE(restored2->quantizer_param().enable_rotate()); -} \ No newline at end of file diff --git a/tests/db/index/common/manifest_codec_test.cc b/tests/db/index/common/manifest_codec_test.cc deleted file mode 100644 index 3e917bbd6..000000000 --- a/tests/db/index/common/manifest_codec_test.cc +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Cross-checks ManifestCodec against the protobuf library. -//! -//! While libprotobuf is still available, every message is encoded with both -//! implementations and the resulting bytes must be identical. This is what -//! guarantees that dropping the protobuf dependency does not change the -//! on-disk manifest format. -//! -//! These tests are removed together with the protobuf dependency; the -//! golden-file tests in manifest_codec_golden_test.cc take over from there. - -#include "db/index/common/manifest_codec.h" -#include -#include -#include -#include -#include "db/index/common/proto_converter.h" -#include "db/index/common/type_helper.h" - -using namespace zvec; - -namespace { - -//! Encodes index params with the new codec. -std::string EncodeNew(const IndexParams *params) { - std::string out; - ManifestCodec::EncodeIndexParams(params, &out); - return out; -} - -//! Encodes index params through the protobuf library. -std::string EncodePb(const IndexParams *params) { - return ProtoConverter::ToPb(params).SerializeAsString(); -} - -//! Asserts both implementations produce the same bytes, and that each side can -//! read what the other produced. -void ExpectIndexParamsRoundTrip(const IndexParams *params) { - const std::string bytes_new = EncodeNew(params); - const std::string bytes_pb = EncodePb(params); - ASSERT_EQ(bytes_new, bytes_pb) << "encoded bytes differ for index type " - << IndexTypeCodeBook::AsString(params->type()); - - // New decoder reads protobuf output. - auto from_pb_bytes = ManifestCodec::DecodeIndexParams(bytes_pb); - ASSERT_NE(from_pb_bytes, nullptr); - EXPECT_EQ(from_pb_bytes->type(), params->type()); - - // protobuf reads new encoder output. - proto::IndexParams pb; - ASSERT_TRUE(pb.ParseFromString(bytes_new)); - auto from_new_bytes = ProtoConverter::FromPb(pb); - ASSERT_NE(from_new_bytes, nullptr); - EXPECT_EQ(from_new_bytes->type(), params->type()); -} - -} // namespace - -TEST(ManifestCodecCrossCheck, InvertIndexParams) { - InvertIndexParams enabled(true); - ExpectIndexParamsRoundTrip(&enabled); - InvertIndexParams disabled(false); - ExpectIndexParamsRoundTrip(&disabled); -} - -TEST(ManifestCodecCrossCheck, HnswIndexParams) { - // All-default values exercise proto3 "skip defaults" plus the always - // present base/quantizer_param sub-messages. - HnswIndexParams defaults(MetricType::UNDEFINED, 0, 0, QuantizeType::UNDEFINED, - false, QuantizerParam()); - ExpectIndexParamsRoundTrip(&defaults); - - HnswIndexParams full(MetricType::COSINE, 32, 200, QuantizeType::INT8, true, - QuantizerParam(true)); - ExpectIndexParamsRoundTrip(&full); -} - -TEST(ManifestCodecCrossCheck, HnswRabitqIndexParams) { - // NOTE: this is the one index type whose quantizer_param sub-message stays - // absent on the wire. - HnswRabitqIndexParams defaults(MetricType::UNDEFINED, 0, 0, 0, 0, 0); - ExpectIndexParamsRoundTrip(&defaults); - - HnswRabitqIndexParams full(MetricType::L2, 7, 4096, 48, 300, 100000); - ExpectIndexParamsRoundTrip(&full); -} - -TEST(ManifestCodecCrossCheck, FlatIndexParams) { - FlatIndexParams defaults(MetricType::UNDEFINED, QuantizeType::UNDEFINED, - QuantizerParam()); - ExpectIndexParamsRoundTrip(&defaults); - - FlatIndexParams full(MetricType::IP, QuantizeType::FP16, - QuantizerParam(true)); - ExpectIndexParamsRoundTrip(&full); -} - -TEST(ManifestCodecCrossCheck, IVFIndexParams) { - IVFIndexParams defaults(MetricType::UNDEFINED, 0, 0, false, - QuantizeType::UNDEFINED, QuantizerParam()); - ExpectIndexParamsRoundTrip(&defaults); - - IVFIndexParams full(MetricType::L2, 2048, 25, true, QuantizeType::INT4, - QuantizerParam(true)); - ExpectIndexParamsRoundTrip(&full); -} - -TEST(ManifestCodecCrossCheck, DiskAnnIndexParams) { - DiskAnnIndexParams defaults(MetricType::UNDEFINED, 0, 0, 0, - QuantizeType::UNDEFINED, QuantizerParam()); - ExpectIndexParamsRoundTrip(&defaults); - - DiskAnnIndexParams full(MetricType::COSINE, 96, 128, 32, QuantizeType::INT8, - QuantizerParam(true)); - ExpectIndexParamsRoundTrip(&full); -} - -TEST(ManifestCodecCrossCheck, VamanaIndexParams) { - // alpha is the only fixed32 field in the whole format. - VamanaIndexParams defaults(MetricType::UNDEFINED, 0, 0, 0.0f, false, false, - false, QuantizeType::UNDEFINED, QuantizerParam()); - ExpectIndexParamsRoundTrip(&defaults); - - VamanaIndexParams full(MetricType::L2, 64, 128, 1.2f, true, true, true, - QuantizeType::RABITQ, QuantizerParam(true)); - ExpectIndexParamsRoundTrip(&full); - - VamanaIndexParams negative_alpha(MetricType::IP, 8, 16, -2.5f, false, false, - false, QuantizeType::UNDEFINED, - QuantizerParam()); - ExpectIndexParamsRoundTrip(&negative_alpha); -} - -TEST(ManifestCodecCrossCheck, FtsIndexParams) { - FtsIndexParams defaults("", {}, ""); - ExpectIndexParamsRoundTrip(&defaults); - - // Repeated strings, including an empty element which must still be written. - FtsIndexParams full("jieba", {"lowercase", "", "stop"}, R"({"k1":1.2})"); - ExpectIndexParamsRoundTrip(&full); - - auto decoded = ManifestCodec::DecodeIndexParams(EncodeNew(&full)); - ASSERT_NE(decoded, nullptr); - auto *fts = dynamic_cast(decoded.get()); - ASSERT_NE(fts, nullptr); - EXPECT_EQ(fts->tokenizer_name(), "jieba"); - ASSERT_EQ(fts->filters().size(), 3u); - EXPECT_EQ(fts->filters()[0], "lowercase"); - EXPECT_EQ(fts->filters()[1], ""); - EXPECT_EQ(fts->filters()[2], "stop"); - EXPECT_EQ(fts->extra_params(), R"({"k1":1.2})"); -} - -TEST(ManifestCodecCrossCheck, FieldSchema) { - // Without index params. - FieldSchema plain; - plain.set_name("id"); - plain.set_data_type(DataType::UINT64); - plain.set_nullable(false); - { - std::string bytes_new; - ManifestCodec::EncodeFieldSchema(plain, &bytes_new); - EXPECT_EQ(bytes_new, ProtoConverter::ToPb(plain).SerializeAsString()); - } - - // With index params and a non-zero dimension. - FieldSchema vector_field; - vector_field.set_name("dense"); - vector_field.set_data_type(DataType::VECTOR_FP32); - vector_field.set_dimension(128); - vector_field.set_nullable(true); - vector_field.set_index_params(std::make_shared( - MetricType::IP, 16, 100, QuantizeType::UNDEFINED, false, - QuantizerParam())); - { - std::string bytes_new; - ManifestCodec::EncodeFieldSchema(vector_field, &bytes_new); - const std::string bytes_pb = - ProtoConverter::ToPb(vector_field).SerializeAsString(); - ASSERT_EQ(bytes_new, bytes_pb); - - auto decoded = ManifestCodec::DecodeFieldSchema(bytes_pb); - ASSERT_NE(decoded, nullptr); - EXPECT_EQ(decoded->name(), "dense"); - EXPECT_EQ(decoded->data_type(), DataType::VECTOR_FP32); - EXPECT_EQ(decoded->dimension(), 128u); - EXPECT_TRUE(decoded->nullable()); - ASSERT_NE(decoded->index_params(), nullptr); - EXPECT_EQ(decoded->index_params()->type(), IndexType::HNSW); - } -} - -TEST(ManifestCodecCrossCheck, CollectionSchema) { - CollectionSchema empty; - { - std::string bytes_new; - ManifestCodec::EncodeCollectionSchema(empty, &bytes_new); - EXPECT_EQ(bytes_new, ProtoConverter::ToPb(empty).SerializeAsString()); - } - - CollectionSchema schema; - schema.set_name("test_collection"); - schema.set_max_doc_count_per_segment(100000); - - auto pk = std::make_shared(); - pk->set_name("pk"); - pk->set_data_type(DataType::STRING); - schema.add_field(pk); - - auto dense = std::make_shared(); - dense->set_name("dense"); - dense->set_data_type(DataType::VECTOR_FP32); - dense->set_dimension(64); - dense->set_index_params(std::make_shared( - MetricType::IP, QuantizeType::UNDEFINED, QuantizerParam())); - schema.add_field(dense); - - auto text = std::make_shared(); - text->set_name("text"); - text->set_data_type(DataType::STRING); - text->set_index_params(std::make_shared( - "standard", std::vector{"lowercase"}, "")); - schema.add_field(text); - - std::string bytes_new; - ManifestCodec::EncodeCollectionSchema(schema, &bytes_new); - const std::string bytes_pb = ProtoConverter::ToPb(schema).SerializeAsString(); - ASSERT_EQ(bytes_new, bytes_pb); - - auto decoded = ManifestCodec::DecodeCollectionSchema(bytes_pb); - ASSERT_NE(decoded, nullptr); - EXPECT_EQ(decoded->name(), "test_collection"); - EXPECT_EQ(decoded->max_doc_count_per_segment(), 100000u); - ASSERT_EQ(decoded->fields().size(), 3u); - EXPECT_EQ(decoded->fields()[1]->index_params()->type(), IndexType::FLAT); - EXPECT_EQ(decoded->fields()[2]->index_params()->type(), IndexType::FTS); -} - -TEST(ManifestCodecCrossCheck, BlockMeta) { - BlockMeta empty; - { - std::string bytes_new; - ManifestCodec::EncodeBlockMeta(empty, &bytes_new); - EXPECT_EQ(bytes_new, ProtoConverter::ToPb(empty).SerializeAsString()); - } - - BlockMeta meta; - meta.set_id(7); - meta.set_type(BlockType::VECTOR_INDEX_QUANTIZE); - meta.set_min_doc_id(1); - meta.set_max_doc_id(1ULL << 40); - meta.set_doc_count(4096); - meta.add_column("dense"); - meta.add_column("sparse"); - - std::string bytes_new; - ManifestCodec::EncodeBlockMeta(meta, &bytes_new); - const std::string bytes_pb = ProtoConverter::ToPb(meta).SerializeAsString(); - ASSERT_EQ(bytes_new, bytes_pb); - - auto decoded = ManifestCodec::DecodeBlockMeta(bytes_pb); - ASSERT_NE(decoded, nullptr); - EXPECT_EQ(decoded->id(), 7u); - EXPECT_EQ(decoded->type(), BlockType::VECTOR_INDEX_QUANTIZE); - EXPECT_EQ(decoded->min_doc_id(), 1u); - EXPECT_EQ(decoded->max_doc_id(), 1ULL << 40); - EXPECT_EQ(decoded->doc_count(), 4096u); - ASSERT_EQ(decoded->columns().size(), 2u); - EXPECT_EQ(decoded->columns()[0], "dense"); -} - -TEST(ManifestCodecCrossCheck, SegmentMeta) { - SegmentMeta empty(0); - { - std::string bytes_new; - ManifestCodec::EncodeSegmentMeta(empty, &bytes_new); - EXPECT_EQ(bytes_new, ProtoConverter::ToPb(empty).SerializeAsString()); - } - - SegmentMeta meta(3); - BlockMeta scalar; - scalar.set_id(0); - scalar.set_type(BlockType::SCALAR); - scalar.set_doc_count(10); - scalar.add_column("pk"); - meta.add_persisted_block(scalar); - - BlockMeta index_block; - index_block.set_id(1); - index_block.set_type(BlockType::VECTOR_INDEX); - index_block.add_column("dense"); - meta.add_persisted_block(index_block); - - BlockMeta writing; - writing.set_id(2); - writing.set_type(BlockType::SCALAR); - meta.set_writing_forward_block(writing); - - meta.add_indexed_vector_field("dense"); - meta.add_indexed_vector_field("sparse"); - - std::string bytes_new; - ManifestCodec::EncodeSegmentMeta(meta, &bytes_new); - const std::string bytes_pb = ProtoConverter::ToPb(meta).SerializeAsString(); - ASSERT_EQ(bytes_new, bytes_pb); - - auto decoded = ManifestCodec::DecodeSegmentMeta(bytes_pb); - ASSERT_NE(decoded, nullptr); - EXPECT_EQ(decoded->id(), 3u); - EXPECT_EQ(decoded->persisted_blocks().size(), 2u); - ASSERT_TRUE(decoded->has_writing_forward_block()); - EXPECT_EQ(decoded->writing_forward_block()->id(), 2u); - EXPECT_EQ(decoded->indexed_vector_fields().size(), 2u); -} - -namespace { - -//! Builds a manifest covering every oneof branch and both segment slots. -ManifestData MakeRichManifest() { - ManifestData data; - data.enable_mmap = true; - data.id_map_path_suffix = 5; - data.delete_snapshot_path_suffix = 9; - data.next_segment_id = 12; - - auto schema = std::make_shared(); - schema->set_name("rich"); - schema->set_max_doc_count_per_segment(1u << 20); - - struct FieldSpec { - const char *name; - DataType type; - uint32_t dimension; - IndexParams::Ptr params; - }; - const std::vector specs = { - {"f_invert", DataType::INT64, 0, - std::make_shared(true)}, - {"f_hnsw", DataType::VECTOR_FP32, 128, - std::make_shared(MetricType::L2, 16, 200, - QuantizeType::INT8, true, - QuantizerParam(true))}, - {"f_flat", DataType::VECTOR_FP16, 64, - std::make_shared(MetricType::IP, QuantizeType::FP16, - QuantizerParam())}, - {"f_ivf", DataType::VECTOR_INT8, 32, - std::make_shared(MetricType::COSINE, 512, 12, true, - QuantizeType::INT4, - QuantizerParam(true))}, - {"f_rabitq", DataType::VECTOR_FP32, 256, - std::make_shared(MetricType::L2, 5, 1024, 24, 150, - 50000)}, - {"f_vamana", DataType::VECTOR_FP32, 96, - std::make_shared(MetricType::L2, 48, 96, 1.35f, true, - false, true, QuantizeType::RABITQ, - QuantizerParam(true))}, - {"f_fts", DataType::STRING, 0, - std::make_shared( - "jieba", std::vector{"lowercase", "stop"}, "{}")}, - {"f_diskann", DataType::VECTOR_FP32, 512, - std::make_shared( - MetricType::L2, 64, 128, 16, QuantizeType::INT8, QuantizerParam())}, - {"f_plain", DataType::ARRAY_DOUBLE, 0, nullptr}, - }; - - for (const auto &spec : specs) { - auto field = std::make_shared(); - field->set_name(spec.name); - field->set_data_type(spec.type); - field->set_dimension(spec.dimension); - field->set_nullable(true); - if (spec.params) { - field->set_index_params(spec.params); - } - schema->add_field(field); - } - data.schema = schema; - - for (uint32_t id = 0; id < 2; ++id) { - auto segment = std::make_shared(id); - BlockMeta block; - block.set_id(id); - block.set_type(BlockType::SCALAR); - block.set_min_doc_id(id * 1000); - block.set_max_doc_id(id * 1000 + 999); - block.set_doc_count(1000); - block.add_column("pk"); - segment->add_persisted_block(block); - segment->add_indexed_vector_field("f_hnsw"); - data.persisted_segment_metas.push_back(segment); - } - - auto writing = std::make_shared(2); - BlockMeta writing_block; - writing_block.set_id(0); - writing_block.set_type(BlockType::SCALAR); - writing->set_writing_forward_block(writing_block); - data.writing_segment_meta = writing; - - return data; -} - -//! Serializes a manifest through the protobuf library, mirroring what -//! Version::Save used to do. -std::string EncodeManifestPb(const ManifestData &data) { - proto::Manifest manifest; - auto schema_pb = ProtoConverter::ToPb(*data.schema); - manifest.mutable_schema()->Swap(&schema_pb); - manifest.set_enable_mmap(data.enable_mmap); - for (const auto &meta : data.persisted_segment_metas) { - auto meta_pb = ProtoConverter::ToPb(*meta); - manifest.add_persisted_segment_metas()->Swap(&meta_pb); - } - if (data.writing_segment_meta) { - auto meta_pb = ProtoConverter::ToPb(*data.writing_segment_meta); - manifest.mutable_writing_segment_meta()->Swap(&meta_pb); - } - manifest.set_id_map_path_suffix(data.id_map_path_suffix); - manifest.set_delete_snapshot_path_suffix(data.delete_snapshot_path_suffix); - manifest.set_next_segment_id(data.next_segment_id); - return manifest.SerializeAsString(); -} - -} // namespace - -TEST(ManifestCodecCrossCheck, FullManifestBytesMatch) { - const ManifestData data = MakeRichManifest(); - - std::string bytes_new; - ASSERT_TRUE(ManifestCodec::Encode(data, &bytes_new).ok()); - const std::string bytes_pb = EncodeManifestPb(data); - ASSERT_EQ(bytes_new, bytes_pb); - - // protobuf must accept our output. - proto::Manifest parsed; - ASSERT_TRUE(parsed.ParseFromString(bytes_new)); - EXPECT_EQ(parsed.schema().fields_size(), 9); - EXPECT_TRUE(parsed.enable_mmap()); - EXPECT_EQ(parsed.next_segment_id(), 12u); - - // And we must accept protobuf's output, preserving every field. - ManifestData decoded; - ASSERT_TRUE(ManifestCodec::Decode(bytes_pb, &decoded).ok()); - EXPECT_TRUE(decoded.enable_mmap); - EXPECT_EQ(decoded.id_map_path_suffix, 5u); - EXPECT_EQ(decoded.delete_snapshot_path_suffix, 9u); - EXPECT_EQ(decoded.next_segment_id, 12u); - ASSERT_NE(decoded.schema, nullptr); - EXPECT_EQ(decoded.schema->name(), "rich"); - ASSERT_EQ(decoded.schema->fields().size(), 9u); - EXPECT_EQ(decoded.persisted_segment_metas.size(), 2u); - ASSERT_NE(decoded.writing_segment_meta, nullptr); - EXPECT_EQ(decoded.writing_segment_meta->id(), 2u); -} - -TEST(ManifestCodecCrossCheck, EmptyManifest) { - ManifestData data; - data.schema = std::make_shared(); - - std::string bytes_new; - ASSERT_TRUE(ManifestCodec::Encode(data, &bytes_new).ok()); - EXPECT_EQ(bytes_new, EncodeManifestPb(data)); -} - -TEST(ManifestCodecCrossCheck, IndexParamsOneofLastWins) { - // protobuf oneof semantics: when several branches appear on the wire the - // last one wins. Craft such a buffer by concatenating two encodings. - FlatIndexParams flat(MetricType::IP, QuantizeType::UNDEFINED, - QuantizerParam()); - HnswIndexParams hnsw(MetricType::L2, 16, 100, QuantizeType::UNDEFINED, false, - QuantizerParam()); - const std::string combined = EncodeNew(&flat) + EncodeNew(&hnsw); - - auto ours = ManifestCodec::DecodeIndexParams(combined); - ASSERT_NE(ours, nullptr); - - proto::IndexParams pb; - ASSERT_TRUE(pb.ParseFromString(combined)); - auto theirs = ProtoConverter::FromPb(pb); - ASSERT_NE(theirs, nullptr); - - EXPECT_EQ(ours->type(), theirs->type()); -} diff --git a/tests/db/index/common/version_manager_test.cc b/tests/db/index/common/version_manager_test.cc index 08cdf4e53..a9320b395 100644 --- a/tests/db/index/common/version_manager_test.cc +++ b/tests/db/index/common/version_manager_test.cc @@ -18,7 +18,6 @@ #include #include "db/common/file_helper.h" #include "db/index/common/meta.h" -#include "proto/zvec.pb.h" #include "zvec/db/schema.h" namespace zvec { @@ -254,27 +253,23 @@ TEST_F(VersionManagerTest, ErrorConditions) { EXPECT_FALSE(version_manager->remove_persisted_segment_meta(999).ok()); } -// Test conversion between protobuf and internal schema +// Test that a schema round-trips through a Version object. TEST_F(VersionManagerTest, SchemaConversion) { - // Create protobuf schema - zvec::proto::CollectionSchema pb_schema; - pb_schema.set_name("test_collection"); - - auto pb_field = pb_schema.add_fields(); - pb_field->set_name("vector_field"); - pb_field->set_data_type(zvec::proto::DataType::DT_VECTOR_FP32); - pb_field->set_dimension(128); - - // Convert to internal schema (this would be done in the Load method) CollectionSchema internal_schema; - internal_schema.set_name(pb_schema.name()); - // In a real implementation, fields would be converted here + internal_schema.set_name("test_collection"); + + auto field = std::make_shared(); + field->set_name("vector_field"); + field->set_data_type(DataType::VECTOR_FP32); + field->set_dimension(128); + internal_schema.add_field(field); - // Test that we can set and retrieve the schema Version version; version.set_schema(internal_schema); EXPECT_EQ(version.schema().name(), "test_collection"); + ASSERT_EQ(version.schema().fields().size(), 1u); + EXPECT_EQ(version.schema().fields()[0]->dimension(), 128u); } // Test SegmentMeta functionality diff --git a/tests/db/sqlengine/CMakeLists.txt b/tests/db/sqlengine/CMakeLists.txt index edfc86540..42a3ee260 100644 --- a/tests/db/sqlengine/CMakeLists.txt +++ b/tests/db/sqlengine/CMakeLists.txt @@ -23,7 +23,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) cc_gmock( NAME ${CC_TARGET} STRICT LIBS zvec_common - zvec_proto zvec_sqlengine zvec zvec_ailego diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a66850c32..42aab5a2f 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -38,7 +38,6 @@ add_subdirectory(arrow arrow EXCLUDE_FROM_ALL) add_subdirectory(gflags gflags EXCLUDE_FROM_ALL) add_subdirectory(glog glog EXCLUDE_FROM_ALL) add_subdirectory(yaml-cpp yaml-cpp EXCLUDE_FROM_ALL) -add_subdirectory(protobuf protobuf EXCLUDE_FROM_ALL) add_subdirectory(antlr antlr EXCLUDE_FROM_ALL) add_subdirectory(lz4 lz4 EXCLUDE_FROM_ALL) add_subdirectory(rocksdb rocksdb EXCLUDE_FROM_ALL) diff --git a/thirdparty/protobuf/CMakeLists.txt b/thirdparty/protobuf/CMakeLists.txt deleted file mode 100644 index d8875324e..000000000 --- a/thirdparty/protobuf/CMakeLists.txt +++ /dev/null @@ -1,76 +0,0 @@ -set(protobuf_BUILD_TESTS OFF CACHE BOOL "Disable testing in protobuf" FORCE) -set(protobuf_WITH_ZLIB ON CACHE BOOL "Disable zlib support in protobuf" FORCE) -if(MSVC) - set(protobuf_MSVC_STATIC_RUNTIME ${ZVEC_USE_STATIC_CRT} CACHE BOOL "" FORCE) -endif() - -if(MSVC) - # Remove /MP for sccache compatibility (sccache cannot cache multi-file compilations). - set(PROTOBUF_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/protobuf-3.21.12) - set(PROTOBUF_WINDOWS_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/protobuf.windows.patch) - apply_patch_once("protobuf_windows_fix" "${PROTOBUF_SRC_DIR}" "${PROTOBUF_WINDOWS_PATCH}") -endif() - -set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) -add_subdirectory(protobuf-3.21.12/cmake protobuf-3.21.12) -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) -unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) - -mark_target_includes_system(libprotobuf libprotobuf-lite libprotoc) - -if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_compile_options(libprotobuf PRIVATE - -Wno-deprecated-declarations - -Wno-invalid-noreturn - -Wno-unused-function - ) - target_compile_options(libprotoc PRIVATE - -Wno-unused-private-field - -Wno-unused-function - ) - target_compile_options(protoc PRIVATE - -Wno-unused-private-field - -Wno-unused-function - ) -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(libprotobuf PRIVATE - -Wno-deprecated-declarations - -Wno-unused-function - -Wno-maybe-uninitialized - -Wno-sign-compare - -Wno-return-type - -Wno-stringop-overflow - -Wno-stringop-overread - -Wno-array-bounds - ) - target_compile_options(libprotoc PRIVATE - -Wno-unused-private-field - -Wno-unused-function - -Wno-unused-but-set-variable - -Wno-sign-compare - ) - target_compile_options(protoc PRIVATE - -Wno-unused-private-field - -Wno-unused-function - -Wno-unused-but-set-variable - -Wno-sign-compare - ) -endif() - -get_target_property(libprotobuf_SOURCE_DIR libprotobuf SOURCE_DIR) -get_filename_component(libprotobuf_INCLUDE_DIR ${libprotobuf_SOURCE_DIR}/../src ABSOLUTE) - -set(PROTOBUF_FOUND TRUE PARENT_SCOPE) -set(PROTOBUF_INCLUDE_DIR ${libprotobuf_INCLUDE_DIR} PARENT_SCOPE) -set(PROTOBUF_INCLUDE_DIRS ${libprotobuf_INCLUDE_DIR} PARENT_SCOPE) - -set(PROTOBUF_LIBRARY $ PARENT_SCOPE) -set(PROTOBUF_LIBRARIES $ PARENT_SCOPE) - -set(PROTOBUF_LITE_LIBRARY $ PARENT_SCOPE) -set(PROTOBUF_LITE_LIBRARIES $ PARENT_SCOPE) - -set(PROTOBUF_PROTOC_LIBRARY $ PARENT_SCOPE) -set(PROTOBUF_PROTOC_LIBRARIES $ PARENT_SCOPE) -set(PROTOBUF_PROTOC_EXECUTABLE $ PARENT_SCOPE) diff --git a/thirdparty/protobuf/protobuf-3.21.12 b/thirdparty/protobuf/protobuf-3.21.12 deleted file mode 160000 index f0dc78d7e..000000000 --- a/thirdparty/protobuf/protobuf-3.21.12 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c diff --git a/thirdparty/protobuf/protobuf.windows.patch b/thirdparty/protobuf/protobuf.windows.patch deleted file mode 100644 index cf08e96c1..000000000 --- a/thirdparty/protobuf/protobuf.windows.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 1234567..abcdefg 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -241,10 +241,6 @@ - - if (MSVC) -- if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") -- # Build with multiple processes -- add_definitions(/MP) -- endif() - # Set source file and execution character sets to UTF-8 - add_definitions(/utf-8) - # MSVC warning suppressions - add_definitions( From e08091f99a624a1011c76035096fa86ddd51346b Mon Sep 17 00:00:00 2001 From: lc285652 Date: Thu, 30 Jul 2026 17:47:55 +0800 Subject: [PATCH 10/10] chore(db): remove the now-unused zvec.proto zvec.proto was retained as documentation when the protobuf dependency was dropped, but the manifest format is now fully defined by manifest_codec.{h,cc} (field numbers) and manifest_enum.h (enum values). Delete the file and the src/db/proto directory, and repoint the doc comments that referenced it to the codec instead. Also drop the stale clang-tidy header cache entries for build/src/db/proto and the src/db/proto/*.proto hashFiles input, which no longer exist. --- .github/workflows/clang_tidy.yml | 6 +- src/db/CMakeLists.txt | 6 +- src/db/index/common/manifest_codec.cc | 2 +- src/db/index/common/manifest_codec.h | 7 +- src/db/index/common/manifest_enum.h | 5 +- src/db/index/common/pb_wire.h | 11 +- src/db/proto/zvec.proto | 240 ------------------ .../common/manifest_codec_golden_test.cc | 2 +- 8 files changed, 19 insertions(+), 260 deletions(-) delete mode 100644 src/db/proto/zvec.proto diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 5ab872d58..161188d82 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -134,8 +134,7 @@ jobs: path: | build/external build/thirdparty - build/src/db/proto - key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }} + key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch') }}-${{ steps.submodule_hash.outputs.value }} - name: Build generated headers only if: steps.tidy_files.outputs.any_tidy_files == 'true' && steps.cache_headers.outputs.cache-hit != 'true' @@ -149,8 +148,7 @@ jobs: path: | build/external build/thirdparty - build/src/db/proto - key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }} + key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch') }}-${{ steps.submodule_hash.outputs.value }} - name: Run clang-tidy on changed files (parallel) if: steps.tidy_files.outputs.any_tidy_files == 'true' diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt index c6603bab2..741f105ce 100644 --- a/src/db/CMakeLists.txt +++ b/src/db/CMakeLists.txt @@ -1,9 +1,9 @@ include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) include(${PROJECT_ROOT_DIR}/cmake/option.cmake) -# NOTE: proto/zvec.proto is no longer compiled. It is kept as the -# authoritative documentation of the manifest on-disk format, which is now -# read and written by src/db/index/common/manifest_codec.{h,cc}. +# NOTE: the manifest on-disk format is read and written by +# src/db/index/common/manifest_codec.{h,cc}; there is no protobuf/protoc +# dependency any more. cc_directory(common) cc_directory(index) diff --git a/src/db/index/common/manifest_codec.cc b/src/db/index/common/manifest_codec.cc index fca36d759..098957639 100644 --- a/src/db/index/common/manifest_codec.cc +++ b/src/db/index/common/manifest_codec.cc @@ -24,7 +24,7 @@ namespace { using pbwire::Reader; using pbwire::Writer; -// Field numbers, mirroring src/db/proto/zvec.proto. Changing any of these +// Field numbers of the manifest wire format. Changing any of these // breaks compatibility with existing manifest files. namespace f_quantizer { constexpr uint32_t kEnableRotate = 1; diff --git a/src/db/index/common/manifest_codec.h b/src/db/index/common/manifest_codec.h index 82e7b8810..e01a8277d 100644 --- a/src/db/index/common/manifest_codec.h +++ b/src/db/index/common/manifest_codec.h @@ -14,9 +14,10 @@ //! Encoder/decoder for the collection manifest. //! -//! The on-disk layout is the protobuf wire format described by -//! src/db/proto/zvec.proto, which remains the authoritative documentation of -//! the format. Field numbers here must stay in sync with that file. +//! The on-disk layout is the protobuf wire format. This file and +//! manifest_codec.cc are the authoritative definition of that format: the +//! field numbers assigned here and the enum values in manifest_enum.h are +//! part of the persisted format and must never change (only append). //! //! Compared with the generated protobuf code this converts directly between //! zvec's own C++ types and bytes, without an intermediate message object. diff --git a/src/db/index/common/manifest_enum.h b/src/db/index/common/manifest_enum.h index 89beca152..a7cd0c392 100644 --- a/src/db/index/common/manifest_enum.h +++ b/src/db/index/common/manifest_enum.h @@ -14,9 +14,8 @@ //! On-disk enum values of the manifest format. //! -//! These mirror the enums declared in src/db/proto/zvec.proto one-to-one. -//! The numeric values are part of the persisted manifest format and must -//! never change; only append new entries. +//! These enum values are part of the persisted manifest format. The numeric +//! values must never change; only append new entries. #pragma once diff --git a/src/db/index/common/pb_wire.h b/src/db/index/common/pb_wire.h index 5183560e0..bb3ee2ef2 100644 --- a/src/db/index/common/pb_wire.h +++ b/src/db/index/common/pb_wire.h @@ -14,15 +14,16 @@ //! Minimal protobuf wire-format reader/writer. //! -//! zvec persists its manifest in protobuf wire format (see -//! src/db/proto/zvec.proto, kept as the authoritative format documentation). -//! This header implements just enough of the encoding to read and write that -//! format so that the project does not need to depend on libprotobuf/protoc. +//! zvec persists its manifest in protobuf wire format. This header implements +//! just enough of the encoding to read and write that format so that the +//! project does not need to depend on libprotobuf/protoc. The concrete +//! message layout lives in manifest_codec.{h,cc} (field numbers) and +//! manifest_enum.h (enum values), which together define the format. //! //! Wire format reference: //! https://protobuf.dev/programming-guides/encoding/ //! -//! Only the wire types actually used by zvec.proto are supported for reading +//! Only the wire types actually used by the manifest are supported for reading //! values (varint, 64-bit, length-delimited, 32-bit); deprecated groups //! (wire types 3 and 4) are rejected as corrupt input. diff --git a/src/db/proto/zvec.proto b/src/db/proto/zvec.proto deleted file mode 100644 index 4fdd5ea84..000000000 --- a/src/db/proto/zvec.proto +++ /dev/null @@ -1,240 +0,0 @@ -syntax = "proto3"; - -package zvec.proto; - -option cc_enable_arenas = true; -option optimize_for = LITE_RUNTIME; - -// The Go package name, refers to -// https://developers.google.com/protocol-buffers/docs/reference/go-generated#package -option go_package = "proxima/zvec/proto"; - -/*! Types of Data - */ -enum DataType { - DT_UNDEFINED = 0; - - DT_BINARY = 1; - DT_STRING = 2; - DT_BOOL = 3; - DT_INT32 = 4; - DT_INT64 = 5; - DT_UINT32 = 6; - DT_UINT64 = 7; - DT_FLOAT = 8; - DT_DOUBLE = 9; - - DT_VECTOR_BINARY32 = 20; - DT_VECTOR_BINARY64 = 21; - DT_VECTOR_FP16 = 22; - DT_VECTOR_FP32 = 23; - DT_VECTOR_FP64 = 24; - DT_VECTOR_INT4 = 25; - DT_VECTOR_INT8 = 26; - DT_VECTOR_INT16 = 27; - - DT_SPARSE_VECTOR_FP16 = 30; - DT_SPARSE_VECTOR_FP32 = 31; - - // ARRAY - DT_ARRAY_BINARY = 40; - DT_ARRAY_STRING = 41; - DT_ARRAY_BOOL = 42; - DT_ARRAY_INT32 = 43; - DT_ARRAY_INT64 = 44; - DT_ARRAY_UINT32 = 45; - DT_ARRAY_UINT64 = 46; - DT_ARRAY_FLOAT = 47; - DT_ARRAY_DOUBLE = 48; -}; - -enum IndexType { - // Undefined - IT_UNDEFINED = 0; - // Proxima HNSW Index - IT_HNSW = 1; - // Proxima IVF Index - IT_IVF = 2; - // Proxima FLAT Index - IT_FLAT = 3; - // Proxima HNSW RABITQ Index - IT_HNSW_RABITQ = 4; - // Proxima Vamana Index - IT_VAMANA = 5; - // Proxima DiskAnn Index - IT_DISKANN = 6; - // Invert Index - IT_INVERT = 10; - // Full-Text Search Index - IT_FTS = 11; -}; - -enum QuantizeType { - QT_UNDEFINED = 0; - QT_FP16 = 1; - QT_INT8 = 2; - QT_INT4 = 3; - QT_RABITQ = 4; -}; - -enum MetricType { - MT_UNDEFINED = 0; - MT_L2 = 1; - MT_IP = 2; - MT_COSINE = 3; -}; - -message InvertIndexParams { - bool enable_range_optimization = 1; -}; - -// Quantizer-related parameters for vector indexes. -// Designed for future extensibility. -message QuantizerParam { - // When enabled, vectors are rotated before INT8 quantization to reduce - // quantization error. Only effective with quantize_type=INT8. - bool enable_rotate = 1; -}; - -message BaseIndexParams { - MetricType metric_type = 1; - QuantizeType quantize_type = 2; - // Quantizer parameters (enable_rotate, etc.) - QuantizerParam quantizer_param = 4; -}; - -message HnswIndexParams { - BaseIndexParams base = 1; - int32 m = 2; - int32 ef_construction = 3; - // When enabled, the HNSW streamer allocates a single contiguous memory - // arena for all graph nodes, which improves cache locality / search - // throughput at the cost of peak memory usage. Defaults to false. - bool use_contiguous_memory = 4; -} - -message HnswRabitqIndexParams { - BaseIndexParams base = 1; - int32 m = 2; - int32 ef_construction = 3; - int32 total_bits = 4; - int32 num_clusters = 5; - int32 sample_count = 6; -} - -message FlatIndexParams { - BaseIndexParams base = 1; -} - -message IVFIndexParams { - BaseIndexParams base = 1; - int32 n_list = 2; - int32 n_iters = 3; - bool use_soar = 4; -} - -message DiskAnnIndexParams { - BaseIndexParams base = 1; - int32 max_degree = 2; - int32 list_size = 3; - int32 pq_chunk_num = 4; -} - -message VamanaIndexParams { - BaseIndexParams base = 1; - int32 max_degree = 2; - int32 search_list_size = 3; - float alpha = 4; - bool saturate_graph = 5; - // When enabled, the Vamana streamer allocates a single contiguous memory - // arena for all graph nodes, which improves cache locality / search - // throughput at the cost of peak memory usage. Defaults to false. - bool use_contiguous_memory = 6; - bool use_id_map = 7; -} - -message FtsIndexParams { - string tokenizer_name = 1; - repeated string filters = 2; - string extra_params = 3; -}; - -message IndexParams { - oneof params { - InvertIndexParams invert = 1; - HnswIndexParams hnsw = 2; - FlatIndexParams flat = 3; - IVFIndexParams ivf = 4; - HnswRabitqIndexParams hnsw_rabitq = 5; - VamanaIndexParams vamana = 6; - FtsIndexParams fts = 7; - DiskAnnIndexParams diskann = 8; - }; -}; - -message FieldSchema { - string name = 1; - DataType data_type = 2; - uint32 dimension = 3; - bool nullable = 4; - IndexParams index_params = 5; -}; - -message CollectionSchema { - string name = 1; - repeated FieldSchema fields = 2; - uint64 max_doc_count_per_segment = 3; -}; - -enum BlockType { - BT_UNDEFINED = 0; - BT_SCALAR = 1; - BT_SCALAR_INDEX = 2; - BT_VECTOR_INDEX = 3; - BT_VECTOR_INDEX_QUANTIZE = 4; - BT_FTS_INDEX = 5; -}; - -message BlockMeta { - uint32 block_id = 1; - BlockType block_type = 2; // for getting filename prefix - uint64 min_doc_id = 3; - uint64 max_doc_id = 4; - uint64 doc_count = 5; - repeated string columns = 6; // columns contained in this block -}; - -// message AlterColumnMeta { -// string old_column_name = 1; -// FieldSchema new_schema = 2; -// }; - -message SegmentMeta { - uint32 segment_id = 1; - // scalar data, vector data and vector index - repeated BlockMeta persisted_blocks = 2; - - BlockMeta writing_forward_block = 3; - - // if indexed, index_params can be retrieved from schema - // if not indexed, index_params is default index_params(flat) - repeated string indexed_vector_fields = 4; - // repeated AlterColumnMeta alter_columns = 10; -}; - -message Manifest { - uint32 version = 1; - - CollectionSchema schema = 2; - - bool enable_mmap = 3; - - repeated SegmentMeta persisted_segment_metas = 4; - - SegmentMeta writing_segment_meta = 5; - - uint32 id_map_path_suffix = 6; - uint32 delete_snapshot_path_suffix = 7; - - uint32 next_segment_id = 8; -}; diff --git a/tests/db/index/common/manifest_codec_golden_test.cc b/tests/db/index/common/manifest_codec_golden_test.cc index 5e4803eb1..0ab61634b 100644 --- a/tests/db/index/common/manifest_codec_golden_test.cc +++ b/tests/db/index/common/manifest_codec_golden_test.cc @@ -254,7 +254,7 @@ TEST(ManifestCodecGolden, CorruptInputIsRejected) { } TEST(ManifestCodecGolden, WireReaderRejectsGroups) { - // Wire types 3 and 4 (deprecated groups) are never produced by zvec.proto + // Wire types 3 and 4 (deprecated groups) are never produced by the manifest // and must be treated as corrupt input rather than silently skipped. for (uint8_t type : {3, 4}) { std::string buf;