Added PQ INT8 quantizer in Turbo

src/core/interface/index.cc

@@ -290,7 +290,11 @@ int Index::Init(const BaseIndexParam &param) {
     return core::IndexError_Runtime;
   }
 
-  if (CreateAndInitConverterReformer(param.quantizer_param, param) != 0) {
+  // an absent quantizer param behaves the same as a kNone one
+  const auto quantizer_param = param.quantizer_param
+                                   ? param.quantizer_param
+                                   : std::make_shared<QuantizerParam>();
+  if (CreateAndInitConverterReformer(*quantizer_param, param) != 0) {
     LOG_ERROR("Failed to create and init converter");
     return core::IndexError_Runtime;
   }

src/core/interface/index_param.cc

@@ -185,9 +185,15 @@ ailego::JsonObject BaseIndexParam::SerializeToJsonObject(
   // if (preprocess_param) {
   //   json.set("preprocess_param", preprocess_param->SerializeToJson());
   // }
-  if (!omit_empty_value || quantizer_param.type != QuantizerType::kNone) {
+  if (quantizer_param) {
+    if (!omit_empty_value || quantizer_param->type != QuantizerType::kNone) {
+      json_obj.set("quantizer_param",
+                   quantizer_param->SerializeToJsonObject(omit_empty_value));
+    }
+  } else if (!omit_empty_value) {
+    // no quantizer configured, keep the default(kNone) object as before
     json_obj.set("quantizer_param",
-                 quantizer_param.SerializeToJsonObject(omit_empty_value));
+                 QuantizerParam().SerializeToJsonObject(false));
   }
   // if (refiner_param) {
   //   json.set("refiner_param", refiner_param->SerializeToJson());
@@ -239,7 +245,20 @@ bool BaseIndexParam::DeserializeFromJsonObject(
   if (json_obj.has("quantizer_param")) {
     if (json_obj.get("quantizer_param", &tmp_json_value);
         tmp_json_value.is_object()) {
-      quantizer_param.DeserializeFromJsonObject(tmp_json_value.as_object());
+      const auto &quantizer_json_obj = tmp_json_value.as_object();
+      // the concrete param type is determined by the quantizer type
+      auto quantizer_type = QuantizerType::kNone;
+      ailego::JsonValue type_json_value;
+      if (!extract_enum_from_json<QuantizerType>(
+              quantizer_json_obj, "type", quantizer_type, type_json_value)) {
+        LOG_ERROR("Error when deserialize json - field:quantizer_param.type");
+        return false;
+      }
+      auto quantizer = QuantizerParam::Create(quantizer_type);
+      if (!quantizer->DeserializeFromJsonObject(quantizer_json_obj)) {
+        LOG_ERROR("Error when deserialize json - field:quantizer_param");
+      }
+      quantizer_param = std::move(quantizer);
     }
   }
 
@@ -393,6 +412,33 @@ bool QuantizerParam::DeserializeFromJsonObject(
   return true;
 }
 
+QuantizerParam::Pointer QuantizerParam::Create(QuantizerType t) {
+  switch (t) {
+    case QuantizerType::kPQ:
+      return std::make_shared<PqQuantizerParam>();
+    default:
+      return std::make_shared<QuantizerParam>(t);
+  }
+}
+
+ailego::JsonObject PqQuantizerParam::SerializeToJsonObject(
+    bool omit_empty_value) const {
+  auto json_obj = QuantizerParam::SerializeToJsonObject(omit_empty_value);
+  json_obj.set("num_chunk", ailego::JsonValue(num_chunk));
+  json_obj.set("num_bits", ailego::JsonValue(num_bits));
+  return json_obj;
+}
+
+bool PqQuantizerParam::DeserializeFromJsonObject(
+    const ailego::JsonObject &json_obj) {
+  if (!QuantizerParam::DeserializeFromJsonObject(json_obj)) {
+    return false;
+  }
+  DESERIALIZE_VALUE_FIELD(json_obj, num_chunk);
+  DESERIALIZE_VALUE_FIELD(json_obj, num_bits);
+  return true;
+}
+
 // bool BaseIndexQueryParam::DeserializeFromJsonObject(
 //     const ailego::JsonObject &json_obj) {
 //   DESERIALIZE_ENUM_FIELD(json_obj, index_type, IndexType);

src/db/index/column/vector_column/engine_helper.hpp

@@ -357,7 +357,7 @@ class ProximaEngineHelper {
             convert_to_engine_quantize_type(db_index_params->quantize_type());
         quantize_type.has_value()) {
       index_param_builder->WithQuantizerParam(
-          core_interface::QuantizerParam(quantize_type.value()));
+          core_interface::QuantizerParam::Create(quantize_type.value()));
     } else {
       return tl::make_unexpected(
           Status::InvalidArgument("unsupported quantize type"));

src/include/zvec/core/interface/index_param.h

@@ -119,23 +119,49 @@ struct ZVEC_CORE_API SerializableBase {
       const ailego::JsonObject &json_obj) = 0;
 };
 
-// TODO: maybe a base class for quantizer?
+//! Common quantizer params shared by all quantizer types
 struct ZVEC_CORE_API QuantizerParam : public SerializableBase {
+  using Pointer = std::shared_ptr<QuantizerParam>;
+
   QuantizerType type = QuantizerType::kNone;
-  int num_subquantizers = 8;  // M
-  int num_bits = 8;           // bits per subquantizer
   bool enable_rotate =
       false;  // rotate vectors before quantization to reduce error
 
   // Constructors
-  // QuantizerParam() = default;
-  QuantizerParam(QuantizerType t = QuantizerType::kNone, int subquantizers = 8,
-                 int bits = 8, bool rotate = false)
-      : type(t),
-        num_subquantizers(subquantizers),
-        num_bits(bits),
-        enable_rotate(rotate) {}
+  QuantizerParam(QuantizerType t = QuantizerType::kNone, bool rotate = false)
+      : type(t), enable_rotate(rotate) {}
+  virtual ~QuantizerParam() = default;
+
+  //! Duplicate the param object, keeping the concrete type
+  virtual Pointer Clone() const {
+    return std::make_shared<QuantizerParam>(*this);
+  }
+
+  //! Create the param object matching the quantizer type
+  static Pointer Create(QuantizerType t);
+
+ protected:
+  friend class BaseIndexParam;
+  ailego::JsonObject SerializeToJsonObject(
+      bool omit_empty_value = false) const override;
+
+  bool DeserializeFromJsonObject(const ailego::JsonObject &json_obj) override;
+};
+
+//! Product-Quantization specific params
+struct PqQuantizerParam : public QuantizerParam {
+  int num_chunk = 8;  // M: number of sub-quantizers
+  int num_bits = 8;   // bits per sub-quantizer
+
+  // Constructors
+  PqQuantizerParam(int chunks = 8, int bits = 8, bool rotate = false)
+      : QuantizerParam(QuantizerType::kPQ, rotate),
+        num_chunk(chunks),
+        num_bits(bits) {}
 
+  QuantizerParam::Pointer Clone() const override {
+    return std::make_shared<PqQuantizerParam>(*this);
+  }
 
  protected:
   friend class BaseIndexParam;
@@ -306,7 +332,16 @@ class ZVEC_CORE_API BaseIndexParam : public SerializableBase {
 
   // pipeline
   PreprocessorParam preprocess_param;
-  QuantizerParam quantizer_param;
+  //! nullptr means no quantizer is configured (equivalent to kNone)
+  QuantizerParam::Pointer quantizer_param{nullptr};
+
+  QuantizerType quantizer_type() const {
+    return quantizer_param ? quantizer_param->type : QuantizerType::kNone;
+  }
+
+  bool enable_rotate() const {
+    return quantizer_param && quantizer_param->enable_rotate;
+  }
 
   BaseIndexQueryParam::Pointer default_query_param = nullptr;
   // virtual std::shared_ptr<BaseIndexQueryParam> GetDefaultQueryParam() const

src/include/zvec/core/interface/index_param_builders.h

@@ -59,7 +59,13 @@ class BaseIndexParamBuilder {  //  : public
   }
   ActualIndexParamBuilderType &WithQuantizerParam(
       const QuantizerParam &quantizer_param) {
-    param->quantizer_param = quantizer_param;
+    param->quantizer_param = quantizer_param.Clone();
+    return static_cast<ActualIndexParamBuilderType &>(*this);
+  }
+  ActualIndexParamBuilderType &WithQuantizerParam(
+      const QuantizerParam::Pointer &quantizer_param) {
+    param->quantizer_param =
+        quantizer_param ? quantizer_param->Clone() : nullptr;
     return static_cast<ActualIndexParamBuilderType &>(*this);
   }
   // ActualIndexParamBuilderType &WithRefinerParam(
@@ -88,7 +94,12 @@ class BaseIndexParamBuilder {  //  : public
   }
 
   ActualIndexParamBuilderType &WithEnableRotate(bool enable_rotate) {
-    param->quantizer_param.enable_rotate = enable_rotate;
+    // copy-on-write: never mutate a param object shared with others
+    auto quantizer_param = param->quantizer_param
+                               ? param->quantizer_param->Clone()
+                               : std::make_shared<QuantizerParam>();
+    quantizer_param->enable_rotate = enable_rotate;
+    param->quantizer_param = std::move(quantizer_param);
     return static_cast<ActualIndexParamBuilderType &>(*this);
   }
 

src/include/zvec/turbo/turbo.h

@@ -65,12 +65,42 @@ using RotateFunc = void (*)(const float *in, float *out, size_t in_dim,
 using UnrotateFunc = void (*)(const float *in, float *out, size_t in_dim,
                               size_t out_dim, void *ctx);
 
+// PQ kernel function pointer types.
+//
+// ADC: LUT look-up distance between a PQ code and a query (via LUT).
+//   pq_code:           [num_chunk] uint8_t
+//   lut:               [num_chunk * 256] float
+// Uses void* to match DistanceFunc signature for direct assignment.
+using PqAdcDistanceFunc = void (*)(const void *pq_code, const void *lut,
+                                   size_t num_chunk, float *out);
+
+// SDC kernel: centroid-to-centroid distance between two PQ codes.
+//   a, b:              [num_chunk] uint8_t
+//   dist_table:        [num_chunk * 256 * 256] float
+// Uses void* for consistency with DistanceFunc / PqAdcDistanceFunc.
+using PqSdcKernelFunc = void (*)(const void *a, const void *b,
+                                 const void *dist_table, size_t num_chunk,
+                                 float *out);
+
+// Batch ADC: compute distances for multiple PQ codes against a shared LUT.
+// Signature matches BatchDistanceFunc for direct assignment (no lambda).
+using PqBatchAdcFunc = void (*)(const void **candidates, const void *lut,
+                                size_t num, size_t num_chunk, float *out);
+
 // ISA-dispatched rotate/unrotate kernels.
 struct RotatorKernels {
   RotateFunc rotate = nullptr;
   UnrotateFunc unrotate = nullptr;
 };
 
+// data_type selects the code packing layout:
+//   kInt8: one uint8 per sub-quantizer (256 centroids, stride=256)
+struct PqKernels {
+  PqAdcDistanceFunc adc_distance = nullptr;
+  PqSdcKernelFunc sdc_distance = nullptr;
+  PqBatchAdcFunc batch_adc_distance = nullptr;
+};
+
 enum class MetricType {
   kSquaredEuclidean,
   kCosine,
@@ -142,4 +172,10 @@ get_uniform_quantize_func(DataType data_type);
 RotatorKernels get_rotator_kernels(
     RotateType rotate_type, CpuArchType cpu_arch_type = CpuArchType::kAuto);
 
+// Returns all PQ kernels dispatched for the given data_type, quantize_type
+// and CPU arch.
+PqKernels get_pq_kernels(DataType data_type,
+                         QuantizeType quantize_type = QuantizeType::kPQ,
+                         CpuArchType cpu_arch_type = CpuArchType::kAuto);
+
 }  // namespace zvec::turbo