Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/core/algorithm/vamana/vamana_streamer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ int VamanaStreamer::open(IndexStorage::Pointer stg) {
metric_params.merge(meta_.metric_params());
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
// Propagate reformer info from stored meta (needed for quantizers
// whose reformer params are computed during training, e.g. UniformInt8)
// whose reformer params are computed during training, e.g. UniformUint7)
if (!index_meta.reformer_name().empty()) {
meta_.set_reformer(index_meta.reformer_name(), 0,
index_meta.reformer_params());
Expand Down
9 changes: 6 additions & 3 deletions src/core/interface/index.cc
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,11 @@ int Index::CreateAndInitConverterReformer(const QuantizerParam &param,
case QuantizerType::kRabitq:
// no converter here
return 0;
case QuantizerType::kUniformInt8:
converter_name = "UniformInt8StreamingConverter";
case QuantizerType::kUniformUint7:
converter_name = "UniformUint7StreamingConverter";
break;
case QuantizerType::kUniformUint8:
converter_name = "UniformUint8StreamingConverter";
break;
default:
LOG_ERROR("Unsupported quantizer type: ");
Expand Down Expand Up @@ -371,7 +374,7 @@ int Index::Open(const std::string &file_path, StorageOptions storage_options) {
}

// If a converter exists but reformer was not created during Init()
// (converters like UniformInt8 whose reformer params are only available
// (uniform quantizers whose reformer params are only available
// after train()), create it now from the persisted meta that the streamer
// has loaded. When there is no converter (QuantizerType::kNone), reformer_
// is nullptr by design — skip this block entirely.
Expand Down
12 changes: 8 additions & 4 deletions src/core/metric/metric_params.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,13 @@ static const std::string QUANTIZED_INTEGER_METRIC_ORIGIN_METRIC_NAME =
static const std::string QUANTIZED_INTEGER_METRIC_ORIGIN_METRIC_PARAMS =
"proxima.quantized_integer.metric.origin_metric_params";

//! UniformInt8 Metric
static const std::string UNIFORM_INT8_METRIC_ORIGIN_METRIC_NAME =
"proxima.uniform_int8.metric.origin_metric_name";
//! UniformUint7 Metric
static const std::string UNIFORM_UINT7_METRIC_ORIGIN_METRIC_NAME =
"proxima.uniform_uint7.metric.origin_metric_name";

//! UniformUint8 Metric
static const std::string UNIFORM_UINT8_METRIC_ORIGIN_METRIC_NAME =
"proxima.uniform_uint8.metric.origin_metric_name";

} // namespace core
} // namespace zvec
} // namespace zvec
Original file line number Diff line number Diff line change
Expand Up @@ -16,46 +16,52 @@
#include <ailego/math_batch/euclidean_distance_batch.h>
#include <zvec/core/framework/index_error.h>
#include <zvec/core/framework/index_factory.h>
#include <zvec/core/interface/index_param.h>
#include <zvec/turbo/turbo.h>
#include "metric_params.h"

namespace zvec {
namespace core {

/*! Index Metric for Uniform Int8 Quantization (Global Scale)
/*! Index Metric for Uniform Uint7 Quantization (Global Scale)
*
* Uses direct int8 L2 distance computation. Since all vectors share
* a single global scale/bias, no per-vector reconstruction is needed.
* This is the key benefit: distance = sum((a[i] - b[i])^2) on raw int8
* values, with optional post-scaling by 1/scale^2 for real L2 distances.
*/
class UniformInt8Metric : public IndexMetric {
class UniformUint7Metric : public IndexMetric {
public:
//! Initialize Metric
int init(const IndexMeta &meta, const ailego::Params &index_params) override {
if (meta.data_type() != IndexMeta::DataType::DT_INT8) {
LOG_ERROR("UniformInt8Metric: unsupported type %d", meta.data_type());
LOG_ERROR("UniformUint7Metric: unsupported type %d", meta.data_type());
return IndexError_Unsupported;
}
if (meta.dimension() == 0 || meta.dimension() > MAX_DIMENSION) {
LOG_ERROR("UniformUint7Metric: dimension=%u must be in [1, %d]",
meta.dimension(), MAX_DIMENSION);
return IndexError_InvalidArgument;
}

std::string metric_name;
index_params.get(UNIFORM_INT8_METRIC_ORIGIN_METRIC_NAME, &metric_name);
index_params.get(UNIFORM_UINT7_METRIC_ORIGIN_METRIC_NAME, &metric_name);
if (metric_name.empty()) {
LOG_ERROR("UniformInt8Metric: param %s is required",
UNIFORM_INT8_METRIC_ORIGIN_METRIC_NAME.c_str());
LOG_ERROR("UniformUint7Metric: param %s is required",
UNIFORM_UINT7_METRIC_ORIGIN_METRIC_NAME.c_str());
return IndexError_InvalidArgument;
}

if (metric_name != "SquaredEuclidean") {
LOG_ERROR("UniformInt8Metric: only SquaredEuclidean supported, got %s",
LOG_ERROR("UniformUint7Metric: only SquaredEuclidean supported, got %s",
metric_name.c_str());
return IndexError_Unsupported;
}

meta_ = meta;
params_ = index_params;

LOG_INFO("UniformInt8Metric initialized: dimension=%u", meta_.dimension());
LOG_INFO("UniformUint7Metric initialized: dimension=%u", meta_.dimension());
return 0;
}

Expand All @@ -67,15 +73,16 @@ class UniformInt8Metric : public IndexMetric {
//! Retrieve if it matched
bool is_matched(const IndexMeta &meta) const override {
return meta.data_type() == meta_.data_type() &&
meta.unit_size() == meta_.unit_size();
meta.unit_size() == meta_.unit_size() &&
meta.dimension() == meta_.dimension();
}

//! Retrieve if it matched
bool is_matched(const IndexMeta &meta,
const IndexQueryMeta &qmeta) const override {
return qmeta.data_type() == meta_.data_type() &&
return is_matched(meta) && qmeta.data_type() == meta_.data_type() &&
qmeta.unit_size() == meta_.unit_size() &&
qmeta.dimension() == meta.dimension();
qmeta.dimension() == meta_.dimension();
}

//! Retrieve distance function for query (1x1)
Expand Down Expand Up @@ -152,7 +159,7 @@ class UniformInt8Metric : public IndexMetric {
ailego::Params params_{};
};

INDEX_FACTORY_REGISTER_METRIC_ALIAS(UniformInt8, UniformInt8Metric);
INDEX_FACTORY_REGISTER_METRIC_ALIAS(UniformUint7, UniformUint7Metric);

} // namespace core
} // namespace zvec
Loading
Loading