Skip to content

Commit c11286c

Browse files
Neiko2002Neiko2002
committed
deterministic normal distribution for our regression tests
1 parent 0ec78c6 commit c11286c

8 files changed

Lines changed: 139 additions & 99 deletions

File tree

‎.github/workflows/cpp-ci.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ jobs:
2222
steps:
2323
- uses: actions/checkout@v4
2424

25+
- name: Install Build Tools (Linux)
26+
if: runner.os == 'Linux'
27+
run: |
28+
apt-get update
29+
apt-get install -y cmake g++
30+
2531
- name: Configure CMake
2632
run: cmake -B build -S cpp -DFORCE_AVX2=OFF -DENABLE_BENCHMARKS=OFF
2733

‎cpp/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ if (FORCE_AVX2)
3737

3838
# setup compiler flags
3939
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
40-
target_compile_options(compile-options INTERFACE -O3 -mavx2 -mfma -fpic -w -fopenmp -pthread -ftree-vectorize)
40+
target_compile_options(compile-options INTERFACE -O3 -mavx2 -mfma -fpic -w -pthread -ftree-vectorize)
4141
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
4242
target_compile_options(compile-options INTERFACE -O2 -fpic)
4343

@@ -70,7 +70,7 @@ else() # Native build
7070
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
7171
target_compile_options(compile-options INTERFACE -O2 -DNDEBUG -march=native -fpic)
7272
elseif(MSVC)
73-
target_compile_options(compile-options INTERFACE /W4 /EHsc /fp:precise $<$<CONFIG:Release>:/O2>)
73+
target_compile_options(compile-options INTERFACE /W4 /EHsc $<$<CONFIG:Release>:/O2>)
7474
if(SUPPORT_AVX512F)
7575
target_compile_options(compile-options INTERFACE /arch:AVX512)
7676
elseif(SUPPORT_AVX2)

‎cpp/deglib/include/distance/fp32_ip.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,15 +92,17 @@ namespace deglib::distances::fp32_ip {
9292

9393
const float *last = a + size;
9494

95-
__m256 sum256 = _mm256_setzero_ps();
95+
__m256 sum256_1 = _mm256_setzero_ps();
96+
__m256 sum256_2 = _mm256_setzero_ps();
9697
while (a < last) {
97-
sum256 = _mm256_fmadd_ps(_mm256_loadu_ps(a), _mm256_loadu_ps(b), sum256);
98+
sum256_1 = _mm256_fmadd_ps(_mm256_loadu_ps(a), _mm256_loadu_ps(b), sum256_1);
9899
a += 8;
99100
b += 8;
100-
sum256 = _mm256_fmadd_ps(_mm256_loadu_ps(a), _mm256_loadu_ps(b), sum256);
101+
sum256_2 = _mm256_fmadd_ps(_mm256_loadu_ps(a), _mm256_loadu_ps(b), sum256_2);
101102
a += 8;
102103
b += 8;
103104
}
105+
__m256 sum256 = _mm256_add_ps(sum256_1, sum256_2);
104106
__m128 sum128 = _mm_add_ps(_mm256_extractf128_ps(sum256, 0), _mm256_extractf128_ps(sum256, 1));
105107
alignas(32) float f[4];
106108
_mm_store_ps(f, sum128);

‎cpp/test/src/regression/deterministic_normal_distribution.h‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#pragma once
2+
3+
#include <cmath>
4+
#include <cstdint>
5+
#include <random>
6+
7+
// Portable normal distribution using Box-Muller transform.
8+
// std::normal_distribution is NOT portable across compilers/platforms:
9+
// - MSVC STL, libstdc++, and libc++ each use different algorithms
10+
// - The same seed produces different sequences on different platforms
11+
// - This makes regression tests non-deterministic across CI runners
12+
//
13+
// This implementation uses the Box-Muller transform with std::mt19937
14+
// (which IS portable — same seed produces same uniform sequence) and
15+
// a manual uniform_real_distribution to ensure portability.
16+
//
17+
// Reference: Box, G.E.P. and Muller, M.E. (1958), "A Note on the Generation
18+
// of Random Normal Deviates", Annals of Mathematical Statistics, 29(2), pp.610-611.
19+
class DeterministicNormalDistribution {
20+
public:
21+
explicit DeterministicNormalDistribution(float mean, float stddev)
22+
: mean_(mean), stddev_(stddev) {}
23+
24+
// Generate a single normally-distributed float using the provided RNG.
25+
// Always consumes exactly 2 RNG values per call (no caching/spare state)
26+
// to ensure deterministic behavior regardless of how multiple
27+
// DeterministicNormalDistribution instances share the same RNG.
28+
float operator()(std::mt19937& rng) {
29+
// Box-Muller transform: generate one standard normal from two uniforms.
30+
// We discard the second value (z1) to avoid cross-instance state issues.
31+
float u1, u2;
32+
do {
33+
u1 = static_cast<float>(static_cast<double>(rng()) / (static_cast<double>(rng.max()) + 1.0));
34+
} while (u1 <= 0.0f);
35+
u2 = static_cast<float>(static_cast<double>(rng()) / (static_cast<double>(rng.max()) + 1.0));
36+
37+
// Box-Muller formula: z0 = sqrt(-2 * ln(u1)) * cos(2 * pi * u2)
38+
float mag = stddev_ * std::sqrt(-2.0f * std::log(u1));
39+
return mean_ + mag * std::cos(2.0f * 3.14159265358979323846f * u2);
40+
}
41+
42+
private:
43+
float mean_;
44+
float stddev_;
45+
};

‎cpp/test/src/regression/metric/test_innerproduct_regression.cpp‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,27 +11,30 @@ TEST(DeglibRegressionIP, MultiInstructionSetBenchmark)
1111
std::vector<float> query_data;
1212
generate_synthetic_clustered_dataset(base_count, dim, base_data, query_data, query_count, num_clusters);
1313

14+
// Compute ground-truth using the scalar InnerProductFloat::compare() implementation.
1415
auto gt_data = compute_groundtruth_innerproduct(base_data, base_count, query_data, query_count, dim, 10);
1516

16-
// Explicitly test each distance variant with its own performance thresholds
17+
// Test each distance variant with its own performance thresholds
18+
// num_runs=50 extends the search measurement window to ~280ms per run,
19+
// reducing QPS noise from OS jitter and CPU power-state transitions.
1720
if (deglib::cpu::has_avx512()) {
18-
run_regression_test("AVX512_16Ext", deglib::Metric::InnerProduct, 18000.0, 9.0, 0.797,
21+
run_regression_test("AVX512_16Ext", deglib::Metric::InnerProduct, 18000.0, 10.3, 0.774,
1922
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
20-
deglib::distances::fp32_ip::InnerProductFloat16Ext_AVX512{});
23+
deglib::distances::fp32_ip::InnerProductFloat16Ext_AVX512{}, 50);
2124
}
2225
if (deglib::cpu::has_avx2()) {
23-
run_regression_test("AVX2_16Ext", deglib::Metric::InnerProduct, 18000.0, 9.2, 0.744,
26+
run_regression_test("AVX2_16Ext", deglib::Metric::InnerProduct, 18000.0, 10.2, 0.774,
2427
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
25-
deglib::distances::fp32_ip::InnerProductFloat16Ext_AVX2{});
28+
deglib::distances::fp32_ip::InnerProductFloat16Ext_AVX2{}, 50);
2629
}
2730
if (deglib::cpu::has_sse42()) {
28-
run_regression_test("SSE_16Ext", deglib::Metric::InnerProduct, 15000.0, 10.0, 0.757,
31+
run_regression_test("SSE_16Ext", deglib::Metric::InnerProduct, 15000.0, 11.8, 0.774,
2932
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
30-
deglib::distances::fp32_ip::InnerProductFloat16Ext_SSE{});
33+
deglib::distances::fp32_ip::InnerProductFloat16Ext_SSE{}, 50);
3134
}
32-
run_regression_test("Scalar", deglib::Metric::InnerProduct, 12000.0, 15.0, 0.784,
35+
run_regression_test("Scalar", deglib::Metric::InnerProduct, 12000.0, 15.4, 0.78,
3336
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
34-
deglib::distances::fp32_ip::InnerProductFloat{});
37+
deglib::distances::fp32_ip::InnerProductFloat{}, 50);
3538
}
3639

3740
TEST(DeglibRegressionIP, DistanceRecallAllVariantsSameDataset)

‎cpp/test/src/regression/metric/test_l2_regression.cpp‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,27 +11,30 @@ TEST(DeglibRegressionL2, MultiInstructionSetBenchmark)
1111
std::vector<float> query_data;
1212
generate_synthetic_clustered_dataset(base_count, dim, base_data, query_data, query_count, num_clusters);
1313

14+
// Compute ground-truth using the scalar L2Float::compare() implementation.
1415
auto gt_data = compute_groundtruth_l2(base_data, base_count, query_data, query_count, dim, 10);
1516

16-
// Explicitly test each distance variant with its own performance thresholds
17+
// Test each distance variant with its own performance thresholds
18+
// num_runs=50 extends the search measurement window to ~130ms per run,
19+
// reducing QPS noise from OS jitter and CPU power-state transitions.
1720
if (deglib::cpu::has_avx512()) {
18-
run_regression_test("AVX512_16Ext", deglib::Metric::L2, 41000.0, 6.0, 0.98,
21+
run_regression_test("AVX512_16Ext", deglib::Metric::L2, 38000.0, 6.1, 0.92,
1922
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
20-
deglib::distances::fp32_l2::L2Float16Ext_AVX512{});
23+
deglib::distances::fp32_l2::L2Float16Ext_AVX512{}, 100);
2124
}
2225
if (deglib::cpu::has_avx2()) {
23-
run_regression_test("AVX2_16Ext", deglib::Metric::L2, 38000.0, 6.2, 0.96,
26+
run_regression_test("AVX2_16Ext", deglib::Metric::L2, 38000.0, 6.2, 0.92,
2427
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
25-
deglib::distances::fp32_l2::L2Float16Ext_AVX2{});
28+
deglib::distances::fp32_l2::L2Float16Ext_AVX2{}, 100);
2629
}
2730
if (deglib::cpu::has_sse42()) {
28-
run_regression_test("SSE_16Ext", deglib::Metric::L2, 33000.0, 7.2, 0.96,
31+
run_regression_test("SSE_16Ext", deglib::Metric::L2, 33000.0, 7.0, 0.909,
2932
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
30-
deglib::distances::fp32_l2::L2Float16Ext_SSE{});
33+
deglib::distances::fp32_l2::L2Float16Ext_SSE{}, 100);
3134
}
32-
run_regression_test("Scalar", deglib::Metric::L2, 26000.0, 9.2, 0.989,
35+
run_regression_test("Scalar", deglib::Metric::L2, 28000.0, 9.3, 0.92,
3336
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
34-
deglib::distances::fp32_l2::L2Float{});
37+
deglib::distances::fp32_l2::L2Float{}, 100);
3538
}
3639

3740
TEST(DeglibRegressionL2, DistanceRecallAllVariantsSameDataset)

‎cpp/test/src/regression/metric/test_l2_uint8_regression.cpp‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,26 @@ TEST(DeglibRegressionL2Uint8, MultiInstructionSetBenchmark)
1414
auto gt_data = compute_groundtruth_l2_uint8(base_data, base_count, query_data, query_count, dim, 10);
1515

1616
// Explicitly test each distance variant with its own performance thresholds
17+
// num_runs=50 extends the search measurement window to ~70ms per run,
18+
// reducing QPS noise from OS jitter and CPU power-state transitions.
1719
if (deglib::cpu::has_avx512()) {
18-
run_regression_test("AVX512_Ext32", deglib::Metric::L2_Uint8, 72000.0, 4.7, 0.99,
20+
run_regression_test("AVX512_Ext32", deglib::Metric::L2_Uint8, 73000.0, 4.7, 1,
1921
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
20-
deglib::distances::uint8_l2::L2Uint8Ext32_AVX512{});
22+
deglib::distances::uint8_l2::L2Uint8Ext32_AVX512{}, 100);
2123
}
2224
if (deglib::cpu::has_avx2()) {
23-
run_regression_test("AVX2_Ext32", deglib::Metric::L2_Uint8, 72000.0, 4.8, 0.99,
25+
run_regression_test("AVX2_Ext32", deglib::Metric::L2_Uint8, 74000.0, 4.8, 1,
2426
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
25-
deglib::distances::uint8_l2::L2Uint8Ext32_AVX2{});
27+
deglib::distances::uint8_l2::L2Uint8Ext32_AVX2{}, 100);
2628
}
2729
if (deglib::cpu::has_sse42()) {
28-
run_regression_test("SSE_Ext32", deglib::Metric::L2_Uint8, 67000.0, 5.2, 0.5,
30+
run_regression_test("SSE_Ext32", deglib::Metric::L2_Uint8, 70000.0, 5.2, 1,
2931
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
30-
deglib::distances::uint8_l2::L2Uint8Ext32_SSE{});
32+
deglib::distances::uint8_l2::L2Uint8Ext32_SSE{}, 100);
3133
}
32-
run_regression_test("Scalar", deglib::Metric::L2_Uint8, 56000.0, 5.5, 0.5,
34+
run_regression_test("Scalar", deglib::Metric::L2_Uint8, 63000.0, 5.5, 1,
3335
base_data.data(), query_data.data(), base_count, query_count, dim, gt_data,
34-
deglib::distances::uint8_l2::L2Uint8{});
36+
deglib::distances::uint8_l2::L2Uint8{}, 100);
3537
}
3638

3739
TEST(DeglibRegressionL2Uint8, DistanceRecallAllVariantsSameDataset)

0 commit comments

Comments
 (0)