VectorDB-NTU · gouyt13 · Sep 13, 2026 · Sep 12, 2026
@@ -55,6 +55,44 @@ jobs:
       - name: Run clang-tidy
         run: ./scripts/check-tidy.sh build-tidy
 
+  includes:
+    needs: changes
+    if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.cpp == 'true') }}
+    name: Include cleaner (advisory)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+      - name: Install clang-tidy and build dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y clang-tidy clang libomp-dev cmake ninja-build
+      - name: Configure compilation database
+        run: >-
+          cmake -S . -B build-includes -G Ninja
+          -DCMAKE_CXX_COMPILER=clang++
+          -DRABITQ_BUILD_SAMPLES=OFF
+          -DRABITQ_BUILD_TESTS=OFF
+          -DRABITQ_BUILD_PYTHON_BINDINGS=OFF
+          -DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF
+          -DCMAKE_BUILD_TYPE=Release
+      - name: Analyze library includes
+        id: analysis
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -o pipefail
+          ./scripts/check-includes.sh build-includes 2>&1 | tee include-report.txt
+      - name: Store include report
+        if: ${{ !cancelled() && steps.analysis.outcome != 'skipped' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: include-report
+          path: include-report.txt
+          if-no-files-found: error
+
   shellcheck:
     needs: changes
     if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.shell == 'true') }}

@@ -192,6 +192,65 @@ Focused clang-tidy checks on affected code are sufficient during iteration.
 For first-party C++ changes, run the full check above before merging and report
 whether validation was focused or complete.
 
+### Include dependency reports
+
+The **Include cleaner (advisory)** job uses clang-tidy's `misc-include-cleaner`
+check to report missing and unused includes. CI uses `ubuntu-latest` and
+unversioned distribution packages:
+
+```bash
+sudo apt-get update
+sudo apt-get install clang-tidy clang libomp-dev cmake ninja-build
+```
+
+Clang-tidy 17 or newer is required. On older distributions such as Ubuntu 22.04,
+use [LLVM's APT repository](https://apt.llvm.org/) to install a newer release.
+For example, install LLVM 22 and its matching analysis and OpenMP packages:
+
+```bash
+wget https://apt.llvm.org/llvm.sh
+sudo bash llvm.sh 22
+sudo apt-get install clang-tidy-22 libomp-22-dev cmake ninja-build
+export CLANG_TIDY=clang-tidy-22
+export CXX=clang++-22
+```
+
+Configure and run the same check locally (use a fresh build directory when
+changing compilers):
+
+```bash
+cmake -S . -B build-includes -G Ninja \
+  -DCMAKE_CXX_COMPILER="${CXX:-clang++}" \
+  -DRABITQ_BUILD_SAMPLES=OFF \
+  -DRABITQ_BUILD_TESTS=OFF \
+  -DRABITQ_BUILD_PYTHON_BINDINGS=OFF \
+  -DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF \
+  -DCMAKE_BUILD_TYPE=Release
+./scripts/check-includes.sh build-includes
+```
+
+The script checks library sources using their compilation database and checks
+headers as main files, since this clang-tidy check does not report findings in
+included headers. Private headers are checked with AVX2 and AVX-512 flags.
+Vendored files are excluded. The script also ignores suggestions to include
+Eigen and hnswlib implementation headers behind their existing public headers;
+these vendor snapshots lack the export annotations needed by include-cleaner.
+`INCLUDE_JOBS` controls parallelism (default: 2).
+The script fails for findings or analyzer errors; CI keeps this step advisory
+and uploads the `include-report` artifact without modifying files.
+
+Review suggestions before applying them, especially for templates and public
+forwarding headers. For a source file, automatic fixes can be applied with:
+
+```bash
+"${CLANG_TIDY:-clang-tidy}" -p build-includes --config='{}' \
+  --checks='-*,misc-include-cleaner' --fix src/simd/dispatch.cpp
+```
+
+Review the diff, run formatting, and rebuild and test affected code after fixes.
+Findings can vary between LLVM releases. This include check is independent of
+the existing general clang-tidy job.
+
 ### Focused static analysis
 
 Use a temporary subset of the compilation database with the same wrapper to

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdint>
 #include <limits>
 #include <stdexcept>
 

@@ -5,11 +5,8 @@
 
 #include <algorithm>
 #include <array>
-#include <cassert>
+#include <cstddef>
 #include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
 
 #include "rabitqlib/defines.hpp"
 
@@ -82,36 +79,38 @@ static inline void get_column(
 inline void pack_codes(
     size_t padded_dim, const uint8_t* quantization_code, size_t num, uint8_t* blocks
 ) {
-    size_t num_rd = (num + 31) & ~31;  // round up num of vecs to multiple of batch size(32)
-
     // consider codes is a matrix
     // rows = number of vectors
     // cols = number of uint8_t of one vector's code
     size_t cols = padded_dim / 8;
 
-    std::array<uint8_t, 32> col;    // column of a batch of code, 8 bits
-    std::array<uint8_t, 32> col_0;  // upper 4 bits
-    std::array<uint8_t, 32> col_1;  // lower 4 bits
+    // Full batches can gather directly in output order, without a temporary column.
+    size_t row = 0;
+    for (; num - row >= kBatchSize; row += kBatchSize) {
+        for (size_t i = 0; i < cols; ++i) {
+            for (size_t j = 0; j < 16; ++j) {
+                const size_t first = row + kPerm0[j];
+                const uint8_t a = quantization_code[first * cols + i];
+                const uint8_t b = quantization_code[(first + 16) * cols + i];
+                blocks[j] = (a >> 4) | (b & 0xF0);
+                blocks[j + 16] = (a & 0x0F) | (b << 4);
+            }
+            blocks += 32;
+        }
+    }
 
-    // pack codes batch by batch
-    // each batch contain codes for 32 vectors
-    for (size_t row = 0; row < num_rd; row += kBatchSize) {
-        // get quantization codes for each column for each batch
-        // i.e., we get the codes for 8 dims of 32 vectors and reorganize the data layout
-        // based on the shuffle SIMD instruction used during querying
+    // Only the final partial batch needs a zero-padded column.
+    if (row < num) {
+        std::array<uint8_t, 32> col;
         for (size_t i = 0; i < cols; ++i) {
             get_column(quantization_code, num, cols, row, i, col);
-            for (size_t j = 0; j < 32; ++j) {
-                col_0[j] = col[j] >> 4;
-                col_1[j] = col[j] & 15;
-            }
             for (size_t j = 0; j < 16; ++j) {
                 // the lower 4 bits represent vector 0 to 15
                 // the upper 4 bits represent vector 16 to 31
-                uint8_t val0 = col_0[kPerm0[j]] | (col_0[kPerm0[j] + 16] << 4);
-                uint8_t val1 = col_1[kPerm0[j]] | (col_1[kPerm0[j] + 16] << 4);
-                blocks[j] = val0;
-                blocks[j + 16] = val1;
+                const uint8_t a = col[kPerm0[j]];
+                const uint8_t b = col[kPerm0[j] + 16];
+                blocks[j] = (a >> 4) | (b & 0xF0);
+                blocks[j + 16] = (a & 0x0F) | (b << 4);
             }
             blocks += 32;
         }
@@ -140,4 +139,7 @@ inline void pack_lut(size_t dim, const T* __restrict__ query, T* __restrict__ lu
         query += 4;
     }
 }
+// Float query tables use runtime SIMD dispatch; other types keep the generic path.
+template <>
+void pack_lut<float>(size_t dim, const float* __restrict__ query, float* __restrict__ lut);
 }  // namespace rabitqlib::fastscan
@@ -1,10 +1,7 @@
 #pragma once
 
-#include <cassert>
+#include <cstddef>
 #include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
 
 namespace rabitqlib::fastscan {
 /**

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <array>
+#include <cstddef>
 #include <cstdint>
 
 #include "rabitqlib/defines.hpp"

@@ -1,16 +1,24 @@
 // HNSW is developed from the [HNSW library](https://github.com/nmslib/hnswlib)
 #pragma once
 
-#include <omp.h>
-
+#include <algorithm>
 #include <atomic>
 #include <cassert>
+#include <cmath>
 #include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
 #include <fstream>
+#include <functional>
+#include <ios>
 #include <memory>
 #include <mutex>
+#include <queue>
+#include <random>
 #include <stdexcept>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include "rabitqlib/defines.hpp"
@@ -21,10 +29,12 @@
 #include "rabitqlib/quantization/rabitq.hpp"
 #include "rabitqlib/utils/buffer.hpp"
 #include "rabitqlib/utils/cpu_features.hpp"
+#include "rabitqlib/utils/memory.hpp"
 #include "rabitqlib/utils/rotator.hpp"
 #include "rabitqlib/utils/space.hpp"
 #include "rabitqlib/utils/tools.hpp"
 #include "rabitqlib/utils/visited_pool.hpp"
+#include "rabitqlib/utils/visited_set.hpp"
 
 namespace rabitqlib::hnsw {
 

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <cassert>
+#include <cstddef>
 
 #include "rabitqlib/defines.hpp"
 

@@ -1,11 +1,17 @@
 #pragma once
 
+#include <algorithm>
+#include <atomic>
 #include <cmath>
 #include <cstddef>
 #include <cstring>
+#include <exception>
 #include <fstream>
+#include <mutex>
+#include <queue>
 #include <string>
 #include <thread>
+#include <utility>
 #include <vector>
 
 #include "rabitqlib/defines.hpp"

@@ -1,14 +1,15 @@
 #pragma once
 
-#include <omp.h>
-
 #include <algorithm>
+#include <array>
 #include <cassert>
-#include <cmath>
 #include <cstddef>
 #include <cstdint>
+#include <cstdlib>
 #include <cstring>
 #include <fstream>
+#include <ios>
+#include <limits>
 #include <memory>
 #include <stdexcept>
 #include <utility>

@@ -2,6 +2,8 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <type_traits>
+#include <utility>
 #include <vector>
 
 #include "rabitqlib/fastscan/fastscan.hpp"

@@ -1,10 +1,11 @@
 #pragma once
 
-#include <cmath>
+#include <cstddef>
 #include <cstdint>
 #include <numeric>
 #include <stdexcept>
 #include <utility>
+#include <vector>
 
 #include "rabitqlib/defines.hpp"
 #include "rabitqlib/index/lut.hpp"

@@ -1,12 +1,13 @@
 #pragma once
 
-#include <omp.h>
-
+#include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
 #include <fstream>
+#include <ios>
+#include <limits>
 #include <memory>
 #include <numeric>
 #include <optional>
@@ -19,13 +20,15 @@
 #include "rabitqlib/index/estimator.hpp"
 #include "rabitqlib/index/query.hpp"
 #include "rabitqlib/quantization/data_layout.hpp"
+#include "rabitqlib/quantization/pack_excode.hpp"
 #include "rabitqlib/quantization/rabitq.hpp"
 #include "rabitqlib/utils/array.hpp"
 #include "rabitqlib/utils/buffer.hpp"
 #include "rabitqlib/utils/io.hpp"
 #include "rabitqlib/utils/memory.hpp"
 #include "rabitqlib/utils/rotator.hpp"
 #include "rabitqlib/utils/space.hpp"
+#include "rabitqlib/utils/tools.hpp"
 #include "rabitqlib/utils/visited_set.hpp"
 
 namespace rabitqlib::symqg {

@@ -1,9 +1,13 @@
 #pragma once
 
+#include <omp.h>
+
 #include <algorithm>
 #include <atomic>
-#include <cassert>
+#include <cmath>
+#include <cstddef>
 #include <cstdint>
+#include <limits>
 #include <mutex>
 #include <numeric>
 #include <stdexcept>

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 
 #include "rabitqlib/fastscan/fastscan.hpp"

@@ -1,8 +1,5 @@
 #pragma once
 
-#include <omp.h>
-
-#include <cmath>
 #include <cstddef>
 #include <vector>