From 66a0dd85571e55782a199330f5f0b60d3e33e47d Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 24 Aug 2026 08:31:37 +0000
Subject: [PATCH 1/2] Improve SVE2 optimizations of
 SimdDescrIntCosineDistancesMxNa.

Replace the 4x1 unpack GEMM and missing 4/8-bit direct tiles with
unpredicated 4x4 svdot kernels, vectorized unpack of packed 4-7 bit
data, and transposed B norms. Drop the previous Mx1-only fallback
for depths 4 and 8, extend MxNa tests, and note the change in
release 7.2.165.

Co-authored-by: igor.ermolaev <igor.ermolaev@altezza.org>
---
 docs/2026.html                   |   1 +
 src/Simd/SimdDescrInt.h          |   2 +-
 src/Simd/SimdDescrIntCommon.h    | 102 +++++
 src/Simd/SimdSve2DescrInt.cpp    |   6 +-
 src/Simd/SimdSve2DescrIntCdd.cpp | 663 +++++++++++++++++++------------
 src/Simd/SimdSve2DescrIntCdu.cpp | 317 ++++++++++-----
 src/Test/TestDescrInt.cpp        |   3 +
 7 files changed, 749 insertions(+), 345 deletions(-)

diff --git a/docs/2026.html b/docs/2026.html
index 3521babca5..39b8c90114 100644
--- a/docs/2026.html
+++ b/docs/2026.html
@@ -112,6 +112,7 @@ <h5>Improving</h5>
  <li>SVE2 optimizations of function GetMoments.</li>
  <li>SVE2 optimizations of function GetObjectMoments.</li>
  <li>SVE2 optimizations of function DescrIntCosineDistance.</li>
+ <li>SVE2 optimizations of function DescrIntCosineDistancesMxNa.</li>
 </ul>
 <h5>Renaming</h5>
 <ul>
diff --git a/src/Simd/SimdDescrInt.h b/src/Simd/SimdDescrInt.h
index 78ef0fe32e..43fd4c1eac 100644
--- a/src/Simd/SimdDescrInt.h
+++ b/src/Simd/SimdDescrInt.h
@@ -292,7 +292,7 @@ namespace Simd
         Base::DescrInt::MacroCosineDistancesDirectPtr GetMacroCosineDistancesDirect(size_t depth);
 
         Base::DescrInt::UnpackNormPtr GetUnpackNorm(bool transpose);
-        Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth);
+        Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth, bool transpose);
         Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t depth);
 
         //-------------------------------------------------------------------------------------------------
diff --git a/src/Simd/SimdDescrIntCommon.h b/src/Simd/SimdDescrIntCommon.h
index 1144f13828..0db8b1de17 100644
--- a/src/Simd/SimdDescrIntCommon.h
+++ b/src/Simd/SimdDescrIntCommon.h
@@ -721,5 +721,107 @@ namespace Simd
         }
     }
 #endif
+
+#ifdef SIMD_SVE2_ENABLE
+    namespace Sve2
+    {
+        const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
+        const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
+        const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
+        const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
+        const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
+        const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };
+
+        SIMD_INLINE svuint8_t UnpackTbl(const uint8_t* tbl16, uint8_t bits)
+        {
+            const svbool_t all = svptrue_b8();
+            svuint8_t base = svld1rq_u8(all, tbl16);
+            svuint8_t index = svindex_u8(0, 1);
+            svuint8_t group = svlsr_n_u8_x(all, index, 4);
+            return svadd_u8_x(all, base, svmul_n_u8_x(all, group, bits));
+        }
+
+        SIMD_INLINE svuint16_t UnpackShr(const uint16_t* shr8)
+        {
+            return svld1rq_u16(svptrue_b16(), shr8);
+        }
+
+        template<int bits> SIMD_INLINE svuint8_t UnpackTbl();
+        template<int bits> SIMD_INLINE svuint16_t UnpackShr();
+
+        template<> SIMD_INLINE svuint8_t UnpackTbl<5>() { return UnpackTbl(C5_TBL, 5); }
+        template<> SIMD_INLINE svuint16_t UnpackShr<5>() { return UnpackShr(C5_SHR); }
+        template<> SIMD_INLINE svuint8_t UnpackTbl<6>() { return UnpackTbl(C6_TBL, 6); }
+        template<> SIMD_INLINE svuint16_t UnpackShr<6>() { return UnpackShr(C6_SHR); }
+        template<> SIMD_INLINE svuint8_t UnpackTbl<7>() { return UnpackTbl(C7_TBL, 7); }
+        template<> SIMD_INLINE svuint16_t UnpackShr<7>() { return UnpackShr(C7_SHR); }
+
+        SIMD_INLINE svuint16_t UnpackTo16(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr, uint16_t mask)
+        {
+            svuint8_t raw = svld1_u8(svwhilelt_b8((size_t)0, packed), src);
+            svuint16_t wide = svreinterpret_u16_u8(svtbl_u8(raw, tbl));
+            const svbool_t all = svptrue_b16();
+            return svand_n_u16_x(all, svlsr_u16_x(all, wide, shr), mask);
+        }
+
+        template<int bits> SIMD_INLINE svuint8_t UnpackTo8(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr)
+        {
+            const size_t packedHalf = svcnth() * bits / 8;
+            const uint16_t mask = (uint16_t)((1 << bits) - 1);
+            size_t packedLo = packed < packedHalf ? packed : packedHalf;
+            size_t packedHi = packed - packedLo;
+            svuint16_t lo = UnpackTo16(src, packedLo, tbl, shr, mask);
+            svuint16_t hi = UnpackTo16(src + packedLo, packedHi, tbl, shr, mask);
+            return svuzp1_u8(svreinterpret_u8_u16(lo), svreinterpret_u8_u16(hi));
+        }
+
+        SIMD_INLINE void DecodeCosineDistances1x4(const uint8_t* a, const uint8_t* const* B,
+            svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
+        {
+            float ab[4] = {
+                (float)svaddv_u32(svptrue_b32(), s0),
+                (float)svaddv_u32(svptrue_b32(), s1),
+                (float)svaddv_u32(svptrue_b32(), s2),
+                (float)svaddv_u32(svptrue_b32(), s3)
+            };
+#ifdef SIMD_NEON_ENABLE
+            Neon::DecodeCosineDistances1x4(a, B, vld1q_f32(ab), distances);
+#else
+            Base::DecodeCosineDistance(a, B[0], ab[0], distances + 0);
+            Base::DecodeCosineDistance(a, B[1], ab[1], distances + 1);
+            Base::DecodeCosineDistance(a, B[2], ab[2], distances + 2);
+            Base::DecodeCosineDistance(a, B[3], ab[3], distances + 3);
+#endif
+        }
+
+        SIMD_INLINE void DecodeCosineDistances1x4(const float* a, const float* b, size_t stride,
+            svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
+        {
+            uint32_t ab[4] = {
+                (uint32_t)svaddv_u32(svptrue_b32(), s0),
+                (uint32_t)svaddv_u32(svptrue_b32(), s1),
+                (uint32_t)svaddv_u32(svptrue_b32(), s2),
+                (uint32_t)svaddv_u32(svptrue_b32(), s3)
+            };
+#ifdef SIMD_NEON_ENABLE
+            Neon::DecodeCosineDistances1x4(a, b, stride, vld1q_u32(ab), distances);
+#else
+            for (size_t j = 0; j < 4; ++j)
+            {
+                float bb[4] = { b[j + 0 * stride], b[j + 1 * stride], b[j + 2 * stride], b[j + 3 * stride] };
+                float abSum = (float)ab[j] * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
+                distances[j] = Simd::RestrictRange(1.0f - abSum / (a[3] * bb[3]), 0.0f, 2.0f);
+            }
+#endif
+        }
+
+        SIMD_INLINE void DecodeCosineDistance(const float* a, const float* b, size_t stride, uint32_t abSum, float* distance)
+        {
+            float bb[4] = { b[0], b[stride], b[2 * stride], b[3 * stride] };
+            float ab = (float)abSum * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
+            distance[0] = Simd::RestrictRange(1.0f - ab / (a[3] * bb[3]), 0.0f, 2.0f);
+        }
+    }
+#endif
 }
 #endif//__SimdDescrIntCommon_h__
diff --git a/src/Simd/SimdSve2DescrInt.cpp b/src/Simd/SimdSve2DescrInt.cpp
index da6821e7d1..318824f2ac 100644
--- a/src/Simd/SimdSve2DescrInt.cpp
+++ b/src/Simd/SimdSve2DescrInt.cpp
@@ -49,12 +49,12 @@ namespace Simd
 
             _unpackNormA = GetUnpackNorm(false);
             _unpackNormB = GetUnpackNorm(true);
-            _unpackDataA = GetUnpackData(_depth);
-            _unpackDataB = GetUnpackData(_depth);
+            _unpackDataA = GetUnpackData(_depth, false);
+            _unpackDataB = GetUnpackData(_depth, true);
             _macroCosineDistancesUnpack = GetMacroCosineDistancesUnpack(_depth);
             _unpSize = _size;
             _microMu = 4;
-            _microNu = 1;
+            _microNu = 4;
         }
 
         //-------------------------------------------------------------------------------------------------
diff --git a/src/Simd/SimdSve2DescrIntCdd.cpp b/src/Simd/SimdSve2DescrIntCdd.cpp
index 300235ff11..40abed74bd 100644
--- a/src/Simd/SimdSve2DescrIntCdd.cpp
+++ b/src/Simd/SimdSve2DescrIntCdd.cpp
@@ -32,56 +32,6 @@ namespace Simd
 #ifdef SIMD_SVE2_ENABLE
     namespace Sve2
     {
-        const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
-        const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
-        const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
-        const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
-        const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
-        const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };
-
-        SIMD_INLINE svuint8_t UnpackTbl(const uint8_t* tbl16, uint8_t bits)
-        {
-            const svbool_t all = svptrue_b8();
-            svuint8_t base = svld1rq_u8(all, tbl16);
-            svuint8_t index = svindex_u8(0, 1);
-            svuint8_t group = svlsr_n_u8_x(all, index, 4);
-            return svadd_u8_x(all, base, svmul_n_u8_x(all, group, bits));
-        }
-
-        SIMD_INLINE svuint16_t UnpackShr(const uint16_t* shr8)
-        {
-            return svld1rq_u16(svptrue_b16(), shr8);
-        }
-
-        template<int bits> SIMD_INLINE svuint8_t UnpackTbl();
-        template<int bits> SIMD_INLINE svuint16_t UnpackShr();
-
-        template<> SIMD_INLINE svuint8_t UnpackTbl<5>() { return UnpackTbl(C5_TBL, 5); }
-        template<> SIMD_INLINE svuint16_t UnpackShr<5>() { return UnpackShr(C5_SHR); }
-        template<> SIMD_INLINE svuint8_t UnpackTbl<6>() { return UnpackTbl(C6_TBL, 6); }
-        template<> SIMD_INLINE svuint16_t UnpackShr<6>() { return UnpackShr(C6_SHR); }
-        template<> SIMD_INLINE svuint8_t UnpackTbl<7>() { return UnpackTbl(C7_TBL, 7); }
-        template<> SIMD_INLINE svuint16_t UnpackShr<7>() { return UnpackShr(C7_SHR); }
-
-        SIMD_INLINE svuint16_t UnpackTo16(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr, uint16_t mask)
-        {
-            svuint8_t raw = svld1_u8(svwhilelt_b8((size_t)0, packed), src);
-            svuint16_t wide = svreinterpret_u16_u8(svtbl_u8(raw, tbl));
-            const svbool_t all = svptrue_b16();
-            return svand_n_u16_x(all, svlsr_u16_x(all, wide, shr), mask);
-        }
-
-        template<int bits> SIMD_INLINE svuint8_t UnpackTo8(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr)
-        {
-            const size_t packedHalf = svcnth() * bits / 8;
-            const uint16_t mask = (uint16_t)((1 << bits) - 1);
-            size_t packedLo = packed < packedHalf ? packed : packedHalf;
-            size_t packedHi = packed - packedLo;
-            svuint16_t lo = UnpackTo16(src, packedLo, tbl, shr, mask);
-            svuint16_t hi = UnpackTo16(src + packedLo, packedHi, tbl, shr, mask);
-            return svuzp1_u8(svreinterpret_u8_u16(lo), svreinterpret_u8_u16(hi));
-        }
-
         template<int bits> int32_t Correlation(const uint8_t* a, const uint8_t* b, size_t size)
         {
             assert(size % 8 == 0 && size >= 8);
@@ -111,13 +61,23 @@ namespace Simd
         {
             assert(size % 8 == 0 && size >= 8);
             const size_t byteSize = size / 2;
-            const svuint8_t zero = svdup_n_u8(0);
+            const size_t width = svcntb();
+            const size_t main = byteSize & ~(width - 1);
+            const svbool_t all = svptrue_b8();
             svuint32_t sums = svdup_n_u32(0);
-            for (size_t i = 0; i < byteSize; i += svcntb())
+            size_t i = 0;
+            for (; i < main; i += width)
+            {
+                svuint8_t _a = svld1_u8(all, a + i);
+                svuint8_t _b = svld1_u8(all, b + i);
+                sums = svdot_u32(sums, svand_n_u8_x(all, _a, 0x0F), svand_n_u8_x(all, _b, 0x0F));
+                sums = svdot_u32(sums, svlsr_n_u8_x(all, _a, 4), svlsr_n_u8_x(all, _b, 4));
+            }
+            if (i < byteSize)
             {
                 svbool_t mask = svwhilelt_b8(i, byteSize);
-                svuint8_t _a = svsel_u8(mask, svld1_u8(mask, a + i), zero);
-                svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
+                svuint8_t _a = svld1_u8(mask, a + i);
+                svuint8_t _b = svld1_u8(mask, b + i);
                 sums = svdot_u32(sums, svand_n_u8_z(mask, _a, 0x0F), svand_n_u8_z(mask, _b, 0x0F));
                 sums = svdot_u32(sums, svlsr_n_u8_z(mask, _a, 4), svlsr_n_u8_z(mask, _b, 4));
             }
@@ -127,14 +87,17 @@ namespace Simd
         template<> int32_t Correlation<8>(const uint8_t* a, const uint8_t* b, size_t size)
         {
             assert(size % 8 == 0 && size >= 8);
-            const svuint8_t zero = svdup_n_u8(0);
+            const size_t width = svcntb();
+            const size_t main = size & ~(width - 1);
+            const svbool_t all = svptrue_b8();
             svuint32_t sums = svdup_n_u32(0);
-            for (size_t i = 0; i < size; i += svcntb())
+            size_t i = 0;
+            for (; i < main; i += width)
+                sums = svdot_u32(sums, svld1_u8(all, a + i), svld1_u8(all, b + i));
+            if (i < size)
             {
                 svbool_t mask = svwhilelt_b8(i, size);
-                svuint8_t _a = svsel_u8(mask, svld1_u8(mask, a + i), zero);
-                svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
-                sums = svdot_u32(sums, _a, _b);
+                sums = svdot_u32(sums, svld1_u8(mask, a + i), svld1_u8(mask, b + i));
             }
             return (int32_t)svaddv_u32(svptrue_b32(), sums);
         }
@@ -145,131 +108,338 @@ namespace Simd
             Base::DecodeCosineDistance(a, b, abSum, distance);
         }
 
-        template<int bits, int M> SIMD_INLINE void MicroCosineDistancesDirectMx4(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
+        template<int bits> struct DirectMx4
         {
-            const size_t valuesPerVec = svcntb();
-            const size_t packedPerVec = valuesPerVec * bits / 8;
-            const svuint8_t tbl = UnpackTbl<bits>();
-            const svuint16_t shr = UnpackShr<bits>();
-            svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
-            svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
-            svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
-            svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
-            size_t i = 0, o = 16;
-            for (; i + valuesPerVec <= size; i += valuesPerVec, o += packedPerVec)
+            template<int M> static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
             {
-                svuint8_t b0 = UnpackTo8<bits>(B[0] + o, packedPerVec, tbl, shr);
-                svuint8_t b1 = UnpackTo8<bits>(B[1] + o, packedPerVec, tbl, shr);
-                svuint8_t b2 = UnpackTo8<bits>(B[2] + o, packedPerVec, tbl, shr);
-                svuint8_t b3 = UnpackTo8<bits>(B[3] + o, packedPerVec, tbl, shr);
-                if (M > 0)
-                {
-                    svuint8_t a0 = UnpackTo8<bits>(A[0] + o, packedPerVec, tbl, shr);
-                    ab00 = svdot_u32(ab00, a0, b0);
-                    ab01 = svdot_u32(ab01, a0, b1);
-                    ab02 = svdot_u32(ab02, a0, b2);
-                    ab03 = svdot_u32(ab03, a0, b3);
-                }
-                if (M > 1)
-                {
-                    svuint8_t a1 = UnpackTo8<bits>(A[1] + o, packedPerVec, tbl, shr);
-                    ab10 = svdot_u32(ab10, a1, b0);
-                    ab11 = svdot_u32(ab11, a1, b1);
-                    ab12 = svdot_u32(ab12, a1, b2);
-                    ab13 = svdot_u32(ab13, a1, b3);
-                }
-                if (M > 2)
+                const size_t valuesPerVec = svcntb();
+                const size_t packedPerVec = valuesPerVec * bits / 8;
+                const svuint8_t tbl = UnpackTbl<bits>();
+                const svuint16_t shr = UnpackShr<bits>();
+                svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+                svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+                svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+                svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+                size_t i = 0, o = 16;
+                for (; i + valuesPerVec <= size; i += valuesPerVec, o += packedPerVec)
                 {
-                    svuint8_t a2 = UnpackTo8<bits>(A[2] + o, packedPerVec, tbl, shr);
-                    ab20 = svdot_u32(ab20, a2, b0);
-                    ab21 = svdot_u32(ab21, a2, b1);
-                    ab22 = svdot_u32(ab22, a2, b2);
-                    ab23 = svdot_u32(ab23, a2, b3);
+                    svuint8_t b0 = UnpackTo8<bits>(B[0] + o, packedPerVec, tbl, shr);
+                    svuint8_t b1 = UnpackTo8<bits>(B[1] + o, packedPerVec, tbl, shr);
+                    svuint8_t b2 = UnpackTo8<bits>(B[2] + o, packedPerVec, tbl, shr);
+                    svuint8_t b3 = UnpackTo8<bits>(B[3] + o, packedPerVec, tbl, shr);
+                    if (M > 0)
+                    {
+                        svuint8_t a0 = UnpackTo8<bits>(A[0] + o, packedPerVec, tbl, shr);
+                        ab00 = svdot_u32(ab00, a0, b0);
+                        ab01 = svdot_u32(ab01, a0, b1);
+                        ab02 = svdot_u32(ab02, a0, b2);
+                        ab03 = svdot_u32(ab03, a0, b3);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t a1 = UnpackTo8<bits>(A[1] + o, packedPerVec, tbl, shr);
+                        ab10 = svdot_u32(ab10, a1, b0);
+                        ab11 = svdot_u32(ab11, a1, b1);
+                        ab12 = svdot_u32(ab12, a1, b2);
+                        ab13 = svdot_u32(ab13, a1, b3);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t a2 = UnpackTo8<bits>(A[2] + o, packedPerVec, tbl, shr);
+                        ab20 = svdot_u32(ab20, a2, b0);
+                        ab21 = svdot_u32(ab21, a2, b1);
+                        ab22 = svdot_u32(ab22, a2, b2);
+                        ab23 = svdot_u32(ab23, a2, b3);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t a3 = UnpackTo8<bits>(A[3] + o, packedPerVec, tbl, shr);
+                        ab30 = svdot_u32(ab30, a3, b0);
+                        ab31 = svdot_u32(ab31, a3, b1);
+                        ab32 = svdot_u32(ab32, a3, b2);
+                        ab33 = svdot_u32(ab33, a3, b3);
+                    }
                 }
-                if (M > 3)
+                if (i < size)
                 {
-                    svuint8_t a3 = UnpackTo8<bits>(A[3] + o, packedPerVec, tbl, shr);
-                    ab30 = svdot_u32(ab30, a3, b0);
-                    ab31 = svdot_u32(ab31, a3, b1);
-                    ab32 = svdot_u32(ab32, a3, b2);
-                    ab33 = svdot_u32(ab33, a3, b3);
+                    size_t packedTail = (size - i) * bits / 8;
+                    svuint8_t b0 = UnpackTo8<bits>(B[0] + o, packedTail, tbl, shr);
+                    svuint8_t b1 = UnpackTo8<bits>(B[1] + o, packedTail, tbl, shr);
+                    svuint8_t b2 = UnpackTo8<bits>(B[2] + o, packedTail, tbl, shr);
+                    svuint8_t b3 = UnpackTo8<bits>(B[3] + o, packedTail, tbl, shr);
+                    if (M > 0)
+                    {
+                        svuint8_t a0 = UnpackTo8<bits>(A[0] + o, packedTail, tbl, shr);
+                        ab00 = svdot_u32(ab00, a0, b0);
+                        ab01 = svdot_u32(ab01, a0, b1);
+                        ab02 = svdot_u32(ab02, a0, b2);
+                        ab03 = svdot_u32(ab03, a0, b3);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t a1 = UnpackTo8<bits>(A[1] + o, packedTail, tbl, shr);
+                        ab10 = svdot_u32(ab10, a1, b0);
+                        ab11 = svdot_u32(ab11, a1, b1);
+                        ab12 = svdot_u32(ab12, a1, b2);
+                        ab13 = svdot_u32(ab13, a1, b3);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t a2 = UnpackTo8<bits>(A[2] + o, packedTail, tbl, shr);
+                        ab20 = svdot_u32(ab20, a2, b0);
+                        ab21 = svdot_u32(ab21, a2, b1);
+                        ab22 = svdot_u32(ab22, a2, b2);
+                        ab23 = svdot_u32(ab23, a2, b3);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t a3 = UnpackTo8<bits>(A[3] + o, packedTail, tbl, shr);
+                        ab30 = svdot_u32(ab30, a3, b0);
+                        ab31 = svdot_u32(ab31, a3, b1);
+                        ab32 = svdot_u32(ab32, a3, b2);
+                        ab33 = svdot_u32(ab33, a3, b3);
+                    }
                 }
+                if (M > 0) DecodeCosineDistances1x4(A[0], B, ab00, ab01, ab02, ab03, distances + 0 * stride);
+                if (M > 1) DecodeCosineDistances1x4(A[1], B, ab10, ab11, ab12, ab13, distances + 1 * stride);
+                if (M > 2) DecodeCosineDistances1x4(A[2], B, ab20, ab21, ab22, ab23, distances + 2 * stride);
+                if (M > 3) DecodeCosineDistances1x4(A[3], B, ab30, ab31, ab32, ab33, distances + 3 * stride);
             }
-            if (i < size)
+        };
+
+        template<> struct DirectMx4<4>
+        {
+            template<int M> static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
             {
-                size_t packedTail = (size - i) * bits / 8;
-                svuint8_t b0 = UnpackTo8<bits>(B[0] + o, packedTail, tbl, shr);
-                svuint8_t b1 = UnpackTo8<bits>(B[1] + o, packedTail, tbl, shr);
-                svuint8_t b2 = UnpackTo8<bits>(B[2] + o, packedTail, tbl, shr);
-                svuint8_t b3 = UnpackTo8<bits>(B[3] + o, packedTail, tbl, shr);
-                if (M > 0)
+                const size_t width = svcntb();
+                const size_t byteSize = size / 2;
+                const size_t main = byteSize & ~(width - 1);
+                const svbool_t all = svptrue_b8();
+                svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+                svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+                svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+                svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+                const uint8_t* a0 = A[0] + 16;
+                const uint8_t* a1 = M > 1 ? A[1] + 16 : a0;
+                const uint8_t* a2 = M > 2 ? A[2] + 16 : a0;
+                const uint8_t* a3 = M > 3 ? A[3] + 16 : a0;
+                const uint8_t* b0 = B[0] + 16;
+                const uint8_t* b1 = B[1] + 16;
+                const uint8_t* b2 = B[2] + 16;
+                const uint8_t* b3 = B[3] + 16;
+                size_t i = 0;
+                for (; i < main; i += width)
                 {
-                    svuint8_t a0 = UnpackTo8<bits>(A[0] + o, packedTail, tbl, shr);
-                    ab00 = svdot_u32(ab00, a0, b0);
-                    ab01 = svdot_u32(ab01, a0, b1);
-                    ab02 = svdot_u32(ab02, a0, b2);
-                    ab03 = svdot_u32(ab03, a0, b3);
+                    svuint8_t B0 = svld1_u8(all, b0 + i);
+                    svuint8_t B1 = svld1_u8(all, b1 + i);
+                    svuint8_t B2 = svld1_u8(all, b2 + i);
+                    svuint8_t B3 = svld1_u8(all, b3 + i);
+                    svuint8_t B0l = svand_n_u8_x(all, B0, 0x0F), B0h = svlsr_n_u8_x(all, B0, 4);
+                    svuint8_t B1l = svand_n_u8_x(all, B1, 0x0F), B1h = svlsr_n_u8_x(all, B1, 4);
+                    svuint8_t B2l = svand_n_u8_x(all, B2, 0x0F), B2h = svlsr_n_u8_x(all, B2, 4);
+                    svuint8_t B3l = svand_n_u8_x(all, B3, 0x0F), B3h = svlsr_n_u8_x(all, B3, 4);
+                    if (M > 0)
+                    {
+                        svuint8_t A0 = svld1_u8(all, a0 + i);
+                        svuint8_t A0l = svand_n_u8_x(all, A0, 0x0F), A0h = svlsr_n_u8_x(all, A0, 4);
+                        ab00 = svdot_u32(svdot_u32(ab00, A0l, B0l), A0h, B0h);
+                        ab01 = svdot_u32(svdot_u32(ab01, A0l, B1l), A0h, B1h);
+                        ab02 = svdot_u32(svdot_u32(ab02, A0l, B2l), A0h, B2h);
+                        ab03 = svdot_u32(svdot_u32(ab03, A0l, B3l), A0h, B3h);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t A1 = svld1_u8(all, a1 + i);
+                        svuint8_t A1l = svand_n_u8_x(all, A1, 0x0F), A1h = svlsr_n_u8_x(all, A1, 4);
+                        ab10 = svdot_u32(svdot_u32(ab10, A1l, B0l), A1h, B0h);
+                        ab11 = svdot_u32(svdot_u32(ab11, A1l, B1l), A1h, B1h);
+                        ab12 = svdot_u32(svdot_u32(ab12, A1l, B2l), A1h, B2h);
+                        ab13 = svdot_u32(svdot_u32(ab13, A1l, B3l), A1h, B3h);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t A2 = svld1_u8(all, a2 + i);
+                        svuint8_t A2l = svand_n_u8_x(all, A2, 0x0F), A2h = svlsr_n_u8_x(all, A2, 4);
+                        ab20 = svdot_u32(svdot_u32(ab20, A2l, B0l), A2h, B0h);
+                        ab21 = svdot_u32(svdot_u32(ab21, A2l, B1l), A2h, B1h);
+                        ab22 = svdot_u32(svdot_u32(ab22, A2l, B2l), A2h, B2h);
+                        ab23 = svdot_u32(svdot_u32(ab23, A2l, B3l), A2h, B3h);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t A3 = svld1_u8(all, a3 + i);
+                        svuint8_t A3l = svand_n_u8_x(all, A3, 0x0F), A3h = svlsr_n_u8_x(all, A3, 4);
+                        ab30 = svdot_u32(svdot_u32(ab30, A3l, B0l), A3h, B0h);
+                        ab31 = svdot_u32(svdot_u32(ab31, A3l, B1l), A3h, B1h);
+                        ab32 = svdot_u32(svdot_u32(ab32, A3l, B2l), A3h, B2h);
+                        ab33 = svdot_u32(svdot_u32(ab33, A3l, B3l), A3h, B3h);
+                    }
                 }
-                if (M > 1)
+                if (i < byteSize)
                 {
-                    svuint8_t a1 = UnpackTo8<bits>(A[1] + o, packedTail, tbl, shr);
-                    ab10 = svdot_u32(ab10, a1, b0);
-                    ab11 = svdot_u32(ab11, a1, b1);
-                    ab12 = svdot_u32(ab12, a1, b2);
-                    ab13 = svdot_u32(ab13, a1, b3);
+                    svbool_t mask = svwhilelt_b8(i, byteSize);
+                    svuint8_t B0 = svld1_u8(mask, b0 + i);
+                    svuint8_t B1 = svld1_u8(mask, b1 + i);
+                    svuint8_t B2 = svld1_u8(mask, b2 + i);
+                    svuint8_t B3 = svld1_u8(mask, b3 + i);
+                    svuint8_t B0l = svand_n_u8_z(mask, B0, 0x0F), B0h = svlsr_n_u8_z(mask, B0, 4);
+                    svuint8_t B1l = svand_n_u8_z(mask, B1, 0x0F), B1h = svlsr_n_u8_z(mask, B1, 4);
+                    svuint8_t B2l = svand_n_u8_z(mask, B2, 0x0F), B2h = svlsr_n_u8_z(mask, B2, 4);
+                    svuint8_t B3l = svand_n_u8_z(mask, B3, 0x0F), B3h = svlsr_n_u8_z(mask, B3, 4);
+                    if (M > 0)
+                    {
+                        svuint8_t A0 = svld1_u8(mask, a0 + i);
+                        svuint8_t A0l = svand_n_u8_z(mask, A0, 0x0F), A0h = svlsr_n_u8_z(mask, A0, 4);
+                        ab00 = svdot_u32(svdot_u32(ab00, A0l, B0l), A0h, B0h);
+                        ab01 = svdot_u32(svdot_u32(ab01, A0l, B1l), A0h, B1h);
+                        ab02 = svdot_u32(svdot_u32(ab02, A0l, B2l), A0h, B2h);
+                        ab03 = svdot_u32(svdot_u32(ab03, A0l, B3l), A0h, B3h);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t A1 = svld1_u8(mask, a1 + i);
+                        svuint8_t A1l = svand_n_u8_z(mask, A1, 0x0F), A1h = svlsr_n_u8_z(mask, A1, 4);
+                        ab10 = svdot_u32(svdot_u32(ab10, A1l, B0l), A1h, B0h);
+                        ab11 = svdot_u32(svdot_u32(ab11, A1l, B1l), A1h, B1h);
+                        ab12 = svdot_u32(svdot_u32(ab12, A1l, B2l), A1h, B2h);
+                        ab13 = svdot_u32(svdot_u32(ab13, A1l, B3l), A1h, B3h);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t A2 = svld1_u8(mask, a2 + i);
+                        svuint8_t A2l = svand_n_u8_z(mask, A2, 0x0F), A2h = svlsr_n_u8_z(mask, A2, 4);
+                        ab20 = svdot_u32(svdot_u32(ab20, A2l, B0l), A2h, B0h);
+                        ab21 = svdot_u32(svdot_u32(ab21, A2l, B1l), A2h, B1h);
+                        ab22 = svdot_u32(svdot_u32(ab22, A2l, B2l), A2h, B2h);
+                        ab23 = svdot_u32(svdot_u32(ab23, A2l, B3l), A2h, B3h);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t A3 = svld1_u8(mask, a3 + i);
+                        svuint8_t A3l = svand_n_u8_z(mask, A3, 0x0F), A3h = svlsr_n_u8_z(mask, A3, 4);
+                        ab30 = svdot_u32(svdot_u32(ab30, A3l, B0l), A3h, B0h);
+                        ab31 = svdot_u32(svdot_u32(ab31, A3l, B1l), A3h, B1h);
+                        ab32 = svdot_u32(svdot_u32(ab32, A3l, B2l), A3h, B2h);
+                        ab33 = svdot_u32(svdot_u32(ab33, A3l, B3l), A3h, B3h);
+                    }
                 }
-                if (M > 2)
+                if (M > 0) DecodeCosineDistances1x4(A[0], B, ab00, ab01, ab02, ab03, distances + 0 * stride);
+                if (M > 1) DecodeCosineDistances1x4(A[1], B, ab10, ab11, ab12, ab13, distances + 1 * stride);
+                if (M > 2) DecodeCosineDistances1x4(A[2], B, ab20, ab21, ab22, ab23, distances + 2 * stride);
+                if (M > 3) DecodeCosineDistances1x4(A[3], B, ab30, ab31, ab32, ab33, distances + 3 * stride);
+            }
+        };
+
+        template<> struct DirectMx4<8>
+        {
+            template<int M> static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
+            {
+                const size_t width = svcntb();
+                const size_t main = size & ~(width - 1);
+                const svbool_t all = svptrue_b8();
+                svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+                svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+                svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+                svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+                const uint8_t* a0 = A[0] + 16;
+                const uint8_t* a1 = M > 1 ? A[1] + 16 : a0;
+                const uint8_t* a2 = M > 2 ? A[2] + 16 : a0;
+                const uint8_t* a3 = M > 3 ? A[3] + 16 : a0;
+                const uint8_t* b0 = B[0] + 16;
+                const uint8_t* b1 = B[1] + 16;
+                const uint8_t* b2 = B[2] + 16;
+                const uint8_t* b3 = B[3] + 16;
+                size_t i = 0;
+                for (; i < main; i += width)
                 {
-                    svuint8_t a2 = UnpackTo8<bits>(A[2] + o, packedTail, tbl, shr);
-                    ab20 = svdot_u32(ab20, a2, b0);
-                    ab21 = svdot_u32(ab21, a2, b1);
-                    ab22 = svdot_u32(ab22, a2, b2);
-                    ab23 = svdot_u32(ab23, a2, b3);
+                    svuint8_t B0 = svld1_u8(all, b0 + i);
+                    svuint8_t B1 = svld1_u8(all, b1 + i);
+                    svuint8_t B2 = svld1_u8(all, b2 + i);
+                    svuint8_t B3 = svld1_u8(all, b3 + i);
+                    if (M > 0)
+                    {
+                        svuint8_t A0 = svld1_u8(all, a0 + i);
+                        ab00 = svdot_u32(ab00, A0, B0);
+                        ab01 = svdot_u32(ab01, A0, B1);
+                        ab02 = svdot_u32(ab02, A0, B2);
+                        ab03 = svdot_u32(ab03, A0, B3);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t A1 = svld1_u8(all, a1 + i);
+                        ab10 = svdot_u32(ab10, A1, B0);
+                        ab11 = svdot_u32(ab11, A1, B1);
+                        ab12 = svdot_u32(ab12, A1, B2);
+                        ab13 = svdot_u32(ab13, A1, B3);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t A2 = svld1_u8(all, a2 + i);
+                        ab20 = svdot_u32(ab20, A2, B0);
+                        ab21 = svdot_u32(ab21, A2, B1);
+                        ab22 = svdot_u32(ab22, A2, B2);
+                        ab23 = svdot_u32(ab23, A2, B3);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t A3 = svld1_u8(all, a3 + i);
+                        ab30 = svdot_u32(ab30, A3, B0);
+                        ab31 = svdot_u32(ab31, A3, B1);
+                        ab32 = svdot_u32(ab32, A3, B2);
+                        ab33 = svdot_u32(ab33, A3, B3);
+                    }
                 }
-                if (M > 3)
+                if (i < size)
                 {
-                    svuint8_t a3 = UnpackTo8<bits>(A[3] + o, packedTail, tbl, shr);
-                    ab30 = svdot_u32(ab30, a3, b0);
-                    ab31 = svdot_u32(ab31, a3, b1);
-                    ab32 = svdot_u32(ab32, a3, b2);
-                    ab33 = svdot_u32(ab33, a3, b3);
+                    svbool_t mask = svwhilelt_b8(i, size);
+                    svuint8_t B0 = svld1_u8(mask, b0 + i);
+                    svuint8_t B1 = svld1_u8(mask, b1 + i);
+                    svuint8_t B2 = svld1_u8(mask, b2 + i);
+                    svuint8_t B3 = svld1_u8(mask, b3 + i);
+                    if (M > 0)
+                    {
+                        svuint8_t A0 = svld1_u8(mask, a0 + i);
+                        ab00 = svdot_u32(ab00, A0, B0);
+                        ab01 = svdot_u32(ab01, A0, B1);
+                        ab02 = svdot_u32(ab02, A0, B2);
+                        ab03 = svdot_u32(ab03, A0, B3);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t A1 = svld1_u8(mask, a1 + i);
+                        ab10 = svdot_u32(ab10, A1, B0);
+                        ab11 = svdot_u32(ab11, A1, B1);
+                        ab12 = svdot_u32(ab12, A1, B2);
+                        ab13 = svdot_u32(ab13, A1, B3);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t A2 = svld1_u8(mask, a2 + i);
+                        ab20 = svdot_u32(ab20, A2, B0);
+                        ab21 = svdot_u32(ab21, A2, B1);
+                        ab22 = svdot_u32(ab22, A2, B2);
+                        ab23 = svdot_u32(ab23, A2, B3);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t A3 = svld1_u8(mask, a3 + i);
+                        ab30 = svdot_u32(ab30, A3, B0);
+                        ab31 = svdot_u32(ab31, A3, B1);
+                        ab32 = svdot_u32(ab32, A3, B2);
+                        ab33 = svdot_u32(ab33, A3, B3);
+                    }
                 }
+                if (M > 0) DecodeCosineDistances1x4(A[0], B, ab00, ab01, ab02, ab03, distances + 0 * stride);
+                if (M > 1) DecodeCosineDistances1x4(A[1], B, ab10, ab11, ab12, ab13, distances + 1 * stride);
+                if (M > 2) DecodeCosineDistances1x4(A[2], B, ab20, ab21, ab22, ab23, distances + 2 * stride);
+                if (M > 3) DecodeCosineDistances1x4(A[3], B, ab30, ab31, ab32, ab33, distances + 3 * stride);
             }
-            if (M > 0)
-            {
-                Base::DecodeCosineDistance(A[0], B[0], (float)svaddv_u32(svptrue_b32(), ab00), distances + 0 * stride + 0);
-                Base::DecodeCosineDistance(A[0], B[1], (float)svaddv_u32(svptrue_b32(), ab01), distances + 0 * stride + 1);
-                Base::DecodeCosineDistance(A[0], B[2], (float)svaddv_u32(svptrue_b32(), ab02), distances + 0 * stride + 2);
-                Base::DecodeCosineDistance(A[0], B[3], (float)svaddv_u32(svptrue_b32(), ab03), distances + 0 * stride + 3);
-            }
-            if (M > 1)
-            {
-                Base::DecodeCosineDistance(A[1], B[0], (float)svaddv_u32(svptrue_b32(), ab10), distances + 1 * stride + 0);
-                Base::DecodeCosineDistance(A[1], B[1], (float)svaddv_u32(svptrue_b32(), ab11), distances + 1 * stride + 1);
-                Base::DecodeCosineDistance(A[1], B[2], (float)svaddv_u32(svptrue_b32(), ab12), distances + 1 * stride + 2);
-                Base::DecodeCosineDistance(A[1], B[3], (float)svaddv_u32(svptrue_b32(), ab13), distances + 1 * stride + 3);
-            }
-            if (M > 2)
-            {
-                Base::DecodeCosineDistance(A[2], B[0], (float)svaddv_u32(svptrue_b32(), ab20), distances + 2 * stride + 0);
-                Base::DecodeCosineDistance(A[2], B[1], (float)svaddv_u32(svptrue_b32(), ab21), distances + 2 * stride + 1);
-                Base::DecodeCosineDistance(A[2], B[2], (float)svaddv_u32(svptrue_b32(), ab22), distances + 2 * stride + 2);
-                Base::DecodeCosineDistance(A[2], B[3], (float)svaddv_u32(svptrue_b32(), ab23), distances + 2 * stride + 3);
-            }
-            if (M > 3)
-            {
-                Base::DecodeCosineDistance(A[3], B[0], (float)svaddv_u32(svptrue_b32(), ab30), distances + 3 * stride + 0);
-                Base::DecodeCosineDistance(A[3], B[1], (float)svaddv_u32(svptrue_b32(), ab31), distances + 3 * stride + 1);
-                Base::DecodeCosineDistance(A[3], B[2], (float)svaddv_u32(svptrue_b32(), ab32), distances + 3 * stride + 2);
-                Base::DecodeCosineDistance(A[3], B[3], (float)svaddv_u32(svptrue_b32(), ab33), distances + 3 * stride + 3);
-            }
-        }
+        };
 
-        template<int bits> struct CorrelationsMx1
+        template<int bits> struct DirectMx1
         {
             template<int M> static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* B, size_t size, uint32_t* ab)
             {
-                assert(size % 8 == 0 && size >= 8);
                 const size_t valuesPerVec = svcntb();
                 const size_t packedPerVec = valuesPerVec * bits / 8;
                 const svuint8_t tbl = UnpackTbl<bits>();
@@ -280,27 +450,19 @@ namespace Simd
                 for (; i + valuesPerVec <= size; i += valuesPerVec, packed += packedPerVec)
                 {
                     svuint8_t _b = UnpackTo8<bits>(b + packed, packedPerVec, tbl, shr);
-                    if (M > 0)
-                        sums0 = svdot_u32(sums0, UnpackTo8<bits>(A[0] + 16 + packed, packedPerVec, tbl, shr), _b);
-                    if (M > 1)
-                        sums1 = svdot_u32(sums1, UnpackTo8<bits>(A[1] + 16 + packed, packedPerVec, tbl, shr), _b);
-                    if (M > 2)
-                        sums2 = svdot_u32(sums2, UnpackTo8<bits>(A[2] + 16 + packed, packedPerVec, tbl, shr), _b);
-                    if (M > 3)
-                        sums3 = svdot_u32(sums3, UnpackTo8<bits>(A[3] + 16 + packed, packedPerVec, tbl, shr), _b);
+                    if (M > 0) sums0 = svdot_u32(sums0, UnpackTo8<bits>(A[0] + 16 + packed, packedPerVec, tbl, shr), _b);
+                    if (M > 1) sums1 = svdot_u32(sums1, UnpackTo8<bits>(A[1] + 16 + packed, packedPerVec, tbl, shr), _b);
+                    if (M > 2) sums2 = svdot_u32(sums2, UnpackTo8<bits>(A[2] + 16 + packed, packedPerVec, tbl, shr), _b);
+                    if (M > 3) sums3 = svdot_u32(sums3, UnpackTo8<bits>(A[3] + 16 + packed, packedPerVec, tbl, shr), _b);
                 }
                 if (i < size)
                 {
                     size_t packedTail = (size - i) * bits / 8;
                     svuint8_t _b = UnpackTo8<bits>(b + packed, packedTail, tbl, shr);
-                    if (M > 0)
-                        sums0 = svdot_u32(sums0, UnpackTo8<bits>(A[0] + 16 + packed, packedTail, tbl, shr), _b);
-                    if (M > 1)
-                        sums1 = svdot_u32(sums1, UnpackTo8<bits>(A[1] + 16 + packed, packedTail, tbl, shr), _b);
-                    if (M > 2)
-                        sums2 = svdot_u32(sums2, UnpackTo8<bits>(A[2] + 16 + packed, packedTail, tbl, shr), _b);
-                    if (M > 3)
-                        sums3 = svdot_u32(sums3, UnpackTo8<bits>(A[3] + 16 + packed, packedTail, tbl, shr), _b);
+                    if (M > 0) sums0 = svdot_u32(sums0, UnpackTo8<bits>(A[0] + 16 + packed, packedTail, tbl, shr), _b);
+                    if (M > 1) sums1 = svdot_u32(sums1, UnpackTo8<bits>(A[1] + 16 + packed, packedTail, tbl, shr), _b);
+                    if (M > 2) sums2 = svdot_u32(sums2, UnpackTo8<bits>(A[2] + 16 + packed, packedTail, tbl, shr), _b);
+                    if (M > 3) sums3 = svdot_u32(sums3, UnpackTo8<bits>(A[3] + 16 + packed, packedTail, tbl, shr), _b);
                 }
                 if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
                 if (M > 1) ab[1] = (uint32_t)svaddv_u32(svptrue_b32(), sums1);
@@ -309,44 +471,68 @@ namespace Simd
             }
         };
 
-        template<> struct CorrelationsMx1<4>
+        template<> struct DirectMx1<4>
         {
             template<int M> static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* B, size_t size, uint32_t* ab)
             {
-                assert(size % 8 == 0 && size >= 8);
+                const size_t width = svcntb();
                 const size_t byteSize = size / 2;
-                const svuint8_t zero = svdup_n_u8(0);
+                const size_t main = byteSize & ~(width - 1);
+                const svbool_t all = svptrue_b8();
                 svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
                 const uint8_t* b = B + 16;
-                for (size_t i = 0; i < byteSize; i += svcntb())
+                size_t i = 0;
+                for (; i < main; i += width)
+                {
+                    svuint8_t _b = svld1_u8(all, b + i);
+                    svuint8_t b0 = svand_n_u8_x(all, _b, 0x0F);
+                    svuint8_t b1 = svlsr_n_u8_x(all, _b, 4);
+                    if (M > 0)
+                    {
+                        svuint8_t _a = svld1_u8(all, A[0] + 16 + i);
+                        sums0 = svdot_u32(svdot_u32(sums0, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+                    }
+                    if (M > 1)
+                    {
+                        svuint8_t _a = svld1_u8(all, A[1] + 16 + i);
+                        sums1 = svdot_u32(svdot_u32(sums1, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+                    }
+                    if (M > 2)
+                    {
+                        svuint8_t _a = svld1_u8(all, A[2] + 16 + i);
+                        sums2 = svdot_u32(svdot_u32(sums2, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+                    }
+                    if (M > 3)
+                    {
+                        svuint8_t _a = svld1_u8(all, A[3] + 16 + i);
+                        sums3 = svdot_u32(svdot_u32(sums3, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+                    }
+                }
+                if (i < byteSize)
                 {
                     svbool_t mask = svwhilelt_b8(i, byteSize);
-                    svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
+                    svuint8_t _b = svld1_u8(mask, b + i);
                     svuint8_t b0 = svand_n_u8_z(mask, _b, 0x0F);
                     svuint8_t b1 = svlsr_n_u8_z(mask, _b, 4);
                     if (M > 0)
                     {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[0] + 16 + i), zero);
-                        sums0 = svdot_u32(sums0, svand_n_u8_z(mask, _a, 0x0F), b0);
-                        sums0 = svdot_u32(sums0, svlsr_n_u8_z(mask, _a, 4), b1);
+                        svuint8_t _a = svld1_u8(mask, A[0] + 16 + i);
+                        sums0 = svdot_u32(svdot_u32(sums0, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
                     }
                     if (M > 1)
                     {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[1] + 16 + i), zero);
-                        sums1 = svdot_u32(sums1, svand_n_u8_z(mask, _a, 0x0F), b0);
-                        sums1 = svdot_u32(sums1, svlsr_n_u8_z(mask, _a, 4), b1);
+                        svuint8_t _a = svld1_u8(mask, A[1] + 16 + i);
+                        sums1 = svdot_u32(svdot_u32(sums1, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
                     }
                     if (M > 2)
                     {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[2] + 16 + i), zero);
-                        sums2 = svdot_u32(sums2, svand_n_u8_z(mask, _a, 0x0F), b0);
-                        sums2 = svdot_u32(sums2, svlsr_n_u8_z(mask, _a, 4), b1);
+                        svuint8_t _a = svld1_u8(mask, A[2] + 16 + i);
+                        sums2 = svdot_u32(svdot_u32(sums2, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
                     }
                     if (M > 3)
                     {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[3] + 16 + i), zero);
-                        sums3 = svdot_u32(sums3, svand_n_u8_z(mask, _a, 0x0F), b0);
-                        sums3 = svdot_u32(sums3, svlsr_n_u8_z(mask, _a, 4), b1);
+                        svuint8_t _a = svld1_u8(mask, A[3] + 16 + i);
+                        sums3 = svdot_u32(svdot_u32(sums3, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
                     }
                 }
                 if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
@@ -356,38 +542,32 @@ namespace Simd
             }
         };
 
-        template<> struct CorrelationsMx1<8>
+        template<> struct DirectMx1<8>
         {
             template<int M> static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* B, size_t size, uint32_t* ab)
             {
-                assert(size % 8 == 0 && size >= 8);
-                const svuint8_t zero = svdup_n_u8(0);
+                const size_t width = svcntb();
+                const size_t main = size & ~(width - 1);
+                const svbool_t all = svptrue_b8();
                 svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
                 const uint8_t* b = B + 16;
-                for (size_t i = 0; i < size; i += svcntb())
+                size_t i = 0;
+                for (; i < main; i += width)
+                {
+                    svuint8_t _b = svld1_u8(all, b + i);
+                    if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(all, A[0] + 16 + i), _b);
+                    if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(all, A[1] + 16 + i), _b);
+                    if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(all, A[2] + 16 + i), _b);
+                    if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(all, A[3] + 16 + i), _b);
+                }
+                if (i < size)
                 {
                     svbool_t mask = svwhilelt_b8(i, size);
-                    svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
-                    if (M > 0)
-                    {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[0] + 16 + i), zero);
-                        sums0 = svdot_u32(sums0, _a, _b);
-                    }
-                    if (M > 1)
-                    {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[1] + 16 + i), zero);
-                        sums1 = svdot_u32(sums1, _a, _b);
-                    }
-                    if (M > 2)
-                    {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[2] + 16 + i), zero);
-                        sums2 = svdot_u32(sums2, _a, _b);
-                    }
-                    if (M > 3)
-                    {
-                        svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[3] + 16 + i), zero);
-                        sums3 = svdot_u32(sums3, _a, _b);
-                    }
+                    svuint8_t _b = svld1_u8(mask, b + i);
+                    if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(mask, A[0] + 16 + i), _b);
+                    if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(mask, A[1] + 16 + i), _b);
+                    if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(mask, A[2] + 16 + i), _b);
+                    if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(mask, A[3] + 16 + i), _b);
                 }
                 if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
                 if (M > 1) ab[1] = (uint32_t)svaddv_u32(svptrue_b32(), sums1);
@@ -399,36 +579,13 @@ namespace Simd
         template<int bits, int M> SIMD_INLINE void MicroCosineDistancesDirectMx1(const uint8_t* const* A, const uint8_t* B, size_t size, float* distances, size_t stride)
         {
             uint32_t ab[4];
-            CorrelationsMx1<bits>::template Run<M>(A, B, size, ab);
+            DirectMx1<bits>::template Run<M>(A, B, size, ab);
             if (M > 0) Base::DecodeCosineDistance(A[0], B, (float)ab[0], distances + 0 * stride);
             if (M > 1) Base::DecodeCosineDistance(A[1], B, (float)ab[1], distances + 1 * stride);
             if (M > 2) Base::DecodeCosineDistance(A[2], B, (float)ab[2], distances + 2 * stride);
             if (M > 3) Base::DecodeCosineDistance(A[3], B, (float)ab[3], distances + 3 * stride);
         }
 
-        template<int bits> void MacroCosineDistancesDirectMx1(size_t M, size_t N, const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
-        {
-            size_t M4 = AlignLoAny(M, 4), i = 0;
-            for (; i < M4; i += 4)
-            {
-                for (size_t j = 0; j < N; ++j)
-                    MicroCosineDistancesDirectMx1<bits, 4>(A + i, B[j], size, distances + j, stride);
-                distances += 4 * stride;
-            }
-            if (i < M)
-            {
-                for (size_t j = 0; j < N; ++j)
-                {
-                    switch (M - i)
-                    {
-                    case 1: MicroCosineDistancesDirectMx1<bits, 1>(A + i, B[j], size, distances + j, stride); break;
-                    case 2: MicroCosineDistancesDirectMx1<bits, 2>(A + i, B[j], size, distances + j, stride); break;
-                    case 3: MicroCosineDistancesDirectMx1<bits, 3>(A + i, B[j], size, distances + j, stride); break;
-                    }
-                }
-            }
-        }
-
         template<int bits> void MacroCosineDistancesDirect(size_t M, size_t N, const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
         {
             size_t M4 = AlignLoAny(M, 4), N4 = AlignLo(N, 4), i = 0;
@@ -436,7 +593,7 @@ namespace Simd
             {
                 size_t j = 0;
                 for (; j < N4; j += 4)
-                    MicroCosineDistancesDirectMx4<bits, 4>(A + i, B + j, size, distances + j, stride);
+                    DirectMx4<bits>::template Run<4>(A + i, B + j, size, distances + j, stride);
                 for (; j < N; ++j)
                     MicroCosineDistancesDirectMx1<bits, 4>(A + i, B[j], size, distances + j, stride);
                 distances += 4 * stride;
@@ -448,11 +605,11 @@ namespace Simd
                 for (; j < N4; j += 4)
                 {
                     if (m == 1)
-                        MicroCosineDistancesDirectMx4<bits, 1>(A + i, B + j, size, distances + j, stride);
+                        DirectMx4<bits>::template Run<1>(A + i, B + j, size, distances + j, stride);
                     else if (m == 2)
-                        MicroCosineDistancesDirectMx4<bits, 2>(A + i, B + j, size, distances + j, stride);
+                        DirectMx4<bits>::template Run<2>(A + i, B + j, size, distances + j, stride);
                     else
-                        MicroCosineDistancesDirectMx4<bits, 3>(A + i, B + j, size, distances + j, stride);
+                        DirectMx4<bits>::template Run<3>(A + i, B + j, size, distances + j, stride);
                 }
                 for (; j < N; ++j)
                 {
@@ -475,7 +632,11 @@ namespace Simd
             case 6: return CosineDistance<6>;
             case 7: return CosineDistance<7>;
             case 8: return CosineDistance<8>;
+#ifdef SIMD_NEON_ENABLE
             default: return Neon::GetCosineDistance(depth);
+#else
+            default: return Base::GetCosineDistance(depth);
+#endif
             }
         }
 
@@ -483,11 +644,11 @@ namespace Simd
         {
             switch (depth)
             {
-            case 4: return MacroCosineDistancesDirectMx1<4>;
+            case 4: return MacroCosineDistancesDirect<4>;
             case 5: return MacroCosineDistancesDirect<5>;
             case 6: return MacroCosineDistancesDirect<6>;
             case 7: return MacroCosineDistancesDirect<7>;
-            case 8: return MacroCosineDistancesDirectMx1<8>;
+            case 8: return MacroCosineDistancesDirect<8>;
             default: return NULL;
             }
         }
diff --git a/src/Simd/SimdSve2DescrIntCdu.cpp b/src/Simd/SimdSve2DescrIntCdu.cpp
index 7579bc49be..701f59e4ca 100644
--- a/src/Simd/SimdSve2DescrIntCdu.cpp
+++ b/src/Simd/SimdSve2DescrIntCdu.cpp
@@ -32,73 +32,125 @@ namespace Simd
 #ifdef SIMD_SVE2_ENABLE
     namespace Sve2
     {
-        static void UnpackNorm(size_t count, const uint8_t* const* src, float* dst, size_t stride)
+        static void UnpackNormA(size_t count, const uint8_t* const* src, float* dst, size_t)
         {
             for (size_t i = 0; i < count; ++i)
             {
                 const float* ps = (const float*)src[i];
-                float* pd = dst + i * 4;
-                pd[0] = ps[0];
-                pd[1] = ps[1];
-                pd[2] = ps[2];
-                pd[3] = ps[3];
+                dst[0] = ps[0];
+                dst[1] = ps[1];
+                dst[2] = ps[2];
+                dst[3] = ps[3];
+                dst += 4;
+            }
+        }
+
+        static void UnpackNormB(size_t count, const uint8_t* const* src, float* dst, size_t stride)
+        {
+            size_t count4 = AlignLo(count, 4), i = 0;
+#ifdef SIMD_NEON_ENABLE
+            for (; i < count4; i += 4, src += 4)
+            {
+                float32x4x2_t a0, a1, b0, b1;
+                a0.val[0] = vld1q_f32((const float*)src[0]);
+                a0.val[1] = vld1q_f32((const float*)src[1]);
+                a1.val[0] = vld1q_f32((const float*)src[2]);
+                a1.val[1] = vld1q_f32((const float*)src[3]);
+                b0 = vzipq_f32(a0.val[0], a1.val[0]);
+                b1 = vzipq_f32(a0.val[1], a1.val[1]);
+                a0 = vzipq_f32(b0.val[0], b1.val[0]);
+                a1 = vzipq_f32(b0.val[1], b1.val[1]);
+                vst1q_f32(dst + 0 * stride + i, a0.val[0]);
+                vst1q_f32(dst + 1 * stride + i, a0.val[1]);
+                vst1q_f32(dst + 2 * stride + i, a1.val[0]);
+                vst1q_f32(dst + 3 * stride + i, a1.val[1]);
+            }
+#endif
+            for (; i < count; ++i)
+            {
+                const float* ps = (const float*)src[i];
+                dst[0 * stride + i] = ps[0];
+                dst[1 * stride + i] = ps[1];
+                dst[2 * stride + i] = ps[2];
+                dst[3 * stride + i] = ps[3];
             }
         }
 
         Base::DescrInt::UnpackNormPtr GetUnpackNorm(bool transpose)
         {
-            return UnpackNorm;
+            return transpose ? UnpackNormB : UnpackNormA;
         }
 
         //-------------------------------------------------------------------------------------------------
 
-        template<int bits> SIMD_INLINE uint64_t LoadBitsBlock(const uint8_t* src)
+        template<int bits> void UnpackData(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t)
         {
-            uint64_t value = 0;
-            for (size_t i = 0; i < bits; ++i)
-                value |= uint64_t(src[i]) << (8 * i);
-            return value;
+            const size_t valuesPerVec = svcntb();
+            const size_t packedPerVec = valuesPerVec * bits / 8;
+            const svuint8_t tbl = UnpackTbl<bits>();
+            const svuint16_t shr = UnpackShr<bits>();
+            const svbool_t all = svptrue_b8();
+            for (size_t i = 0; i < count; ++i)
+            {
+                const uint8_t* ps = src[i] + 16;
+                uint8_t* pd = dst + i * size;
+                size_t j = 0, packed = 0;
+                for (; j + valuesPerVec <= size; j += valuesPerVec, packed += packedPerVec)
+                    svst1_u8(all, pd + j, UnpackTo8<bits>(ps + packed, packedPerVec, tbl, shr));
+                if (j < size)
+                {
+                    size_t packedTail = (size - j) * bits / 8;
+                    svst1_u8(svwhilelt_b8(j, size), pd + j, UnpackTo8<bits>(ps + packed, packedTail, tbl, shr));
+                }
+            }
         }
 
-        template<int bits> void UnpackData(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t stride)
+        template<> void UnpackData<4>(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t)
         {
-            const uint64_t mask = (uint64_t(1) << bits) - 1;
+            const size_t width = svcntb();
+            const size_t byteSize = size / 2;
+            const svbool_t all = svptrue_b8();
             for (size_t i = 0; i < count; ++i)
             {
                 const uint8_t* ps = src[i] + 16;
                 uint8_t* pd = dst + i * size;
-                for (size_t j = 0; j < size; j += 8, ps += bits, pd += 8)
+                size_t j = 0;
+                for (; j + width <= byteSize; j += width, ps += width, pd += 2 * width)
                 {
-                    uint64_t value = LoadBitsBlock<bits>(ps);
-                    pd[0] = uint8_t((value >> (0 * bits)) & mask);
-                    pd[1] = uint8_t((value >> (1 * bits)) & mask);
-                    pd[2] = uint8_t((value >> (2 * bits)) & mask);
-                    pd[3] = uint8_t((value >> (3 * bits)) & mask);
-                    pd[4] = uint8_t((value >> (4 * bits)) & mask);
-                    pd[5] = uint8_t((value >> (5 * bits)) & mask);
-                    pd[6] = uint8_t((value >> (6 * bits)) & mask);
-                    pd[7] = uint8_t((value >> (7 * bits)) & mask);
+                    svuint8_t value = svld1_u8(all, ps);
+                    svuint8_t lo = svand_n_u8_x(all, value, 0x0F);
+                    svuint8_t hi = svlsr_n_u8_x(all, value, 4);
+                    svst1_u8(all, pd, svzip1_u8(lo, hi));
+                    svst1_u8(all, pd + width, svzip2_u8(lo, hi));
+                }
+                for (; j < byteSize; ++j, ++ps, pd += 2)
+                {
+                    pd[0] = uint8_t(ps[0] & 0x0F);
+                    pd[1] = uint8_t(ps[0] >> 4);
                 }
             }
         }
 
-        template<> void UnpackData<8>(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t stride)
+        template<> void UnpackData<8>(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t)
         {
-            const svuint8_t zero = svdup_n_u8(0);
+            const svbool_t all = svptrue_b8();
+            const size_t width = svcntb();
             for (size_t i = 0; i < count; ++i)
             {
                 const uint8_t* ps = src[i] + 16;
                 uint8_t* pd = dst + i * size;
-                for (size_t j = 0; j < size; j += svcntb())
+                size_t j = 0;
+                for (; j + width <= size; j += width)
+                    svst1_u8(all, pd + j, svld1_u8(all, ps + j));
+                if (j < size)
                 {
                     svbool_t mask = svwhilelt_b8(j, size);
-                    svuint8_t value = svsel_u8(mask, svld1_u8(mask, ps + j), zero);
-                    svst1_u8(mask, pd + j, value);
+                    svst1_u8(mask, pd + j, svld1_u8(mask, ps + j));
                 }
             }
         }
 
-        Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth)
+        Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth, bool)
         {
             switch (depth)
             {
@@ -113,97 +165,182 @@ namespace Simd
 
         //-------------------------------------------------------------------------------------------------
 
-        SIMD_INLINE uint32_t Correlation8u(const uint8_t* a, const uint8_t* b, size_t size)
+        template<int M> SIMD_INLINE void CorrelationMx4(size_t K, const uint8_t* ad, const float* an,
+            const uint8_t* bd, const float* bn, size_t bnStride, float* distances, size_t stride)
         {
-            const svuint8_t zero = svdup_n_u8(0);
-            svuint32_t sums = svdup_n_u32(0);
-            for (size_t i = 0; i < size; i += svcntb())
+            const size_t width = svcntb();
+            const size_t main = K & ~(width - 1);
+            const svbool_t all = svptrue_b8();
+            svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+            svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+            svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+            svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+            const uint8_t* ad0 = ad;
+            const uint8_t* ad1 = ad + K;
+            const uint8_t* ad2 = ad + 2 * K;
+            const uint8_t* ad3 = ad + 3 * K;
+            const uint8_t* bd0 = bd;
+            const uint8_t* bd1 = bd + K;
+            const uint8_t* bd2 = bd + 2 * K;
+            const uint8_t* bd3 = bd + 3 * K;
+            size_t k = 0;
+            for (; k < main; k += width)
             {
-                svbool_t mask = svwhilelt_b8(i, size);
-                svuint8_t _a = svsel_u8(mask, svld1_u8(mask, a + i), zero);
-                svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
-                sums = svdot_u32(sums, _a, _b);
+                svuint8_t b0 = svld1_u8(all, bd0 + k);
+                svuint8_t b1 = svld1_u8(all, bd1 + k);
+                svuint8_t b2 = svld1_u8(all, bd2 + k);
+                svuint8_t b3 = svld1_u8(all, bd3 + k);
+                if (M > 0)
+                {
+                    svuint8_t a0 = svld1_u8(all, ad0 + k);
+                    ab00 = svdot_u32(ab00, a0, b0);
+                    ab01 = svdot_u32(ab01, a0, b1);
+                    ab02 = svdot_u32(ab02, a0, b2);
+                    ab03 = svdot_u32(ab03, a0, b3);
+                }
+                if (M > 1)
+                {
+                    svuint8_t a1 = svld1_u8(all, ad1 + k);
+                    ab10 = svdot_u32(ab10, a1, b0);
+                    ab11 = svdot_u32(ab11, a1, b1);
+                    ab12 = svdot_u32(ab12, a1, b2);
+                    ab13 = svdot_u32(ab13, a1, b3);
+                }
+                if (M > 2)
+                {
+                    svuint8_t a2 = svld1_u8(all, ad2 + k);
+                    ab20 = svdot_u32(ab20, a2, b0);
+                    ab21 = svdot_u32(ab21, a2, b1);
+                    ab22 = svdot_u32(ab22, a2, b2);
+                    ab23 = svdot_u32(ab23, a2, b3);
+                }
+                if (M > 3)
+                {
+                    svuint8_t a3 = svld1_u8(all, ad3 + k);
+                    ab30 = svdot_u32(ab30, a3, b0);
+                    ab31 = svdot_u32(ab31, a3, b1);
+                    ab32 = svdot_u32(ab32, a3, b2);
+                    ab33 = svdot_u32(ab33, a3, b3);
+                }
             }
-            return (uint32_t)svaddv_u32(svptrue_b32(), sums);
-        }
-
-        SIMD_INLINE float DecodeCosineDistance(const float* a, const float* b, uint32_t abSum)
-        {
-            float ab = float(abSum) * a[0] * b[0] + a[2] * b[1] + b[2] * a[1];
-            return Simd::RestrictRange(1.0f - ab / (a[3] * b[3]), 0.0f, 2.0f);
-        }
-
-        template<int M> SIMD_INLINE void CorrelationsMx1(size_t K, const uint8_t* ad, const uint8_t* bd, uint32_t* ab)
-        {
-            const svuint8_t zero = svdup_n_u8(0);
-            svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
-            for (size_t i = 0; i < K; i += svcntb())
+            if (k < K)
             {
-                svbool_t mask = svwhilelt_b8(i, K);
-                svuint8_t b = svsel_u8(mask, svld1_u8(mask, bd + i), zero);
+                svbool_t mask = svwhilelt_b8(k, K);
+                svuint8_t b0 = svld1_u8(mask, bd0 + k);
+                svuint8_t b1 = svld1_u8(mask, bd1 + k);
+                svuint8_t b2 = svld1_u8(mask, bd2 + k);
+                svuint8_t b3 = svld1_u8(mask, bd3 + k);
                 if (M > 0)
                 {
-                    svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 0 * K + i), zero);
-                    sums0 = svdot_u32(sums0, a, b);
+                    svuint8_t a0 = svld1_u8(mask, ad0 + k);
+                    ab00 = svdot_u32(ab00, a0, b0);
+                    ab01 = svdot_u32(ab01, a0, b1);
+                    ab02 = svdot_u32(ab02, a0, b2);
+                    ab03 = svdot_u32(ab03, a0, b3);
                 }
                 if (M > 1)
                 {
-                    svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 1 * K + i), zero);
-                    sums1 = svdot_u32(sums1, a, b);
+                    svuint8_t a1 = svld1_u8(mask, ad1 + k);
+                    ab10 = svdot_u32(ab10, a1, b0);
+                    ab11 = svdot_u32(ab11, a1, b1);
+                    ab12 = svdot_u32(ab12, a1, b2);
+                    ab13 = svdot_u32(ab13, a1, b3);
                 }
                 if (M > 2)
                 {
-                    svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 2 * K + i), zero);
-                    sums2 = svdot_u32(sums2, a, b);
+                    svuint8_t a2 = svld1_u8(mask, ad2 + k);
+                    ab20 = svdot_u32(ab20, a2, b0);
+                    ab21 = svdot_u32(ab21, a2, b1);
+                    ab22 = svdot_u32(ab22, a2, b2);
+                    ab23 = svdot_u32(ab23, a2, b3);
                 }
                 if (M > 3)
                 {
-                    svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 3 * K + i), zero);
-                    sums3 = svdot_u32(sums3, a, b);
+                    svuint8_t a3 = svld1_u8(mask, ad3 + k);
+                    ab30 = svdot_u32(ab30, a3, b0);
+                    ab31 = svdot_u32(ab31, a3, b1);
+                    ab32 = svdot_u32(ab32, a3, b2);
+                    ab33 = svdot_u32(ab33, a3, b3);
                 }
             }
-            if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
-            if (M > 1) ab[1] = (uint32_t)svaddv_u32(svptrue_b32(), sums1);
-            if (M > 2) ab[2] = (uint32_t)svaddv_u32(svptrue_b32(), sums2);
-            if (M > 3) ab[3] = (uint32_t)svaddv_u32(svptrue_b32(), sums3);
+            if (M > 0) DecodeCosineDistances1x4(an + 0 * 4, bn, bnStride, ab00, ab01, ab02, ab03, distances + 0 * stride);
+            if (M > 1) DecodeCosineDistances1x4(an + 1 * 4, bn, bnStride, ab10, ab11, ab12, ab13, distances + 1 * stride);
+            if (M > 2) DecodeCosineDistances1x4(an + 2 * 4, bn, bnStride, ab20, ab21, ab22, ab23, distances + 2 * stride);
+            if (M > 3) DecodeCosineDistances1x4(an + 3 * 4, bn, bnStride, ab30, ab31, ab32, ab33, distances + 3 * stride);
         }
 
-        template<int M> SIMD_INLINE void MacroCorrelationMx1(size_t K, const uint8_t* ad, const float* an, const uint8_t* bd, const float* bn, float* distances, size_t stride)
+        template<int M> SIMD_INLINE void CorrelationMx1(size_t K, const uint8_t* ad, const float* an,
+            const uint8_t* bd, const float* bn, size_t bnStride, float* distances, size_t stride)
         {
-            uint32_t ab[4];
-            CorrelationsMx1<M>(K, ad, bd, ab);
-            if (M > 0) distances[0 * stride] = DecodeCosineDistance(an + 0 * 4, bn, ab[0]);
-            if (M > 1) distances[1 * stride] = DecodeCosineDistance(an + 1 * 4, bn, ab[1]);
-            if (M > 2) distances[2 * stride] = DecodeCosineDistance(an + 2 * 4, bn, ab[2]);
-            if (M > 3) distances[3 * stride] = DecodeCosineDistance(an + 3 * 4, bn, ab[3]);
+            const size_t width = svcntb();
+            const size_t main = K & ~(width - 1);
+            const svbool_t all = svptrue_b8();
+            svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
+            size_t k = 0;
+            for (; k < main; k += width)
+            {
+                svuint8_t b = svld1_u8(all, bd + k);
+                if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(all, ad + 0 * K + k), b);
+                if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(all, ad + 1 * K + k), b);
+                if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(all, ad + 2 * K + k), b);
+                if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(all, ad + 3 * K + k), b);
+            }
+            if (k < K)
+            {
+                svbool_t mask = svwhilelt_b8(k, K);
+                svuint8_t b = svld1_u8(mask, bd + k);
+                if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(mask, ad + 0 * K + k), b);
+                if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(mask, ad + 1 * K + k), b);
+                if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(mask, ad + 2 * K + k), b);
+                if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(mask, ad + 3 * K + k), b);
+            }
+            if (M > 0) DecodeCosineDistance(an + 0 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums0), distances + 0 * stride);
+            if (M > 1) DecodeCosineDistance(an + 1 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums1), distances + 1 * stride);
+            if (M > 2) DecodeCosineDistance(an + 2 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums2), distances + 2 * stride);
+            if (M > 3) DecodeCosineDistance(an + 3 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums3), distances + 3 * stride);
         }
 
-        void MacroCorrelation(size_t M, size_t N, size_t K, const uint8_t* ad, const float* an, const uint8_t* bd, const float* bn, float* distances, size_t stride)
+        void MacroCorrelation(size_t M, size_t N, size_t K, const uint8_t* ad, const float* an,
+            const uint8_t* bd, const float* bn, float* distances, size_t stride)
         {
-            size_t M4 = AlignLoAny(M, 4);
-            for (size_t j = 0; j < N; ++j)
+            size_t M4 = AlignLoAny(M, 4), N4 = AlignLo(N, 4), i = 0;
+            for (; i < M4; i += 4)
+            {
+                size_t j = 0;
+                for (; j < N4; j += 4)
+                    CorrelationMx4<4>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+                for (; j < N; ++j)
+                    CorrelationMx1<4>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+            }
+            if (i < M)
             {
-                size_t i = 0;
-                const uint8_t* b = bd + j * K;
-                const float* nb = bn + j * 4;
-                for (; i < M4; i += 4)
-                    MacroCorrelationMx1<4>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride);
-                if (i < M)
+                size_t m = M - i;
+                size_t j = 0;
+                for (; j < N4; j += 4)
+                {
+                    if (m == 1)
+                        CorrelationMx4<1>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+                    else if (m == 2)
+                        CorrelationMx4<2>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+                    else
+                        CorrelationMx4<3>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+                }
+                for (; j < N; ++j)
                 {
-                    switch (M - i)
-                    {
-                    case 1: MacroCorrelationMx1<1>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride); break;
-                    case 2: MacroCorrelationMx1<2>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride); break;
-                    case 3: MacroCorrelationMx1<3>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride); break;
-                    }
+                    if (m == 1)
+                        CorrelationMx1<1>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+                    else if (m == 2)
+                        CorrelationMx1<2>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+                    else
+                        CorrelationMx1<3>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
                 }
             }
         }
 
-        Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t depth)
+        Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t)
         {
             return MacroCorrelation;
         }
     }
-#endif// SIMD_SVE2_ENABLE
+#endif
 }
diff --git a/src/Test/TestDescrInt.cpp b/src/Test/TestDescrInt.cpp
index db2fd4822c..4d186b61a6 100644
--- a/src/Test/TestDescrInt.cpp
+++ b/src/Test/TestDescrInt.cpp
@@ -635,6 +635,9 @@ namespace Test
 
         for (size_t depth = 4; depth <= 8; depth++)
         {
+            result = result && DescrIntCosineDistancesMxNaAutoTest(8, 8, 24, depth, f1, f2);
+            result = result && DescrIntCosineDistancesMxNaAutoTest(4, 12, 256, depth, f1, f2);
+            result = result && DescrIntCosineDistancesMxNaAutoTest(5, 7, 128, depth, f1, f2);
             result = result && DescrIntCosineDistancesMxNaAutoTest(256, 128, 256, depth, f1, f2);
             result = result && DescrIntCosineDistancesMxNaAutoTest(128, 128, 512, depth, f1, f2);
 #if !(defined(__GNUC__) && defined(__clang__))

From 7fd5641ef11992aa94152f6f80e47f334f140dfc Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 24 Aug 2026 08:58:14 +0000
Subject: [PATCH 2/2] Fix SVE2 DescrIntCosineDistancesMxNa SIGSEGV on
 5x7x128-4.

UnpackNormB used vst1q_f32 with stride N, which is not 16-byte aligned
when N=7. Decode used the same NEON 16-byte loads on that layout and on
unaligned stack temps. Those accesses are new on the SVE2 unpack path
(NEON keeps N<8 on the direct path) and can raise a segment violation.

Copy headers with memcpy, decode 4-wide tiles with scalar loads, align
svld1rq tables, and bind SVE2 function pointers explicitly.

Co-authored-by: igor.ermolaev <igor.ermolaev@altezza.org>
---
 src/Simd/SimdDescrIntCommon.h    | 64 +++++++++++---------------------
 src/Simd/SimdSve2DescrInt.cpp    | 22 +++++------
 src/Simd/SimdSve2DescrIntCdu.cpp | 48 +++++++-----------------
 3 files changed, 46 insertions(+), 88 deletions(-)

diff --git a/src/Simd/SimdDescrIntCommon.h b/src/Simd/SimdDescrIntCommon.h
index 0db8b1de17..9a0770de94 100644
--- a/src/Simd/SimdDescrIntCommon.h
+++ b/src/Simd/SimdDescrIntCommon.h
@@ -725,12 +725,12 @@ namespace Simd
 #ifdef SIMD_SVE2_ENABLE
     namespace Sve2
     {
-        const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
-        const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
-        const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
-        const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
-        const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
-        const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };
+        SIMD_ALIGNED(16) const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
+        SIMD_ALIGNED(16) const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
+        SIMD_ALIGNED(16) const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
+        SIMD_ALIGNED(16) const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
+        SIMD_ALIGNED(16) const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
+        SIMD_ALIGNED(16) const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };
 
         SIMD_INLINE svuint8_t UnpackTbl(const uint8_t* tbl16, uint8_t bits)
         {
@@ -775,51 +775,29 @@ namespace Simd
             return svuzp1_u8(svreinterpret_u8_u16(lo), svreinterpret_u8_u16(hi));
         }
 
-        SIMD_INLINE void DecodeCosineDistances1x4(const uint8_t* a, const uint8_t* const* B,
-            svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
+        SIMD_INLINE void DecodeCosineDistance(const float* a, const float* b, size_t stride, uint32_t abSum, float* distance)
         {
-            float ab[4] = {
-                (float)svaddv_u32(svptrue_b32(), s0),
-                (float)svaddv_u32(svptrue_b32(), s1),
-                (float)svaddv_u32(svptrue_b32(), s2),
-                (float)svaddv_u32(svptrue_b32(), s3)
-            };
-#ifdef SIMD_NEON_ENABLE
-            Neon::DecodeCosineDistances1x4(a, B, vld1q_f32(ab), distances);
-#else
-            Base::DecodeCosineDistance(a, B[0], ab[0], distances + 0);
-            Base::DecodeCosineDistance(a, B[1], ab[1], distances + 1);
-            Base::DecodeCosineDistance(a, B[2], ab[2], distances + 2);
-            Base::DecodeCosineDistance(a, B[3], ab[3], distances + 3);
-#endif
+            float bb[4] = { b[0], b[stride], b[2 * stride], b[3 * stride] };
+            float ab = (float)abSum * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
+            distance[0] = Simd::RestrictRange(1.0f - ab / (a[3] * bb[3]), 0.0f, 2.0f);
         }
 
-        SIMD_INLINE void DecodeCosineDistances1x4(const float* a, const float* b, size_t stride,
+        SIMD_INLINE void DecodeCosineDistances1x4(const uint8_t* a, const uint8_t* const* B,
             svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
         {
-            uint32_t ab[4] = {
-                (uint32_t)svaddv_u32(svptrue_b32(), s0),
-                (uint32_t)svaddv_u32(svptrue_b32(), s1),
-                (uint32_t)svaddv_u32(svptrue_b32(), s2),
-                (uint32_t)svaddv_u32(svptrue_b32(), s3)
-            };
-#ifdef SIMD_NEON_ENABLE
-            Neon::DecodeCosineDistances1x4(a, b, stride, vld1q_u32(ab), distances);
-#else
-            for (size_t j = 0; j < 4; ++j)
-            {
-                float bb[4] = { b[j + 0 * stride], b[j + 1 * stride], b[j + 2 * stride], b[j + 3 * stride] };
-                float abSum = (float)ab[j] * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
-                distances[j] = Simd::RestrictRange(1.0f - abSum / (a[3] * bb[3]), 0.0f, 2.0f);
-            }
-#endif
+            Base::DecodeCosineDistance(a, B[0], (float)svaddv_u32(svptrue_b32(), s0), distances + 0);
+            Base::DecodeCosineDistance(a, B[1], (float)svaddv_u32(svptrue_b32(), s1), distances + 1);
+            Base::DecodeCosineDistance(a, B[2], (float)svaddv_u32(svptrue_b32(), s2), distances + 2);
+            Base::DecodeCosineDistance(a, B[3], (float)svaddv_u32(svptrue_b32(), s3), distances + 3);
         }
 
-        SIMD_INLINE void DecodeCosineDistance(const float* a, const float* b, size_t stride, uint32_t abSum, float* distance)
+        SIMD_INLINE void DecodeCosineDistances1x4(const float* a, const float* b, size_t stride,
+            svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
         {
-            float bb[4] = { b[0], b[stride], b[2 * stride], b[3 * stride] };
-            float ab = (float)abSum * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
-            distance[0] = Simd::RestrictRange(1.0f - ab / (a[3] * bb[3]), 0.0f, 2.0f);
+            DecodeCosineDistance(a, b + 0, stride, (uint32_t)svaddv_u32(svptrue_b32(), s0), distances + 0);
+            DecodeCosineDistance(a, b + 1, stride, (uint32_t)svaddv_u32(svptrue_b32(), s1), distances + 1);
+            DecodeCosineDistance(a, b + 2, stride, (uint32_t)svaddv_u32(svptrue_b32(), s2), distances + 2);
+            DecodeCosineDistance(a, b + 3, stride, (uint32_t)svaddv_u32(svptrue_b32(), s3), distances + 3);
         }
     }
 #endif
diff --git a/src/Simd/SimdSve2DescrInt.cpp b/src/Simd/SimdSve2DescrInt.cpp
index 318824f2ac..14eae1e284 100644
--- a/src/Simd/SimdSve2DescrInt.cpp
+++ b/src/Simd/SimdSve2DescrInt.cpp
@@ -37,21 +37,21 @@ namespace Simd
             : Base::DescrInt(size, depth)
 #endif
         {
-            _encode32f = GetEncode32f(_depth);
-            _encode16f = GetEncode16f(_depth);
-            _decode32f = GetDecode32f(_depth);
-            _decode16f = GetDecode16f(_depth);
+            _encode32f = Sve2::GetEncode32f(_depth);
+            _encode16f = Sve2::GetEncode16f(_depth);
+            _decode32f = Sve2::GetDecode32f(_depth);
+            _decode16f = Sve2::GetDecode16f(_depth);
 
-            _cosineDistance = GetCosineDistance(_depth);
-            _macroCosineDistancesDirect = GetMacroCosineDistancesDirect(_depth);
+            _cosineDistance = Sve2::GetCosineDistance(_depth);
+            _macroCosineDistancesDirect = Sve2::GetMacroCosineDistancesDirect(_depth);
             _microMd = 4;
             _microNd = 4;
 
-            _unpackNormA = GetUnpackNorm(false);
-            _unpackNormB = GetUnpackNorm(true);
-            _unpackDataA = GetUnpackData(_depth, false);
-            _unpackDataB = GetUnpackData(_depth, true);
-            _macroCosineDistancesUnpack = GetMacroCosineDistancesUnpack(_depth);
+            _unpackNormA = Sve2::GetUnpackNorm(false);
+            _unpackNormB = Sve2::GetUnpackNorm(true);
+            _unpackDataA = Sve2::GetUnpackData(_depth, false);
+            _unpackDataB = Sve2::GetUnpackData(_depth, true);
+            _macroCosineDistancesUnpack = Sve2::GetMacroCosineDistancesUnpack(_depth);
             _unpSize = _size;
             _microMu = 4;
             _microNu = 4;
diff --git a/src/Simd/SimdSve2DescrIntCdu.cpp b/src/Simd/SimdSve2DescrIntCdu.cpp
index 701f59e4ca..45e16e7575 100644
--- a/src/Simd/SimdSve2DescrIntCdu.cpp
+++ b/src/Simd/SimdSve2DescrIntCdu.cpp
@@ -32,47 +32,27 @@ namespace Simd
 #ifdef SIMD_SVE2_ENABLE
     namespace Sve2
     {
+        SIMD_INLINE void LoadHeader(const uint8_t* src, float* dst)
+        {
+            memcpy(dst, src, 16);
+        }
+
         static void UnpackNormA(size_t count, const uint8_t* const* src, float* dst, size_t)
         {
-            for (size_t i = 0; i < count; ++i)
-            {
-                const float* ps = (const float*)src[i];
-                dst[0] = ps[0];
-                dst[1] = ps[1];
-                dst[2] = ps[2];
-                dst[3] = ps[3];
-                dst += 4;
-            }
+            for (size_t i = 0; i < count; ++i, dst += 4)
+                LoadHeader(src[i], dst);
         }
 
         static void UnpackNormB(size_t count, const uint8_t* const* src, float* dst, size_t stride)
         {
-            size_t count4 = AlignLo(count, 4), i = 0;
-#ifdef SIMD_NEON_ENABLE
-            for (; i < count4; i += 4, src += 4)
-            {
-                float32x4x2_t a0, a1, b0, b1;
-                a0.val[0] = vld1q_f32((const float*)src[0]);
-                a0.val[1] = vld1q_f32((const float*)src[1]);
-                a1.val[0] = vld1q_f32((const float*)src[2]);
-                a1.val[1] = vld1q_f32((const float*)src[3]);
-                b0 = vzipq_f32(a0.val[0], a1.val[0]);
-                b1 = vzipq_f32(a0.val[1], a1.val[1]);
-                a0 = vzipq_f32(b0.val[0], b1.val[0]);
-                a1 = vzipq_f32(b0.val[1], b1.val[1]);
-                vst1q_f32(dst + 0 * stride + i, a0.val[0]);
-                vst1q_f32(dst + 1 * stride + i, a0.val[1]);
-                vst1q_f32(dst + 2 * stride + i, a1.val[0]);
-                vst1q_f32(dst + 3 * stride + i, a1.val[1]);
-            }
-#endif
-            for (; i < count; ++i)
+            for (size_t i = 0; i < count; ++i)
             {
-                const float* ps = (const float*)src[i];
-                dst[0 * stride + i] = ps[0];
-                dst[1 * stride + i] = ps[1];
-                dst[2 * stride + i] = ps[2];
-                dst[3 * stride + i] = ps[3];
+                float header[4];
+                LoadHeader(src[i], header);
+                dst[0 * stride + i] = header[0];
+                dst[1 * stride + i] = header[1];
+                dst[2 * stride + i] = header[2];
+                dst[3 * stride + i] = header[3];
             }
         }