diff --git a/docs/2026.html b/docs/2026.html
index 3521babca5..39b8c90114 100644
--- a/docs/2026.html
+++ b/docs/2026.html
@@ -112,6 +112,7 @@ Improving
SVE2 optimizations of function GetMoments.
SVE2 optimizations of function GetObjectMoments.
SVE2 optimizations of function DescrIntCosineDistance.
+ SVE2 optimizations of function DescrIntCosineDistancesMxNa.
Renaming
diff --git a/src/Simd/SimdDescrInt.h b/src/Simd/SimdDescrInt.h
index 78ef0fe32e..43fd4c1eac 100644
--- a/src/Simd/SimdDescrInt.h
+++ b/src/Simd/SimdDescrInt.h
@@ -292,7 +292,7 @@ namespace Simd
Base::DescrInt::MacroCosineDistancesDirectPtr GetMacroCosineDistancesDirect(size_t depth);
Base::DescrInt::UnpackNormPtr GetUnpackNorm(bool transpose);
- Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth);
+ Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth, bool transpose);
Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t depth);
//-------------------------------------------------------------------------------------------------
diff --git a/src/Simd/SimdDescrIntCommon.h b/src/Simd/SimdDescrIntCommon.h
index 1144f13828..9a0770de94 100644
--- a/src/Simd/SimdDescrIntCommon.h
+++ b/src/Simd/SimdDescrIntCommon.h
@@ -721,5 +721,85 @@ namespace Simd
}
}
#endif
+
+#ifdef SIMD_SVE2_ENABLE
+ namespace Sve2
+ {
+ SIMD_ALIGNED(16) const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
+ SIMD_ALIGNED(16) const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
+ SIMD_ALIGNED(16) const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
+ SIMD_ALIGNED(16) const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
+ SIMD_ALIGNED(16) const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
+ SIMD_ALIGNED(16) const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };
+
+ SIMD_INLINE svuint8_t UnpackTbl(const uint8_t* tbl16, uint8_t bits)
+ {
+ const svbool_t all = svptrue_b8();
+ svuint8_t base = svld1rq_u8(all, tbl16);
+ svuint8_t index = svindex_u8(0, 1);
+ svuint8_t group = svlsr_n_u8_x(all, index, 4);
+ return svadd_u8_x(all, base, svmul_n_u8_x(all, group, bits));
+ }
+
+ SIMD_INLINE svuint16_t UnpackShr(const uint16_t* shr8)
+ {
+ return svld1rq_u16(svptrue_b16(), shr8);
+ }
+
+ template SIMD_INLINE svuint8_t UnpackTbl();
+ template SIMD_INLINE svuint16_t UnpackShr();
+
+ template<> SIMD_INLINE svuint8_t UnpackTbl<5>() { return UnpackTbl(C5_TBL, 5); }
+ template<> SIMD_INLINE svuint16_t UnpackShr<5>() { return UnpackShr(C5_SHR); }
+ template<> SIMD_INLINE svuint8_t UnpackTbl<6>() { return UnpackTbl(C6_TBL, 6); }
+ template<> SIMD_INLINE svuint16_t UnpackShr<6>() { return UnpackShr(C6_SHR); }
+ template<> SIMD_INLINE svuint8_t UnpackTbl<7>() { return UnpackTbl(C7_TBL, 7); }
+ template<> SIMD_INLINE svuint16_t UnpackShr<7>() { return UnpackShr(C7_SHR); }
+
+ SIMD_INLINE svuint16_t UnpackTo16(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr, uint16_t mask)
+ {
+ svuint8_t raw = svld1_u8(svwhilelt_b8((size_t)0, packed), src);
+ svuint16_t wide = svreinterpret_u16_u8(svtbl_u8(raw, tbl));
+ const svbool_t all = svptrue_b16();
+ return svand_n_u16_x(all, svlsr_u16_x(all, wide, shr), mask);
+ }
+
+ template SIMD_INLINE svuint8_t UnpackTo8(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr)
+ {
+ const size_t packedHalf = svcnth() * bits / 8;
+ const uint16_t mask = (uint16_t)((1 << bits) - 1);
+ size_t packedLo = packed < packedHalf ? packed : packedHalf;
+ size_t packedHi = packed - packedLo;
+ svuint16_t lo = UnpackTo16(src, packedLo, tbl, shr, mask);
+ svuint16_t hi = UnpackTo16(src + packedLo, packedHi, tbl, shr, mask);
+ return svuzp1_u8(svreinterpret_u8_u16(lo), svreinterpret_u8_u16(hi));
+ }
+
+ SIMD_INLINE void DecodeCosineDistance(const float* a, const float* b, size_t stride, uint32_t abSum, float* distance)
+ {
+ float bb[4] = { b[0], b[stride], b[2 * stride], b[3 * stride] };
+ float ab = (float)abSum * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
+ distance[0] = Simd::RestrictRange(1.0f - ab / (a[3] * bb[3]), 0.0f, 2.0f);
+ }
+
+ SIMD_INLINE void DecodeCosineDistances1x4(const uint8_t* a, const uint8_t* const* B,
+ svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
+ {
+ Base::DecodeCosineDistance(a, B[0], (float)svaddv_u32(svptrue_b32(), s0), distances + 0);
+ Base::DecodeCosineDistance(a, B[1], (float)svaddv_u32(svptrue_b32(), s1), distances + 1);
+ Base::DecodeCosineDistance(a, B[2], (float)svaddv_u32(svptrue_b32(), s2), distances + 2);
+ Base::DecodeCosineDistance(a, B[3], (float)svaddv_u32(svptrue_b32(), s3), distances + 3);
+ }
+
+ SIMD_INLINE void DecodeCosineDistances1x4(const float* a, const float* b, size_t stride,
+ svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
+ {
+ DecodeCosineDistance(a, b + 0, stride, (uint32_t)svaddv_u32(svptrue_b32(), s0), distances + 0);
+ DecodeCosineDistance(a, b + 1, stride, (uint32_t)svaddv_u32(svptrue_b32(), s1), distances + 1);
+ DecodeCosineDistance(a, b + 2, stride, (uint32_t)svaddv_u32(svptrue_b32(), s2), distances + 2);
+ DecodeCosineDistance(a, b + 3, stride, (uint32_t)svaddv_u32(svptrue_b32(), s3), distances + 3);
+ }
+ }
+#endif
}
#endif//__SimdDescrIntCommon_h__
diff --git a/src/Simd/SimdSve2DescrInt.cpp b/src/Simd/SimdSve2DescrInt.cpp
index da6821e7d1..14eae1e284 100644
--- a/src/Simd/SimdSve2DescrInt.cpp
+++ b/src/Simd/SimdSve2DescrInt.cpp
@@ -37,24 +37,24 @@ namespace Simd
: Base::DescrInt(size, depth)
#endif
{
- _encode32f = GetEncode32f(_depth);
- _encode16f = GetEncode16f(_depth);
- _decode32f = GetDecode32f(_depth);
- _decode16f = GetDecode16f(_depth);
+ _encode32f = Sve2::GetEncode32f(_depth);
+ _encode16f = Sve2::GetEncode16f(_depth);
+ _decode32f = Sve2::GetDecode32f(_depth);
+ _decode16f = Sve2::GetDecode16f(_depth);
- _cosineDistance = GetCosineDistance(_depth);
- _macroCosineDistancesDirect = GetMacroCosineDistancesDirect(_depth);
+ _cosineDistance = Sve2::GetCosineDistance(_depth);
+ _macroCosineDistancesDirect = Sve2::GetMacroCosineDistancesDirect(_depth);
_microMd = 4;
_microNd = 4;
- _unpackNormA = GetUnpackNorm(false);
- _unpackNormB = GetUnpackNorm(true);
- _unpackDataA = GetUnpackData(_depth);
- _unpackDataB = GetUnpackData(_depth);
- _macroCosineDistancesUnpack = GetMacroCosineDistancesUnpack(_depth);
+ _unpackNormA = Sve2::GetUnpackNorm(false);
+ _unpackNormB = Sve2::GetUnpackNorm(true);
+ _unpackDataA = Sve2::GetUnpackData(_depth, false);
+ _unpackDataB = Sve2::GetUnpackData(_depth, true);
+ _macroCosineDistancesUnpack = Sve2::GetMacroCosineDistancesUnpack(_depth);
_unpSize = _size;
_microMu = 4;
- _microNu = 1;
+ _microNu = 4;
}
//-------------------------------------------------------------------------------------------------
diff --git a/src/Simd/SimdSve2DescrIntCdd.cpp b/src/Simd/SimdSve2DescrIntCdd.cpp
index 300235ff11..40abed74bd 100644
--- a/src/Simd/SimdSve2DescrIntCdd.cpp
+++ b/src/Simd/SimdSve2DescrIntCdd.cpp
@@ -32,56 +32,6 @@ namespace Simd
#ifdef SIMD_SVE2_ENABLE
namespace Sve2
{
- const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
- const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
- const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
- const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
- const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
- const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };
-
- SIMD_INLINE svuint8_t UnpackTbl(const uint8_t* tbl16, uint8_t bits)
- {
- const svbool_t all = svptrue_b8();
- svuint8_t base = svld1rq_u8(all, tbl16);
- svuint8_t index = svindex_u8(0, 1);
- svuint8_t group = svlsr_n_u8_x(all, index, 4);
- return svadd_u8_x(all, base, svmul_n_u8_x(all, group, bits));
- }
-
- SIMD_INLINE svuint16_t UnpackShr(const uint16_t* shr8)
- {
- return svld1rq_u16(svptrue_b16(), shr8);
- }
-
- template SIMD_INLINE svuint8_t UnpackTbl();
- template SIMD_INLINE svuint16_t UnpackShr();
-
- template<> SIMD_INLINE svuint8_t UnpackTbl<5>() { return UnpackTbl(C5_TBL, 5); }
- template<> SIMD_INLINE svuint16_t UnpackShr<5>() { return UnpackShr(C5_SHR); }
- template<> SIMD_INLINE svuint8_t UnpackTbl<6>() { return UnpackTbl(C6_TBL, 6); }
- template<> SIMD_INLINE svuint16_t UnpackShr<6>() { return UnpackShr(C6_SHR); }
- template<> SIMD_INLINE svuint8_t UnpackTbl<7>() { return UnpackTbl(C7_TBL, 7); }
- template<> SIMD_INLINE svuint16_t UnpackShr<7>() { return UnpackShr(C7_SHR); }
-
- SIMD_INLINE svuint16_t UnpackTo16(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr, uint16_t mask)
- {
- svuint8_t raw = svld1_u8(svwhilelt_b8((size_t)0, packed), src);
- svuint16_t wide = svreinterpret_u16_u8(svtbl_u8(raw, tbl));
- const svbool_t all = svptrue_b16();
- return svand_n_u16_x(all, svlsr_u16_x(all, wide, shr), mask);
- }
-
- template SIMD_INLINE svuint8_t UnpackTo8(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr)
- {
- const size_t packedHalf = svcnth() * bits / 8;
- const uint16_t mask = (uint16_t)((1 << bits) - 1);
- size_t packedLo = packed < packedHalf ? packed : packedHalf;
- size_t packedHi = packed - packedLo;
- svuint16_t lo = UnpackTo16(src, packedLo, tbl, shr, mask);
- svuint16_t hi = UnpackTo16(src + packedLo, packedHi, tbl, shr, mask);
- return svuzp1_u8(svreinterpret_u8_u16(lo), svreinterpret_u8_u16(hi));
- }
-
template int32_t Correlation(const uint8_t* a, const uint8_t* b, size_t size)
{
assert(size % 8 == 0 && size >= 8);
@@ -111,13 +61,23 @@ namespace Simd
{
assert(size % 8 == 0 && size >= 8);
const size_t byteSize = size / 2;
- const svuint8_t zero = svdup_n_u8(0);
+ const size_t width = svcntb();
+ const size_t main = byteSize & ~(width - 1);
+ const svbool_t all = svptrue_b8();
svuint32_t sums = svdup_n_u32(0);
- for (size_t i = 0; i < byteSize; i += svcntb())
+ size_t i = 0;
+ for (; i < main; i += width)
+ {
+ svuint8_t _a = svld1_u8(all, a + i);
+ svuint8_t _b = svld1_u8(all, b + i);
+ sums = svdot_u32(sums, svand_n_u8_x(all, _a, 0x0F), svand_n_u8_x(all, _b, 0x0F));
+ sums = svdot_u32(sums, svlsr_n_u8_x(all, _a, 4), svlsr_n_u8_x(all, _b, 4));
+ }
+ if (i < byteSize)
{
svbool_t mask = svwhilelt_b8(i, byteSize);
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, a + i), zero);
- svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
+ svuint8_t _a = svld1_u8(mask, a + i);
+ svuint8_t _b = svld1_u8(mask, b + i);
sums = svdot_u32(sums, svand_n_u8_z(mask, _a, 0x0F), svand_n_u8_z(mask, _b, 0x0F));
sums = svdot_u32(sums, svlsr_n_u8_z(mask, _a, 4), svlsr_n_u8_z(mask, _b, 4));
}
@@ -127,14 +87,17 @@ namespace Simd
template<> int32_t Correlation<8>(const uint8_t* a, const uint8_t* b, size_t size)
{
assert(size % 8 == 0 && size >= 8);
- const svuint8_t zero = svdup_n_u8(0);
+ const size_t width = svcntb();
+ const size_t main = size & ~(width - 1);
+ const svbool_t all = svptrue_b8();
svuint32_t sums = svdup_n_u32(0);
- for (size_t i = 0; i < size; i += svcntb())
+ size_t i = 0;
+ for (; i < main; i += width)
+ sums = svdot_u32(sums, svld1_u8(all, a + i), svld1_u8(all, b + i));
+ if (i < size)
{
svbool_t mask = svwhilelt_b8(i, size);
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, a + i), zero);
- svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
- sums = svdot_u32(sums, _a, _b);
+ sums = svdot_u32(sums, svld1_u8(mask, a + i), svld1_u8(mask, b + i));
}
return (int32_t)svaddv_u32(svptrue_b32(), sums);
}
@@ -145,131 +108,338 @@ namespace Simd
Base::DecodeCosineDistance(a, b, abSum, distance);
}
- template SIMD_INLINE void MicroCosineDistancesDirectMx4(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
+ template struct DirectMx4
{
- const size_t valuesPerVec = svcntb();
- const size_t packedPerVec = valuesPerVec * bits / 8;
- const svuint8_t tbl = UnpackTbl();
- const svuint16_t shr = UnpackShr();
- svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
- svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
- svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
- svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
- size_t i = 0, o = 16;
- for (; i + valuesPerVec <= size; i += valuesPerVec, o += packedPerVec)
+ template static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
{
- svuint8_t b0 = UnpackTo8(B[0] + o, packedPerVec, tbl, shr);
- svuint8_t b1 = UnpackTo8(B[1] + o, packedPerVec, tbl, shr);
- svuint8_t b2 = UnpackTo8(B[2] + o, packedPerVec, tbl, shr);
- svuint8_t b3 = UnpackTo8(B[3] + o, packedPerVec, tbl, shr);
- if (M > 0)
- {
- svuint8_t a0 = UnpackTo8(A[0] + o, packedPerVec, tbl, shr);
- ab00 = svdot_u32(ab00, a0, b0);
- ab01 = svdot_u32(ab01, a0, b1);
- ab02 = svdot_u32(ab02, a0, b2);
- ab03 = svdot_u32(ab03, a0, b3);
- }
- if (M > 1)
- {
- svuint8_t a1 = UnpackTo8(A[1] + o, packedPerVec, tbl, shr);
- ab10 = svdot_u32(ab10, a1, b0);
- ab11 = svdot_u32(ab11, a1, b1);
- ab12 = svdot_u32(ab12, a1, b2);
- ab13 = svdot_u32(ab13, a1, b3);
- }
- if (M > 2)
+ const size_t valuesPerVec = svcntb();
+ const size_t packedPerVec = valuesPerVec * bits / 8;
+ const svuint8_t tbl = UnpackTbl();
+ const svuint16_t shr = UnpackShr();
+ svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+ svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+ svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+ svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+ size_t i = 0, o = 16;
+ for (; i + valuesPerVec <= size; i += valuesPerVec, o += packedPerVec)
{
- svuint8_t a2 = UnpackTo8(A[2] + o, packedPerVec, tbl, shr);
- ab20 = svdot_u32(ab20, a2, b0);
- ab21 = svdot_u32(ab21, a2, b1);
- ab22 = svdot_u32(ab22, a2, b2);
- ab23 = svdot_u32(ab23, a2, b3);
+ svuint8_t b0 = UnpackTo8(B[0] + o, packedPerVec, tbl, shr);
+ svuint8_t b1 = UnpackTo8(B[1] + o, packedPerVec, tbl, shr);
+ svuint8_t b2 = UnpackTo8(B[2] + o, packedPerVec, tbl, shr);
+ svuint8_t b3 = UnpackTo8(B[3] + o, packedPerVec, tbl, shr);
+ if (M > 0)
+ {
+ svuint8_t a0 = UnpackTo8(A[0] + o, packedPerVec, tbl, shr);
+ ab00 = svdot_u32(ab00, a0, b0);
+ ab01 = svdot_u32(ab01, a0, b1);
+ ab02 = svdot_u32(ab02, a0, b2);
+ ab03 = svdot_u32(ab03, a0, b3);
+ }
+ if (M > 1)
+ {
+ svuint8_t a1 = UnpackTo8(A[1] + o, packedPerVec, tbl, shr);
+ ab10 = svdot_u32(ab10, a1, b0);
+ ab11 = svdot_u32(ab11, a1, b1);
+ ab12 = svdot_u32(ab12, a1, b2);
+ ab13 = svdot_u32(ab13, a1, b3);
+ }
+ if (M > 2)
+ {
+ svuint8_t a2 = UnpackTo8(A[2] + o, packedPerVec, tbl, shr);
+ ab20 = svdot_u32(ab20, a2, b0);
+ ab21 = svdot_u32(ab21, a2, b1);
+ ab22 = svdot_u32(ab22, a2, b2);
+ ab23 = svdot_u32(ab23, a2, b3);
+ }
+ if (M > 3)
+ {
+ svuint8_t a3 = UnpackTo8(A[3] + o, packedPerVec, tbl, shr);
+ ab30 = svdot_u32(ab30, a3, b0);
+ ab31 = svdot_u32(ab31, a3, b1);
+ ab32 = svdot_u32(ab32, a3, b2);
+ ab33 = svdot_u32(ab33, a3, b3);
+ }
}
- if (M > 3)
+ if (i < size)
{
- svuint8_t a3 = UnpackTo8(A[3] + o, packedPerVec, tbl, shr);
- ab30 = svdot_u32(ab30, a3, b0);
- ab31 = svdot_u32(ab31, a3, b1);
- ab32 = svdot_u32(ab32, a3, b2);
- ab33 = svdot_u32(ab33, a3, b3);
+ size_t packedTail = (size - i) * bits / 8;
+ svuint8_t b0 = UnpackTo8(B[0] + o, packedTail, tbl, shr);
+ svuint8_t b1 = UnpackTo8(B[1] + o, packedTail, tbl, shr);
+ svuint8_t b2 = UnpackTo8(B[2] + o, packedTail, tbl, shr);
+ svuint8_t b3 = UnpackTo8(B[3] + o, packedTail, tbl, shr);
+ if (M > 0)
+ {
+ svuint8_t a0 = UnpackTo8(A[0] + o, packedTail, tbl, shr);
+ ab00 = svdot_u32(ab00, a0, b0);
+ ab01 = svdot_u32(ab01, a0, b1);
+ ab02 = svdot_u32(ab02, a0, b2);
+ ab03 = svdot_u32(ab03, a0, b3);
+ }
+ if (M > 1)
+ {
+ svuint8_t a1 = UnpackTo8(A[1] + o, packedTail, tbl, shr);
+ ab10 = svdot_u32(ab10, a1, b0);
+ ab11 = svdot_u32(ab11, a1, b1);
+ ab12 = svdot_u32(ab12, a1, b2);
+ ab13 = svdot_u32(ab13, a1, b3);
+ }
+ if (M > 2)
+ {
+ svuint8_t a2 = UnpackTo8(A[2] + o, packedTail, tbl, shr);
+ ab20 = svdot_u32(ab20, a2, b0);
+ ab21 = svdot_u32(ab21, a2, b1);
+ ab22 = svdot_u32(ab22, a2, b2);
+ ab23 = svdot_u32(ab23, a2, b3);
+ }
+ if (M > 3)
+ {
+ svuint8_t a3 = UnpackTo8(A[3] + o, packedTail, tbl, shr);
+ ab30 = svdot_u32(ab30, a3, b0);
+ ab31 = svdot_u32(ab31, a3, b1);
+ ab32 = svdot_u32(ab32, a3, b2);
+ ab33 = svdot_u32(ab33, a3, b3);
+ }
}
+ if (M > 0) DecodeCosineDistances1x4(A[0], B, ab00, ab01, ab02, ab03, distances + 0 * stride);
+ if (M > 1) DecodeCosineDistances1x4(A[1], B, ab10, ab11, ab12, ab13, distances + 1 * stride);
+ if (M > 2) DecodeCosineDistances1x4(A[2], B, ab20, ab21, ab22, ab23, distances + 2 * stride);
+ if (M > 3) DecodeCosineDistances1x4(A[3], B, ab30, ab31, ab32, ab33, distances + 3 * stride);
}
- if (i < size)
+ };
+
+ template<> struct DirectMx4<4>
+ {
+ template static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
{
- size_t packedTail = (size - i) * bits / 8;
- svuint8_t b0 = UnpackTo8(B[0] + o, packedTail, tbl, shr);
- svuint8_t b1 = UnpackTo8(B[1] + o, packedTail, tbl, shr);
- svuint8_t b2 = UnpackTo8(B[2] + o, packedTail, tbl, shr);
- svuint8_t b3 = UnpackTo8(B[3] + o, packedTail, tbl, shr);
- if (M > 0)
+ const size_t width = svcntb();
+ const size_t byteSize = size / 2;
+ const size_t main = byteSize & ~(width - 1);
+ const svbool_t all = svptrue_b8();
+ svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+ svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+ svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+ svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+ const uint8_t* a0 = A[0] + 16;
+ const uint8_t* a1 = M > 1 ? A[1] + 16 : a0;
+ const uint8_t* a2 = M > 2 ? A[2] + 16 : a0;
+ const uint8_t* a3 = M > 3 ? A[3] + 16 : a0;
+ const uint8_t* b0 = B[0] + 16;
+ const uint8_t* b1 = B[1] + 16;
+ const uint8_t* b2 = B[2] + 16;
+ const uint8_t* b3 = B[3] + 16;
+ size_t i = 0;
+ for (; i < main; i += width)
{
- svuint8_t a0 = UnpackTo8(A[0] + o, packedTail, tbl, shr);
- ab00 = svdot_u32(ab00, a0, b0);
- ab01 = svdot_u32(ab01, a0, b1);
- ab02 = svdot_u32(ab02, a0, b2);
- ab03 = svdot_u32(ab03, a0, b3);
+ svuint8_t B0 = svld1_u8(all, b0 + i);
+ svuint8_t B1 = svld1_u8(all, b1 + i);
+ svuint8_t B2 = svld1_u8(all, b2 + i);
+ svuint8_t B3 = svld1_u8(all, b3 + i);
+ svuint8_t B0l = svand_n_u8_x(all, B0, 0x0F), B0h = svlsr_n_u8_x(all, B0, 4);
+ svuint8_t B1l = svand_n_u8_x(all, B1, 0x0F), B1h = svlsr_n_u8_x(all, B1, 4);
+ svuint8_t B2l = svand_n_u8_x(all, B2, 0x0F), B2h = svlsr_n_u8_x(all, B2, 4);
+ svuint8_t B3l = svand_n_u8_x(all, B3, 0x0F), B3h = svlsr_n_u8_x(all, B3, 4);
+ if (M > 0)
+ {
+ svuint8_t A0 = svld1_u8(all, a0 + i);
+ svuint8_t A0l = svand_n_u8_x(all, A0, 0x0F), A0h = svlsr_n_u8_x(all, A0, 4);
+ ab00 = svdot_u32(svdot_u32(ab00, A0l, B0l), A0h, B0h);
+ ab01 = svdot_u32(svdot_u32(ab01, A0l, B1l), A0h, B1h);
+ ab02 = svdot_u32(svdot_u32(ab02, A0l, B2l), A0h, B2h);
+ ab03 = svdot_u32(svdot_u32(ab03, A0l, B3l), A0h, B3h);
+ }
+ if (M > 1)
+ {
+ svuint8_t A1 = svld1_u8(all, a1 + i);
+ svuint8_t A1l = svand_n_u8_x(all, A1, 0x0F), A1h = svlsr_n_u8_x(all, A1, 4);
+ ab10 = svdot_u32(svdot_u32(ab10, A1l, B0l), A1h, B0h);
+ ab11 = svdot_u32(svdot_u32(ab11, A1l, B1l), A1h, B1h);
+ ab12 = svdot_u32(svdot_u32(ab12, A1l, B2l), A1h, B2h);
+ ab13 = svdot_u32(svdot_u32(ab13, A1l, B3l), A1h, B3h);
+ }
+ if (M > 2)
+ {
+ svuint8_t A2 = svld1_u8(all, a2 + i);
+ svuint8_t A2l = svand_n_u8_x(all, A2, 0x0F), A2h = svlsr_n_u8_x(all, A2, 4);
+ ab20 = svdot_u32(svdot_u32(ab20, A2l, B0l), A2h, B0h);
+ ab21 = svdot_u32(svdot_u32(ab21, A2l, B1l), A2h, B1h);
+ ab22 = svdot_u32(svdot_u32(ab22, A2l, B2l), A2h, B2h);
+ ab23 = svdot_u32(svdot_u32(ab23, A2l, B3l), A2h, B3h);
+ }
+ if (M > 3)
+ {
+ svuint8_t A3 = svld1_u8(all, a3 + i);
+ svuint8_t A3l = svand_n_u8_x(all, A3, 0x0F), A3h = svlsr_n_u8_x(all, A3, 4);
+ ab30 = svdot_u32(svdot_u32(ab30, A3l, B0l), A3h, B0h);
+ ab31 = svdot_u32(svdot_u32(ab31, A3l, B1l), A3h, B1h);
+ ab32 = svdot_u32(svdot_u32(ab32, A3l, B2l), A3h, B2h);
+ ab33 = svdot_u32(svdot_u32(ab33, A3l, B3l), A3h, B3h);
+ }
}
- if (M > 1)
+ if (i < byteSize)
{
- svuint8_t a1 = UnpackTo8(A[1] + o, packedTail, tbl, shr);
- ab10 = svdot_u32(ab10, a1, b0);
- ab11 = svdot_u32(ab11, a1, b1);
- ab12 = svdot_u32(ab12, a1, b2);
- ab13 = svdot_u32(ab13, a1, b3);
+ svbool_t mask = svwhilelt_b8(i, byteSize);
+ svuint8_t B0 = svld1_u8(mask, b0 + i);
+ svuint8_t B1 = svld1_u8(mask, b1 + i);
+ svuint8_t B2 = svld1_u8(mask, b2 + i);
+ svuint8_t B3 = svld1_u8(mask, b3 + i);
+ svuint8_t B0l = svand_n_u8_z(mask, B0, 0x0F), B0h = svlsr_n_u8_z(mask, B0, 4);
+ svuint8_t B1l = svand_n_u8_z(mask, B1, 0x0F), B1h = svlsr_n_u8_z(mask, B1, 4);
+ svuint8_t B2l = svand_n_u8_z(mask, B2, 0x0F), B2h = svlsr_n_u8_z(mask, B2, 4);
+ svuint8_t B3l = svand_n_u8_z(mask, B3, 0x0F), B3h = svlsr_n_u8_z(mask, B3, 4);
+ if (M > 0)
+ {
+ svuint8_t A0 = svld1_u8(mask, a0 + i);
+ svuint8_t A0l = svand_n_u8_z(mask, A0, 0x0F), A0h = svlsr_n_u8_z(mask, A0, 4);
+ ab00 = svdot_u32(svdot_u32(ab00, A0l, B0l), A0h, B0h);
+ ab01 = svdot_u32(svdot_u32(ab01, A0l, B1l), A0h, B1h);
+ ab02 = svdot_u32(svdot_u32(ab02, A0l, B2l), A0h, B2h);
+ ab03 = svdot_u32(svdot_u32(ab03, A0l, B3l), A0h, B3h);
+ }
+ if (M > 1)
+ {
+ svuint8_t A1 = svld1_u8(mask, a1 + i);
+ svuint8_t A1l = svand_n_u8_z(mask, A1, 0x0F), A1h = svlsr_n_u8_z(mask, A1, 4);
+ ab10 = svdot_u32(svdot_u32(ab10, A1l, B0l), A1h, B0h);
+ ab11 = svdot_u32(svdot_u32(ab11, A1l, B1l), A1h, B1h);
+ ab12 = svdot_u32(svdot_u32(ab12, A1l, B2l), A1h, B2h);
+ ab13 = svdot_u32(svdot_u32(ab13, A1l, B3l), A1h, B3h);
+ }
+ if (M > 2)
+ {
+ svuint8_t A2 = svld1_u8(mask, a2 + i);
+ svuint8_t A2l = svand_n_u8_z(mask, A2, 0x0F), A2h = svlsr_n_u8_z(mask, A2, 4);
+ ab20 = svdot_u32(svdot_u32(ab20, A2l, B0l), A2h, B0h);
+ ab21 = svdot_u32(svdot_u32(ab21, A2l, B1l), A2h, B1h);
+ ab22 = svdot_u32(svdot_u32(ab22, A2l, B2l), A2h, B2h);
+ ab23 = svdot_u32(svdot_u32(ab23, A2l, B3l), A2h, B3h);
+ }
+ if (M > 3)
+ {
+ svuint8_t A3 = svld1_u8(mask, a3 + i);
+ svuint8_t A3l = svand_n_u8_z(mask, A3, 0x0F), A3h = svlsr_n_u8_z(mask, A3, 4);
+ ab30 = svdot_u32(svdot_u32(ab30, A3l, B0l), A3h, B0h);
+ ab31 = svdot_u32(svdot_u32(ab31, A3l, B1l), A3h, B1h);
+ ab32 = svdot_u32(svdot_u32(ab32, A3l, B2l), A3h, B2h);
+ ab33 = svdot_u32(svdot_u32(ab33, A3l, B3l), A3h, B3h);
+ }
}
- if (M > 2)
+ if (M > 0) DecodeCosineDistances1x4(A[0], B, ab00, ab01, ab02, ab03, distances + 0 * stride);
+ if (M > 1) DecodeCosineDistances1x4(A[1], B, ab10, ab11, ab12, ab13, distances + 1 * stride);
+ if (M > 2) DecodeCosineDistances1x4(A[2], B, ab20, ab21, ab22, ab23, distances + 2 * stride);
+ if (M > 3) DecodeCosineDistances1x4(A[3], B, ab30, ab31, ab32, ab33, distances + 3 * stride);
+ }
+ };
+
+ template<> struct DirectMx4<8>
+ {
+ template static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
+ {
+ const size_t width = svcntb();
+ const size_t main = size & ~(width - 1);
+ const svbool_t all = svptrue_b8();
+ svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+ svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+ svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+ svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+ const uint8_t* a0 = A[0] + 16;
+ const uint8_t* a1 = M > 1 ? A[1] + 16 : a0;
+ const uint8_t* a2 = M > 2 ? A[2] + 16 : a0;
+ const uint8_t* a3 = M > 3 ? A[3] + 16 : a0;
+ const uint8_t* b0 = B[0] + 16;
+ const uint8_t* b1 = B[1] + 16;
+ const uint8_t* b2 = B[2] + 16;
+ const uint8_t* b3 = B[3] + 16;
+ size_t i = 0;
+ for (; i < main; i += width)
{
- svuint8_t a2 = UnpackTo8(A[2] + o, packedTail, tbl, shr);
- ab20 = svdot_u32(ab20, a2, b0);
- ab21 = svdot_u32(ab21, a2, b1);
- ab22 = svdot_u32(ab22, a2, b2);
- ab23 = svdot_u32(ab23, a2, b3);
+ svuint8_t B0 = svld1_u8(all, b0 + i);
+ svuint8_t B1 = svld1_u8(all, b1 + i);
+ svuint8_t B2 = svld1_u8(all, b2 + i);
+ svuint8_t B3 = svld1_u8(all, b3 + i);
+ if (M > 0)
+ {
+ svuint8_t A0 = svld1_u8(all, a0 + i);
+ ab00 = svdot_u32(ab00, A0, B0);
+ ab01 = svdot_u32(ab01, A0, B1);
+ ab02 = svdot_u32(ab02, A0, B2);
+ ab03 = svdot_u32(ab03, A0, B3);
+ }
+ if (M > 1)
+ {
+ svuint8_t A1 = svld1_u8(all, a1 + i);
+ ab10 = svdot_u32(ab10, A1, B0);
+ ab11 = svdot_u32(ab11, A1, B1);
+ ab12 = svdot_u32(ab12, A1, B2);
+ ab13 = svdot_u32(ab13, A1, B3);
+ }
+ if (M > 2)
+ {
+ svuint8_t A2 = svld1_u8(all, a2 + i);
+ ab20 = svdot_u32(ab20, A2, B0);
+ ab21 = svdot_u32(ab21, A2, B1);
+ ab22 = svdot_u32(ab22, A2, B2);
+ ab23 = svdot_u32(ab23, A2, B3);
+ }
+ if (M > 3)
+ {
+ svuint8_t A3 = svld1_u8(all, a3 + i);
+ ab30 = svdot_u32(ab30, A3, B0);
+ ab31 = svdot_u32(ab31, A3, B1);
+ ab32 = svdot_u32(ab32, A3, B2);
+ ab33 = svdot_u32(ab33, A3, B3);
+ }
}
- if (M > 3)
+ if (i < size)
{
- svuint8_t a3 = UnpackTo8(A[3] + o, packedTail, tbl, shr);
- ab30 = svdot_u32(ab30, a3, b0);
- ab31 = svdot_u32(ab31, a3, b1);
- ab32 = svdot_u32(ab32, a3, b2);
- ab33 = svdot_u32(ab33, a3, b3);
+ svbool_t mask = svwhilelt_b8(i, size);
+ svuint8_t B0 = svld1_u8(mask, b0 + i);
+ svuint8_t B1 = svld1_u8(mask, b1 + i);
+ svuint8_t B2 = svld1_u8(mask, b2 + i);
+ svuint8_t B3 = svld1_u8(mask, b3 + i);
+ if (M > 0)
+ {
+ svuint8_t A0 = svld1_u8(mask, a0 + i);
+ ab00 = svdot_u32(ab00, A0, B0);
+ ab01 = svdot_u32(ab01, A0, B1);
+ ab02 = svdot_u32(ab02, A0, B2);
+ ab03 = svdot_u32(ab03, A0, B3);
+ }
+ if (M > 1)
+ {
+ svuint8_t A1 = svld1_u8(mask, a1 + i);
+ ab10 = svdot_u32(ab10, A1, B0);
+ ab11 = svdot_u32(ab11, A1, B1);
+ ab12 = svdot_u32(ab12, A1, B2);
+ ab13 = svdot_u32(ab13, A1, B3);
+ }
+ if (M > 2)
+ {
+ svuint8_t A2 = svld1_u8(mask, a2 + i);
+ ab20 = svdot_u32(ab20, A2, B0);
+ ab21 = svdot_u32(ab21, A2, B1);
+ ab22 = svdot_u32(ab22, A2, B2);
+ ab23 = svdot_u32(ab23, A2, B3);
+ }
+ if (M > 3)
+ {
+ svuint8_t A3 = svld1_u8(mask, a3 + i);
+ ab30 = svdot_u32(ab30, A3, B0);
+ ab31 = svdot_u32(ab31, A3, B1);
+ ab32 = svdot_u32(ab32, A3, B2);
+ ab33 = svdot_u32(ab33, A3, B3);
+ }
}
+ if (M > 0) DecodeCosineDistances1x4(A[0], B, ab00, ab01, ab02, ab03, distances + 0 * stride);
+ if (M > 1) DecodeCosineDistances1x4(A[1], B, ab10, ab11, ab12, ab13, distances + 1 * stride);
+ if (M > 2) DecodeCosineDistances1x4(A[2], B, ab20, ab21, ab22, ab23, distances + 2 * stride);
+ if (M > 3) DecodeCosineDistances1x4(A[3], B, ab30, ab31, ab32, ab33, distances + 3 * stride);
}
- if (M > 0)
- {
- Base::DecodeCosineDistance(A[0], B[0], (float)svaddv_u32(svptrue_b32(), ab00), distances + 0 * stride + 0);
- Base::DecodeCosineDistance(A[0], B[1], (float)svaddv_u32(svptrue_b32(), ab01), distances + 0 * stride + 1);
- Base::DecodeCosineDistance(A[0], B[2], (float)svaddv_u32(svptrue_b32(), ab02), distances + 0 * stride + 2);
- Base::DecodeCosineDistance(A[0], B[3], (float)svaddv_u32(svptrue_b32(), ab03), distances + 0 * stride + 3);
- }
- if (M > 1)
- {
- Base::DecodeCosineDistance(A[1], B[0], (float)svaddv_u32(svptrue_b32(), ab10), distances + 1 * stride + 0);
- Base::DecodeCosineDistance(A[1], B[1], (float)svaddv_u32(svptrue_b32(), ab11), distances + 1 * stride + 1);
- Base::DecodeCosineDistance(A[1], B[2], (float)svaddv_u32(svptrue_b32(), ab12), distances + 1 * stride + 2);
- Base::DecodeCosineDistance(A[1], B[3], (float)svaddv_u32(svptrue_b32(), ab13), distances + 1 * stride + 3);
- }
- if (M > 2)
- {
- Base::DecodeCosineDistance(A[2], B[0], (float)svaddv_u32(svptrue_b32(), ab20), distances + 2 * stride + 0);
- Base::DecodeCosineDistance(A[2], B[1], (float)svaddv_u32(svptrue_b32(), ab21), distances + 2 * stride + 1);
- Base::DecodeCosineDistance(A[2], B[2], (float)svaddv_u32(svptrue_b32(), ab22), distances + 2 * stride + 2);
- Base::DecodeCosineDistance(A[2], B[3], (float)svaddv_u32(svptrue_b32(), ab23), distances + 2 * stride + 3);
- }
- if (M > 3)
- {
- Base::DecodeCosineDistance(A[3], B[0], (float)svaddv_u32(svptrue_b32(), ab30), distances + 3 * stride + 0);
- Base::DecodeCosineDistance(A[3], B[1], (float)svaddv_u32(svptrue_b32(), ab31), distances + 3 * stride + 1);
- Base::DecodeCosineDistance(A[3], B[2], (float)svaddv_u32(svptrue_b32(), ab32), distances + 3 * stride + 2);
- Base::DecodeCosineDistance(A[3], B[3], (float)svaddv_u32(svptrue_b32(), ab33), distances + 3 * stride + 3);
- }
- }
+ };
- template struct CorrelationsMx1
+ template struct DirectMx1
{
template static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* B, size_t size, uint32_t* ab)
{
- assert(size % 8 == 0 && size >= 8);
const size_t valuesPerVec = svcntb();
const size_t packedPerVec = valuesPerVec * bits / 8;
const svuint8_t tbl = UnpackTbl();
@@ -280,27 +450,19 @@ namespace Simd
for (; i + valuesPerVec <= size; i += valuesPerVec, packed += packedPerVec)
{
svuint8_t _b = UnpackTo8(b + packed, packedPerVec, tbl, shr);
- if (M > 0)
- sums0 = svdot_u32(sums0, UnpackTo8(A[0] + 16 + packed, packedPerVec, tbl, shr), _b);
- if (M > 1)
- sums1 = svdot_u32(sums1, UnpackTo8(A[1] + 16 + packed, packedPerVec, tbl, shr), _b);
- if (M > 2)
- sums2 = svdot_u32(sums2, UnpackTo8(A[2] + 16 + packed, packedPerVec, tbl, shr), _b);
- if (M > 3)
- sums3 = svdot_u32(sums3, UnpackTo8(A[3] + 16 + packed, packedPerVec, tbl, shr), _b);
+ if (M > 0) sums0 = svdot_u32(sums0, UnpackTo8(A[0] + 16 + packed, packedPerVec, tbl, shr), _b);
+ if (M > 1) sums1 = svdot_u32(sums1, UnpackTo8(A[1] + 16 + packed, packedPerVec, tbl, shr), _b);
+ if (M > 2) sums2 = svdot_u32(sums2, UnpackTo8(A[2] + 16 + packed, packedPerVec, tbl, shr), _b);
+ if (M > 3) sums3 = svdot_u32(sums3, UnpackTo8(A[3] + 16 + packed, packedPerVec, tbl, shr), _b);
}
if (i < size)
{
size_t packedTail = (size - i) * bits / 8;
svuint8_t _b = UnpackTo8(b + packed, packedTail, tbl, shr);
- if (M > 0)
- sums0 = svdot_u32(sums0, UnpackTo8(A[0] + 16 + packed, packedTail, tbl, shr), _b);
- if (M > 1)
- sums1 = svdot_u32(sums1, UnpackTo8(A[1] + 16 + packed, packedTail, tbl, shr), _b);
- if (M > 2)
- sums2 = svdot_u32(sums2, UnpackTo8(A[2] + 16 + packed, packedTail, tbl, shr), _b);
- if (M > 3)
- sums3 = svdot_u32(sums3, UnpackTo8(A[3] + 16 + packed, packedTail, tbl, shr), _b);
+ if (M > 0) sums0 = svdot_u32(sums0, UnpackTo8(A[0] + 16 + packed, packedTail, tbl, shr), _b);
+ if (M > 1) sums1 = svdot_u32(sums1, UnpackTo8(A[1] + 16 + packed, packedTail, tbl, shr), _b);
+ if (M > 2) sums2 = svdot_u32(sums2, UnpackTo8(A[2] + 16 + packed, packedTail, tbl, shr), _b);
+ if (M > 3) sums3 = svdot_u32(sums3, UnpackTo8(A[3] + 16 + packed, packedTail, tbl, shr), _b);
}
if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
if (M > 1) ab[1] = (uint32_t)svaddv_u32(svptrue_b32(), sums1);
@@ -309,44 +471,68 @@ namespace Simd
}
};
- template<> struct CorrelationsMx1<4>
+ template<> struct DirectMx1<4>
{
template static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* B, size_t size, uint32_t* ab)
{
- assert(size % 8 == 0 && size >= 8);
+ const size_t width = svcntb();
const size_t byteSize = size / 2;
- const svuint8_t zero = svdup_n_u8(0);
+ const size_t main = byteSize & ~(width - 1);
+ const svbool_t all = svptrue_b8();
svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
const uint8_t* b = B + 16;
- for (size_t i = 0; i < byteSize; i += svcntb())
+ size_t i = 0;
+ for (; i < main; i += width)
+ {
+ svuint8_t _b = svld1_u8(all, b + i);
+ svuint8_t b0 = svand_n_u8_x(all, _b, 0x0F);
+ svuint8_t b1 = svlsr_n_u8_x(all, _b, 4);
+ if (M > 0)
+ {
+ svuint8_t _a = svld1_u8(all, A[0] + 16 + i);
+ sums0 = svdot_u32(svdot_u32(sums0, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+ }
+ if (M > 1)
+ {
+ svuint8_t _a = svld1_u8(all, A[1] + 16 + i);
+ sums1 = svdot_u32(svdot_u32(sums1, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+ }
+ if (M > 2)
+ {
+ svuint8_t _a = svld1_u8(all, A[2] + 16 + i);
+ sums2 = svdot_u32(svdot_u32(sums2, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+ }
+ if (M > 3)
+ {
+ svuint8_t _a = svld1_u8(all, A[3] + 16 + i);
+ sums3 = svdot_u32(svdot_u32(sums3, svand_n_u8_x(all, _a, 0x0F), b0), svlsr_n_u8_x(all, _a, 4), b1);
+ }
+ }
+ if (i < byteSize)
{
svbool_t mask = svwhilelt_b8(i, byteSize);
- svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
+ svuint8_t _b = svld1_u8(mask, b + i);
svuint8_t b0 = svand_n_u8_z(mask, _b, 0x0F);
svuint8_t b1 = svlsr_n_u8_z(mask, _b, 4);
if (M > 0)
{
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[0] + 16 + i), zero);
- sums0 = svdot_u32(sums0, svand_n_u8_z(mask, _a, 0x0F), b0);
- sums0 = svdot_u32(sums0, svlsr_n_u8_z(mask, _a, 4), b1);
+ svuint8_t _a = svld1_u8(mask, A[0] + 16 + i);
+ sums0 = svdot_u32(svdot_u32(sums0, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
}
if (M > 1)
{
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[1] + 16 + i), zero);
- sums1 = svdot_u32(sums1, svand_n_u8_z(mask, _a, 0x0F), b0);
- sums1 = svdot_u32(sums1, svlsr_n_u8_z(mask, _a, 4), b1);
+ svuint8_t _a = svld1_u8(mask, A[1] + 16 + i);
+ sums1 = svdot_u32(svdot_u32(sums1, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
}
if (M > 2)
{
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[2] + 16 + i), zero);
- sums2 = svdot_u32(sums2, svand_n_u8_z(mask, _a, 0x0F), b0);
- sums2 = svdot_u32(sums2, svlsr_n_u8_z(mask, _a, 4), b1);
+ svuint8_t _a = svld1_u8(mask, A[2] + 16 + i);
+ sums2 = svdot_u32(svdot_u32(sums2, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
}
if (M > 3)
{
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[3] + 16 + i), zero);
- sums3 = svdot_u32(sums3, svand_n_u8_z(mask, _a, 0x0F), b0);
- sums3 = svdot_u32(sums3, svlsr_n_u8_z(mask, _a, 4), b1);
+ svuint8_t _a = svld1_u8(mask, A[3] + 16 + i);
+ sums3 = svdot_u32(svdot_u32(sums3, svand_n_u8_z(mask, _a, 0x0F), b0), svlsr_n_u8_z(mask, _a, 4), b1);
}
}
if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
@@ -356,38 +542,32 @@ namespace Simd
}
};
- template<> struct CorrelationsMx1<8>
+ template<> struct DirectMx1<8>
{
template static SIMD_INLINE void Run(const uint8_t* const* A, const uint8_t* B, size_t size, uint32_t* ab)
{
- assert(size % 8 == 0 && size >= 8);
- const svuint8_t zero = svdup_n_u8(0);
+ const size_t width = svcntb();
+ const size_t main = size & ~(width - 1);
+ const svbool_t all = svptrue_b8();
svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
const uint8_t* b = B + 16;
- for (size_t i = 0; i < size; i += svcntb())
+ size_t i = 0;
+ for (; i < main; i += width)
+ {
+ svuint8_t _b = svld1_u8(all, b + i);
+ if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(all, A[0] + 16 + i), _b);
+ if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(all, A[1] + 16 + i), _b);
+ if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(all, A[2] + 16 + i), _b);
+ if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(all, A[3] + 16 + i), _b);
+ }
+ if (i < size)
{
svbool_t mask = svwhilelt_b8(i, size);
- svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
- if (M > 0)
- {
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[0] + 16 + i), zero);
- sums0 = svdot_u32(sums0, _a, _b);
- }
- if (M > 1)
- {
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[1] + 16 + i), zero);
- sums1 = svdot_u32(sums1, _a, _b);
- }
- if (M > 2)
- {
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[2] + 16 + i), zero);
- sums2 = svdot_u32(sums2, _a, _b);
- }
- if (M > 3)
- {
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, A[3] + 16 + i), zero);
- sums3 = svdot_u32(sums3, _a, _b);
- }
+ svuint8_t _b = svld1_u8(mask, b + i);
+ if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(mask, A[0] + 16 + i), _b);
+ if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(mask, A[1] + 16 + i), _b);
+ if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(mask, A[2] + 16 + i), _b);
+ if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(mask, A[3] + 16 + i), _b);
}
if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
if (M > 1) ab[1] = (uint32_t)svaddv_u32(svptrue_b32(), sums1);
@@ -399,36 +579,13 @@ namespace Simd
template SIMD_INLINE void MicroCosineDistancesDirectMx1(const uint8_t* const* A, const uint8_t* B, size_t size, float* distances, size_t stride)
{
uint32_t ab[4];
- CorrelationsMx1::template Run(A, B, size, ab);
+ DirectMx1::template Run(A, B, size, ab);
if (M > 0) Base::DecodeCosineDistance(A[0], B, (float)ab[0], distances + 0 * stride);
if (M > 1) Base::DecodeCosineDistance(A[1], B, (float)ab[1], distances + 1 * stride);
if (M > 2) Base::DecodeCosineDistance(A[2], B, (float)ab[2], distances + 2 * stride);
if (M > 3) Base::DecodeCosineDistance(A[3], B, (float)ab[3], distances + 3 * stride);
}
- template void MacroCosineDistancesDirectMx1(size_t M, size_t N, const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
- {
- size_t M4 = AlignLoAny(M, 4), i = 0;
- for (; i < M4; i += 4)
- {
- for (size_t j = 0; j < N; ++j)
- MicroCosineDistancesDirectMx1(A + i, B[j], size, distances + j, stride);
- distances += 4 * stride;
- }
- if (i < M)
- {
- for (size_t j = 0; j < N; ++j)
- {
- switch (M - i)
- {
- case 1: MicroCosineDistancesDirectMx1(A + i, B[j], size, distances + j, stride); break;
- case 2: MicroCosineDistancesDirectMx1(A + i, B[j], size, distances + j, stride); break;
- case 3: MicroCosineDistancesDirectMx1(A + i, B[j], size, distances + j, stride); break;
- }
- }
- }
- }
-
template void MacroCosineDistancesDirect(size_t M, size_t N, const uint8_t* const* A, const uint8_t* const* B, size_t size, float* distances, size_t stride)
{
size_t M4 = AlignLoAny(M, 4), N4 = AlignLo(N, 4), i = 0;
@@ -436,7 +593,7 @@ namespace Simd
{
size_t j = 0;
for (; j < N4; j += 4)
- MicroCosineDistancesDirectMx4(A + i, B + j, size, distances + j, stride);
+ DirectMx4::template Run<4>(A + i, B + j, size, distances + j, stride);
for (; j < N; ++j)
MicroCosineDistancesDirectMx1(A + i, B[j], size, distances + j, stride);
distances += 4 * stride;
@@ -448,11 +605,11 @@ namespace Simd
for (; j < N4; j += 4)
{
if (m == 1)
- MicroCosineDistancesDirectMx4(A + i, B + j, size, distances + j, stride);
+ DirectMx4::template Run<1>(A + i, B + j, size, distances + j, stride);
else if (m == 2)
- MicroCosineDistancesDirectMx4(A + i, B + j, size, distances + j, stride);
+ DirectMx4::template Run<2>(A + i, B + j, size, distances + j, stride);
else
- MicroCosineDistancesDirectMx4(A + i, B + j, size, distances + j, stride);
+ DirectMx4::template Run<3>(A + i, B + j, size, distances + j, stride);
}
for (; j < N; ++j)
{
@@ -475,7 +632,11 @@ namespace Simd
case 6: return CosineDistance<6>;
case 7: return CosineDistance<7>;
case 8: return CosineDistance<8>;
+#ifdef SIMD_NEON_ENABLE
default: return Neon::GetCosineDistance(depth);
+#else
+ default: return Base::GetCosineDistance(depth);
+#endif
}
}
@@ -483,11 +644,11 @@ namespace Simd
{
switch (depth)
{
- case 4: return MacroCosineDistancesDirectMx1<4>;
+ case 4: return MacroCosineDistancesDirect<4>;
case 5: return MacroCosineDistancesDirect<5>;
case 6: return MacroCosineDistancesDirect<6>;
case 7: return MacroCosineDistancesDirect<7>;
- case 8: return MacroCosineDistancesDirectMx1<8>;
+ case 8: return MacroCosineDistancesDirect<8>;
default: return NULL;
}
}
diff --git a/src/Simd/SimdSve2DescrIntCdu.cpp b/src/Simd/SimdSve2DescrIntCdu.cpp
index 7579bc49be..45e16e7575 100644
--- a/src/Simd/SimdSve2DescrIntCdu.cpp
+++ b/src/Simd/SimdSve2DescrIntCdu.cpp
@@ -32,73 +32,105 @@ namespace Simd
#ifdef SIMD_SVE2_ENABLE
namespace Sve2
{
- static void UnpackNorm(size_t count, const uint8_t* const* src, float* dst, size_t stride)
+ SIMD_INLINE void LoadHeader(const uint8_t* src, float* dst)
+ {
+ memcpy(dst, src, 16);
+ }
+
+ static void UnpackNormA(size_t count, const uint8_t* const* src, float* dst, size_t)
+ {
+ for (size_t i = 0; i < count; ++i, dst += 4)
+ LoadHeader(src[i], dst);
+ }
+
+ static void UnpackNormB(size_t count, const uint8_t* const* src, float* dst, size_t stride)
{
for (size_t i = 0; i < count; ++i)
{
- const float* ps = (const float*)src[i];
- float* pd = dst + i * 4;
- pd[0] = ps[0];
- pd[1] = ps[1];
- pd[2] = ps[2];
- pd[3] = ps[3];
+ float header[4];
+ LoadHeader(src[i], header);
+ dst[0 * stride + i] = header[0];
+ dst[1 * stride + i] = header[1];
+ dst[2 * stride + i] = header[2];
+ dst[3 * stride + i] = header[3];
}
}
Base::DescrInt::UnpackNormPtr GetUnpackNorm(bool transpose)
{
- return UnpackNorm;
+ return transpose ? UnpackNormB : UnpackNormA;
}
//-------------------------------------------------------------------------------------------------
- template SIMD_INLINE uint64_t LoadBitsBlock(const uint8_t* src)
+ template void UnpackData(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t)
{
- uint64_t value = 0;
- for (size_t i = 0; i < bits; ++i)
- value |= uint64_t(src[i]) << (8 * i);
- return value;
+ const size_t valuesPerVec = svcntb();
+ const size_t packedPerVec = valuesPerVec * bits / 8;
+ const svuint8_t tbl = UnpackTbl();
+ const svuint16_t shr = UnpackShr();
+ const svbool_t all = svptrue_b8();
+ for (size_t i = 0; i < count; ++i)
+ {
+ const uint8_t* ps = src[i] + 16;
+ uint8_t* pd = dst + i * size;
+ size_t j = 0, packed = 0;
+ for (; j + valuesPerVec <= size; j += valuesPerVec, packed += packedPerVec)
+ svst1_u8(all, pd + j, UnpackTo8(ps + packed, packedPerVec, tbl, shr));
+ if (j < size)
+ {
+ size_t packedTail = (size - j) * bits / 8;
+ svst1_u8(svwhilelt_b8(j, size), pd + j, UnpackTo8(ps + packed, packedTail, tbl, shr));
+ }
+ }
}
- template void UnpackData(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t stride)
+ template<> void UnpackData<4>(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t)
{
- const uint64_t mask = (uint64_t(1) << bits) - 1;
+ const size_t width = svcntb();
+ const size_t byteSize = size / 2;
+ const svbool_t all = svptrue_b8();
for (size_t i = 0; i < count; ++i)
{
const uint8_t* ps = src[i] + 16;
uint8_t* pd = dst + i * size;
- for (size_t j = 0; j < size; j += 8, ps += bits, pd += 8)
+ size_t j = 0;
+ for (; j + width <= byteSize; j += width, ps += width, pd += 2 * width)
+ {
+ svuint8_t value = svld1_u8(all, ps);
+ svuint8_t lo = svand_n_u8_x(all, value, 0x0F);
+ svuint8_t hi = svlsr_n_u8_x(all, value, 4);
+ svst1_u8(all, pd, svzip1_u8(lo, hi));
+ svst1_u8(all, pd + width, svzip2_u8(lo, hi));
+ }
+ for (; j < byteSize; ++j, ++ps, pd += 2)
{
- uint64_t value = LoadBitsBlock(ps);
- pd[0] = uint8_t((value >> (0 * bits)) & mask);
- pd[1] = uint8_t((value >> (1 * bits)) & mask);
- pd[2] = uint8_t((value >> (2 * bits)) & mask);
- pd[3] = uint8_t((value >> (3 * bits)) & mask);
- pd[4] = uint8_t((value >> (4 * bits)) & mask);
- pd[5] = uint8_t((value >> (5 * bits)) & mask);
- pd[6] = uint8_t((value >> (6 * bits)) & mask);
- pd[7] = uint8_t((value >> (7 * bits)) & mask);
+ pd[0] = uint8_t(ps[0] & 0x0F);
+ pd[1] = uint8_t(ps[0] >> 4);
}
}
}
- template<> void UnpackData<8>(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t stride)
+ template<> void UnpackData<8>(size_t count, const uint8_t* const* src, size_t size, uint8_t* dst, size_t)
{
- const svuint8_t zero = svdup_n_u8(0);
+ const svbool_t all = svptrue_b8();
+ const size_t width = svcntb();
for (size_t i = 0; i < count; ++i)
{
const uint8_t* ps = src[i] + 16;
uint8_t* pd = dst + i * size;
- for (size_t j = 0; j < size; j += svcntb())
+ size_t j = 0;
+ for (; j + width <= size; j += width)
+ svst1_u8(all, pd + j, svld1_u8(all, ps + j));
+ if (j < size)
{
svbool_t mask = svwhilelt_b8(j, size);
- svuint8_t value = svsel_u8(mask, svld1_u8(mask, ps + j), zero);
- svst1_u8(mask, pd + j, value);
+ svst1_u8(mask, pd + j, svld1_u8(mask, ps + j));
}
}
}
- Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth)
+ Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth, bool)
{
switch (depth)
{
@@ -113,97 +145,182 @@ namespace Simd
//-------------------------------------------------------------------------------------------------
- SIMD_INLINE uint32_t Correlation8u(const uint8_t* a, const uint8_t* b, size_t size)
+ template SIMD_INLINE void CorrelationMx4(size_t K, const uint8_t* ad, const float* an,
+ const uint8_t* bd, const float* bn, size_t bnStride, float* distances, size_t stride)
{
- const svuint8_t zero = svdup_n_u8(0);
- svuint32_t sums = svdup_n_u32(0);
- for (size_t i = 0; i < size; i += svcntb())
+ const size_t width = svcntb();
+ const size_t main = K & ~(width - 1);
+ const svbool_t all = svptrue_b8();
+ svuint32_t ab00 = svdup_n_u32(0), ab01 = ab00, ab02 = ab00, ab03 = ab00;
+ svuint32_t ab10 = ab00, ab11 = ab00, ab12 = ab00, ab13 = ab00;
+ svuint32_t ab20 = ab00, ab21 = ab00, ab22 = ab00, ab23 = ab00;
+ svuint32_t ab30 = ab00, ab31 = ab00, ab32 = ab00, ab33 = ab00;
+ const uint8_t* ad0 = ad;
+ const uint8_t* ad1 = ad + K;
+ const uint8_t* ad2 = ad + 2 * K;
+ const uint8_t* ad3 = ad + 3 * K;
+ const uint8_t* bd0 = bd;
+ const uint8_t* bd1 = bd + K;
+ const uint8_t* bd2 = bd + 2 * K;
+ const uint8_t* bd3 = bd + 3 * K;
+ size_t k = 0;
+ for (; k < main; k += width)
{
- svbool_t mask = svwhilelt_b8(i, size);
- svuint8_t _a = svsel_u8(mask, svld1_u8(mask, a + i), zero);
- svuint8_t _b = svsel_u8(mask, svld1_u8(mask, b + i), zero);
- sums = svdot_u32(sums, _a, _b);
+ svuint8_t b0 = svld1_u8(all, bd0 + k);
+ svuint8_t b1 = svld1_u8(all, bd1 + k);
+ svuint8_t b2 = svld1_u8(all, bd2 + k);
+ svuint8_t b3 = svld1_u8(all, bd3 + k);
+ if (M > 0)
+ {
+ svuint8_t a0 = svld1_u8(all, ad0 + k);
+ ab00 = svdot_u32(ab00, a0, b0);
+ ab01 = svdot_u32(ab01, a0, b1);
+ ab02 = svdot_u32(ab02, a0, b2);
+ ab03 = svdot_u32(ab03, a0, b3);
+ }
+ if (M > 1)
+ {
+ svuint8_t a1 = svld1_u8(all, ad1 + k);
+ ab10 = svdot_u32(ab10, a1, b0);
+ ab11 = svdot_u32(ab11, a1, b1);
+ ab12 = svdot_u32(ab12, a1, b2);
+ ab13 = svdot_u32(ab13, a1, b3);
+ }
+ if (M > 2)
+ {
+ svuint8_t a2 = svld1_u8(all, ad2 + k);
+ ab20 = svdot_u32(ab20, a2, b0);
+ ab21 = svdot_u32(ab21, a2, b1);
+ ab22 = svdot_u32(ab22, a2, b2);
+ ab23 = svdot_u32(ab23, a2, b3);
+ }
+ if (M > 3)
+ {
+ svuint8_t a3 = svld1_u8(all, ad3 + k);
+ ab30 = svdot_u32(ab30, a3, b0);
+ ab31 = svdot_u32(ab31, a3, b1);
+ ab32 = svdot_u32(ab32, a3, b2);
+ ab33 = svdot_u32(ab33, a3, b3);
+ }
}
- return (uint32_t)svaddv_u32(svptrue_b32(), sums);
- }
-
- SIMD_INLINE float DecodeCosineDistance(const float* a, const float* b, uint32_t abSum)
- {
- float ab = float(abSum) * a[0] * b[0] + a[2] * b[1] + b[2] * a[1];
- return Simd::RestrictRange(1.0f - ab / (a[3] * b[3]), 0.0f, 2.0f);
- }
-
- template SIMD_INLINE void CorrelationsMx1(size_t K, const uint8_t* ad, const uint8_t* bd, uint32_t* ab)
- {
- const svuint8_t zero = svdup_n_u8(0);
- svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
- for (size_t i = 0; i < K; i += svcntb())
+ if (k < K)
{
- svbool_t mask = svwhilelt_b8(i, K);
- svuint8_t b = svsel_u8(mask, svld1_u8(mask, bd + i), zero);
+ svbool_t mask = svwhilelt_b8(k, K);
+ svuint8_t b0 = svld1_u8(mask, bd0 + k);
+ svuint8_t b1 = svld1_u8(mask, bd1 + k);
+ svuint8_t b2 = svld1_u8(mask, bd2 + k);
+ svuint8_t b3 = svld1_u8(mask, bd3 + k);
if (M > 0)
{
- svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 0 * K + i), zero);
- sums0 = svdot_u32(sums0, a, b);
+ svuint8_t a0 = svld1_u8(mask, ad0 + k);
+ ab00 = svdot_u32(ab00, a0, b0);
+ ab01 = svdot_u32(ab01, a0, b1);
+ ab02 = svdot_u32(ab02, a0, b2);
+ ab03 = svdot_u32(ab03, a0, b3);
}
if (M > 1)
{
- svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 1 * K + i), zero);
- sums1 = svdot_u32(sums1, a, b);
+ svuint8_t a1 = svld1_u8(mask, ad1 + k);
+ ab10 = svdot_u32(ab10, a1, b0);
+ ab11 = svdot_u32(ab11, a1, b1);
+ ab12 = svdot_u32(ab12, a1, b2);
+ ab13 = svdot_u32(ab13, a1, b3);
}
if (M > 2)
{
- svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 2 * K + i), zero);
- sums2 = svdot_u32(sums2, a, b);
+ svuint8_t a2 = svld1_u8(mask, ad2 + k);
+ ab20 = svdot_u32(ab20, a2, b0);
+ ab21 = svdot_u32(ab21, a2, b1);
+ ab22 = svdot_u32(ab22, a2, b2);
+ ab23 = svdot_u32(ab23, a2, b3);
}
if (M > 3)
{
- svuint8_t a = svsel_u8(mask, svld1_u8(mask, ad + 3 * K + i), zero);
- sums3 = svdot_u32(sums3, a, b);
+ svuint8_t a3 = svld1_u8(mask, ad3 + k);
+ ab30 = svdot_u32(ab30, a3, b0);
+ ab31 = svdot_u32(ab31, a3, b1);
+ ab32 = svdot_u32(ab32, a3, b2);
+ ab33 = svdot_u32(ab33, a3, b3);
}
}
- if (M > 0) ab[0] = (uint32_t)svaddv_u32(svptrue_b32(), sums0);
- if (M > 1) ab[1] = (uint32_t)svaddv_u32(svptrue_b32(), sums1);
- if (M > 2) ab[2] = (uint32_t)svaddv_u32(svptrue_b32(), sums2);
- if (M > 3) ab[3] = (uint32_t)svaddv_u32(svptrue_b32(), sums3);
+ if (M > 0) DecodeCosineDistances1x4(an + 0 * 4, bn, bnStride, ab00, ab01, ab02, ab03, distances + 0 * stride);
+ if (M > 1) DecodeCosineDistances1x4(an + 1 * 4, bn, bnStride, ab10, ab11, ab12, ab13, distances + 1 * stride);
+ if (M > 2) DecodeCosineDistances1x4(an + 2 * 4, bn, bnStride, ab20, ab21, ab22, ab23, distances + 2 * stride);
+ if (M > 3) DecodeCosineDistances1x4(an + 3 * 4, bn, bnStride, ab30, ab31, ab32, ab33, distances + 3 * stride);
}
- template SIMD_INLINE void MacroCorrelationMx1(size_t K, const uint8_t* ad, const float* an, const uint8_t* bd, const float* bn, float* distances, size_t stride)
+ template SIMD_INLINE void CorrelationMx1(size_t K, const uint8_t* ad, const float* an,
+ const uint8_t* bd, const float* bn, size_t bnStride, float* distances, size_t stride)
{
- uint32_t ab[4];
- CorrelationsMx1(K, ad, bd, ab);
- if (M > 0) distances[0 * stride] = DecodeCosineDistance(an + 0 * 4, bn, ab[0]);
- if (M > 1) distances[1 * stride] = DecodeCosineDistance(an + 1 * 4, bn, ab[1]);
- if (M > 2) distances[2 * stride] = DecodeCosineDistance(an + 2 * 4, bn, ab[2]);
- if (M > 3) distances[3 * stride] = DecodeCosineDistance(an + 3 * 4, bn, ab[3]);
+ const size_t width = svcntb();
+ const size_t main = K & ~(width - 1);
+ const svbool_t all = svptrue_b8();
+ svuint32_t sums0 = svdup_n_u32(0), sums1 = sums0, sums2 = sums0, sums3 = sums0;
+ size_t k = 0;
+ for (; k < main; k += width)
+ {
+ svuint8_t b = svld1_u8(all, bd + k);
+ if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(all, ad + 0 * K + k), b);
+ if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(all, ad + 1 * K + k), b);
+ if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(all, ad + 2 * K + k), b);
+ if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(all, ad + 3 * K + k), b);
+ }
+ if (k < K)
+ {
+ svbool_t mask = svwhilelt_b8(k, K);
+ svuint8_t b = svld1_u8(mask, bd + k);
+ if (M > 0) sums0 = svdot_u32(sums0, svld1_u8(mask, ad + 0 * K + k), b);
+ if (M > 1) sums1 = svdot_u32(sums1, svld1_u8(mask, ad + 1 * K + k), b);
+ if (M > 2) sums2 = svdot_u32(sums2, svld1_u8(mask, ad + 2 * K + k), b);
+ if (M > 3) sums3 = svdot_u32(sums3, svld1_u8(mask, ad + 3 * K + k), b);
+ }
+ if (M > 0) DecodeCosineDistance(an + 0 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums0), distances + 0 * stride);
+ if (M > 1) DecodeCosineDistance(an + 1 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums1), distances + 1 * stride);
+ if (M > 2) DecodeCosineDistance(an + 2 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums2), distances + 2 * stride);
+ if (M > 3) DecodeCosineDistance(an + 3 * 4, bn, bnStride, (uint32_t)svaddv_u32(svptrue_b32(), sums3), distances + 3 * stride);
}
- void MacroCorrelation(size_t M, size_t N, size_t K, const uint8_t* ad, const float* an, const uint8_t* bd, const float* bn, float* distances, size_t stride)
+ void MacroCorrelation(size_t M, size_t N, size_t K, const uint8_t* ad, const float* an,
+ const uint8_t* bd, const float* bn, float* distances, size_t stride)
{
- size_t M4 = AlignLoAny(M, 4);
- for (size_t j = 0; j < N; ++j)
+ size_t M4 = AlignLoAny(M, 4), N4 = AlignLo(N, 4), i = 0;
+ for (; i < M4; i += 4)
+ {
+ size_t j = 0;
+ for (; j < N4; j += 4)
+ CorrelationMx4<4>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ for (; j < N; ++j)
+ CorrelationMx1<4>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ }
+ if (i < M)
{
- size_t i = 0;
- const uint8_t* b = bd + j * K;
- const float* nb = bn + j * 4;
- for (; i < M4; i += 4)
- MacroCorrelationMx1<4>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride);
- if (i < M)
+ size_t m = M - i;
+ size_t j = 0;
+ for (; j < N4; j += 4)
+ {
+ if (m == 1)
+ CorrelationMx4<1>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ else if (m == 2)
+ CorrelationMx4<2>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ else
+ CorrelationMx4<3>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ }
+ for (; j < N; ++j)
{
- switch (M - i)
- {
- case 1: MacroCorrelationMx1<1>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride); break;
- case 2: MacroCorrelationMx1<2>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride); break;
- case 3: MacroCorrelationMx1<3>(K, ad + i * K, an + i * 4, b, nb, distances + i * stride + j, stride); break;
- }
+ if (m == 1)
+ CorrelationMx1<1>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ else if (m == 2)
+ CorrelationMx1<2>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
+ else
+ CorrelationMx1<3>(K, ad + i * K, an + i * 4, bd + j * K, bn + j, N, distances + i * stride + j, stride);
}
}
}
- Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t depth)
+ Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t)
{
return MacroCorrelation;
}
}
-#endif// SIMD_SVE2_ENABLE
+#endif
}
diff --git a/src/Test/TestDescrInt.cpp b/src/Test/TestDescrInt.cpp
index db2fd4822c..4d186b61a6 100644
--- a/src/Test/TestDescrInt.cpp
+++ b/src/Test/TestDescrInt.cpp
@@ -635,6 +635,9 @@ namespace Test
for (size_t depth = 4; depth <= 8; depth++)
{
+ result = result && DescrIntCosineDistancesMxNaAutoTest(8, 8, 24, depth, f1, f2);
+ result = result && DescrIntCosineDistancesMxNaAutoTest(4, 12, 256, depth, f1, f2);
+ result = result && DescrIntCosineDistancesMxNaAutoTest(5, 7, 128, depth, f1, f2);
result = result && DescrIntCosineDistancesMxNaAutoTest(256, 128, 256, depth, f1, f2);
result = result && DescrIntCosineDistancesMxNaAutoTest(128, 128, 512, depth, f1, f2);
#if !(defined(__GNUC__) && defined(__clang__))