Skip to content

Improve SVE2 optimizations of SimdDescrIntCosineDistancesMxNa #905

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ermig1979 merged 2 commits into from
Aug 24, 2026
Merged

Improve SVE2 optimizations of SimdDescrIntCosineDistancesMxNa #905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
66a0dd8
Improve SVE2 optimizations of SimdDescrIntCosineDistancesMxNa.
cursoragent Aug 24, 2026
7fd5641
Fix SVE2 DescrIntCosineDistancesMxNa SIGSEGV on 5x7x128-4.
cursoragent Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/2026.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ <h5>Improving</h5>
<li>SVE2 optimizations of function GetMoments.</li>
<li>SVE2 optimizations of function GetObjectMoments.</li>
<li>SVE2 optimizations of function DescrIntCosineDistance.</li>
<li>SVE2 optimizations of function DescrIntCosineDistancesMxNa.</li>
</ul>
<h5>Renaming</h5>
<ul>
Expand Down
2 changes: 1 addition & 1 deletion src/Simd/SimdDescrInt.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ namespace Simd
Base::DescrInt::MacroCosineDistancesDirectPtr GetMacroCosineDistancesDirect(size_t depth);

Base::DescrInt::UnpackNormPtr GetUnpackNorm(bool transpose);
Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth);
Base::DescrInt::UnpackDataPtr GetUnpackData(size_t depth, bool transpose);
Base::DescrInt::MacroCosineDistancesUnpackPtr GetMacroCosineDistancesUnpack(size_t depth);

//-------------------------------------------------------------------------------------------------
Expand Down
80 changes: 80 additions & 0 deletions src/Simd/SimdDescrIntCommon.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,5 +721,85 @@ namespace Simd
}
}
#endif

#ifdef SIMD_SVE2_ENABLE
namespace Sve2
{
SIMD_ALIGNED(16) const uint8_t C5_TBL[16] = { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4 };
SIMD_ALIGNED(16) const uint16_t C5_SHR[8] = { 8, 5, 10, 7, 4, 9, 6, 11 };
SIMD_ALIGNED(16) const uint8_t C6_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5 };
SIMD_ALIGNED(16) const uint16_t C6_SHR[8] = { 8, 6, 4, 2, 8, 6, 4, 2 };
SIMD_ALIGNED(16) const uint8_t C7_TBL[16] = { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6 };
SIMD_ALIGNED(16) const uint16_t C7_SHR[8] = { 8, 7, 6, 5, 4, 3, 2, 1 };

SIMD_INLINE svuint8_t UnpackTbl(const uint8_t* tbl16, uint8_t bits)
{
const svbool_t all = svptrue_b8();
svuint8_t base = svld1rq_u8(all, tbl16);
svuint8_t index = svindex_u8(0, 1);
svuint8_t group = svlsr_n_u8_x(all, index, 4);
return svadd_u8_x(all, base, svmul_n_u8_x(all, group, bits));
}

SIMD_INLINE svuint16_t UnpackShr(const uint16_t* shr8)
{
return svld1rq_u16(svptrue_b16(), shr8);
}

template<int bits> SIMD_INLINE svuint8_t UnpackTbl();
template<int bits> SIMD_INLINE svuint16_t UnpackShr();

template<> SIMD_INLINE svuint8_t UnpackTbl<5>() { return UnpackTbl(C5_TBL, 5); }
template<> SIMD_INLINE svuint16_t UnpackShr<5>() { return UnpackShr(C5_SHR); }
template<> SIMD_INLINE svuint8_t UnpackTbl<6>() { return UnpackTbl(C6_TBL, 6); }
template<> SIMD_INLINE svuint16_t UnpackShr<6>() { return UnpackShr(C6_SHR); }
template<> SIMD_INLINE svuint8_t UnpackTbl<7>() { return UnpackTbl(C7_TBL, 7); }
template<> SIMD_INLINE svuint16_t UnpackShr<7>() { return UnpackShr(C7_SHR); }

SIMD_INLINE svuint16_t UnpackTo16(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr, uint16_t mask)
{
svuint8_t raw = svld1_u8(svwhilelt_b8((size_t)0, packed), src);
svuint16_t wide = svreinterpret_u16_u8(svtbl_u8(raw, tbl));
const svbool_t all = svptrue_b16();
return svand_n_u16_x(all, svlsr_u16_x(all, wide, shr), mask);
}

template<int bits> SIMD_INLINE svuint8_t UnpackTo8(const uint8_t* src, size_t packed, const svuint8_t& tbl, const svuint16_t& shr)
{
const size_t packedHalf = svcnth() * bits / 8;
const uint16_t mask = (uint16_t)((1 << bits) - 1);
size_t packedLo = packed < packedHalf ? packed : packedHalf;
size_t packedHi = packed - packedLo;
svuint16_t lo = UnpackTo16(src, packedLo, tbl, shr, mask);
svuint16_t hi = UnpackTo16(src + packedLo, packedHi, tbl, shr, mask);
return svuzp1_u8(svreinterpret_u8_u16(lo), svreinterpret_u8_u16(hi));
}

SIMD_INLINE void DecodeCosineDistance(const float* a, const float* b, size_t stride, uint32_t abSum, float* distance)
{
float bb[4] = { b[0], b[stride], b[2 * stride], b[3 * stride] };
float ab = (float)abSum * a[0] * bb[0] + a[2] * bb[1] + bb[2] * a[1];
distance[0] = Simd::RestrictRange(1.0f - ab / (a[3] * bb[3]), 0.0f, 2.0f);
}

SIMD_INLINE void DecodeCosineDistances1x4(const uint8_t* a, const uint8_t* const* B,
svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
{
Base::DecodeCosineDistance(a, B[0], (float)svaddv_u32(svptrue_b32(), s0), distances + 0);
Base::DecodeCosineDistance(a, B[1], (float)svaddv_u32(svptrue_b32(), s1), distances + 1);
Base::DecodeCosineDistance(a, B[2], (float)svaddv_u32(svptrue_b32(), s2), distances + 2);
Base::DecodeCosineDistance(a, B[3], (float)svaddv_u32(svptrue_b32(), s3), distances + 3);
}

SIMD_INLINE void DecodeCosineDistances1x4(const float* a, const float* b, size_t stride,
svuint32_t s0, svuint32_t s1, svuint32_t s2, svuint32_t s3, float* distances)
{
DecodeCosineDistance(a, b + 0, stride, (uint32_t)svaddv_u32(svptrue_b32(), s0), distances + 0);
DecodeCosineDistance(a, b + 1, stride, (uint32_t)svaddv_u32(svptrue_b32(), s1), distances + 1);
DecodeCosineDistance(a, b + 2, stride, (uint32_t)svaddv_u32(svptrue_b32(), s2), distances + 2);
DecodeCosineDistance(a, b + 3, stride, (uint32_t)svaddv_u32(svptrue_b32(), s3), distances + 3);
}
}
#endif
}
#endif//__SimdDescrIntCommon_h__
24 changes: 12 additions & 12 deletions src/Simd/SimdSve2DescrInt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,24 +37,24 @@ namespace Simd
: Base::DescrInt(size, depth)
#endif
{
_encode32f = GetEncode32f(_depth);
_encode16f = GetEncode16f(_depth);
_decode32f = GetDecode32f(_depth);
_decode16f = GetDecode16f(_depth);
_encode32f = Sve2::GetEncode32f(_depth);
_encode16f = Sve2::GetEncode16f(_depth);
_decode32f = Sve2::GetDecode32f(_depth);
_decode16f = Sve2::GetDecode16f(_depth);

_cosineDistance = GetCosineDistance(_depth);
_macroCosineDistancesDirect = GetMacroCosineDistancesDirect(_depth);
_cosineDistance = Sve2::GetCosineDistance(_depth);
_macroCosineDistancesDirect = Sve2::GetMacroCosineDistancesDirect(_depth);
_microMd = 4;
_microNd = 4;

_unpackNormA = GetUnpackNorm(false);
_unpackNormB = GetUnpackNorm(true);
_unpackDataA = GetUnpackData(_depth);
_unpackDataB = GetUnpackData(_depth);
_macroCosineDistancesUnpack = GetMacroCosineDistancesUnpack(_depth);
_unpackNormA = Sve2::GetUnpackNorm(false);
_unpackNormB = Sve2::GetUnpackNorm(true);
_unpackDataA = Sve2::GetUnpackData(_depth, false);
_unpackDataB = Sve2::GetUnpackData(_depth, true);
_macroCosineDistancesUnpack = Sve2::GetMacroCosineDistancesUnpack(_depth);
_unpSize = _size;
_microMu = 4;
_microNu = 1;
_microNu = 4;
}

//-------------------------------------------------------------------------------------------------
Expand Down
Loading