Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 15 additions & 24 deletions src/VecSim/spaces/IP/IP.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -146,28 +146,18 @@ float SQ8_SQ8_InnerProduct_Impl(const void *pVect1v, const void *pVect2v, size_t
const auto *pVect1 = static_cast<const uint8_t *>(pVect1v);
const auto *pVect2 = static_cast<const uint8_t *>(pVect2v);

// Compute inner product of quantized values: Σ(q1[i]*q2[i])
float product = 0;
// Inner product of the quantized bytes: Σ(a[i]*b[i]). Exact in integer arithmetic.
uint64_t q_dot = 0;
for (size_t i = 0; i < dimension; i++) {
product += pVect1[i] * pVect2[i];
q_dot += static_cast<uint32_t>(pVect1[i]) * static_cast<uint32_t>(pVect2[i]);
}

// Metadata follows byte payloads and is not necessarily float-aligned.
const auto *params1 = pVect1 + dimension;
const float min_val1 = load_unaligned<float>(params1 + sq8::MIN_VAL * sizeof(float));
const float delta1 = load_unaligned<float>(params1 + sq8::DELTA * sizeof(float));
const float sum1 = load_unaligned<float>(params1 + sq8::SUM * sizeof(float));

// Get quantization parameters from pVect2
const auto *params2 = pVect2 + dimension;
const float min_val2 = load_unaligned<float>(params2 + sq8::MIN_VAL * sizeof(float));
const float delta2 = load_unaligned<float>(params2 + sq8::DELTA * sizeof(float));
const float sum2 = load_unaligned<float>(params2 + sq8::SUM * sizeof(float));

// Apply the algebraic formula using precomputed sums:
// IP = min1*sum2 + min2*sum1 + delta1*delta2*Σ(q1[i]*q2[i]) - dim*min1*min2
return min_val1 * sum2 + min_val2 * sum1 - static_cast<float>(dimension) * min_val1 * min_val2 +
delta1 * delta2 * product;
// Both operands describe their reconstructions, so every term is derived from the stored
// integer sums rather than from sums over the original inputs. See sq8.h.
return static_cast<float>(sq8::reconstructed_ip(
sq8::min_val_of(pVect1, dimension), sq8::delta_of(pVect1, dimension),
sq8::q_sum_of(pVect1, dimension), sq8::min_val_of(pVect2, dimension),
sq8::delta_of(pVect2, dimension), sq8::q_sum_of(pVect2, dimension), q_dot, dimension));
}

// SQ8-to-SQ8: Both vectors are uint8 quantized with precomputed sum
Expand Down Expand Up @@ -238,11 +228,12 @@ float FP16_InnerProduct(const void *pVect1, const void *pVect2, size_t dimension
}

// Return type for the inner product functions.
// The type should be able to hold `dimension * MAX_VAL(int_elem_t) * MAX_VAL(int_elem_t)`.
// To support dimension up to 2^16, we need the difference between the type and int_elem_t to be at
// least 2 bytes. We assert that in the implementation.
// The type must hold `dimension * MAX_VAL(int_elem_t) * MAX_VAL(int_elem_t)`. For uint8 that is
// 65025 * dimension, which overflows a 32-bit int from dimension 33,026, so this is 64-bit for
// every element type. Keeping it signed also means the `1 - ip` in the wrappers below stays
// signed arithmetic and cannot underflow. We assert the width in the implementation.
template <typename int_elem_t>
using ret_t = std::conditional_t<sizeof(int_elem_t) == 1, int, long long>;
using ret_t = long long;

template <typename int_elem_t>
static inline ret_t<int_elem_t>
Expand Down Expand Up @@ -273,7 +264,7 @@ float INT8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension) {
float UINT8_InnerProduct(const void *pVect1v, const void *pVect2v, size_t dimension) {
const auto *pVect1 = static_cast<const uint8_t *>(pVect1v);
const auto *pVect2 = static_cast<const uint8_t *>(pVect2v);
return 1 - INTEGER_InnerProductImp(pVect1, pVect2, dimension);
return 1.0f - static_cast<float>(INTEGER_InnerProductImp(pVect1, pVect2, dimension));
}

float UINT8_Cosine(const void *pVect1v, const void *pVect2v, size_t dimension) {
Expand Down
27 changes: 7 additions & 20 deletions src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_SQ8_SQ8.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,29 +38,16 @@ using sq8 = vecsim_types::sq8;
// Uses UINT8_InnerProductImp for efficient dot product computation with VNNI
template <unsigned char residual> // 0..63
float SQ8_SQ8_InnerProductImp(const void *pVec1v, const void *pVec2v, size_t dimension) {
// Compute raw dot product using efficient UINT8 AVX512 VNNI implementation
// UINT8_InnerProductImp uses _mm512_dpwssd_epi32 for native integer dot product
int dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
// Exact integer dot product of the quantized bytes.
const uint64_t q_dot = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);

// Get dequantization parameters and precomputed values from the end of vectors
// Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]
// Every term describes the reconstructions, derived from the stored integer sums. See sq8.h.
const uint8_t *pVec1 = static_cast<const uint8_t *>(pVec1v);
const uint8_t *pVec2 = static_cast<const uint8_t *>(pVec2v);

const auto *params1 = pVec1 + dimension;
const float min1 = load_unaligned<float>(params1 + sq8::MIN_VAL * sizeof(float));
const float delta1 = load_unaligned<float>(params1 + sq8::DELTA * sizeof(float));
const float sum1 = load_unaligned<float>(params1 + sq8::SUM * sizeof(float));

const auto *params2 = pVec2 + dimension;
const float min2 = load_unaligned<float>(params2 + sq8::MIN_VAL * sizeof(float));
const float delta2 = load_unaligned<float>(params2 + sq8::DELTA * sizeof(float));
const float sum2 = load_unaligned<float>(params2 + sq8::SUM * sizeof(float));

// Apply the algebraic formula using precomputed sums:
// IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1[i]*q2[i]) - dim*min1*min2
return min1 * sum2 + min2 * sum1 + delta1 * delta2 * static_cast<float>(dot_product) -
static_cast<float>(dimension) * min1 * min2;
return static_cast<float>(sq8::reconstructed_ip(
sq8::min_val_of(pVec1, dimension), sq8::delta_of(pVec1, dimension),
sq8::q_sum_of(pVec1, dimension), sq8::min_val_of(pVec2, dimension),
sq8::delta_of(pVec2, dimension), sq8::q_sum_of(pVec2, dimension), q_dot, dimension));
}

// SQ8-to-SQ8 Inner Product distance function
Expand Down
15 changes: 10 additions & 5 deletions src/VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_UINT8.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ static inline void InnerProductStep(uint8_t *&pVect1, uint8_t *&pVect2, __m512i
}

template <unsigned char residual> // 0..63
static inline int UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v,
size_t dimension) {
static inline uint64_t UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v,
size_t dimension) {
uint8_t *pVect1 = (uint8_t *)pVect1v;
uint8_t *pVect2 = (uint8_t *)pVect2v;

Expand Down Expand Up @@ -87,19 +87,24 @@ static inline int UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v
} while (pVect1 < pEnd1);
}

return _mm512_reduce_add_epi32(sum);
// Reduce in 64-bit. Each int32 lane is in range, but their total reaches 255*255*dim, which
// passes INT_MAX from dimension 33,027, so a 32-bit horizontal sum would wrap. Callers rely
// on this being the exact integer dot product.
const __m512i lo = _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64(sum, 0));
const __m512i hi = _mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64(sum, 1));
return static_cast<uint64_t>(_mm512_reduce_add_epi64(_mm512_add_epi64(lo, hi)));
}

template <unsigned char residual> // 0..63
float UINT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v,
size_t dimension) {

return 1 - UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
return 1.0f - static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
}
template <unsigned char residual> // 0..63
float UINT8_CosineSIMD64_AVX512F_BW_VL_VNNI(const void *pVect1v, const void *pVect2v,
size_t dimension) {
float ip = UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
float ip = static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
return 1.0f - ip / (norm_v1 * norm_v2);
Expand Down
27 changes: 7 additions & 20 deletions src/VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,29 +39,16 @@ using sq8 = vecsim_types::sq8;
template <unsigned char residual> // 0..63
float SQ8_SQ8_InnerProductSIMD64_NEON_DOTPROD_IMP(const void *pVec1v, const void *pVec2v,
size_t dimension) {
// Compute raw dot product using efficient UINT8 DOTPROD implementation
// UINT8_InnerProductImp uses vdotq_u32 for native uint8 dot product
float dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
// Exact integer dot product of the quantized bytes.
const uint64_t q_dot = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);

// Get dequantization parameters and precomputed values from the end of vectors
// Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]
// Every term describes the reconstructions, derived from the stored integer sums. See sq8.h.
const uint8_t *pVec1 = static_cast<const uint8_t *>(pVec1v);
const uint8_t *pVec2 = static_cast<const uint8_t *>(pVec2v);

const auto *params1 = pVec1 + dimension;
const float min1 = load_unaligned<float>(params1 + sq8::MIN_VAL * sizeof(float));
const float delta1 = load_unaligned<float>(params1 + sq8::DELTA * sizeof(float));
const float sum1 = load_unaligned<float>(params1 + sq8::SUM * sizeof(float));

const auto *params2 = pVec2 + dimension;
const float min2 = load_unaligned<float>(params2 + sq8::MIN_VAL * sizeof(float));
const float delta2 = load_unaligned<float>(params2 + sq8::DELTA * sizeof(float));
const float sum2 = load_unaligned<float>(params2 + sq8::SUM * sizeof(float));

// Apply algebraic formula using precomputed sums:
// IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1*q2) - dim*min1*min2
return min1 * sum2 + min2 * sum1 + delta1 * delta2 * dot_product -
static_cast<float>(dimension) * min1 * min2;
return static_cast<float>(sq8::reconstructed_ip(
sq8::min_val_of(pVec1, dimension), sq8::delta_of(pVec1, dimension),
sq8::q_sum_of(pVec1, dimension), sq8::min_val_of(pVec2, dimension),
sq8::delta_of(pVec2, dimension), sq8::q_sum_of(pVec2, dimension), q_dot, dimension));
}

// SQ8-to-SQ8 Inner Product distance function
Expand Down
12 changes: 6 additions & 6 deletions src/VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ InnerProductStep(uint8_t *&pVect1, uint8_t *&pVect2, uint32x4_t &sum) {
}

template <unsigned char residual> // 0..63
float UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
uint64_t UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
uint8_t *pVect1 = (uint8_t *)pVect1v;
uint8_t *pVect2 = (uint8_t *)pVect2v;

Expand Down Expand Up @@ -97,20 +97,20 @@ float UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dim

uint32x4_t total_sum = vaddq_u32(sum0, sum1);

int32_t result = vaddvq_u32(total_sum);

return static_cast<float>(result);
// Widening horizontal sum: the lane total reaches 255*255*dim, so reducing into 32 bits
// would wrap from dimension 33,027. Callers rely on this being the exact integer dot.
return vaddlvq_u32(total_sum);
}

template <unsigned char residual> // 0..63
float UINT8_InnerProductSIMD16_NEON_DOTPROD(const void *pVect1v, const void *pVect2v,
size_t dimension) {
return 1.0f - UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
return 1.0f - static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
}

template <unsigned char residual> // 0..63
float UINT8_CosineSIMD_NEON_DOTPROD(const void *pVect1v, const void *pVect2v, size_t dimension) {
float ip = UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
float ip = static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
return 1.0f - ip / (norm_v1 * norm_v2);
Expand Down
28 changes: 7 additions & 21 deletions src/VecSim/spaces/IP/IP_NEON_SQ8_SQ8.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,30 +39,16 @@ using sq8 = vecsim_types::sq8;
template <unsigned char residual> // 0..63
float SQ8_SQ8_InnerProductSIMD64_NEON_IMP(const void *pVec1v, const void *pVec2v,
size_t dimension) {
// Compute raw dot product using efficient UINT8 implementation
// UINT8_InnerProductImp processes 16 elements at a time using native uint8 instructions
float dot_product = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);
// Exact integer dot product of the quantized bytes.
const uint64_t q_dot = UINT8_InnerProductImp<residual>(pVec1v, pVec2v, dimension);

// Get dequantization parameters and precomputed values from the end of pVec1
// Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]
// Every term describes the reconstructions, derived from the stored integer sums. See sq8.h.
const uint8_t *pVec1 = static_cast<const uint8_t *>(pVec1v);
const uint8_t *pVec2 = static_cast<const uint8_t *>(pVec2v);

const auto *params1 = pVec1 + dimension;
const float min1 = load_unaligned<float>(params1 + sq8::MIN_VAL * sizeof(float));
const float delta1 = load_unaligned<float>(params1 + sq8::DELTA * sizeof(float));
const float sum1 = load_unaligned<float>(params1 + sq8::SUM * sizeof(float));

// Get dequantization parameters and precomputed values from the end of pVec2
const auto *params2 = pVec2 + dimension;
const float min2 = load_unaligned<float>(params2 + sq8::MIN_VAL * sizeof(float));
const float delta2 = load_unaligned<float>(params2 + sq8::DELTA * sizeof(float));
const float sum2 = load_unaligned<float>(params2 + sq8::SUM * sizeof(float));

// Apply algebraic formula using precomputed sums:
// IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1*q2) - dim*min1*min2
return min1 * sum2 + min2 * sum1 + delta1 * delta2 * dot_product -
static_cast<float>(dimension) * min1 * min2;
return static_cast<float>(sq8::reconstructed_ip(
sq8::min_val_of(pVec1, dimension), sq8::delta_of(pVec1, dimension),
sq8::q_sum_of(pVec1, dimension), sq8::min_val_of(pVec2, dimension),
sq8::delta_of(pVec2, dimension), sq8::q_sum_of(pVec2, dimension), q_dot, dimension));
}

// SQ8-to-SQ8 Inner Product distance function
Expand Down
13 changes: 6 additions & 7 deletions src/VecSim/spaces/IP/IP_NEON_UINT8.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ InnerProductStep(uint8_t *&pVect1, uint8_t *&pVect2, uint32x4_t &sum) {
}

template <unsigned char residual> // 0..63
float UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
uint64_t UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dimension) {
uint8_t *pVect1 = (uint8_t *)pVect1v;
uint8_t *pVect2 = (uint8_t *)pVect2v;

Expand Down Expand Up @@ -105,20 +105,19 @@ float UINT8_InnerProductImp(const void *pVect1v, const void *pVect2v, size_t dim

uint32x4_t total_sum = vaddq_u32(sum0, sum1);

// Horizontal sum of the 4 elements in the combined sum register
int32_t result = vaddvq_u32(total_sum);

return static_cast<float>(result);
// Widening horizontal sum: the lane total reaches 255*255*dim, so reducing into 32 bits
// would wrap from dimension 33,027. Callers rely on this being the exact integer dot.
return vaddlvq_u32(total_sum);
}

template <unsigned char residual> // 0..15
float UINT8_InnerProductSIMD16_NEON(const void *pVect1v, const void *pVect2v, size_t dimension) {
return 1.0f - UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
return 1.0f - static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
}

template <unsigned char residual> // 0..63
float UINT8_CosineSIMD_NEON(const void *pVect1v, const void *pVect2v, size_t dimension) {
float ip = UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension);
float ip = static_cast<float>(UINT8_InnerProductImp<residual>(pVect1v, pVect2v, dimension));
const float norm_v1 = load_unaligned<float>(static_cast<const uint8_t *>(pVect1v) + dimension);
const float norm_v2 = load_unaligned<float>(static_cast<const uint8_t *>(pVect2v) + dimension);
return 1.0f - ip / (norm_v1 * norm_v2);
Expand Down
27 changes: 7 additions & 20 deletions src/VecSim/spaces/IP/IP_SVE_SQ8_SQ8.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,17 @@ using sq8 = vecsim_types::sq8;
// Uses UINT8_InnerProductImp for efficient dot product computation with SVE
template <bool partial_chunk, unsigned char additional_steps>
float SQ8_SQ8_InnerProductSIMD_SVE_IMP(const void *pVec1v, const void *pVec2v, size_t dimension) {
// Compute raw dot product using efficient UINT8 SVE implementation
// UINT8_InnerProductImp uses svdot_u32 for native uint8 dot product
float dot_product =
// Exact integer dot product of the quantized bytes.
const uint64_t q_dot =
UINT8_InnerProductImp<partial_chunk, additional_steps>(pVec1v, pVec2v, dimension);

// Get dequantization parameters and precomputed values from the end of vectors
// Layout: [data (dim)] [min (float)] [delta (float)] [sum (float)]
// Every term describes the reconstructions, derived from the stored integer sums. See sq8.h.
const uint8_t *pVec1 = static_cast<const uint8_t *>(pVec1v);
const uint8_t *pVec2 = static_cast<const uint8_t *>(pVec2v);

const auto *params1 = pVec1 + dimension;
const float min1 = load_unaligned<float>(params1 + sq8::MIN_VAL * sizeof(float));
const float delta1 = load_unaligned<float>(params1 + sq8::DELTA * sizeof(float));
const float sum1 = load_unaligned<float>(params1 + sq8::SUM * sizeof(float));

const auto *params2 = pVec2 + dimension;
const float min2 = load_unaligned<float>(params2 + sq8::MIN_VAL * sizeof(float));
const float delta2 = load_unaligned<float>(params2 + sq8::DELTA * sizeof(float));
const float sum2 = load_unaligned<float>(params2 + sq8::SUM * sizeof(float));

// Apply algebraic formula with float conversion only at the end:
// IP = min1*sum2 + min2*sum1 + δ1*δ2 * Σ(q1*q2) - dim*min1*min2
return min1 * sum2 + min2 * sum1 + delta1 * delta2 * dot_product -
static_cast<float>(dimension) * min1 * min2;
return static_cast<float>(sq8::reconstructed_ip(
sq8::min_val_of(pVec1, dimension), sq8::delta_of(pVec1, dimension),
sq8::q_sum_of(pVec1, dimension), sq8::min_val_of(pVec2, dimension),
sq8::delta_of(pVec2, dimension), sq8::q_sum_of(pVec2, dimension), q_dot, dimension));
}

// SQ8-to-SQ8 Inner Product distance function
Expand Down
Loading
Loading