diff --git a/src/VecSim/spaces/functions/AVX.cpp b/src/VecSim/spaces/functions/AVX.cpp index c3c993b6b..01de32303 100644 --- a/src/VecSim/spaces/functions/AVX.cpp +++ b/src/VecSim/spaces/functions/AVX.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_AVX_FP32.h" #include "VecSim/spaces/L2/L2_AVX_FP64.h" @@ -16,32 +17,24 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP32_IP_implementation_AVX(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP32_InnerProductSIMD16_AVX); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP32_InnerProductSIMD16_AVX; }); } dist_func_t Choose_FP64_IP_implementation_AVX(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 8, FP64_InnerProductSIMD8_AVX); - return ret_dist_func; + return dispatch_by_residual, 8>( + dim, []() { return FP64_InnerProductSIMD8_AVX; }); } dist_func_t Choose_FP32_L2_implementation_AVX(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP32_L2SqrSIMD16_AVX); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP32_L2SqrSIMD16_AVX; }); } dist_func_t Choose_FP64_L2_implementation_AVX(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 8, FP64_L2SqrSIMD8_AVX); - return ret_dist_func; + return dispatch_by_residual, 8>( + dim, []() { return FP64_L2SqrSIMD8_AVX; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX2.cpp b/src/VecSim/spaces/functions/AVX2.cpp index 1a9c41b6f..7dded0acb 100644 --- a/src/VecSim/spaces/functions/AVX2.cpp +++ b/src/VecSim/spaces/functions/AVX2.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX2.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_AVX2_BF16.h" #include "VecSim/spaces/L2/L2_AVX2_BF16.h" @@ -15,38 +16,29 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_BF16_IP_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_InnerProductSIMD32_AVX2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_InnerProductSIMD32_AVX2; }); } dist_func_t Choose_BF16_L2_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_L2SqrSIMD32_AVX2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_L2SqrSIMD32_AVX2; }); } dist_func_t Choose_SQ8_FP32_IP_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_InnerProductSIMD16_AVX2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_InnerProductSIMD16_AVX2; }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_CosineSIMD16_AVX2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_CosineSIMD16_AVX2; }); } dist_func_t Choose_SQ8_FP32_L2_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_L2SqrSIMD16_AVX2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_L2SqrSIMD16_AVX2; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX2_F16C.cpp b/src/VecSim/spaces/functions/AVX2_F16C.cpp index 3d298e81b..c6555e6d3 100644 --- a/src/VecSim/spaces/functions/AVX2_F16C.cpp +++ b/src/VecSim/spaces/functions/AVX2_F16C.cpp @@ -7,29 +7,23 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX2_F16C.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_AVX2_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_AVX2_SQ8_FP16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_SQ8_FP16_IP_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_AVX2); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_InnerProductSIMD16_AVX2; }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_AVX2); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_CosineSIMD16_AVX2; }); } dist_func_t Choose_SQ8_FP16_L2_implementation_AVX2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_AVX2); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_L2SqrSIMD16_AVX2; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX2_FMA.cpp b/src/VecSim/spaces/functions/AVX2_FMA.cpp index 0eaacd070..7c11b4c07 100644 --- a/src/VecSim/spaces/functions/AVX2_FMA.cpp +++ b/src/VecSim/spaces/functions/AVX2_FMA.cpp @@ -7,30 +7,25 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX2_FMA.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP32.h" #include "VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP32.h" namespace spaces { -#include "implementation_chooser.h" // FMA optimized implementations dist_func_t Choose_SQ8_FP32_IP_implementation_AVX2_FMA(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_InnerProductSIMD16_AVX2_FMA); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_InnerProductSIMD16_AVX2_FMA; }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_AVX2_FMA(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_CosineSIMD16_AVX2_FMA); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_CosineSIMD16_AVX2_FMA; }); } dist_func_t Choose_SQ8_FP32_L2_implementation_AVX2_FMA(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_L2SqrSIMD16_AVX2_FMA); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_L2SqrSIMD16_AVX2_FMA; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX2_FMA_F16C.cpp b/src/VecSim/spaces/functions/AVX2_FMA_F16C.cpp index 4e9dd8131..a76e217aa 100644 --- a/src/VecSim/spaces/functions/AVX2_FMA_F16C.cpp +++ b/src/VecSim/spaces/functions/AVX2_FMA_F16C.cpp @@ -7,29 +7,23 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX2_FMA_F16C.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_AVX2_FMA_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_AVX2_FMA_SQ8_FP16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_SQ8_FP16_IP_implementation_AVX2_FMA(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_AVX2_FMA); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_InnerProductSIMD16_AVX2_FMA; }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_AVX2_FMA(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_AVX2_FMA); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_CosineSIMD16_AVX2_FMA; }); } dist_func_t Choose_SQ8_FP16_L2_implementation_AVX2_FMA(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_AVX2_FMA); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_L2SqrSIMD16_AVX2_FMA; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512BF16_VL.cpp b/src/VecSim/spaces/functions/AVX512BF16_VL.cpp index 8ed623205..b376d08d2 100644 --- a/src/VecSim/spaces/functions/AVX512BF16_VL.cpp +++ b/src/VecSim/spaces/functions/AVX512BF16_VL.cpp @@ -7,19 +7,15 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX512BF16_VL.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_AVX512_BF16_VL_BF16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_BF16_IP_implementation_AVX512BF16_VL(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_InnerProductSIMD32_AVX512BF16_VL); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_InnerProductSIMD32_AVX512BF16_VL; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512BW_VBMI2.cpp b/src/VecSim/spaces/functions/AVX512BW_VBMI2.cpp index 532cb1882..3de22c46e 100644 --- a/src/VecSim/spaces/functions/AVX512BW_VBMI2.cpp +++ b/src/VecSim/spaces/functions/AVX512BW_VBMI2.cpp @@ -7,26 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX512BW_VBMI2.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_AVX512BW_VBMI2_BF16.h" #include "VecSim/spaces/L2/L2_AVX512BW_VBMI2_BF16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_BF16_IP_implementation_AVX512BW_VBMI2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_InnerProductSIMD32_AVX512BW_VBMI2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_InnerProductSIMD32_AVX512BW_VBMI2; }); } dist_func_t Choose_BF16_L2_implementation_AVX512BW_VBMI2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_L2SqrSIMD32_AVX512BW_VBMI2); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_L2SqrSIMD32_AVX512BW_VBMI2; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512F.cpp b/src/VecSim/spaces/functions/AVX512F.cpp index 0b68d505e..bdc769b34 100644 --- a/src/VecSim/spaces/functions/AVX512F.cpp +++ b/src/VecSim/spaces/functions/AVX512F.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX512F.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_AVX512F_FP16.h" #include "VecSim/spaces/L2/L2_AVX512F_FP32.h" @@ -20,63 +21,50 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP32_IP_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP32_InnerProductSIMD16_AVX512); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP32_InnerProductSIMD16_AVX512; }); } dist_func_t Choose_FP64_IP_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, FP64_InnerProductSIMD8_AVX512); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return FP64_InnerProductSIMD8_AVX512; }); } dist_func_t Choose_FP32_L2_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP32_L2SqrSIMD16_AVX512); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP32_L2SqrSIMD16_AVX512; }); } dist_func_t Choose_FP64_L2_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, FP64_L2SqrSIMD8_AVX512); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return FP64_L2SqrSIMD8_AVX512; }); } dist_func_t Choose_FP16_IP_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_InnerProductSIMD32_AVX512); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_InnerProductSIMD32_AVX512; }); } dist_func_t Choose_FP16_L2_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_L2SqrSIMD32_AVX512); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_L2SqrSIMD32_AVX512; }); } // SQ8↔FP16 kernels only use AVX-512F (cvtph_ps + FMA), so they register here rather than under // the VNNI tier — CPUs with AVX-512F but no VNNI (Skylake-X, some Cascade Lake variants) can use // these kernels. dist_func_t Choose_SQ8_FP16_IP_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_AVX512F); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_InnerProductSIMD16_AVX512F; }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_AVX512F); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_CosineSIMD16_AVX512F; }); } dist_func_t Choose_SQ8_FP16_L2_implementation_AVX512F(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_AVX512F); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_L2SqrSIMD16_AVX512F; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512FP16_VL.cpp b/src/VecSim/spaces/functions/AVX512FP16_VL.cpp index 93549b589..7e3bf213a 100644 --- a/src/VecSim/spaces/functions/AVX512FP16_VL.cpp +++ b/src/VecSim/spaces/functions/AVX512FP16_VL.cpp @@ -7,26 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX512FP16_VL.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_AVX512FP16_VL_FP16.h" #include "VecSim/spaces/L2/L2_AVX512FP16_VL_FP16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP16_IP_implementation_AVX512FP16_VL(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_InnerProductSIMD32_AVX512FP16_VL); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_InnerProductSIMD32_AVX512FP16_VL; }); } dist_func_t Choose_FP16_L2_implementation_AVX512FP16_VL(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_L2SqrSIMD32_AVX512FP16_VL); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_L2SqrSIMD32_AVX512FP16_VL; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp b/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp index 97da55546..0ed47439c 100644 --- a/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp +++ b/src/VecSim/spaces/functions/AVX512F_BW_VL_VNNI.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "AVX512F_BW_VL_VNNI.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_AVX512F_BW_VL_VNNI_INT8.h" #include "VecSim/spaces/IP/IP_AVX512F_BW_VL_VNNI_INT8.h" @@ -22,78 +23,62 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_INT8_L2_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_L2SqrSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_L2SqrSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_INT8_IP_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_INT8_Cosine_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_CosineSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_CosineSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_UINT8_L2_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_L2SqrSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_L2SqrSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_UINT8_IP_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_InnerProductSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_UINT8_Cosine_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_CosineSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_CosineSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_SQ8_FP32_IP_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_InnerProductSIMD16_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_InnerProductSIMD16_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_CosineSIMD16_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_CosineSIMD16_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_SQ8_FP32_L2_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, SQ8_FP32_L2SqrSIMD16_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return SQ8_FP32_L2SqrSIMD16_AVX512F_BW_VL_VNNI; }); } // SQ8-to-SQ8 distance functions (both vectors are uint8 quantized with precomputed sum) dist_func_t Choose_SQ8_SQ8_IP_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_InnerProductSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_InnerProductSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_SQ8_SQ8_Cosine_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_CosineSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_CosineSIMD64_AVX512F_BW_VL_VNNI; }); } dist_func_t Choose_SQ8_SQ8_L2_implementation_AVX512F_BW_VL_VNNI(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_L2SqrSIMD64_AVX512F_BW_VL_VNNI); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_L2SqrSIMD64_AVX512F_BW_VL_VNNI; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/F16C.cpp b/src/VecSim/spaces/functions/F16C.cpp index 7a4127adc..92a8f019e 100644 --- a/src/VecSim/spaces/functions/F16C.cpp +++ b/src/VecSim/spaces/functions/F16C.cpp @@ -7,26 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "F16C.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_F16C_FP16.h" #include "VecSim/spaces/L2/L2_F16C_FP16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP16_IP_implementation_F16C(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_InnerProductSIMD32_F16C); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_InnerProductSIMD32_F16C; }); } dist_func_t Choose_FP16_L2_implementation_F16C(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_L2SqrSIMD32_F16C); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_L2SqrSIMD32_F16C; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON.cpp b/src/VecSim/spaces/functions/NEON.cpp index 0c9a286e3..002b077f7 100644 --- a/src/VecSim/spaces/functions/NEON.cpp +++ b/src/VecSim/spaces/functions/NEON.cpp @@ -7,6 +7,8 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON.h" +#include "VecSim/spaces/functions/residual_dispatch.h" + #include "VecSim/spaces/L2/L2_NEON_FP32.h" #include "VecSim/spaces/IP/IP_NEON_FP32.h" #include "VecSim/spaces/L2/L2_NEON_INT8.h" @@ -22,105 +24,85 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_INT8_IP_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_InnerProductSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_InnerProductSIMD16_NEON; }); } dist_func_t Choose_UINT8_IP_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_InnerProductSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_InnerProductSIMD16_NEON; }); } dist_func_t Choose_FP32_IP_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, FP32_InnerProductSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return FP32_InnerProductSIMD16_NEON; }); } dist_func_t Choose_FP64_IP_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 8, FP64_InnerProductSIMD8_NEON); - return ret_dist_func; + return dispatch_by_residual, 8>( + dim, []() { return FP64_InnerProductSIMD8_NEON; }); } dist_func_t Choose_INT8_Cosine_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_CosineSIMD_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_CosineSIMD_NEON; }); } dist_func_t Choose_UINT8_Cosine_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_CosineSIMD_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_CosineSIMD_NEON; }); } dist_func_t Choose_FP32_L2_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, FP32_L2SqrSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return FP32_L2SqrSIMD16_NEON; }); } dist_func_t Choose_INT8_L2_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_L2SqrSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_L2SqrSIMD16_NEON; }); } dist_func_t Choose_UINT8_L2_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_L2SqrSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_L2SqrSIMD16_NEON; }); } dist_func_t Choose_FP64_L2_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 8, FP64_L2SqrSIMD8_NEON); - return ret_dist_func; + return dispatch_by_residual, 8>( + dim, []() { return FP64_L2SqrSIMD8_NEON; }); } dist_func_t Choose_SQ8_FP32_L2_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP32_L2SqrSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP32_L2SqrSIMD16_NEON; }); } dist_func_t Choose_SQ8_FP32_IP_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP32_InnerProductSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP32_InnerProductSIMD16_NEON; }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP32_CosineSIMD16_NEON); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP32_CosineSIMD16_NEON; }); } // SQ8-to-SQ8 distance functions (both vectors are uint8 quantized with precomputed sum) // Uses 64-element chunking to leverage efficient UINT8_InnerProductImp dist_func_t Choose_SQ8_SQ8_IP_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_InnerProductSIMD64_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_InnerProductSIMD64_NEON; }); } dist_func_t Choose_SQ8_SQ8_Cosine_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_CosineSIMD64_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_CosineSIMD64_NEON; }); } dist_func_t Choose_SQ8_SQ8_L2_implementation_NEON(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_L2SqrSIMD64_NEON); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_L2SqrSIMD64_NEON; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON_BF16.cpp b/src/VecSim/spaces/functions/NEON_BF16.cpp index 4de205bb8..f1bac8694 100644 --- a/src/VecSim/spaces/functions/NEON_BF16.cpp +++ b/src/VecSim/spaces/functions/NEON_BF16.cpp @@ -7,26 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON_BF16.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_NEON_BF16.h" #include "VecSim/spaces/IP/IP_NEON_BF16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_BF16_L2_implementation_NEON_BF16(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_L2Sqr_NEON); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_L2Sqr_NEON; }); } dist_func_t Choose_BF16_IP_implementation_NEON_BF16(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_InnerProduct_NEON); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_InnerProduct_NEON; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON_DOTPROD.cpp b/src/VecSim/spaces/functions/NEON_DOTPROD.cpp index 12f762093..4f38f8c11 100644 --- a/src/VecSim/spaces/functions/NEON_DOTPROD.cpp +++ b/src/VecSim/spaces/functions/NEON_DOTPROD.cpp @@ -7,6 +7,8 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON.h" +#include "VecSim/spaces/functions/residual_dispatch.h" + #include "VecSim/spaces/IP/IP_NEON_DOTPROD_INT8.h" #include "VecSim/spaces/IP/IP_NEON_DOTPROD_UINT8.h" #include "VecSim/spaces/IP/IP_NEON_DOTPROD_SQ8_SQ8.h" @@ -16,63 +18,50 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_INT8_IP_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_InnerProductSIMD16_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_InnerProductSIMD16_NEON_DOTPROD; }); } dist_func_t Choose_UINT8_IP_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_InnerProductSIMD16_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_InnerProductSIMD16_NEON_DOTPROD; }); } dist_func_t Choose_INT8_Cosine_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_CosineSIMD_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_CosineSIMD_NEON_DOTPROD; }); } dist_func_t Choose_UINT8_Cosine_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_CosineSIMD_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_CosineSIMD_NEON_DOTPROD; }); } dist_func_t Choose_INT8_L2_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, INT8_L2SqrSIMD16_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return INT8_L2SqrSIMD16_NEON_DOTPROD; }); } dist_func_t Choose_UINT8_L2_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, UINT8_L2SqrSIMD16_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return UINT8_L2SqrSIMD16_NEON_DOTPROD; }); } // SQ8-to-SQ8 distance functions (both vectors are uint8 quantized with precomputed sum) dist_func_t Choose_SQ8_SQ8_IP_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_InnerProductSIMD64_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_InnerProductSIMD64_NEON_DOTPROD; }); } dist_func_t Choose_SQ8_SQ8_Cosine_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_CosineSIMD64_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_CosineSIMD64_NEON_DOTPROD; }); } dist_func_t Choose_SQ8_SQ8_L2_implementation_NEON_DOTPROD(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 64, SQ8_SQ8_L2SqrSIMD64_NEON_DOTPROD); - return ret_dist_func; + return dispatch_by_residual, 64>( + dim, []() { return SQ8_SQ8_L2SqrSIMD64_NEON_DOTPROD; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/NEON_HP.cpp b/src/VecSim/spaces/functions/NEON_HP.cpp index 15e40ba82..822344e89 100644 --- a/src/VecSim/spaces/functions/NEON_HP.cpp +++ b/src/VecSim/spaces/functions/NEON_HP.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "NEON_HP.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_NEON_FP16.h" #include "VecSim/spaces/IP/IP_NEON_FP16.h" @@ -15,57 +16,45 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP16_L2_implementation_NEON_HP(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_L2Sqr_NEON_HP); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_L2Sqr_NEON_HP; }); } dist_func_t Choose_FP16_IP_implementation_NEON_HP(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, FP16_InnerProduct_NEON_HP); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return FP16_InnerProduct_NEON_HP; }); } dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_HP(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_HP); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_InnerProductSIMD16_NEON_HP; }); } dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_HP(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_HP); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_L2SqrSIMD16_NEON_HP; }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_HP(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_HP); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_CosineSIMD16_NEON_HP; }); } // FMLAL (FEAT_FHM / asimdfhm) variants. dist_func_t Choose_SQ8_FP16_IP_implementation_NEON_FHM(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_NEON_FHM); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_InnerProductSIMD16_NEON_FHM; }); } dist_func_t Choose_SQ8_FP16_L2_implementation_NEON_FHM(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_NEON_FHM); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_L2SqrSIMD16_NEON_FHM; }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_NEON_FHM(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_NEON_FHM); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_CosineSIMD16_NEON_FHM; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SSE.cpp b/src/VecSim/spaces/functions/SSE.cpp index 9eeefe833..3381847ca 100644 --- a/src/VecSim/spaces/functions/SSE.cpp +++ b/src/VecSim/spaces/functions/SSE.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SSE.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_SSE_FP32.h" #include "VecSim/spaces/L2/L2_SSE_FP64.h" @@ -18,32 +19,24 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP32_IP_implementation_SSE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, FP32_InnerProductSIMD16_SSE); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return FP32_InnerProductSIMD16_SSE; }); } dist_func_t Choose_FP64_IP_implementation_SSE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 4, FP64_InnerProductSIMD8_SSE); - return ret_dist_func; + return dispatch_by_residual, 4>( + dim, []() { return FP64_InnerProductSIMD8_SSE; }); } dist_func_t Choose_FP32_L2_implementation_SSE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, FP32_L2SqrSIMD16_SSE); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return FP32_L2SqrSIMD16_SSE; }); } dist_func_t Choose_FP64_L2_implementation_SSE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 4, FP64_L2SqrSIMD8_SSE); - return ret_dist_func; + return dispatch_by_residual, 4>( + dim, []() { return FP64_L2SqrSIMD8_SSE; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SSE3.cpp b/src/VecSim/spaces/functions/SSE3.cpp index 4c60c2e02..5fdd55e98 100644 --- a/src/VecSim/spaces/functions/SSE3.cpp +++ b/src/VecSim/spaces/functions/SSE3.cpp @@ -7,26 +7,21 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SSE3.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_SSE3_BF16.h" #include "VecSim/spaces/L2/L2_SSE3_BF16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_BF16_IP_implementation_SSE3(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_InnerProductSIMD32_SSE3); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_InnerProductSIMD32_SSE3; }); } dist_func_t Choose_BF16_L2_implementation_SSE3(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 32, BF16_L2SqrSIMD32_SSE3); - return ret_dist_func; + return dispatch_by_residual, 32>( + dim, []() { return BF16_L2SqrSIMD32_SSE3; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SSE4.cpp b/src/VecSim/spaces/functions/SSE4.cpp index 5f5bbc1ba..6fdf85d9e 100644 --- a/src/VecSim/spaces/functions/SSE4.cpp +++ b/src/VecSim/spaces/functions/SSE4.cpp @@ -7,31 +7,25 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SSE4.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_SSE4_SQ8_FP32.h" #include "VecSim/spaces/L2/L2_SSE4_SQ8_FP32.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_SQ8_FP32_IP_implementation_SSE4(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP32_InnerProductSIMD16_SSE4); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP32_InnerProductSIMD16_SSE4; }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_SSE4(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP32_CosineSIMD16_SSE4); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP32_CosineSIMD16_SSE4; }); } dist_func_t Choose_SQ8_FP32_L2_implementation_SSE4(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP32_L2SqrSIMD16_SSE4); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP32_L2SqrSIMD16_SSE4; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SSE4_F16C.cpp b/src/VecSim/spaces/functions/SSE4_F16C.cpp index 91a11885f..ac2fad1c5 100644 --- a/src/VecSim/spaces/functions/SSE4_F16C.cpp +++ b/src/VecSim/spaces/functions/SSE4_F16C.cpp @@ -7,29 +7,23 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SSE4_F16C.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_SSE4_SQ8_FP16.h" #include "VecSim/spaces/L2/L2_SSE4_SQ8_FP16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_SQ8_FP16_IP_implementation_SSE4(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_InnerProductSIMD16_SSE4); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_InnerProductSIMD16_SSE4; }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_SSE4(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_CosineSIMD16_SSE4); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_CosineSIMD16_SSE4; }); } dist_func_t Choose_SQ8_FP16_L2_implementation_SSE4(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_IMPLEMENTATION(ret_dist_func, dim, 16, SQ8_FP16_L2SqrSIMD16_SSE4); - return ret_dist_func; + return dispatch_by_residual, 16>( + dim, []() { return SQ8_FP16_L2SqrSIMD16_SSE4; }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SVE.cpp b/src/VecSim/spaces/functions/SVE.cpp index bd197c84c..fec91ae2b 100644 --- a/src/VecSim/spaces/functions/SVE.cpp +++ b/src/VecSim/spaces/functions/SVE.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SVE.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_SVE_FP32.h" #include "VecSim/spaces/IP/IP_SVE_FP32.h" @@ -33,133 +34,150 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP32_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP32_InnerProductSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return FP32_InnerProductSIMD_SVE; + }); } dist_func_t Choose_FP32_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP32_L2SqrSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return FP32_L2SqrSIMD_SVE; + }); } dist_func_t Choose_FP16_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP16_InnerProduct_SVE, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return FP16_InnerProduct_SVE; + }); } dist_func_t Choose_FP16_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP16_L2Sqr_SVE, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return FP16_L2Sqr_SVE; + }); } dist_func_t Choose_FP64_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP64_InnerProductSIMD_SVE, dim, svcntd); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntd(), []() { + return FP64_InnerProductSIMD_SVE; + }); } dist_func_t Choose_FP64_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP64_L2SqrSIMD_SVE, dim, svcntd); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntd(), []() { + return FP64_L2SqrSIMD_SVE; + }); } dist_func_t Choose_INT8_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, INT8_L2SqrSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return INT8_L2SqrSIMD_SVE; + }); } dist_func_t Choose_INT8_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, INT8_InnerProductSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return INT8_InnerProductSIMD_SVE; + }); } dist_func_t Choose_INT8_Cosine_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, INT8_CosineSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return INT8_CosineSIMD_SVE; + }); } dist_func_t Choose_UINT8_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, UINT8_L2SqrSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return UINT8_L2SqrSIMD_SVE; + }); } dist_func_t Choose_UINT8_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, UINT8_InnerProductSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return UINT8_InnerProductSIMD_SVE; + }); } dist_func_t Choose_UINT8_Cosine_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, UINT8_CosineSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return UINT8_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP32_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP32_InnerProductSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP32_InnerProductSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP32_CosineSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP32_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP32_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP32_L2SqrSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP32_L2SqrSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP16_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP16_InnerProductSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP16_InnerProductSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP16_CosineSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP16_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP16_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP16_L2SqrSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP16_L2SqrSIMD_SVE; + }); } // SQ8-to-SQ8 distance functions (both vectors are uint8 quantized with precomputed sum) // Note: Use svcntb for uint8 elements (not svcntw which is for 32-bit elements) dist_func_t Choose_SQ8_SQ8_IP_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_SQ8_InnerProductSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return SQ8_SQ8_InnerProductSIMD_SVE; + }); } dist_func_t Choose_SQ8_SQ8_Cosine_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_SQ8_CosineSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return SQ8_SQ8_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_SQ8_L2_implementation_SVE(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_SQ8_L2SqrSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return SQ8_SQ8_L2SqrSIMD_SVE; + }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SVE2.cpp b/src/VecSim/spaces/functions/SVE2.cpp index 9eea81523..a4bc6b5f9 100644 --- a/src/VecSim/spaces/functions/SVE2.cpp +++ b/src/VecSim/spaces/functions/SVE2.cpp @@ -7,6 +7,7 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SVE2.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/L2/L2_SVE_FP32.h" #include "VecSim/spaces/IP/IP_SVE_FP32.h" @@ -29,133 +30,150 @@ namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_FP32_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP32_InnerProductSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return FP32_InnerProductSIMD_SVE; + }); } dist_func_t Choose_FP32_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP32_L2SqrSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return FP32_L2SqrSIMD_SVE; + }); } dist_func_t Choose_FP16_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP16_InnerProduct_SVE, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return FP16_InnerProduct_SVE; + }); } dist_func_t Choose_FP16_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP16_L2Sqr_SVE, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return FP16_L2Sqr_SVE; + }); } dist_func_t Choose_FP64_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP64_InnerProductSIMD_SVE, dim, svcntd); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntd(), []() { + return FP64_InnerProductSIMD_SVE; + }); } dist_func_t Choose_FP64_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, FP64_L2SqrSIMD_SVE, dim, svcntd); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntd(), []() { + return FP64_L2SqrSIMD_SVE; + }); } dist_func_t Choose_INT8_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, INT8_L2SqrSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return INT8_L2SqrSIMD_SVE; + }); } dist_func_t Choose_INT8_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, INT8_InnerProductSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return INT8_InnerProductSIMD_SVE; + }); } dist_func_t Choose_INT8_Cosine_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, INT8_CosineSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return INT8_CosineSIMD_SVE; + }); } dist_func_t Choose_UINT8_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, UINT8_L2SqrSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return UINT8_L2SqrSIMD_SVE; + }); } dist_func_t Choose_UINT8_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, UINT8_InnerProductSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return UINT8_InnerProductSIMD_SVE; + }); } dist_func_t Choose_UINT8_Cosine_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, UINT8_CosineSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return UINT8_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP32_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP32_InnerProductSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP32_InnerProductSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP32_Cosine_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP32_CosineSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP32_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP32_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP32_L2SqrSIMD_SVE, dim, svcntw); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntw(), []() { + return SQ8_FP32_L2SqrSIMD_SVE; + }); } dist_func_t Choose_SQ8_FP16_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP16_InnerProductSIMD_SVE2, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return SQ8_FP16_InnerProductSIMD_SVE2; + }); } dist_func_t Choose_SQ8_FP16_Cosine_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP16_CosineSIMD_SVE2, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return SQ8_FP16_CosineSIMD_SVE2; + }); } dist_func_t Choose_SQ8_FP16_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_FP16_L2SqrSIMD_SVE2, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return SQ8_FP16_L2SqrSIMD_SVE2; + }); } // SQ8-to-SQ8 distance functions (both vectors are uint8 quantized) // Note: Use svcntb for uint8 elements (not svcntw which is for 32-bit elements) dist_func_t Choose_SQ8_SQ8_IP_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_SQ8_InnerProductSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return SQ8_SQ8_InnerProductSIMD_SVE; + }); } dist_func_t Choose_SQ8_SQ8_Cosine_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_SQ8_CosineSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return SQ8_SQ8_CosineSIMD_SVE; + }); } dist_func_t Choose_SQ8_SQ8_L2_implementation_SVE2(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, SQ8_SQ8_L2SqrSIMD_SVE, dim, svcntb); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcntb(), []() { + return SQ8_SQ8_L2SqrSIMD_SVE; + }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/SVE_BF16.cpp b/src/VecSim/spaces/functions/SVE_BF16.cpp index b457cdb7f..6ba2bc944 100644 --- a/src/VecSim/spaces/functions/SVE_BF16.cpp +++ b/src/VecSim/spaces/functions/SVE_BF16.cpp @@ -7,25 +7,24 @@ * GNU Affero General Public License v3 (AGPLv3). */ #include "SVE_BF16.h" +#include "VecSim/spaces/functions/residual_dispatch.h" #include "VecSim/spaces/IP/IP_SVE_BF16.h" #include "VecSim/spaces/L2/L2_SVE_BF16.h" namespace spaces { -#include "implementation_chooser.h" - dist_func_t Choose_BF16_IP_implementation_SVE_BF16(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, BF16_InnerProduct_SVE, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return BF16_InnerProduct_SVE; + }); } dist_func_t Choose_BF16_L2_implementation_SVE_BF16(size_t dim) { - dist_func_t ret_dist_func; - CHOOSE_SVE_IMPLEMENTATION(ret_dist_func, BF16_L2Sqr_SVE, dim, svcnth); - return ret_dist_func; + return dispatch_by_sve_residual>( + dim, svcnth(), []() { + return BF16_L2Sqr_SVE; + }); } -#include "implementation_chooser_cleanup.h" - } // namespace spaces diff --git a/src/VecSim/spaces/functions/implementation_chooser.h b/src/VecSim/spaces/functions/implementation_chooser.h deleted file mode 100644 index a68907dfd..000000000 --- a/src/VecSim/spaces/functions/implementation_chooser.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2006-Present, Redis Ltd. - * All rights reserved. - * - * Licensed under your choice of the Redis Source Available License 2.0 - * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the - * GNU Affero General Public License v3 (AGPLv3). - */ -#pragma once - -/* - * This file contains macros magic to choose the implementation of a function based on the - * dimension's remainder. It is used to collapse large and repetitive switch statements that are - * used to choose and define the templated values of the implementation of the distance functions. - * We assume that we are dealing with 512-bit blocks, so we define a chunk size of 32 for 16-bit - * elements, 16 for 32-bit elements, and a chunk size of 8 for 64-bit elements. The main macro is - * CHOOSE_IMPLEMENTATION, and it's the one that should be used. - */ - -// Macro for a single case. Sets __ret_dist_func to the function with the given remainder. -#define C1(func, N) \ - case (N): \ - __ret_dist_func = func<(N)>; \ - break; - -// Macros for folding cases of a switch statement, for easier readability. -// Each macro expands into a sequence of cases, from 0 to N-1, doubling the previous macro. -#define C2(func, N) C1(func, 2 * (N)) C1(func, 2 * (N) + 1) -#define C4(func, N) C2(func, 2 * (N)) C2(func, 2 * (N) + 1) -#define C8(func, N) C4(func, 2 * (N)) C4(func, 2 * (N) + 1) -#define C16(func, N) C8(func, 2 * (N)) C8(func, 2 * (N) + 1) -#define C32(func, N) C16(func, 2 * (N)) C16(func, 2 * (N) + 1) -#define C64(func, N) C32(func, 2 * (N)) C32(func, 2 * (N) + 1) - -// Macros for 4, 8, 16, 32 and 64 cases. Used to collapse the switch statement. -// Expands into 0-3, 0-7, 0-15, 0-31 or 0-63 cases respectively. -#define CASES4(func) C4(func, 0) -#define CASES8(func) C8(func, 0) -#define CASES16(func) C16(func, 0) -#define CASES32(func) C32(func, 0) -#define CASES64(func) C64(func, 0) - -// Main macro. Expands into a switch statement that chooses the implementation based on the -// dimension's remainder. -// @params: -// out: The output variable that will be set to the chosen implementation. -// dim: The dimension. -// func: The templated function that we want to choose the implementation for. -// chunk: The chunk size. Can be 64, 32, 16, 8 or 4. Should be the number of elements of the -// expected type fitting in the expected register size. -#define CHOOSE_IMPLEMENTATION(out, dim, chunk, func) \ - do { \ - decltype(out) __ret_dist_func; \ - switch ((dim) % (chunk)) { CASES##chunk(func) default : __builtin_unreachable(); } \ - out = __ret_dist_func; \ - } while (0) - -#define SVE_CASE(base_func, N) \ - case (N): \ - if (partial_chunk) \ - __ret_dist_func = base_func; \ - else \ - __ret_dist_func = base_func; \ - break - -#define CHOOSE_SVE_IMPLEMENTATION(out, base_func, dim, chunk_getter) \ - do { \ - decltype(out) __ret_dist_func; \ - size_t chunk = chunk_getter(); \ - bool partial_chunk = dim % chunk; \ - /* Assuming `base_func` has its main loop for 4 steps */ \ - unsigned char additional_steps = (dim / chunk) % 4; \ - switch (additional_steps) { \ - SVE_CASE(base_func, 0); \ - SVE_CASE(base_func, 1); \ - SVE_CASE(base_func, 2); \ - SVE_CASE(base_func, 3); \ - default: \ - __builtin_unreachable(); \ - } \ - out = __ret_dist_func; \ - } while (0) diff --git a/src/VecSim/spaces/functions/implementation_chooser_cleanup.h b/src/VecSim/spaces/functions/implementation_chooser_cleanup.h deleted file mode 100644 index c51b76098..000000000 --- a/src/VecSim/spaces/functions/implementation_chooser_cleanup.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2006-Present, Redis Ltd. - * All rights reserved. - * - * Licensed under your choice of the Redis Source Available License 2.0 - * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the - * GNU Affero General Public License v3 (AGPLv3). - */ -#pragma once - -/* - * Include this file after done using the implementation chooser. - */ - -#undef C1 -#undef C2 -#undef C4 -#undef C8 -#undef C16 -#undef C32 -#undef C64 - -#undef CASES4 -#undef CASES8 -#undef CASES16 -#undef CASES32 -#undef CASES64 - -#undef SVE_CASE - -#undef CHOOSE_IMPLEMENTATION -#undef CHOOSE_SVE_IMPLEMENTATION diff --git a/src/VecSim/spaces/functions/residual_dispatch.h b/src/VecSim/spaces/functions/residual_dispatch.h new file mode 100644 index 000000000..a3f5047d2 --- /dev/null +++ b/src/VecSim/spaces/functions/residual_dispatch.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ +#pragma once + +#include +#include +#include + +/* + * Picks the compile-time specialization of a residual-templated SIMD kernel — the one matching + * `dim % Chunk` — via a static lookup table built once at compile time, instead of a hand-written + * switch. Every kernel in spaces/IP and spaces/L2 is templated on a "residual" (the leftover + * elements after the main SIMD loop, e.g. `template float + * BF16_InnerProductSIMD32_AVX2(...)`), so there is one compiled specialization per possible + * residual value 0..Chunk-1; this picks the right one at runtime with one array index. + * + * Template arguments, using `Choose_BF16_IP_implementation_AVX2` as a worked example: + * Dist the distance-function pointer type being selected, e.g. `dist_func_t` + * (matches the `Choose_*` function's own return type). + * Chunk the kernel's residual period — how many specializations exist (0..Chunk-1) — e.g. + * `32` for `BF16_InnerProductSIMD32_AVX2` (the "32" in its own name). This must match + * the kernel's own `template ` range; get it from the kernel + * header, not by guessing. + * Selector a templated lambda `[]() { return some_kernel; }` — `some_kernel` is + * the actual kernel being dispatched, one call site's worth of "which function is + * this" living entirely in this one line, not spread across a macro. + * + * Chunk is stated exactly once, as this function's own template argument — the table's size and + * the modulus used to index it are the same Chunk inside the same function, so they cannot drift + * apart the way two independently-written literals at a call site could. The table is `static + * constexpr`, built once at compile time into read-only storage; the only runtime work is + * `dim % Chunk` plus one array load. + */ +namespace spaces { + +template +Dist dispatch_by_residual(size_t dim, Selector selector) { + static constexpr std::array table = [&](std::index_sequence) { + return std::array { selector.template operator()()... }; + }(std::make_index_sequence{}); + return table[dim % Chunk]; +} + +/* + * SVE analog of dispatch_by_residual. SVE kernels are templated on instead of a single residual, because the actual chunk size is + * a runtime property of the hardware's vector length (queried via svcntw()/svcnth()/svcntb()/ + * svcntd(), depending on the kernel's element width), not known at compile time — so the kernel + * can't be templated on "the residual" directly the way the fixed-width x86/NEON kernels are. + * `additional_steps` instead counts how many extra passes of the kernel's own 4-step main loop + * are needed (assumed to be in 0..3), and `partial_chunk` says whether there's a leftover partial + * vector at all. + * + * Template argument: + * Dist same meaning as in dispatch_by_residual — the distance-function pointer type. + * + * Runtime arguments: + * dim the vector dimension, same as elsewhere. + * chunk the hardware's actual vector length for this kernel's element type — the caller + * must pass the right one (e.g. `svcntw()` for 32-bit elements), the same as the old + * macro's chunk_getter argument; this can't be hidden in here since it depends on the + * kernel's element type, not on anything this function knows. + * selector a templated lambda `[]() { return some_kernel; }`. + * + * The "4 steps" assumption is not a separately-written literal here — the modulus for + * additional_steps is derived from the table's own row length, so it can't drift from the shape + * the table was actually built with. + */ +template +Dist dispatch_by_sve_residual(size_t dim, size_t chunk, Selector selector) { + static constexpr std::array full_chunk_row = + [&](std::index_sequence) { + return std::array { selector.template operator()()... }; + }(std::make_index_sequence<4>{}); + static constexpr std::array partial_chunk_row = + [&](std::index_sequence) { + return std::array { selector.template operator()()... }; + }(std::make_index_sequence<4>{}); + bool partial_chunk = dim % chunk; + size_t additional_steps = (dim / chunk) % full_chunk_row.size(); + return partial_chunk ? partial_chunk_row[additional_steps] : full_chunk_row[additional_steps]; +} + +} // namespace spaces