From 0026d4f27bc9158fbe4cd6e3a9dc7b01c17e7ecb Mon Sep 17 00:00:00 2001 From: Manodeep Sinha Date: Thu, 13 Aug 2026 22:57:10 +1000 Subject: [PATCH 1/6] Removed avx512_calls.c from the Makefiles --- mocks/DDrppi_mocks/Makefile | 2 +- mocks/DDsmu_mocks/Makefile | 2 +- mocks/DDtheta_mocks/Makefile | 2 +- mocks/vpf_mocks/Makefile | 2 +- theory/DD/Makefile | 2 +- theory/DDrppi/Makefile | 2 +- theory/DDsmu/Makefile | 2 +- theory/vpf/Makefile | 2 +- theory/wp/Makefile | 2 +- theory/xi/Makefile | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mocks/DDrppi_mocks/Makefile b/mocks/DDrppi_mocks/Makefile index 8c36b65f..f6e286dc 100644 --- a/mocks/DDrppi_mocks/Makefile +++ b/mocks/DDrppi_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_rp_pi_mocks.c countpairs_rp_pi_mocks_impl_double.c countpairs_rp_pi_mocks_impl_float.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/DDsmu_mocks/Makefile b/mocks/DDsmu_mocks/Makefile index 3642b932..f5ca84f1 100644 --- a/mocks/DDsmu_mocks/Makefile +++ b/mocks/DDsmu_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_s_mu_mocks.c countpairs_s_mu_mocks_impl_double.c countpairs_s_mu_mocks_impl_float.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/DDtheta_mocks/Makefile b/mocks/DDtheta_mocks/Makefile index 0ef90bbf..61a31924 100644 --- a/mocks/DDtheta_mocks/Makefile +++ b/mocks/DDtheta_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_theta_mocks.c countpairs_theta_mocks_impl_float.c countpairs_theta_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := DDtheta_mocks diff --git a/mocks/vpf_mocks/Makefile b/mocks/vpf_mocks/Makefile index 808cc938..40df0a99 100644 --- a/mocks/vpf_mocks/Makefile +++ b/mocks/vpf_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY:=lib$(LIBNAME).a LIBSRC:=countspheres_mocks.c countspheres_mocks_impl_float.c countspheres_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_impl_float.c $(UTILS_DIR)/gridlink_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/theory/DD/Makefile b/theory/DD/Makefile index 330b9770..d4e0985b 100644 --- a/theory/DD/Makefile +++ b/theory/DD/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs.c countpairs_impl_double.c countpairs_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := DD diff --git a/theory/DDrppi/Makefile b/theory/DDrppi/Makefile index 0f6f5c16..7555a8db 100644 --- a/theory/DDrppi/Makefile +++ b/theory/DDrppi/Makefile @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_rp_pi.a LIBSRC := countpairs_rp_pi.c countpairs_rp_pi_impl_double.c countpairs_rp_pi_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := countpairs_rp_pi.h TARGETSRC := DDrppi.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/DDsmu/Makefile b/theory/DDsmu/Makefile index d38d0947..76168c66 100644 --- a/theory/DDsmu/Makefile +++ b/theory/DDsmu/Makefile @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_s_mu.a LIBSRC := countpairs_s_mu.c countpairs_s_mu_impl_double.c countpairs_s_mu_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := countpairs_s_mu.h TARGETSRC := DDsmu.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/vpf/Makefile b/theory/vpf/Makefile index ad3de9fc..c02c3dab 100644 --- a/theory/vpf/Makefile +++ b/theory/vpf/Makefile @@ -14,7 +14,7 @@ LIBSRC := countspheres.c countspheres_impl_double.c countspheres_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c \ - $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/cpu_features.c TARGET := vpf TARGETSRC := $(TARGET).c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/wp/Makefile b/theory/wp/Makefile index cba558a4..0a6bbc1c 100644 --- a/theory/wp/Makefile +++ b/theory/wp/Makefile @@ -11,7 +11,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_wp.c countpairs_wp_impl_double.c countpairs_wp_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := wp diff --git a/theory/xi/Makefile b/theory/xi/Makefile index 153cefa8..1a4e1926 100644 --- a/theory/xi/Makefile +++ b/theory/xi/Makefile @@ -13,7 +13,7 @@ LIBRARY_HEADERS := $(LIBNAME).h LIBSRC := countpairs_xi.c countpairs_xi_impl_double.c countpairs_xi_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c TARGET := xi TARGETSRC := $(TARGET).c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) From 9c3e9e3fd182e23168f8874f90d442cf4a53ee90 Mon Sep 17 00:00:00 2001 From: Manodeep Sinha Date: Fri, 14 Aug 2026 22:13:42 +1000 Subject: [PATCH 2/6] Replaced the two lookup tables with a popcnt and a bit-shifting method to calculate load-mask (taken from simSIMD) --- mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src | 2 +- mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src | 2 +- mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src | 4 ++-- mocks/vpf_mocks/vpf_mocks_kernels.c.src | 6 +++--- theory/DD/countpairs_kernels.c.src | 4 ++-- theory/DDrppi/countpairs_rp_pi_kernels.c.src | 2 +- theory/DDsmu/countpairs_s_mu_kernels.c.src | 2 +- theory/vpf/vpf_kernels.c.src | 6 +++--- theory/wp/wp_kernels.c.src | 4 ++-- theory/xi/xi_kernels.c.src | 4 ++-- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src b/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src index 407b2ef9..a61ab06c 100644 --- a/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src +++ b/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src @@ -178,7 +178,7 @@ static inline int countpairs_rp_pi_mocks_avx512_intrinsics_DOUBLE(const int64_t const AVX512_FLOATS m_sqr_rpmin = AVX512_SET_FLOAT(sqr_rpmin); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src b/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src index 643fe83b..c6b0e4bf 100644 --- a/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src +++ b/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src @@ -172,7 +172,7 @@ static inline int countpairs_s_mu_mocks_avx512_intrinsics_DOUBLE(const int64_t N const AVX512_FLOATS m_sqr_smin = AVX512_SET_FLOAT(sqr_smin); const AVX512_FLOATS m_sqr_mumax = AVX512_SET_FLOAT(sqr_mumax); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src b/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src index acac3954..13691d4e 100644 --- a/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src +++ b/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src @@ -1027,7 +1027,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t const AVX512_FLOATS m_max_dz = AVX512_SET_FLOAT(max_dz); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -1099,7 +1099,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t // at the beginning of the loop. for(int kbin=nthetabin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_costheta,m_costheta_upp[kbin-1],_CMP_LE_OQ); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_thetaavg || need_weightavg) { m_thetabin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_thetabin, AVX512_SET_INT(kbin)); } diff --git a/mocks/vpf_mocks/vpf_mocks_kernels.c.src b/mocks/vpf_mocks/vpf_mocks_kernels.c.src index b559ef76..79555011 100644 --- a/mocks/vpf_mocks/vpf_mocks_kernels.c.src +++ b/mocks/vpf_mocks/vpf_mocks_kernels.c.src @@ -53,7 +53,7 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j]; const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); @@ -79,12 +79,12 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS); const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS); const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2); - counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y if(m_mask_left == 0) { break; } else if(k==1){ - counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left]; + counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left); } } } diff --git a/theory/DD/countpairs_kernels.c.src b/theory/DD/countpairs_kernels.c.src index c841fab8..83dcc0c9 100644 --- a/theory/DD/countpairs_kernels.c.src +++ b/theory/DD/countpairs_kernels.c.src @@ -147,7 +147,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE * const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -222,7 +222,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE * // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } diff --git a/theory/DDrppi/countpairs_rp_pi_kernels.c.src b/theory/DDrppi/countpairs_rp_pi_kernels.c.src index b0af4624..45a5a62f 100644 --- a/theory/DDrppi/countpairs_rp_pi_kernels.c.src +++ b/theory/DDrppi/countpairs_rp_pi_kernels.c.src @@ -162,7 +162,7 @@ static inline int countpairs_rp_pi_avx512_intrinsics_DOUBLE(const int64_t N0, DO } for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/theory/DDsmu/countpairs_s_mu_kernels.c.src b/theory/DDsmu/countpairs_s_mu_kernels.c.src index cd590cca..9e567b24 100644 --- a/theory/DDsmu/countpairs_s_mu_kernels.c.src +++ b/theory/DDsmu/countpairs_s_mu_kernels.c.src @@ -178,7 +178,7 @@ static inline int countpairs_s_mu_avx512_intrinsics_DOUBLE(const int64_t N0, DOU } for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/theory/vpf/vpf_kernels.c.src b/theory/vpf/vpf_kernels.c.src index 6e0f74c3..c173e99f 100644 --- a/theory/vpf/vpf_kernels.c.src +++ b/theory/vpf/vpf_kernels.c.src @@ -48,7 +48,7 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j]; const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); @@ -75,12 +75,12 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS); const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS); const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2); - counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y if(m_mask_left == 0) { break; } else if(k==1){ - counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left]; + counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left); } } } diff --git a/theory/wp/wp_kernels.c.src b/theory/wp/wp_kernels.c.src index 06b6d172..265791ce 100644 --- a/theory/wp/wp_kernels.c.src +++ b/theory/wp/wp_kernels.c.src @@ -156,7 +156,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -244,7 +244,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } diff --git a/theory/xi/xi_kernels.c.src b/theory/xi/xi_kernels.c.src index a3c404d6..e6ce2aef 100644 --- a/theory/xi/xi_kernels.c.src +++ b/theory/xi/xi_kernels.c.src @@ -146,7 +146,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N2-j]; + AVX512_MASK m_mask_left = (N2 - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[N2-j]; const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz2); @@ -218,7 +218,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OQ); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } From 5a96e4b4c90758a7038e48af872f1cb7c47ea9e3 Mon Sep 17 00:00:00 2001 From: Manodeep Sinha Date: Fri, 14 Aug 2026 22:14:57 +1000 Subject: [PATCH 3/6] Removed the source and removed unused macros and function prototypes from the header --- utils/avx512_calls.c | 40 ---------------------------------------- utils/avx512_calls.h | 18 ------------------ 2 files changed, 58 deletions(-) delete mode 100644 utils/avx512_calls.c diff --git a/utils/avx512_calls.c b/utils/avx512_calls.c deleted file mode 100644 index 6f3f4f6e..00000000 --- a/utils/avx512_calls.c +++ /dev/null @@ -1,40 +0,0 @@ -/* File: avx512_calls.c */ -/* - This file is a part of the Corrfunc package - Copyright (C) 2015-- Manodeep Sinha (manodeep@gmail.com) - License: MIT LICENSE. See LICENSE file under the top-level - directory at https://github.com/manodeep/Corrfunc/ -*/ -#if defined(__AVX512F__) -#include "avx512_calls.h" - - -const int64_t bits_set_in_avx512_mask_float[] = { B16(0) }; -const uint16_t masks_per_misalignment_value_float[] = {0b1111111111111111, - 0b0000000000000001, - 0b0000000000000011, - 0b0000000000000111, - 0b0000000000001111, - 0b0000000000011111, - 0b0000000000111111, - 0b0000000001111111, - 0b0000000011111111, - 0b0000000111111111, - 0b0000001111111111, - 0b0000011111111111, - 0b0000111111111111, - 0b0001111111111111, - 0b0011111111111111, - 0b0111111111111111}; - -const int64_t bits_set_in_avx512_mask_double[] = { B8(0) }; -const uint8_t masks_per_misalignment_value_double[] = {0b11111111, - 0b00000001, - 0b00000011, - 0b00000111, - 0b00001111, - 0b00011111, - 0b00111111, - 0b01111111}; - -#endif diff --git a/utils/avx512_calls.h b/utils/avx512_calls.h index 5ff60f73..2b81296c 100644 --- a/utils/avx512_calls.h +++ b/utils/avx512_calls.h @@ -42,18 +42,6 @@ extern "C" { #define AVX512_MASK_BITWISE_NOT(X,Y) _mm512_knot(X) //~X - - /* For setting up the array that contains the number bits set in a mask */ - // Can be used to generate up to 16 bit lookup tables -# define B2(n) n, n+1, n+1, n+2 -# define B4(n) B2(n), B2(n+1), B2(n+1), B2(n+2) -# define B6(n) B4(n), B4(n+1), B4(n+1), B4(n+2) -# define B8(n) B6(n), B6(n+1), B6(n+1), B6(n+2) -# define B10(n) B8(n), B8(n+1), B8(n+1), B8(n+2) -# define B12(n) B10(n), B10(n+1), B10(n+1), B10(n+2) -# define B14(n) B12(n), B12(n+1), B12(n+1), B12(n+2) -# define B16(n) B14(0),B14(1), B14(1), B14(2) - #ifndef DOUBLE_PREC #define DOUBLE float @@ -366,12 +354,6 @@ static inline AVX512_FLOATS inv_cosine_avx512(const AVX512_FLOATS X, const int o #endif - extern const int64_t bits_set_in_avx512_mask_float[]; - extern const uint16_t masks_per_misalignment_value_float[]; - - extern const int64_t bits_set_in_avx512_mask_double[]; - extern const uint8_t masks_per_misalignment_value_double[]; - union int16 { AVX512_INTS m_ibin; int ibin[AVX512_NVEC]; From ed2db01e3bfe3f09407b80ecdfeaf2458a681639 Mon Sep 17 00:00:00 2001 From: Manodeep Sinha Date: Sat, 15 Aug 2026 07:13:16 +1000 Subject: [PATCH 4/6] Fixed the few instances of masks_per that I overlooked during the whitespace fixing --- mocks/vpf_mocks/vpf_mocks_kernels.c.src | 2 +- theory/vpf/vpf_kernels.c.src | 2 +- theory/xi/xi_kernels.c.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mocks/vpf_mocks/vpf_mocks_kernels.c.src b/mocks/vpf_mocks/vpf_mocks_kernels.c.src index 79555011..bf153df1 100644 --- a/mocks/vpf_mocks/vpf_mocks_kernels.c.src +++ b/mocks/vpf_mocks/vpf_mocks_kernels.c.src @@ -53,7 +53,7 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(np - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); diff --git a/theory/vpf/vpf_kernels.c.src b/theory/vpf/vpf_kernels.c.src index c173e99f..b1bc5a73 100644 --- a/theory/vpf/vpf_kernels.c.src +++ b/theory/vpf/vpf_kernels.c.src @@ -48,7 +48,7 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(np - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); diff --git a/theory/xi/xi_kernels.c.src b/theory/xi/xi_kernels.c.src index e6ce2aef..0dfcef8a 100644 --- a/theory/xi/xi_kernels.c.src +++ b/theory/xi/xi_kernels.c.src @@ -146,7 +146,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[N2-j]; + AVX512_MASK m_mask_left = (N2 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz2); From 2ae4c011b56ea4b62c70c72ffff37c375ed35218 Mon Sep 17 00:00:00 2001 From: Manodeep Sinha Date: Sat, 15 Aug 2026 13:09:37 +1000 Subject: [PATCH 5/6] Fixed typo --- theory/xi/xi_kernels.c.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theory/xi/xi_kernels.c.src b/theory/xi/xi_kernels.c.src index 0dfcef8a..71372fc1 100644 --- a/theory/xi/xi_kernels.c.src +++ b/theory/xi/xi_kernels.c.src @@ -146,7 +146,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); + AVX512_MASK m_mask_left = (N2 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N2 - j)) - 1U); const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz2); From b18178d3a5bfb92207ca8bdfa94adf0d03670399 Mon Sep 17 00:00:00 2001 From: Manodeep Sinha Date: Sun, 16 Aug 2026 22:32:38 +1000 Subject: [PATCH 6/6] Forgot to remove avx512_calls.c from the utils/Makefile (but didn't get caught since utils/ Makefiles never get invoked) --- utils/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/Makefile b/utils/Makefile index 182aa247..2b48145f 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -8,7 +8,7 @@ ROOT_DIR := .. include $(ROOT_DIR)/common.mk TARGETSRC := cosmology_params.c gridlink_impl_double.c gridlink_impl_float.c gridlink_mocks_impl_float.c \ gridlink_mocks_impl_double.c gridlink_utils_double.c gridlink_utils_float.c \ - progressbar.c set_cosmo_dist.c utils.c cpu_features.c avx512_calls.c + progressbar.c set_cosmo_dist.c utils.c cpu_features.c TARGETOBJS := $(TARGETSRC:.c=.o) INCL := avx512_calls.h avx2_calls.h avx_calls.h sse_calls.h defs.h defs.h function_precision.h cosmology_params.h \ cellarray_float.h cellarray_double.h cellarray.h.src \