diff --git a/mocks/DDrppi_mocks/Makefile b/mocks/DDrppi_mocks/Makefile index 8c36b65f..f6e286dc 100644 --- a/mocks/DDrppi_mocks/Makefile +++ b/mocks/DDrppi_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_rp_pi_mocks.c countpairs_rp_pi_mocks_impl_double.c countpairs_rp_pi_mocks_impl_float.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src b/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src index 407b2ef9..a61ab06c 100644 --- a/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src +++ b/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src @@ -178,7 +178,7 @@ static inline int countpairs_rp_pi_mocks_avx512_intrinsics_DOUBLE(const int64_t const AVX512_FLOATS m_sqr_rpmin = AVX512_SET_FLOAT(sqr_rpmin); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/mocks/DDsmu_mocks/Makefile b/mocks/DDsmu_mocks/Makefile index 3642b932..f5ca84f1 100644 --- a/mocks/DDsmu_mocks/Makefile +++ b/mocks/DDsmu_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_s_mu_mocks.c countpairs_s_mu_mocks_impl_double.c countpairs_s_mu_mocks_impl_float.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src b/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src index 643fe83b..c6b0e4bf 100644 --- a/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src +++ b/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src @@ -172,7 +172,7 @@ static inline int countpairs_s_mu_mocks_avx512_intrinsics_DOUBLE(const int64_t N const AVX512_FLOATS m_sqr_smin = AVX512_SET_FLOAT(sqr_smin); const AVX512_FLOATS m_sqr_mumax = AVX512_SET_FLOAT(sqr_mumax); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/mocks/DDtheta_mocks/Makefile b/mocks/DDtheta_mocks/Makefile index 0ef90bbf..61a31924 100644 --- a/mocks/DDtheta_mocks/Makefile +++ b/mocks/DDtheta_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_theta_mocks.c countpairs_theta_mocks_impl_float.c countpairs_theta_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := DDtheta_mocks diff --git a/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src b/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src index acac3954..13691d4e 100644 --- a/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src +++ b/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src @@ -1027,7 +1027,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t const AVX512_FLOATS m_max_dz = AVX512_SET_FLOAT(max_dz); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -1099,7 +1099,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t // at the beginning of the loop. for(int kbin=nthetabin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_costheta,m_costheta_upp[kbin-1],_CMP_LE_OQ); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_thetaavg || need_weightavg) { m_thetabin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_thetabin, AVX512_SET_INT(kbin)); } diff --git a/mocks/vpf_mocks/Makefile b/mocks/vpf_mocks/Makefile index 808cc938..40df0a99 100644 --- a/mocks/vpf_mocks/Makefile +++ b/mocks/vpf_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY:=lib$(LIBNAME).a LIBSRC:=countspheres_mocks.c countspheres_mocks_impl_float.c countspheres_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_impl_float.c $(UTILS_DIR)/gridlink_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/vpf_mocks/vpf_mocks_kernels.c.src b/mocks/vpf_mocks/vpf_mocks_kernels.c.src index b559ef76..bf153df1 100644 --- a/mocks/vpf_mocks/vpf_mocks_kernels.c.src +++ b/mocks/vpf_mocks/vpf_mocks_kernels.c.src @@ -53,7 +53,7 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(np - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); @@ -79,12 +79,12 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS); const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS); const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2); - counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y if(m_mask_left == 0) { break; } else if(k==1){ - counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left]; + counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left); } } } diff --git a/theory/DD/Makefile b/theory/DD/Makefile index 330b9770..d4e0985b 100644 --- a/theory/DD/Makefile +++ b/theory/DD/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs.c countpairs_impl_double.c countpairs_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := DD diff --git a/theory/DD/countpairs_kernels.c.src b/theory/DD/countpairs_kernels.c.src index c841fab8..83dcc0c9 100644 --- a/theory/DD/countpairs_kernels.c.src +++ b/theory/DD/countpairs_kernels.c.src @@ -147,7 +147,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE * const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -222,7 +222,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE * // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } diff --git a/theory/DDrppi/Makefile b/theory/DDrppi/Makefile index 0f6f5c16..7555a8db 100644 --- a/theory/DDrppi/Makefile +++ b/theory/DDrppi/Makefile @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_rp_pi.a LIBSRC := countpairs_rp_pi.c countpairs_rp_pi_impl_double.c countpairs_rp_pi_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := countpairs_rp_pi.h TARGETSRC := DDrppi.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/DDrppi/countpairs_rp_pi_kernels.c.src b/theory/DDrppi/countpairs_rp_pi_kernels.c.src index b0af4624..45a5a62f 100644 --- a/theory/DDrppi/countpairs_rp_pi_kernels.c.src +++ b/theory/DDrppi/countpairs_rp_pi_kernels.c.src @@ -162,7 +162,7 @@ static inline int countpairs_rp_pi_avx512_intrinsics_DOUBLE(const int64_t N0, DO } for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/theory/DDsmu/Makefile b/theory/DDsmu/Makefile index d38d0947..76168c66 100644 --- a/theory/DDsmu/Makefile +++ b/theory/DDsmu/Makefile @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_s_mu.a LIBSRC := countpairs_s_mu.c countpairs_s_mu_impl_double.c countpairs_s_mu_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := countpairs_s_mu.h TARGETSRC := DDsmu.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/DDsmu/countpairs_s_mu_kernels.c.src b/theory/DDsmu/countpairs_s_mu_kernels.c.src index cd590cca..9e567b24 100644 --- a/theory/DDsmu/countpairs_s_mu_kernels.c.src +++ b/theory/DDsmu/countpairs_s_mu_kernels.c.src @@ -178,7 +178,7 @@ static inline int countpairs_s_mu_avx512_intrinsics_DOUBLE(const int64_t N0, DOU } for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/theory/vpf/Makefile b/theory/vpf/Makefile index ad3de9fc..c02c3dab 100644 --- a/theory/vpf/Makefile +++ b/theory/vpf/Makefile @@ -14,7 +14,7 @@ LIBSRC := countspheres.c countspheres_impl_double.c countspheres_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c \ - $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/cpu_features.c TARGET := vpf TARGETSRC := $(TARGET).c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/vpf/vpf_kernels.c.src b/theory/vpf/vpf_kernels.c.src index 6e0f74c3..b1bc5a73 100644 --- a/theory/vpf/vpf_kernels.c.src +++ b/theory/vpf/vpf_kernels.c.src @@ -48,7 +48,7 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(np - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); @@ -75,12 +75,12 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS); const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS); const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2); - counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y if(m_mask_left == 0) { break; } else if(k==1){ - counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left]; + counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left); } } } diff --git a/theory/wp/Makefile b/theory/wp/Makefile index cba558a4..0a6bbc1c 100644 --- a/theory/wp/Makefile +++ b/theory/wp/Makefile @@ -11,7 +11,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_wp.c countpairs_wp_impl_double.c countpairs_wp_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := wp diff --git a/theory/wp/wp_kernels.c.src b/theory/wp/wp_kernels.c.src index 06b6d172..265791ce 100644 --- a/theory/wp/wp_kernels.c.src +++ b/theory/wp/wp_kernels.c.src @@ -156,7 +156,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -244,7 +244,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } diff --git a/theory/xi/Makefile b/theory/xi/Makefile index 153cefa8..1a4e1926 100644 --- a/theory/xi/Makefile +++ b/theory/xi/Makefile @@ -13,7 +13,7 @@ LIBRARY_HEADERS := $(LIBNAME).h LIBSRC := countpairs_xi.c countpairs_xi_impl_double.c countpairs_xi_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c TARGET := xi TARGETSRC := $(TARGET).c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/xi/xi_kernels.c.src b/theory/xi/xi_kernels.c.src index a3c404d6..71372fc1 100644 --- a/theory/xi/xi_kernels.c.src +++ b/theory/xi/xi_kernels.c.src @@ -146,7 +146,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N2-j]; + AVX512_MASK m_mask_left = (N2 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N2 - j)) - 1U); const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz2); @@ -218,7 +218,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OQ); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } diff --git a/utils/Makefile b/utils/Makefile index 182aa247..2b48145f 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -8,7 +8,7 @@ ROOT_DIR := .. include $(ROOT_DIR)/common.mk TARGETSRC := cosmology_params.c gridlink_impl_double.c gridlink_impl_float.c gridlink_mocks_impl_float.c \ gridlink_mocks_impl_double.c gridlink_utils_double.c gridlink_utils_float.c \ - progressbar.c set_cosmo_dist.c utils.c cpu_features.c avx512_calls.c + progressbar.c set_cosmo_dist.c utils.c cpu_features.c TARGETOBJS := $(TARGETSRC:.c=.o) INCL := avx512_calls.h avx2_calls.h avx_calls.h sse_calls.h defs.h defs.h function_precision.h cosmology_params.h \ cellarray_float.h cellarray_double.h cellarray.h.src \ diff --git a/utils/avx512_calls.c b/utils/avx512_calls.c deleted file mode 100644 index 6f3f4f6e..00000000 --- a/utils/avx512_calls.c +++ /dev/null @@ -1,40 +0,0 @@ -/* File: avx512_calls.c */ -/* - This file is a part of the Corrfunc package - Copyright (C) 2015-- Manodeep Sinha (manodeep@gmail.com) - License: MIT LICENSE. See LICENSE file under the top-level - directory at https://github.com/manodeep/Corrfunc/ -*/ -#if defined(__AVX512F__) -#include "avx512_calls.h" - - -const int64_t bits_set_in_avx512_mask_float[] = { B16(0) }; -const uint16_t masks_per_misalignment_value_float[] = {0b1111111111111111, - 0b0000000000000001, - 0b0000000000000011, - 0b0000000000000111, - 0b0000000000001111, - 0b0000000000011111, - 0b0000000000111111, - 0b0000000001111111, - 0b0000000011111111, - 0b0000000111111111, - 0b0000001111111111, - 0b0000011111111111, - 0b0000111111111111, - 0b0001111111111111, - 0b0011111111111111, - 0b0111111111111111}; - -const int64_t bits_set_in_avx512_mask_double[] = { B8(0) }; -const uint8_t masks_per_misalignment_value_double[] = {0b11111111, - 0b00000001, - 0b00000011, - 0b00000111, - 0b00001111, - 0b00011111, - 0b00111111, - 0b01111111}; - -#endif diff --git a/utils/avx512_calls.h b/utils/avx512_calls.h index 5ff60f73..2b81296c 100644 --- a/utils/avx512_calls.h +++ b/utils/avx512_calls.h @@ -42,18 +42,6 @@ extern "C" { #define AVX512_MASK_BITWISE_NOT(X,Y) _mm512_knot(X) //~X - - /* For setting up the array that contains the number bits set in a mask */ - // Can be used to generate up to 16 bit lookup tables -# define B2(n) n, n+1, n+1, n+2 -# define B4(n) B2(n), B2(n+1), B2(n+1), B2(n+2) -# define B6(n) B4(n), B4(n+1), B4(n+1), B4(n+2) -# define B8(n) B6(n), B6(n+1), B6(n+1), B6(n+2) -# define B10(n) B8(n), B8(n+1), B8(n+1), B8(n+2) -# define B12(n) B10(n), B10(n+1), B10(n+1), B10(n+2) -# define B14(n) B12(n), B12(n+1), B12(n+1), B12(n+2) -# define B16(n) B14(0),B14(1), B14(1), B14(2) - #ifndef DOUBLE_PREC #define DOUBLE float @@ -366,12 +354,6 @@ static inline AVX512_FLOATS inv_cosine_avx512(const AVX512_FLOATS X, const int o #endif - extern const int64_t bits_set_in_avx512_mask_float[]; - extern const uint16_t masks_per_misalignment_value_float[]; - - extern const int64_t bits_set_in_avx512_mask_double[]; - extern const uint8_t masks_per_misalignment_value_double[]; - union int16 { AVX512_INTS m_ibin; int ibin[AVX512_NVEC];