From bc7dd851e2aeded923b7c42d95a8bdca1467df64 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 23 Aug 2026 12:18:50 +0200 Subject: [PATCH] x86: on targets that requires SSE, use those registers for ABI --- compiler/rustc_target/src/callconv/mod.rs | 35 ++-- tests/codegen-llvm/README.md | 13 +- tests/codegen-llvm/abi-x86-sse.rs | 5 +- tests/codegen-llvm/debuginfo-dse.rs | 12 +- .../pclmulqdq-target-feature-inlining.rs | 2 +- ...intrinsic-generic-arithmetic-saturating.rs | 160 +++++++++--------- .../simd-intrinsic-transmute-array.rs | 44 ++--- .../simd/packed-simd-alignment.rs | 4 +- tests/codegen-llvm/simd/packed-simd.rs | 20 ++- tests/codegen-llvm/simd/splat.rs | 26 ++- 10 files changed, 185 insertions(+), 136 deletions(-) diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index a06a6a0a69e12..6d3dc0e51973d 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -8,7 +8,7 @@ use rustc_abi::{ use rustc_macros::StableHash; pub use crate::spec::AbiMap; -use crate::spec::{Arch, HasTargetSpec, HasX86AbiOpt}; +use crate::spec::{Arch, HasTargetSpec, HasX86AbiOpt, RustcAbi}; mod aarch64; mod amdgpu; @@ -744,6 +744,23 @@ impl<'a, Ty> FnAbi<'a, Ty> { _ => {} }; + // Decides whether we can pass the given SIMD argument via `PassMode::Direct`. + // May only return `true` if the target will always pass those arguments the same way, + // no matter what the user does with `-Ctarget-feature`! In other words, whatever + // target features are required to pass a SIMD value in registers must be listed in + // the `abi_required_features` for the current target and ABI. + let can_pass_simd_directly = |arg: &ArgAbi<'_, Ty>| match &spec.arch { + // On x86, if we have SSE2 (which we have by default for x86_64), we can always pass up + // to 128-bit-sized vectors. + Arch::X86 if spec.rustc_abi == Some(RustcAbi::X86Sse2) => arg.layout.size.bits() <= 128, + Arch::X86_64 if spec.rustc_abi != Some(RustcAbi::Softfloat) => { + // x86-64 non-softfloat targets all require SSE2 so we can use SSE registers. + arg.layout.size.bits() <= 128 + } + // So far, we haven't implemented this logic for any other target. + _ => false, + }; + for (arg_idx, arg) in self .args .iter_mut() @@ -861,16 +878,12 @@ impl<'a, Ty> FnAbi<'a, Ty> { // enabled but the callee does, then passing an AVX argument // across this boundary would cause corrupt data to show up. // - // This problem is fixed by unconditionally passing SIMD - // arguments through memory between callers and callees - // which should get them all to agree on ABI regardless of - // target feature sets. Some more information about this - // issue can be found in #44367. - // - // We *could* do better in some cases, e.g. on x86_64 targets where SSE2 is - // required. However, it turns out that that makes LLVM worse at optimizing this - // code, so we pass things indirectly even there. See #139029 for more on that. - if spec.simd_types_indirect { + // This problem is fixed by passing most SIMD arguments through memory between + // callers and callees which should get them all to agree on ABI regardless of + // target feature sets, except for those that rely on target features that we + // know to be always available. Some more information about this issue can be + // found in #44367 and the comment on `can_pass_simd_directly`. + if spec.simd_types_indirect && !can_pass_simd_directly(arg) { arg.make_indirect(); } } diff --git a/tests/codegen-llvm/README.md b/tests/codegen-llvm/README.md index 8f2daaafcc77a..e3d0f4746ec12 100644 --- a/tests/codegen-llvm/README.md +++ b/tests/codegen-llvm/README.md @@ -8,7 +8,7 @@ revisions annotation, like so: ```rust // revisions: aaa bbb -// [bbb] compile-flags: --flags-for-bbb +// [bbb] compile-flags: -Csomething=true ``` After specifying those variations, you can write different expected, or @@ -22,3 +22,14 @@ like so: // bbb-NOT: emitted-only-for-aaa // bbb-SAME: emitted-only-for-bbb ``` + +If multiple test revisions want to share some annotations, you can use a setup like this: + +```rust +// revisions: aaa bbb ccc +// [bbb] filecheck-flags: --check-prefix=both +// [ccc] filecheck-flags: --check-prefix=both +``` + +Now `aaa:`, `bbb:`, `ccc:` annotations are only check for those specific revisions, +and `both:` annotations are checked for revisions `bbb` and `ccc`. diff --git a/tests/codegen-llvm/abi-x86-sse.rs b/tests/codegen-llvm/abi-x86-sse.rs index 68d2acfb527b8..21b5eff4f489b 100644 --- a/tests/codegen-llvm/abi-x86-sse.rs +++ b/tests/codegen-llvm/abi-x86-sse.rs @@ -33,9 +33,8 @@ trait Copy {} #[repr(simd)] pub struct Sse([f32; 4]); -// FIXME: due to #139029 we are passing them all indirectly. -// x86-64: void @sse_id(ptr{{( [^,]*)?}} sret([16 x i8]){{( .*)?}}, ptr{{( [^,]*)?}}) -// x86-32: void @sse_id(ptr{{( [^,]*)?}} sret([16 x i8]){{( .*)?}}, ptr{{( [^,]*)?}}) +// x86-64: <4 x float> @sse_id(<4 x float> {{[^,]*}}) +// x86-32: <4 x float> @sse_id(<4 x float> {{[^,]*}}) // x86-32-nosse: void @sse_id(ptr{{( [^,]*)?}} sret([16 x i8]){{( .*)?}}, ptr{{( [^,]*)?}}) #[no_mangle] pub fn sse_id(x: Sse) -> Sse { diff --git a/tests/codegen-llvm/debuginfo-dse.rs b/tests/codegen-llvm/debuginfo-dse.rs index d8780cca34211..7509facdaadc0 100644 --- a/tests/codegen-llvm/debuginfo-dse.rs +++ b/tests/codegen-llvm/debuginfo-dse.rs @@ -55,7 +55,7 @@ struct Aggregate_4xi8(i8, i8, i8, i8); // scalar(i32) // The pass mode is indirect and the backend represent is simd vector. #[repr(simd)] -struct Simd_i32x4([i32; 4]); +struct Simd_i32x8([i32; 8]); unsafe extern "Rust" { #[rustc_nounwind] @@ -169,10 +169,10 @@ fn indirect( tuple_sliceref_scalar: Tuple_SliceRef_Scalar, array: Array, typle_i32_i64_i8: Typle_i32_i64_i8, - simd_i32x4: Simd_i32x4, + simd_i32x8: Simd_i32x8, ) { // CHECK-LABEL: define{{( dso_local)?}} void @indirect - // CHECK-SAME: (ptr{{.*}} %tuple_sliceref_scalar, ptr{{.*}} %array, ptr{{.*}} %typle_i32_i64_i8, ptr{{.*}} %simd_i32x4) + // CHECK-SAME: (ptr{{.*}} %tuple_sliceref_scalar, ptr{{.*}} %array, ptr{{.*}} %typle_i32_i64_i8, ptr{{.*}} %simd_i32x8) // CHECK: call void @opaque_fn() opaque_fn(); // CHECK-NEXT: #dbg_value(ptr %tuple_sliceref_scalar, [[ref_tuple_sliceref_scalar:![0-9]+]], !DIExpression() @@ -183,8 +183,8 @@ fn indirect( let ref_1_array = &array[1]; // CHECK-NEXT: #dbg_value(ptr %typle_i32_i64_i8, [[ref_1_typle_i32_i64_i8:![0-9]+]], !DIExpression() let ref_1_typle_i32_i64_i8 = &typle_i32_i64_i8.1; - // CHECK-NEXT: #dbg_value(ptr %simd_i32x4, [[ref_simd_i32x4:![0-9]+]], !DIExpression() - let ref_simd_i32x4 = &simd_i32x4; + // CHECK-NEXT: #dbg_value(ptr %simd_i32x8, [[ref_simd_i32x8:![0-9]+]], !DIExpression() + let ref_simd_i32x8 = &simd_i32x8; // CHECK: call void @opaque_fn() opaque_fn(); } @@ -334,7 +334,7 @@ fn index(slice: &[i32; 4], idx: usize) -> i32 { // CHECK-DAG: [[ref_1_tuple_sliceref_scalar]] = !DILocalVariable(name: "ref_1_tuple_sliceref_scalar" // CHECK-DAG: [[ref_1_array]] = !DILocalVariable(name: "ref_1_array" // CHECK-DAG: [[ref_1_typle_i32_i64_i8]] = !DILocalVariable(name: "ref_1_typle_i32_i64_i8" -// CHECK-DAG: [[ref_simd_i32x4]] = !DILocalVariable(name: "ref_simd_i32x4" +// CHECK-DAG: [[ref_simd_i32x8]] = !DILocalVariable(name: "ref_simd_i32x8" // CHECK-DAG: [[ref_direct_ref]] = !DILocalVariable(name: "ref_direct_ref" // CHECK-DAG: [[ref_1_direct_ref]] = !DILocalVariable(name: "ref_1_direct_ref" diff --git a/tests/codegen-llvm/pclmulqdq-target-feature-inlining.rs b/tests/codegen-llvm/pclmulqdq-target-feature-inlining.rs index 2cc1dcf29dc23..d95fd7da05212 100644 --- a/tests/codegen-llvm/pclmulqdq-target-feature-inlining.rs +++ b/tests/codegen-llvm/pclmulqdq-target-feature-inlining.rs @@ -11,7 +11,7 @@ use std::arch::x86_64 as arch; // CHECK-LABEL: @reduce128_caller // CHECK-NEXT: start -// CHECK-COUNT-3: load +// CHECK-NOT: load // CHECK-NEXT: call <2 x i64> @llvm.x86.pclmulqdq // CHECK-NEXT: call <2 x i64> @llvm.x86.pclmulqdq #[target_feature(enable = "pclmulqdq", enable = "sse2", enable = "sse4.1")] diff --git a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-arithmetic-saturating.rs b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-arithmetic-saturating.rs index 06d4688971526..380bd23ae94a8 100644 --- a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-arithmetic-saturating.rs +++ b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-arithmetic-saturating.rs @@ -21,559 +21,559 @@ use std::intrinsics::simd::{simd_saturating_add, simd_saturating_sub}; // CHECK-LABEL: @sadd_i8x2 #[no_mangle] pub unsafe fn sadd_i8x2(x: i8x2, y: i8x2) -> i8x2 { - // CHECK: %{{[0-9]+}} = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i8x4 #[no_mangle] pub unsafe fn sadd_i8x4(x: i8x4, y: i8x4) -> i8x4 { - // CHECK: %{{[0-9]+}} = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i8x8 #[no_mangle] pub unsafe fn sadd_i8x8(x: i8x8, y: i8x8) -> i8x8 { - // CHECK: %{{[0-9]+}} = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i8x16 #[no_mangle] pub unsafe fn sadd_i8x16(x: i8x16, y: i8x16) -> i8x16 { - // CHECK: %{{[0-9]+}} = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i8x32 #[no_mangle] pub unsafe fn sadd_i8x32(x: i8x32, y: i8x32) -> i8x32 { - // CHECK: %{{[0-9]+}} = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i8x64 #[no_mangle] pub unsafe fn sadd_i8x64(x: i8x64, y: i8x64) -> i8x64 { - // CHECK: %{{[0-9]+}} = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i16x2 #[no_mangle] pub unsafe fn sadd_i16x2(x: i16x2, y: i16x2) -> i16x2 { - // CHECK: %{{[0-9]+}} = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i16x4 #[no_mangle] pub unsafe fn sadd_i16x4(x: i16x4, y: i16x4) -> i16x4 { - // CHECK: %{{[0-9]+}} = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i16x8 #[no_mangle] pub unsafe fn sadd_i16x8(x: i16x8, y: i16x8) -> i16x8 { - // CHECK: %{{[0-9]+}} = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i16x16 #[no_mangle] pub unsafe fn sadd_i16x16(x: i16x16, y: i16x16) -> i16x16 { - // CHECK: %{{[0-9]+}} = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i16x32 #[no_mangle] pub unsafe fn sadd_i16x32(x: i16x32, y: i16x32) -> i16x32 { - // CHECK: %{{[0-9]+}} = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i32x2 #[no_mangle] pub unsafe fn sadd_i32x2(x: i32x2, y: i32x2) -> i32x2 { - // CHECK: %{{[0-9]+}} = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i32x4 #[no_mangle] pub unsafe fn sadd_i32x4(x: i32x4, y: i32x4) -> i32x4 { - // CHECK: %{{[0-9]+}} = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i32x8 #[no_mangle] pub unsafe fn sadd_i32x8(x: i32x8, y: i32x8) -> i32x8 { - // CHECK: %{{[0-9]+}} = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i32x16 #[no_mangle] pub unsafe fn sadd_i32x16(x: i32x16, y: i32x16) -> i32x16 { - // CHECK: %{{[0-9]+}} = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i64x2 #[no_mangle] pub unsafe fn sadd_i64x2(x: i64x2, y: i64x2) -> i64x2 { - // CHECK: %{{[0-9]+}} = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i64x4 #[no_mangle] pub unsafe fn sadd_i64x4(x: i64x4, y: i64x4) -> i64x4 { - // CHECK: %{{[0-9]+}} = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i64x8 #[no_mangle] pub unsafe fn sadd_i64x8(x: i64x8, y: i64x8) -> i64x8 { - // CHECK: %{{[0-9]+}} = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i128x2 #[no_mangle] pub unsafe fn sadd_i128x2(x: i128x2, y: i128x2) -> i128x2 { - // CHECK: %{{[0-9]+}} = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @sadd_i128x4 #[no_mangle] pub unsafe fn sadd_i128x4(x: i128x4, y: i128x4) -> i128x4 { - // CHECK: %{{[0-9]+}} = call <4 x i128> @llvm.sadd.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i128> @llvm.sadd.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u8x2 #[no_mangle] pub unsafe fn uadd_u8x2(x: u8x2, y: u8x2) -> u8x2 { - // CHECK: %{{[0-9]+}} = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u8x4 #[no_mangle] pub unsafe fn uadd_u8x4(x: u8x4, y: u8x4) -> u8x4 { - // CHECK: %{{[0-9]+}} = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u8x8 #[no_mangle] pub unsafe fn uadd_u8x8(x: u8x8, y: u8x8) -> u8x8 { - // CHECK: %{{[0-9]+}} = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u8x16 #[no_mangle] pub unsafe fn uadd_u8x16(x: u8x16, y: u8x16) -> u8x16 { - // CHECK: %{{[0-9]+}} = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u8x32 #[no_mangle] pub unsafe fn uadd_u8x32(x: u8x32, y: u8x32) -> u8x32 { - // CHECK: %{{[0-9]+}} = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u8x64 #[no_mangle] pub unsafe fn uadd_u8x64(x: u8x64, y: u8x64) -> u8x64 { - // CHECK: %{{[0-9]+}} = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u16x2 #[no_mangle] pub unsafe fn uadd_u16x2(x: u16x2, y: u16x2) -> u16x2 { - // CHECK: %{{[0-9]+}} = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u16x4 #[no_mangle] pub unsafe fn uadd_u16x4(x: u16x4, y: u16x4) -> u16x4 { - // CHECK: %{{[0-9]+}} = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u16x8 #[no_mangle] pub unsafe fn uadd_u16x8(x: u16x8, y: u16x8) -> u16x8 { - // CHECK: %{{[0-9]+}} = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u16x16 #[no_mangle] pub unsafe fn uadd_u16x16(x: u16x16, y: u16x16) -> u16x16 { - // CHECK: %{{[0-9]+}} = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u16x32 #[no_mangle] pub unsafe fn uadd_u16x32(x: u16x32, y: u16x32) -> u16x32 { - // CHECK: %{{[0-9]+}} = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u32x2 #[no_mangle] pub unsafe fn uadd_u32x2(x: u32x2, y: u32x2) -> u32x2 { - // CHECK: %{{[0-9]+}} = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u32x4 #[no_mangle] pub unsafe fn uadd_u32x4(x: u32x4, y: u32x4) -> u32x4 { - // CHECK: %{{[0-9]+}} = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u32x8 #[no_mangle] pub unsafe fn uadd_u32x8(x: u32x8, y: u32x8) -> u32x8 { - // CHECK: %{{[0-9]+}} = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u32x16 #[no_mangle] pub unsafe fn uadd_u32x16(x: u32x16, y: u32x16) -> u32x16 { - // CHECK: %{{[0-9]+}} = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u64x2 #[no_mangle] pub unsafe fn uadd_u64x2(x: u64x2, y: u64x2) -> u64x2 { - // CHECK: %{{[0-9]+}} = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u64x4 #[no_mangle] pub unsafe fn uadd_u64x4(x: u64x4, y: u64x4) -> u64x4 { - // CHECK: %{{[0-9]+}} = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u64x8 #[no_mangle] pub unsafe fn uadd_u64x8(x: u64x8, y: u64x8) -> u64x8 { - // CHECK: %{{[0-9]+}} = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u128x2 #[no_mangle] pub unsafe fn uadd_u128x2(x: u128x2, y: u128x2) -> u128x2 { - // CHECK: %{{[0-9]+}} = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @uadd_u128x4 #[no_mangle] pub unsafe fn uadd_u128x4(x: u128x4, y: u128x4) -> u128x4 { - // CHECK: %{{[0-9]+}} = call <4 x i128> @llvm.uadd.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i128> @llvm.uadd.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) simd_saturating_add(x, y) } // CHECK-LABEL: @ssub_i8x2 #[no_mangle] pub unsafe fn ssub_i8x2(x: i8x2, y: i8x2) -> i8x2 { - // CHECK: %{{[0-9]+}} = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i8x4 #[no_mangle] pub unsafe fn ssub_i8x4(x: i8x4, y: i8x4) -> i8x4 { - // CHECK: %{{[0-9]+}} = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i8x8 #[no_mangle] pub unsafe fn ssub_i8x8(x: i8x8, y: i8x8) -> i8x8 { - // CHECK: %{{[0-9]+}} = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i8x16 #[no_mangle] pub unsafe fn ssub_i8x16(x: i8x16, y: i8x16) -> i8x16 { - // CHECK: %{{[0-9]+}} = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i8x32 #[no_mangle] pub unsafe fn ssub_i8x32(x: i8x32, y: i8x32) -> i8x32 { - // CHECK: %{{[0-9]+}} = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i8x64 #[no_mangle] pub unsafe fn ssub_i8x64(x: i8x64, y: i8x64) -> i8x64 { - // CHECK: %{{[0-9]+}} = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i16x2 #[no_mangle] pub unsafe fn ssub_i16x2(x: i16x2, y: i16x2) -> i16x2 { - // CHECK: %{{[0-9]+}} = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i16x4 #[no_mangle] pub unsafe fn ssub_i16x4(x: i16x4, y: i16x4) -> i16x4 { - // CHECK: %{{[0-9]+}} = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i16x8 #[no_mangle] pub unsafe fn ssub_i16x8(x: i16x8, y: i16x8) -> i16x8 { - // CHECK: %{{[0-9]+}} = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i16x16 #[no_mangle] pub unsafe fn ssub_i16x16(x: i16x16, y: i16x16) -> i16x16 { - // CHECK: %{{[0-9]+}} = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i16x32 #[no_mangle] pub unsafe fn ssub_i16x32(x: i16x32, y: i16x32) -> i16x32 { - // CHECK: %{{[0-9]+}} = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i32x2 #[no_mangle] pub unsafe fn ssub_i32x2(x: i32x2, y: i32x2) -> i32x2 { - // CHECK: %{{[0-9]+}} = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i32x4 #[no_mangle] pub unsafe fn ssub_i32x4(x: i32x4, y: i32x4) -> i32x4 { - // CHECK: %{{[0-9]+}} = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i32x8 #[no_mangle] pub unsafe fn ssub_i32x8(x: i32x8, y: i32x8) -> i32x8 { - // CHECK: %{{[0-9]+}} = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i32x16 #[no_mangle] pub unsafe fn ssub_i32x16(x: i32x16, y: i32x16) -> i32x16 { - // CHECK: %{{[0-9]+}} = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i64x2 #[no_mangle] pub unsafe fn ssub_i64x2(x: i64x2, y: i64x2) -> i64x2 { - // CHECK: %{{[0-9]+}} = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i64x4 #[no_mangle] pub unsafe fn ssub_i64x4(x: i64x4, y: i64x4) -> i64x4 { - // CHECK: %{{[0-9]+}} = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i64x8 #[no_mangle] pub unsafe fn ssub_i64x8(x: i64x8, y: i64x8) -> i64x8 { - // CHECK: %{{[0-9]+}} = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i128x2 #[no_mangle] pub unsafe fn ssub_i128x2(x: i128x2, y: i128x2) -> i128x2 { - // CHECK: %{{[0-9]+}} = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @ssub_i128x4 #[no_mangle] pub unsafe fn ssub_i128x4(x: i128x4, y: i128x4) -> i128x4 { - // CHECK: %{{[0-9]+}} = call <4 x i128> @llvm.ssub.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i128> @llvm.ssub.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u8x2 #[no_mangle] pub unsafe fn usub_u8x2(x: u8x2, y: u8x2) -> u8x2 { - // CHECK: %{{[0-9]+}} = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %{{x|0}}, <2 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u8x4 #[no_mangle] pub unsafe fn usub_u8x4(x: u8x4, y: u8x4) -> u8x4 { - // CHECK: %{{[0-9]+}} = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %{{x|0}}, <4 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u8x8 #[no_mangle] pub unsafe fn usub_u8x8(x: u8x8, y: u8x8) -> u8x8 { - // CHECK: %{{[0-9]+}} = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> %{{x|0}}, <8 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u8x16 #[no_mangle] pub unsafe fn usub_u8x16(x: u8x16, y: u8x16) -> u8x16 { - // CHECK: %{{[0-9]+}} = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %{{x|0}}, <16 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u8x32 #[no_mangle] pub unsafe fn usub_u8x32(x: u8x32, y: u8x32) -> u8x32 { - // CHECK: %{{[0-9]+}} = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %{{x|0}}, <32 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u8x64 #[no_mangle] pub unsafe fn usub_u8x64(x: u8x64, y: u8x64) -> u8x64 { - // CHECK: %{{[0-9]+}} = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %{{x|0}}, <64 x i8> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u16x2 #[no_mangle] pub unsafe fn usub_u16x2(x: u16x2, y: u16x2) -> u16x2 { - // CHECK: %{{[0-9]+}} = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %{{x|0}}, <2 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u16x4 #[no_mangle] pub unsafe fn usub_u16x4(x: u16x4, y: u16x4) -> u16x4 { - // CHECK: %{{[0-9]+}} = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %{{x|0}}, <4 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u16x8 #[no_mangle] pub unsafe fn usub_u16x8(x: u16x8, y: u16x8) -> u16x8 { - // CHECK: %{{[0-9]+}} = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %{{x|0}}, <8 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u16x16 #[no_mangle] pub unsafe fn usub_u16x16(x: u16x16, y: u16x16) -> u16x16 { - // CHECK: %{{[0-9]+}} = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %{{x|0}}, <16 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u16x32 #[no_mangle] pub unsafe fn usub_u16x32(x: u16x32, y: u16x32) -> u16x32 { - // CHECK: %{{[0-9]+}} = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %{{x|0}}, <32 x i16> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u32x2 #[no_mangle] pub unsafe fn usub_u32x2(x: u32x2, y: u32x2) -> u32x2 { - // CHECK: %{{[0-9]+}} = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %{{x|0}}, <2 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u32x4 #[no_mangle] pub unsafe fn usub_u32x4(x: u32x4, y: u32x4) -> u32x4 { - // CHECK: %{{[0-9]+}} = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %{{x|0}}, <4 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u32x8 #[no_mangle] pub unsafe fn usub_u32x8(x: u32x8, y: u32x8) -> u32x8 { - // CHECK: %{{[0-9]+}} = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %{{x|0}}, <8 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u32x16 #[no_mangle] pub unsafe fn usub_u32x16(x: u32x16, y: u32x16) -> u32x16 { - // CHECK: %{{[0-9]+}} = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %{{x|0}}, <16 x i32> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u64x2 #[no_mangle] pub unsafe fn usub_u64x2(x: u64x2, y: u64x2) -> u64x2 { - // CHECK: %{{[0-9]+}} = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %{{x|0}}, <2 x i64> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u64x4 #[no_mangle] pub unsafe fn usub_u64x4(x: u64x4, y: u64x4) -> u64x4 { - // CHECK: %{{[0-9]+}} = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> %{{x|0}}, <4 x i64> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u64x8 #[no_mangle] pub unsafe fn usub_u64x8(x: u64x8, y: u64x8) -> u64x8 { - // CHECK: %{{[0-9]+}} = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> %{{x|0}}, <8 x i64> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u128x2 #[no_mangle] pub unsafe fn usub_u128x2(x: u128x2, y: u128x2) -> u128x2 { - // CHECK: %{{[0-9]+}} = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %{{x|0}}, <2 x i128> %{{y|1}}) simd_saturating_sub(x, y) } // CHECK-LABEL: @usub_u128x4 #[no_mangle] pub unsafe fn usub_u128x4(x: u128x4, y: u128x4) -> u128x4 { - // CHECK: %{{[0-9]+}} = call <4 x i128> @llvm.usub.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) + // CHECK: %{{[a-z0-9_]+}} = call <4 x i128> @llvm.usub.sat.v4i128(<4 x i128> %{{x|0}}, <4 x i128> %{{y|1}}) simd_saturating_sub(x, y) } diff --git a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-transmute-array.rs b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-transmute-array.rs index 05c2f7e1bdfc1..9de887e288986 100644 --- a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-transmute-array.rs +++ b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-transmute-array.rs @@ -4,6 +4,17 @@ //@ ignore-i686-pc-windows-msvc //@ ignore-i686-pc-windows-gnu +// LLVM IR isn't very portable and the one tested here depends on the ABI which is different between +// x86 (where we use SSE registers) and others. `x86-64` and `x86-32-sse2` are identical, but +// compiletest does not support taking the union of multiple `only` annotations. +//@ revisions: x86-64 x86-32-sse2 by-ref +//@[x86-64] only-x86_64 +//@[x86-64] filecheck-flags: --check-prefix=by-val +//@[x86-32-sse2] only-rustc_abi-x86-sse2 +//@[x86-32-sse2] filecheck-flags: --check-prefix=by-val +//@[by-ref] ignore-rustc_abi-x86-sse2 +//@[by-ref] ignore-x86_64 + #![crate_type = "lib"] #![allow(non_camel_case_types)] #![feature(repr_simd, core_intrinsics)] @@ -13,7 +24,6 @@ mod minisimd; use minisimd::*; pub type S = Simd; -pub type T = Simd; // CHECK-LABEL: @array_align( #[no_mangle] @@ -26,33 +36,23 @@ pub fn array_align() -> usize { #[no_mangle] pub fn vector_align() -> usize { // CHECK: ret [[USIZE]] [[VECTOR_ALIGN:[0-9]+]] - const { std::mem::align_of::() } -} - -// CHECK-LABEL: @build_array_s -#[no_mangle] -pub fn build_array_s(x: [f32; 4]) -> S<4> { - // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false) - Simd(x) -} - -// CHECK-LABEL: @build_array_transmute_s -#[no_mangle] -pub fn build_array_transmute_s(x: [f32; 4]) -> S<4> { - // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false) - unsafe { std::mem::transmute(x) } + const { std::mem::align_of::>() } } -// CHECK-LABEL: @build_array_t +// CHECK-LABEL: @build_array #[no_mangle] -pub fn build_array_t(x: [f32; 4]) -> T { - // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false) +pub fn build_array(x: [f32; 4]) -> S<4> { + // by-val: %[[VAL:.+]] = load <4 x float>, ptr %x, align [[ARRAY_ALIGN]] + // by-val: ret <4 x float> %[[VAL:.+]] + // by-ref: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false) Simd(x) } -// CHECK-LABEL: @build_array_transmute_t +// CHECK-LABEL: @build_array_transmute #[no_mangle] -pub fn build_array_transmute_t(x: [f32; 4]) -> T { - // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false) +pub fn build_array_transmute(x: [f32; 4]) -> S<4> { + // by-val: %[[VAL:.+]] = load <4 x float>, ptr %x, align [[ARRAY_ALIGN]] + // by-val: ret <4 x float> %[[VAL:.+]] + // by-ref: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false) unsafe { std::mem::transmute(x) } } diff --git a/tests/codegen-llvm/simd/packed-simd-alignment.rs b/tests/codegen-llvm/simd/packed-simd-alignment.rs index 53e88d8e5cf11..8bac517d0c75d 100644 --- a/tests/codegen-llvm/simd/packed-simd-alignment.rs +++ b/tests/codegen-llvm/simd/packed-simd-alignment.rs @@ -38,7 +38,7 @@ pub fn add_f32x3(x: f32x3, y: f32x3) -> f32x3 { // CHECK-LABEL: add_f32x4 #[no_mangle] -pub fn add_f32x4(x: f32x4, y: f32x4) -> f32x4 { +pub fn add_f32x4(x: &f32x4, y: &f32x4) -> f32x4 { // CHECK: load <4 x float>, ptr %{{[a-z0-9_]*}}, align {{4|8|16}} - unsafe { intrinsics::simd_add(x, y) } + unsafe { intrinsics::simd_add(*x, *y) } } diff --git a/tests/codegen-llvm/simd/packed-simd.rs b/tests/codegen-llvm/simd/packed-simd.rs index 70c03fcc95574..4241a4fd5a52b 100644 --- a/tests/codegen-llvm/simd/packed-simd.rs +++ b/tests/codegen-llvm/simd/packed-simd.rs @@ -26,17 +26,29 @@ fn load(v: PackedSimd) -> FullSimd { } } -// CHECK-LABEL: square_packed_full -// CHECK-SAME: ptr{{[a-z_ ]*}} sret([[RET_TYPE:[^)]+]]) [[RET_ALIGN:align (8|16)]]{{[^%]*}} [[RET_VREG:%[_0-9]*]] +// CHECK-LABEL: define <3 x float> @square_packed_full // CHECK-SAME: ptr{{[a-z_ ]*}} align 4 #[no_mangle] pub fn square_packed_full(x: PackedSimd) -> FullSimd { // CHECK-NEXT: start - // noopt: alloca [[RET_TYPE]], [[RET_ALIGN]] // CHECK: load <3 x float> let x = load(x); // CHECK: [[VREG:%[a-z0-9_]+]] = fmul <3 x float> - // CHECK-NEXT: store <3 x float> [[VREG]], ptr [[RET_VREG]], [[RET_ALIGN]] + // CHECK-NEXT: ret <3 x float> [[VREG]] + unsafe { intrinsics::simd_mul(x, x) } +} + +// Also test a big version that uses in-memory argument passing. +// CHECK-LABEL: square_packed_full_big +// CHECK-SAME: ptr{{[a-z_ ]*}} sret([[RET_TYPE:[^)]+]]) [[RET_ALIGN:align 32]]{{[^%]*}} [[RET_VREG:%[_0-9]*]] +#[no_mangle] +pub fn square_packed_full_big(x: PackedSimd) -> FullSimd { + // CHECK-NEXT: start + // noopt: alloca [[RET_TYPE]], [[RET_ALIGN]] + // CHECK: load <7 x float> + let x = load(x); + // CHECK: [[VREG:%[a-z0-9_]+]] = fmul <7 x float> + // CHECK-NEXT: store <7 x float> [[VREG]], ptr [[RET_VREG]], [[RET_ALIGN]] // CHECK-NEXT: ret void unsafe { intrinsics::simd_mul(x, x) } } diff --git a/tests/codegen-llvm/simd/splat.rs b/tests/codegen-llvm/simd/splat.rs index 7a24162321bdc..3ddf5eec9c3c1 100644 --- a/tests/codegen-llvm/simd/splat.rs +++ b/tests/codegen-llvm/simd/splat.rs @@ -1,4 +1,16 @@ //@ compile-flags: -Copt-level=3 + +// LLVM IR isn't very portable and the one tested here depends on the ABI which is different between +// x86 (where we use SSE registers) and others. `x86-64` and `x86-32-sse2` are identical, but +// compiletest does not support taking the union of multiple `only` annotations. +//@ revisions: x86-64 x86-32-sse2 by-ref +//@[x86-64] only-x86_64 +//@[x86-64] filecheck-flags: --check-prefix=by-val +//@[x86-32-sse2] only-rustc_abi-x86-sse2 +//@[x86-32-sse2] filecheck-flags: --check-prefix=by-val +//@[by-ref] ignore-rustc_abi-x86-sse2 +//@[by-ref] ignore-x86_64 + #![crate_type = "lib"] #![no_std] #![feature(repr_simd, core_intrinsics)] @@ -15,9 +27,10 @@ unsafe fn int(x: u16) -> u16x2 { // CHECK-LABEL: int // CHECK: start: // CHECK-NEXT: %0 = insertelement <2 x i16> poison, i16 %x, i64 0 - // CHECK-NEXT: %1 = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> zeroinitializer - // CHECK-NEXT: store - // CHECK-NEXT: ret + // CHECK-NEXT: [[VREG:%[a-z0-9_]+]] = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> zeroinitializer + // by-ref-NEXT: store <2 x i16> [[VREG]] + // by-ref-NEXT: ret + // by-val-NEXT: ret <2 x i16> [[VREG]] simd_splat(x) } @@ -26,8 +39,9 @@ unsafe fn float(x: f32) -> f32x4 { // CHECK-LABEL: float // CHECK: start: // CHECK-NEXT: %0 = insertelement <4 x float> poison, float %x, i64 0 - // CHECK-NEXT: %1 = shufflevector <4 x float> %0, <4 x float> poison, <4 x i32> zeroinitializer - // CHECK-NEXT: store - // CHECK-NEXT: ret + // CHECK-NEXT: [[VREG:%[a-z0-9_]+]] = shufflevector <4 x float> %0, <4 x float> poison, <4 x i32> zeroinitializer + // by-ref-NEXT: store <4 x float> [[VREG]] + // by-ref-NEXT: ret + // by-val-NEXT: ret <4 x float> [[VREG]] simd_splat(x) }