Skip to content

Make sin, cos, exp, exp2, log, log2, log10 generic #160989

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
N1ark wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Make sin, cos, exp, exp2, log, log2, log10 generic #160989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c1d8319
Cleanup cranelift `codegen_float_intrinsic_call`
N1ark Aug 12, 2026
9062d4d
Fix misplaced comment
N1ark Aug 12, 2026
9878ba8
Make sin, cos, exp, exp2, log, log2, log10 generic
N1ark Aug 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 72 additions & 73 deletions compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,16 +322,6 @@ fn codegen_float_intrinsic_call<'tcx>(
ret: CPlace<'tcx>,
) -> bool {
let (name, arg_count, ty, clif_ty) = match intrinsic {
sym::expf16 => return false, // has a fallback via f32
sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
sym::expf128 => ("expf128", 1, fx.tcx.types.f128, types::F128),

sym::exp2f16 => return false, // has a fallback via f32
sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
sym::exp2f128 => ("exp2f128", 1, fx.tcx.types.f128, types::F128),

sym::sqrtf16 => return false, // has a fallback via f32
sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
Expand All @@ -347,21 +337,6 @@ fn codegen_float_intrinsic_call<'tcx>(
sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128),

sym::logf16 => return false, // has a fallback via f32
sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
sym::logf128 => ("logf128", 1, fx.tcx.types.f128, types::F128),

sym::log2f16 => return false, // has a fallback via f32
sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
sym::log2f128 => ("log2f128", 1, fx.tcx.types.f128, types::F128),

sym::log10f16 => return false, // has a fallback via f32
sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
sym::log10f128 => ("log10f128", 1, fx.tcx.types.f128, types::F128),

sym::fmaf16 => return false, // has a fallback via f32
sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
Expand Down Expand Up @@ -404,16 +379,6 @@ fn codegen_float_intrinsic_call<'tcx>(
sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128),

sym::sinf16 => return false, // has a fallback via f32
sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
sym::sinf128 => ("sinf128", 1, fx.tcx.types.f128, types::F128),

sym::cosf16 => return false, // has a fallback via f32
sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
sym::cosf128 => ("cosf128", 1, fx.tcx.types.f128, types::F128),

_ => return false,
};

Expand Down Expand Up @@ -448,36 +413,16 @@ fn codegen_float_intrinsic_call<'tcx>(
let layout = fx.layout_of(ty);
// FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift operations
// for `f16` and `f128` once the lowerings have been implemented in Cranelift.
let res = match intrinsic {
let val = match intrinsic {
sym::fmaf32 | sym::fmaf64 | sym::fmuladdf32 | sym::fmuladdf64 => {
CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
}
sym::copysignf32 | sym::copysignf64 => {
CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
}
sym::floorf32
| sym::floorf64
| sym::ceilf32
| sym::ceilf64
| sym::truncf32
| sym::truncf64
| sym::round_ties_even_f32
| sym::round_ties_even_f64
| sym::sqrtf32
| sym::sqrtf64 => {
let val = match intrinsic {
sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
sym::round_ties_even_f32 | sym::round_ties_even_f64 => {
fx.bcx.ins().nearest(args[0])
}
sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]),
_ => unreachable!(),
};

CValue::by_val(val, layout)
fx.bcx.ins().fma(args[0], args[1], args[2])
}
sym::copysignf32 | sym::copysignf64 => fx.bcx.ins().fcopysign(args[0], args[1]),
sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
sym::round_ties_even_f32 | sym::round_ties_even_f64 => fx.bcx.ins().nearest(args[0]),
sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]),

// These intrinsics aren't supported natively by Cranelift.
// Lower them to a libcall.
Expand All @@ -492,20 +437,19 @@ fn codegen_float_intrinsic_call<'tcx>(
let input_tys: Vec<_> =
vec![AbiParam::new(clif_ty), lib_call_arg_param(fx.tcx, types::I32, true)];
let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0];
let ret_val = if intrinsic == sym::powif16 {
if intrinsic == sym::powif16 {
codegen_f16_f128::f32_to_f16(fx, ret_val)
} else {
ret_val
};
CValue::by_val(ret_val, fx.layout_of(ty))
}
}
_ => {
let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0];
CValue::by_val(ret_val, fx.layout_of(ty))
fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0]
}
};

let res = CValue::by_val(val, layout);
ret.write_cvalue(fx, res);

true
Expand Down Expand Up @@ -1199,7 +1143,14 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, old);
}

sym::fabs => {
sym::fabs
| sym::exp
| sym::exp2
| sym::log
| sym::log2
| sym::log10
| sym::sin
| sym::cos => {
intrinsic_args!(fx, args => (arg); intrinsic);
let layout = arg.layout();
let ty::Float(float_ty) = layout.ty.kind() else {
Expand All @@ -1209,13 +1160,61 @@ fn codegen_regular_intrinsic_call<'tcx>(
layout.ty
);
};
enum IntrinsicFallback {
Fallback(&'static str),
Codegen(Value),
}
use FloatTy::*;
use IntrinsicFallback::*;
let x = arg.load_scalar(fx);
let val = match float_ty {
FloatTy::F32 | FloatTy::F64 => fx.bcx.ins().fabs(x),
let res = match (intrinsic, float_ty) {
(sym::fabs, F32 | F64) => Codegen(fx.bcx.ins().fabs(x)),
// FIXME(bytecodealliance/wasmtime#8312): Use `fabsf16` once Cranelift
// backend lowerings are implemented.
FloatTy::F16 => codegen_f16_f128::abs_f16(fx, x),
FloatTy::F128 => codegen_f16_f128::abs_f128(fx, x),
(sym::fabs, F16) => Codegen(codegen_f16_f128::abs_f16(fx, x)),
(sym::fabs, F128) => Codegen(codegen_f16_f128::abs_f128(fx, x)),
Comment on lines +1174 to +1175

@folkertdev folkertdev Aug 20, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could use a fallback for these too, it's just zeroing the sign bit

@N1ark N1ark Aug 22, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

they already have a fallback, so yeah sure :)


(sym::exp, F32) => Fallback("expf"),
(sym::exp, F64) => Fallback("exp"),
(sym::exp, F128) => Fallback("expf128"),

(sym::exp2, F32) => Fallback("exp2f"),
(sym::exp2, F64) => Fallback("exp2"),
(sym::exp2, F128) => Fallback("exp2f128"),

(sym::log, F32) => Fallback("logf"),
(sym::log, F64) => Fallback("log"),
(sym::log, F128) => Fallback("logf128"),

(sym::log2, F32) => Fallback("log2f"),
(sym::log2, F64) => Fallback("log2"),
(sym::log2, F128) => Fallback("log2f128"),

(sym::log10, F32) => Fallback("log10f"),
(sym::log10, F64) => Fallback("log10"),
(sym::log10, F128) => Fallback("log10f128"),

(sym::sin, F32) => Fallback("sinf"),
(sym::sin, F64) => Fallback("sin"),
(sym::sin, F128) => Fallback("sinf128"),

(sym::cos, F32) => Fallback("cosf"),
(sym::cos, F64) => Fallback("cos"),
(sym::cos, F128) => Fallback("cosf128"),

(_, F16) => {
// We implement fallbacks for other f16 intrinsics via f32
return Err(Instance::new_raw(instance.def_id(), instance.args));
}

_ => unreachable!(),
};
let val = match res {
Codegen(val) => val,
Fallback(name) => {
let ty = fx.clif_type(layout.ty).unwrap();
fx.lib_call(name, vec![AbiParam::new(ty)], vec![AbiParam::new(ty)], &[x])[0]
}
};
let val = CValue::by_val(val, layout);
ret.write_cvalue(fx, val);
Expand Down
117 changes: 59 additions & 58 deletions compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,22 +62,8 @@ fn get_simple_intrinsic<'gcc, 'tcx>(
sym::sqrtf64 => "sqrt",
sym::powif32 => "__builtin_powif",
sym::powif64 => "__builtin_powi",
sym::sinf32 => "sinf",
sym::sinf64 => "sin",
sym::cosf32 => "cosf",
sym::cosf64 => "cos",
sym::powf32 => "powf",
sym::powf64 => "pow",
sym::expf32 => "expf",
sym::expf64 => "exp",
sym::exp2f32 => "exp2f",
sym::exp2f64 => "exp2",
sym::logf32 => "logf",
sym::logf64 => "log",
sym::log10f32 => "log10f",
sym::log10f64 => "log10",
sym::log2f32 => "log2f",
sym::log2f64 => "log2",
sym::fmaf32 => "fmaf",
sym::fmaf64 => "fma",
// FIXME: calling `fma` from libc without FMA target feature uses expensive software emulation
Expand Down Expand Up @@ -117,16 +103,18 @@ fn get_simple_function_f128<'gcc, 'tcx>(
let f128_type = cx.type_f128();
let func_name = match name {
sym::ceilf128 => "ceilf128",
sym::cos => "cosf128",
sym::fabs => "fabsf128",
sym::expf128 => "expf128",
sym::exp2f128 => "exp2f128",
sym::exp => "expf128",
sym::exp2 => "exp2f128",
sym::floorf128 => "floorf128",
sym::logf128 => "logf128",
sym::log2f128 => "log2f128",
sym::log10f128 => "log10f128",
sym::log => "logf128",
sym::log2 => "log2f128",
sym::log10 => "log10f128",
sym::truncf128 => "truncf128",
sym::roundf128 => "roundf128",
sym::round_ties_even_f128 => "roundevenf128",
sym::sin => "sinf128",
sym::sqrtf128 => "sqrtf128",
_ => span_bug!(span, "used get_simple_function_f128 for non-unary f128 intrinsic"),
};
Expand All @@ -140,24 +128,6 @@ fn get_simple_function_f128<'gcc, 'tcx>(
)
}

fn generic_f16_builtin<'gcc, 'tcx>(
cx: &CodegenCx<'gcc, 'tcx>,
name: Symbol,
args: &[OperandRef<'tcx, RValue<'gcc>>],
) -> RValue<'gcc> {
let f32_type = cx.type_f32();
let builtin_name = match name {
sym::fabs => "fabsf",
_ => unreachable!(),
};

let func = cx.context.get_builtin_function(builtin_name);
let args: Vec<_> =
args.iter().map(|arg| cx.context.new_cast(None, arg.immediate(), f32_type)).collect();
let result = cx.context.new_call(None, func, &args);
cx.context.new_cast(None, result, cx.type_f16())
}

fn f16_builtin<'gcc, 'tcx>(
cx: &CodegenCx<'gcc, 'tcx>,
name: Symbol,
Expand All @@ -167,17 +137,19 @@ fn f16_builtin<'gcc, 'tcx>(
let builtin_name = match name {
sym::ceilf16 => "__builtin_ceilf",
sym::copysignf16 => "__builtin_copysignf",
sym::expf16 => "expf",
sym::exp2f16 => "exp2f",
sym::cos => "cosf",
sym::exp => "expf",
sym::exp2 => "exp2f",
sym::fabs => "fabsf",
sym::floorf16 => "__builtin_floorf",
sym::fmaf16 => "fmaf",

@folkertdev folkertdev Aug 20, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is wrong, fma16 must use the f64 fma to be correctly rounded. We should just use the core fallback for many of the functions instead of re-inventing the wheel here.

@N1ark N1ark Aug 22, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that was pre-existing; should that be done in a separate PR maybe? happy to fix it here otherwise

@folkertdev folkertdev Aug 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm mostly surprised we don't have a test that catches it. I've opened

#161522

We'll handle it there (though this may need to wait for LLVM 21 being dropped, somewhere in October).

sym::logf16 => "logf",
sym::log2f16 => "log2f",
sym::log10f16 => "log10f",
sym::log => "logf",
sym::log2 => "log2f",
sym::log10 => "log10f",
sym::powf16 => "__builtin_powf",
sym::roundf16 => "__builtin_roundf",
sym::round_ties_even_f16 => "__builtin_rintf",
sym::sin => "sinf",
sym::sqrtf16 => "__builtin_sqrtf",
sym::truncf16 => "__builtin_truncf",
_ => unreachable!(),
Expand Down Expand Up @@ -246,13 +218,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
}
sym::ceilf16
| sym::copysignf16
| sym::expf16
| sym::exp2f16
| sym::floorf16
| sym::fmaf16
| sym::logf16
| sym::log2f16
| sym::log10f16
| sym::powf16
| sym::roundf16
| sym::round_ties_even_f16
Expand All @@ -264,11 +231,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
| sym::roundf128
| sym::round_ties_even_f128
| sym::sqrtf128
| sym::expf128
| sym::exp2f128
| sym::logf128
| sym::log2f128
| sym::log10f128
if self.cx.supports_f128_type =>
{
let func = get_simple_function_f128(span, self, name);
Expand Down Expand Up @@ -452,16 +414,55 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
}
}
}
sym::fabs => 'fabs: {
sym::fabs
| sym::exp
| sym::exp2
| sym::log
| sym::log10
| sym::log2
| sym::sin
| sym::cos => 'float_unop: {
let ty = args[0].layout.ty;
let ty::Float(float_ty) = *ty.kind() else {
span_bug!(span, "expected float type for fabs intrinsic: {:?}", ty);
};
let func = match float_ty {
ty::FloatTy::F16 => break 'fabs generic_f16_builtin(self, name, args),
ty::FloatTy::F32 => self.context.get_builtin_function("fabsf"),
ty::FloatTy::F64 => self.context.get_builtin_function("fabs"),
ty::FloatTy::F128 => get_simple_function_f128(span, self, name),
use ty::FloatTy::*;
let func = match (name, float_ty) {
(sym::fabs, F32) => self.context.get_builtin_function("fabsf"),
(sym::fabs, F64) => self.context.get_builtin_function("fabs"),

(sym::exp, F32) => self.context.get_builtin_function("expf"),
(sym::exp, F64) => self.context.get_builtin_function("exp"),

(sym::exp2, F32) => self.context.get_builtin_function("exp2f"),
(sym::exp2, F64) => self.context.get_builtin_function("exp2"),

(sym::log, F32) => self.context.get_builtin_function("logf"),
(sym::log, F64) => self.context.get_builtin_function("log"),

(sym::log10, F32) => self.context.get_builtin_function("log10f"),
(sym::log10, F64) => self.context.get_builtin_function("log10"),

(sym::log2, F32) => self.context.get_builtin_function("log2f"),
(sym::log2, F64) => self.context.get_builtin_function("log2"),

(sym::sin, F32) => self.context.get_builtin_function("sinf"),
(sym::sin, F64) => self.context.get_builtin_function("sin"),

(sym::cos, F32) => self.context.get_builtin_function("cosf"),
(sym::cos, F64) => self.context.get_builtin_function("cos"),

(_, F32 | F64) => unreachable!(),

(_, F16) => break 'float_unop f16_builtin(self, name, args),
(_, F128) => {
if !self.cx.supports_f128_type {
// Fall back to default body
let fallback = Instance::new_raw(instance.def_id(), instance.args);
return IntrinsicResult::Fallback(fallback);
}
get_simple_function_f128(span, self, name)
}
};
self.cx.context.new_call(
self.location,
Expand Down
Loading
Loading