From 493596ec531b2bbd6d4461d2b27ffecf55d5f7f4 Mon Sep 17 00:00:00 2001 From: FractalFir Date: Tue, 5 Aug 2025 18:20:13 +0200 Subject: [PATCH 1/7] Refactored the handling of the indirect return pointer in cg_ssa to allow for more correct ABI handling in cg_gcc --- compiler/rustc_codegen_llvm/src/asm.rs | 4 +- compiler/rustc_codegen_llvm/src/builder.rs | 40 ++++-- compiler/rustc_codegen_llvm/src/intrinsic.rs | 118 ++++++++++++++++-- compiler/rustc_codegen_ssa/src/base.rs | 5 +- compiler/rustc_codegen_ssa/src/mir/block.rs | 47 ++++--- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 1 + compiler/rustc_codegen_ssa/src/size_of_val.rs | 1 + .../rustc_codegen_ssa/src/traits/builder.rs | 2 + 8 files changed, 181 insertions(+), 37 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 31701acd7bf78..2c7bbd124ed77 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -563,9 +563,9 @@ pub(crate) fn inline_asm_call<'ll>( assert!(catch_funclet.is_none()); bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None) } else if let Some((catch, funclet)) = catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None) + bx.invoke(fty, None, None, v, None /*FIXME(FractalFir): I assume that inline assembly does not return via `PassMode::Indirect` - is this correct?*/, inputs, dest.unwrap(), catch, funclet, None) } else { - bx.call(fty, None, None, v, inputs, None, None) + bx.call(fty, None, None, v, None /*FIXME(FractalFir): I assume that inline assembly does not return via `PassMode::Indirect` - is this correct?*/,inputs, None, None) }; // Store mark in a metadata node so we can map LLVM errors diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 87c941cdeb23a..c6cad5ec2d0d0 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -454,15 +454,24 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, + indirect_return_pointer: Option, args: &[&'ll Value], then: &'ll BasicBlock, catch: &'ll BasicBlock, funclet: Option<&Funclet<'ll>>, instance: Option>, ) -> &'ll Value { + let args = match indirect_return_pointer { + None => args.to_vec(), + Some(sret_ptr) => { + let mut args = args.to_vec(); + // Preappend the indirect return pointer + args.insert(0, sret_ptr); + args + } + }; debug!("invoke {:?} with args ({:?})", llfn, args); - - let args = self.check_call("invoke", llty, llfn, args); + let args = self.check_call("invoke", llty, llfn, &args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1463,13 +1472,22 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, + indirect_return_pointer: Option, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, callee_instance: Option>, ) -> &'ll Value { + let args = match indirect_return_pointer { + None => args.to_vec(), + Some(sret_ptr) => { + let mut args = args.to_vec(); + // Preappend the indirect return pointer + args.insert(0, sret_ptr); + args + } + }; debug!("call {:?} with args ({:?})", llfn, args); - - let args = self.check_call("call", llty, llfn, args); + let args = self.check_call("call", llty, llfn, &args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1534,8 +1552,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { funclet: Option<&Self::Funclet>, callee_instance: Option>, ) { - let call = - self.call(llty, caller_attrs, Some(fn_abi), llfn, args, funclet, callee_instance); + let call = self.call( + llty, + caller_attrs, + Some(fn_abi), + llfn, + None, /*FIXME(FractalFir): can tail calls return indrectly?*/ + args, + funclet, + callee_instance, + ); llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); match &fn_abi.ret.mode { @@ -1875,7 +1901,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { args: &[&'ll Value], ) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params); - self.call(ty, None, None, f, args, None, None) + self.call(ty, None, None, f, None/* FIXME(FractalFir): I **assume** that no LLVM intrinsic returns a value via sret. Is this correct?*/, args, None, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index b407b577042a1..4109181253556 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1348,7 +1348,16 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( ) -> &'ll Value { if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.call(try_func_ty, None, None, try_func, &[data], None, None); + bx.call( + try_func_ty, + None, + None, + try_func, + None, /* this function can't return indirectly */ + &[data], + None, + None, + ); // Return 0 unconditionally from the intrinsic call; // we can never unwind. bx.const_bool(false) @@ -1446,7 +1455,18 @@ fn codegen_msvc_try<'ll, 'tcx>( let ptr_align = bx.tcx().data_layout.pointer_align().abi; let slot = bx.alloca(ptr_size, ptr_align); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + None, /* this function can't return indirectly */ + &[data], + normal, + catchswitch, + None, + None, + ); bx.switch_to_block(normal); bx.ret(bx.const_bool(false)); @@ -1494,7 +1514,16 @@ fn codegen_msvc_try<'ll, 'tcx>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_ptr(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + None, /* this function can't return indirectly */ + &[data, ptr], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -1502,7 +1531,16 @@ fn codegen_msvc_try<'ll, 'tcx>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_ptr()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + None, /* this function can't return indirectly */ + &[data, null], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -1511,7 +1549,16 @@ fn codegen_msvc_try<'ll, 'tcx>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + None, /* this function can't return indirectly */ + &[try_func, data, catch_func], + None, + None, + ); ret } @@ -1558,7 +1605,18 @@ fn codegen_wasm_try<'ll, 'tcx>( // } // let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + None, /* this function can't return indirectly */ + &[data], + normal, + catchswitch, + None, + None, + ); bx.switch_to_block(normal); bx.ret(bx.const_bool(false)); @@ -1574,7 +1632,16 @@ fn codegen_wasm_try<'ll, 'tcx>( let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + None, /* this function can't return indirectly */ + &[data, ptr], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -1583,7 +1650,16 @@ fn codegen_wasm_try<'ll, 'tcx>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + None, /* this function can't return indirectly */ + &[try_func, data, catch_func], + None, + None, + ); ret } @@ -1624,7 +1700,18 @@ fn codegen_gnu_try<'ll, 'tcx>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + None, /* this function can't return indirectly */ + &[data], + then, + catch, + None, + None, + ); bx.switch_to_block(then); bx.ret(bx.const_bool(false)); @@ -1642,13 +1729,22 @@ fn codegen_gnu_try<'ll, 'tcx>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, None, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + None, /* this function can't return indirectly */ + &[try_func, data, catch_func], + None, + None, + ); ret } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 0468e3de18d8b..9ba3803667275 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -592,7 +592,10 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ) }; - let result = bx.call(start_ty, None, None, start_fn, &args, None, instance); + let result = bx.call( + start_ty, None, None, start_fn, None, /*The entry fn does not return indirectly.*/ + &args, None, instance, + ); if cx.sess().target.os == Os::Uefi { bx.ret(result); } else { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 7f907bc630b2f..52c2eec947087 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -171,6 +171,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, + indirect_return_pointer: Option, llargs: &[Bx::Value], destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, mut unwind: mir::UnwindAction, @@ -260,6 +261,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, + indirect_return_pointer, llargs, ret_llbb, unwind_block, @@ -291,6 +293,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, + indirect_return_pointer, llargs, self.funclet(fx), instance, @@ -718,6 +721,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, drop_fn, + None, /*Drops always return nothing, so they can't have an indirect return.*/ args, Some((ReturnDest::Nothing, target)), unwind, @@ -822,6 +826,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + None, /*The assert functions don't return anything, so they can't return indirectly*/ &args, None, unwind, @@ -853,6 +858,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + None /*The codegen terminator does not return anything - so it can't have indirect reutns*/, &[], None, mir::UnwindAction::Unreachable, @@ -922,6 +928,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + None, /*Panics return nothing, so they can't have an indirect return.*/ &[msg.0, msg.1], target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), unwind, @@ -1202,21 +1209,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // We still need to call `make_return_dest` even if there's no `target`, since // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, // and `make_return_dest` adds the return-place indirect pointer to `llargs`. - let destination = match kind { + let (destination, indirect_return) = match kind { CallKind::Normal => { - let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs); - target.map(|target| (return_dest, target)) + let (return_dest, indirect_return) = + self.make_return_dest(bx, destination, &fn_abi.ret); + (target.map(|target| (return_dest, target)), indirect_return) } CallKind::Tail => { if fn_abi.ret.is_indirect() { - match self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs) { - ReturnDest::Nothing => {} + match self.make_return_dest(bx, destination, &fn_abi.ret) { + (ReturnDest::Nothing, _) => {} _ => bug!( "tail calls to functions with indirect returns cannot store into a destination" ), } } - None + (None, None) } }; @@ -1441,6 +1449,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, fn_ptr, + indirect_return, &llargs, destination, unwind, @@ -2359,7 +2368,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { let fn_ty = bx.fn_decl_backend_type(fn_abi); - let llret = bx.call(fn_ty, None, Some(fn_abi), fn_ptr, &[], funclet.as_ref(), None); + let llret = bx.call( + fn_ty, + None, + Some(fn_abi), + fn_ptr, + None, /*This function does not return indirectly.*/ + &[], + funclet.as_ref(), + None, + ); bx.apply_attrs_to_cleanup_callsite(llret); } @@ -2395,11 +2413,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, dest: mir::Place<'tcx>, fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, - llargs: &mut Vec, - ) -> ReturnDest<'tcx, Bx::Value> { + ) -> (ReturnDest<'tcx, Bx::Value>, Option) { // If the return is ignored, we can just return a do-nothing `ReturnDest`. if fn_ret.is_ignore() { - return ReturnDest::Nothing; + return (ReturnDest::Nothing, None); } let dest = if let Some(index) = dest.as_local() { match self.locals[index] { @@ -2413,10 +2430,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // but the calling convention has an indirect return. let tmp = PlaceRef::alloca(bx, fn_ret.layout); tmp.storage_live(bx); - llargs.push(tmp.val.llval); - ReturnDest::IndirectOperand(tmp, index) + (ReturnDest::IndirectOperand(tmp, index), Some(tmp.val.llval)) } else { - ReturnDest::DirectOperand(index) + (ReturnDest::DirectOperand(index), None) }; } LocalRef::Operand(_) => { @@ -2436,10 +2452,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // to create a temporary. span_bug!(self.mir.span, "can't directly store to unaligned value"); } - llargs.push(dest.val.llval); - ReturnDest::Nothing + (ReturnDest::Nothing, Some(dest.val.llval)) } else { - ReturnDest::Store(dest) + (ReturnDest::Store(dest), None) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 344a4834862e4..2b1d57d9b0151 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -779,6 +779,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_attrs.as_deref(), Some(fn_abi), fn_ptr, + None, /*The TLS shim does not return indirectly*/ &[], None, Some(instance), diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 0ef0179eac6c4..289597b9e1634 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -78,6 +78,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /* fn_attrs */ None, Some(fn_abi), llfn, + None,/*We know the ABI of this intrinc, and we know it does not require an indirect reutrn pointer*/ &[msg.0, msg.1], None, None, diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 56dc13b832032..092cd6f98562a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -135,6 +135,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: Self::Value, + indirect_return_pointer: Option, args: &[Self::Value], then: Self::BasicBlock, catch: Self::BasicBlock, @@ -652,6 +653,7 @@ pub trait BuilderMethods<'a, 'tcx>: caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, fn_val: Self::Value, + indirect_return_pointer: Option, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, From 72095a6927f69fce95dccb8cfd20cdf2dacebc36 Mon Sep 17 00:00:00 2001 From: FractalFir Date: Tue, 5 Aug 2025 22:12:26 +0200 Subject: [PATCH 2/7] Ported cg_gcc to the new cg_ssa version --- compiler/rustc_codegen_gcc/src/builder.rs | 23 +++++++++-- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 39 ++++++++++++++++--- compiler/rustc_codegen_llvm/src/builder.rs | 17 +++++--- compiler/rustc_codegen_ssa/src/mir/block.rs | 3 ++ compiler/rustc_codegen_ssa/src/size_of_val.rs | 2 +- 5 files changed, 69 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 88049d67964b1..29155abee276c 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -608,6 +608,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + indirect_return_pointer: Option>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, @@ -618,7 +619,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let current_block = self.block; self.block = try_block; - let call = self.call(typ, fn_attrs, fn_abi, func, args, None, instance); // FIXME(antoyo): use funclet here? + let call = + self.call(typ, fn_attrs, fn_abi, func, indirect_return_pointer, args, None, instance); // FIXME(antoyo): use funclet here? self.block = current_block; let return_value = @@ -646,13 +648,15 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + indirect_return_pointer: Option>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>, instance: Option>, ) -> RValue<'gcc> { - let call_site = self.call(typ, fn_attrs, fn_abi, func, args, None, instance); + let call_site = + self.call(typ, fn_attrs, fn_abi, func, indirect_return_pointer, args, None, instance); let condition = self.context.new_rvalue_from_int(self.bool_type, 1); self.llbb().end_with_conditional(self.location, condition, then, catch); if let Some(_fn_abi) = fn_abi { @@ -1779,19 +1783,30 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + indirect_return_pointer: Option>, args: &[RValue<'gcc>], funclet: Option<&Funclet>, _instance: Option>, ) -> RValue<'gcc> { + // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport. + let args = match indirect_return_pointer { + None => args.to_vec(), + Some(sret_ptr) => { + let mut args = args.to_vec(); + // Preappend the indirect return pointer + args.insert(0, sret_ptr); + args + } + }; // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { // FIXME(antoyo): remove when the API supports a different type for functions. let func: Function<'gcc> = self.cx.rvalue_as_function(func); - self.function_call(func, args, funclet) + self.function_call(func, &args, funclet) } else { // If it's a not function that was defined, it's a function pointer. - self.function_ptr_call(typ, fn_abi, func, args, funclet) + self.function_ptr_call(typ, fn_abi, func, &args, funclet) }; if let Some(_fn_abi) = fn_abi { // FIXME(bjorn3): Apply function attributes diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 09ad3254e5714..91f0c8dc29ab4 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -667,7 +667,16 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; - self.call(self.type_void(), None, None, func, &[], None, None); + self.call( + self.type_void(), + None, + None, + func, + None, /* abort does not return, so it can't return indirectly. */ + &[], + None, + None, + ); } fn assume(&mut self, value: Self::Value) { @@ -1348,7 +1357,7 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( let param_type = bx.u8_type.make_pointer(); let fn_type = bx.context.new_function_pointer_type(None, bx.type_void(), &[param_type], false); - bx.call(fn_type, None, None, try_func, &[data], None, None); + bx.call(fn_type, None, None, try_func, None, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. OperandValue::Immediate(bx.const_bool(false)).store(bx, dest); @@ -1421,21 +1430,41 @@ fn codegen_gnu_try<'gcc, 'tcx>( let zero = bx.cx.context.new_rvalue_zero(bx.int_type); let ptr = bx.cx.context.new_call(None, eh_pointer_builtin, &[zero]); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, None, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); // NOTE: the blocks must be filled before adding the try/catch, otherwise gcc will not // generate a try/catch. // FIXME(antoyo): add a check in the libgccjit API to prevent this. bx.switch_to_block(current_block); - bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + None, /* this function can't return indirectly */ + &[data], + then, + catch, + None, + None, + ); }); let func = unsafe { std::mem::transmute::, RValue<'gcc>>(func) }; // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + func, + None, /*This function can't return indirectly*/ + &[try_func, data, catch_func], + None, + None, + ); OperandValue::Immediate(ret).store(bx, dest); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index c6cad5ec2d0d0..752830adec62a 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -454,18 +454,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, - indirect_return_pointer: Option, + indirect_return_pointer: Option<&'ll Value>, args: &[&'ll Value], then: &'ll BasicBlock, catch: &'ll BasicBlock, funclet: Option<&Funclet<'ll>>, instance: Option>, ) -> &'ll Value { + // If this function returns indirectly(`PassMode::Indirect`), + // the `indirect_return_pointer` should be the first argument. let args = match indirect_return_pointer { None => args.to_vec(), Some(sret_ptr) => { let mut args = args.to_vec(); - // Preappend the indirect return pointer + // Preappend the indirect return pointer. args.insert(0, sret_ptr); args } @@ -1472,11 +1474,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, - indirect_return_pointer: Option, + indirect_return_pointer: Option<&'ll Value>, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, callee_instance: Option>, ) -> &'ll Value { + // If this function returns indirectly(`PassMode::Indirect`), + // the `indirect_return_pointer` should be the first argument. let args = match indirect_return_pointer { None => args.to_vec(), Some(sret_ptr) => { @@ -1557,7 +1561,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs, Some(fn_abi), llfn, - None, /*FIXME(FractalFir): can tail calls return indrectly?*/ + None, /* + FIXME(FractalFir): Tail calls don't support indirect returns at the time of writing, but they will do so soon. + Once this support is added, the indirect return pointer ought to be passed here(if present). + */ args, funclet, callee_instance, @@ -1901,7 +1908,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { args: &[&'ll Value], ) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params); - self.call(ty, None, None, f, None/* FIXME(FractalFir): I **assume** that no LLVM intrinsic returns a value via sret. Is this correct?*/, args, None, None) + self.call(ty, None, None, f, None/* (FractalFir): at the time of writing, no LLVM intrinsic retruns data indirectly(via `sret`). So, this is always None.*/, args, None, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 52c2eec947087..306144844d919 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -165,6 +165,9 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional /// return destination `destination` and the unwind action `unwind`. + /// The `indirect_return_pointer` is specified for functions returning + /// via `PassMode::indirect`, and points to a buffer, where the return value + /// shall be stored. fn do_call>( &self, fx: &mut FunctionCx<'a, 'tcx, Bx>, diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 289597b9e1634..5b4da7bb454cc 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -78,7 +78,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /* fn_attrs */ None, Some(fn_abi), llfn, - None,/*We know the ABI of this intrinc, and we know it does not require an indirect reutrn pointer*/ + None,/*We know the ABI of this intrinc, and we know it does not require an indirect return pointer*/ &[msg.0, msg.1], None, None, From 90adad68ac802f2ede2d8f8dc3b11060fc03f9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fractal=20Fir=28Micha=C5=82=20Kostrubiec=29?= Date: Fri, 19 Sep 2025 20:23:52 +0200 Subject: [PATCH 3/7] Apply suggestions from code review Co-authored-by: Jubilee --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_ssa/src/size_of_val.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 29155abee276c..9b74512b6ac88 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1793,7 +1793,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { None => args.to_vec(), Some(sret_ptr) => { let mut args = args.to_vec(); - // Preappend the indirect return pointer + // Prepend the indirect return pointer args.insert(0, sret_ptr); args } diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 5b4da7bb454cc..296f1cb1f8b60 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -78,7 +78,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /* fn_attrs */ None, Some(fn_abi), llfn, - None,/*We know the ABI of this intrinc, and we know it does not require an indirect return pointer*/ + None, // we know the ABI here &[msg.0, msg.1], None, None, From 80c13fad2e20b5367a5820e1e358e943524fc07c Mon Sep 17 00:00:00 2001 From: Antoni Boucher Date: Sat, 1 Nov 2025 11:11:49 -0400 Subject: [PATCH 4/7] Add missing indirect return argument to new calls --- compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 2 +- compiler/rustc_codegen_llvm/src/builder/autodiff.rs | 2 +- .../rustc_codegen_llvm/src/builder/gpu_offload.rs | 12 ++++++------ compiler/rustc_codegen_llvm/src/mono_item.rs | 1 + 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 91f0c8dc29ab4..a69d809fe54d0 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -656,7 +656,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } // FIXME directly use the llvm intrinsic adjustment functions here - let llret = self.call(fn_ty, None, None, fn_ptr, &call_args, None, None); + let llret = self.call(fn_ty, None, None, fn_ptr, None, &call_args, None, None); if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 8cefd8dc489a9..d08ffb5bed418 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -372,7 +372,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( crate::typetree::add_tt(&bx, fn_to_diff, fnc_tree); } - let call = bx.call(enzyme_ty, None, None, ad_fn, &args, None, None); + let call = bx.call(enzyme_ty, None, None, ad_fn, None, &args, None, None); let fn_ret_ty = bx.cx.val_ty(call); if fn_ret_ty == bx.cx.type_void() || fn_ret_ty == bx.cx.type_struct(&[], false) { diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index d20a73e8e6825..f64c8579a58c6 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -109,9 +109,9 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { // } let bb = Builder::append_block(cx, desc_reg_fn, "entry"); let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, register_lib, &[omp_descriptor], None, None); - a.call(init_ty, None, None, init_rtls, &[], None, None); - a.call(atexit, None, None, atexit_fn, &[desc_unreg_fn], None, None); + a.call(reg_lib_decl, None, None, register_lib, None, &[omp_descriptor], None, None); + a.call(init_ty, None, None, init_rtls, None, &[], None, None); + a.call(atexit, None, None, atexit_fn, None, &[desc_unreg_fn], None, None); a.ret_void(); // define internal void @.omp_offloading.descriptor_unreg() section ".text.startup" { @@ -121,7 +121,7 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { // } let bb = Builder::append_block(cx, desc_unreg_fn, "entry"); let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, unregister_lib, &[omp_descriptor], None, None); + a.call(reg_lib_decl, None, None, unregister_lib, None, &[omp_descriptor], None, None); a.ret_void(); // @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }] @@ -755,7 +755,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, None, None, fn_to_call, &args, None, None); + builder.call(fn_ty, None, None, fn_to_call, None, &args, None, None); } // Step 2) @@ -792,7 +792,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let device_id = builder.sext(device_id, cx.type_i64()); let args = vec![s_ident_t, device_id, num_workgroups, threads_per_block, region_id, a5]; - builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, None, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 19d43797a875d..5511db2dec646 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -203,6 +203,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { Some(attrs), Some(fn_abi), aliasee, + None, &args, None, Some(aliasee_instance), From 89e9f095166504797d9aca04773eda3a9977e7c2 Mon Sep 17 00:00:00 2001 From: Antoni Boucher Date: Sun, 19 Jul 2026 17:08:54 -0400 Subject: [PATCH 5/7] Improve the new indirect return API --- compiler/rustc_codegen_gcc/src/builder.rs | 22 +++--- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 24 +++---- compiler/rustc_codegen_llvm/src/asm.rs | 15 +++- compiler/rustc_codegen_llvm/src/builder.rs | 31 ++++---- .../src/builder/autodiff.rs | 4 +- .../src/builder/gpu_offload.rs | 32 +++++++-- compiler/rustc_codegen_llvm/src/intrinsic.rs | 31 +++----- compiler/rustc_codegen_llvm/src/mono_item.rs | 12 +++- compiler/rustc_codegen_ssa/src/base.rs | 6 +- compiler/rustc_codegen_ssa/src/mir/block.rs | 70 ++++++++++++------- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 2 +- compiler/rustc_codegen_ssa/src/size_of_val.rs | 2 +- .../rustc_codegen_ssa/src/traits/builder.rs | 26 +++++-- compiler/rustc_codegen_ssa/src/traits/mod.rs | 2 +- 14 files changed, 166 insertions(+), 113 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 9b74512b6ac88..83f2bcdd3959c 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -18,7 +18,7 @@ use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::{ BackendTypes, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - LayoutTypeCodegenMethods, OverflowOp, StaticBuilderMethods, + LayoutTypeCodegenMethods, OverflowOp, ReturnSlot, StaticBuilderMethods, }; use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; @@ -608,7 +608,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, - indirect_return_pointer: Option>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, @@ -619,8 +619,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let current_block = self.block; self.block = try_block; - let call = - self.call(typ, fn_attrs, fn_abi, func, indirect_return_pointer, args, None, instance); // FIXME(antoyo): use funclet here? + // FIXME(antoyo): use funclet here? + let call = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); self.block = current_block; let return_value = @@ -648,15 +648,14 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, - indirect_return_pointer: Option>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>, instance: Option>, ) -> RValue<'gcc> { - let call_site = - self.call(typ, fn_attrs, fn_abi, func, indirect_return_pointer, args, None, instance); + let call_site = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); let condition = self.context.new_rvalue_from_int(self.bool_type, 1); self.llbb().end_with_conditional(self.location, condition, then, catch); if let Some(_fn_abi) = fn_abi { @@ -1783,15 +1782,15 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, - indirect_return_pointer: Option>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], funclet: Option<&Funclet>, _instance: Option>, ) -> RValue<'gcc> { // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport. - let args = match indirect_return_pointer { - None => args.to_vec(), - Some(sret_ptr) => { + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { let mut args = args.to_vec(); // Prepend the indirect return pointer args.insert(0, sret_ptr); @@ -1820,6 +1819,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, _fn_abi: &FnAbi<'tcx, Ty<'tcx>>, _llfn: Self::Value, + _return_slot: ReturnSlot, _args: &[Self::Value], _funclet: Option<&Self::Funclet>, _instance: Option>, diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index a69d809fe54d0..2caaa0ee24aef 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -16,7 +16,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, ReturnSlot, }; use rustc_codegen_ssa::{MemFlags, RetagInfo}; use rustc_data_structures::fx::FxHashSet; @@ -656,7 +656,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } // FIXME directly use the llvm intrinsic adjustment functions here - let llret = self.call(fn_ty, None, None, fn_ptr, None, &call_args, None, None); + let llret = + self.call(fn_ty, None, None, fn_ptr, ReturnSlot::Direct, &call_args, None, None); if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } @@ -667,16 +668,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; - self.call( - self.type_void(), - None, - None, - func, - None, /* abort does not return, so it can't return indirectly. */ - &[], - None, - None, - ); + self.call(self.type_void(), None, None, func, ReturnSlot::Direct, &[], None, None); } fn assume(&mut self, value: Self::Value) { @@ -1357,7 +1349,7 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( let param_type = bx.u8_type.make_pointer(); let fn_type = bx.context.new_function_pointer_type(None, bx.type_void(), &[param_type], false); - bx.call(fn_type, None, None, try_func, None, &[data], None, None); + bx.call(fn_type, None, None, try_func, ReturnSlot::Direct, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. OperandValue::Immediate(bx.const_bool(false)).store(bx, dest); @@ -1430,7 +1422,7 @@ fn codegen_gnu_try<'gcc, 'tcx>( let zero = bx.cx.context.new_rvalue_zero(bx.int_type); let ptr = bx.cx.context.new_call(None, eh_pointer_builtin, &[zero]); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, None, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); // NOTE: the blocks must be filled before adding the try/catch, otherwise gcc will not @@ -1442,7 +1434,7 @@ fn codegen_gnu_try<'gcc, 'tcx>( None, None, try_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data], then, catch, @@ -1460,7 +1452,7 @@ fn codegen_gnu_try<'gcc, 'tcx>( None, None, func, - None, /*This function can't return indirectly*/ + ReturnSlot::Direct, &[try_func, data, catch_func], None, None, diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 2c7bbd124ed77..a54baeb3fecd9 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -563,9 +563,20 @@ pub(crate) fn inline_asm_call<'ll>( assert!(catch_funclet.is_none()); bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None) } else if let Some((catch, funclet)) = catch_funclet { - bx.invoke(fty, None, None, v, None /*FIXME(FractalFir): I assume that inline assembly does not return via `PassMode::Indirect` - is this correct?*/, inputs, dest.unwrap(), catch, funclet, None) + bx.invoke( + fty, + None, + None, + v, + ReturnSlot::Direct, + inputs, + dest.unwrap(), + catch, + funclet, + None, + ) } else { - bx.call(fty, None, None, v, None /*FIXME(FractalFir): I assume that inline assembly does not return via `PassMode::Indirect` - is this correct?*/,inputs, None, None) + bx.call(fty, None, None, v, ReturnSlot::Direct, inputs, None, None) }; // Store mark in a metadata node so we can map LLVM errors diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 752830adec62a..fe1c7251b1e8f 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -454,7 +454,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, - indirect_return_pointer: Option<&'ll Value>, + return_slot: ReturnSlot<&'ll Value>, args: &[&'ll Value], then: &'ll BasicBlock, catch: &'ll BasicBlock, @@ -462,12 +462,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { instance: Option>, ) -> &'ll Value { // If this function returns indirectly(`PassMode::Indirect`), - // the `indirect_return_pointer` should be the first argument. - let args = match indirect_return_pointer { - None => args.to_vec(), - Some(sret_ptr) => { + // the `return_slot` should be the first argument. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { let mut args = args.to_vec(); - // Preappend the indirect return pointer. args.insert(0, sret_ptr); args } @@ -1474,18 +1473,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, - indirect_return_pointer: Option<&'ll Value>, + return_slot: ReturnSlot<&'ll Value>, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, callee_instance: Option>, ) -> &'ll Value { // If this function returns indirectly(`PassMode::Indirect`), - // the `indirect_return_pointer` should be the first argument. - let args = match indirect_return_pointer { - None => args.to_vec(), - Some(sret_ptr) => { + // the `return_slot` should be the first argument. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { let mut args = args.to_vec(); - // Preappend the indirect return pointer args.insert(0, sret_ptr); args } @@ -1552,6 +1550,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, @@ -1561,10 +1560,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs, Some(fn_abi), llfn, - None, /* - FIXME(FractalFir): Tail calls don't support indirect returns at the time of writing, but they will do so soon. - Once this support is added, the indirect return pointer ought to be passed here(if present). - */ + return_slot, args, funclet, callee_instance, @@ -1908,7 +1904,8 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { args: &[&'ll Value], ) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params); - self.call(ty, None, None, f, None/* (FractalFir): at the time of writing, no LLVM intrinsic retruns data indirectly(via `sret`). So, this is always None.*/, args, None, None) + // No LLVM intrinsic returns its data indirectly (via `sret`). + self.call(ty, None, None, f, ReturnSlot::Direct, args, None, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index d08ffb5bed418..3831f3b0912fd 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -6,7 +6,7 @@ use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::IntrinsicResult; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceValue; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir::attrs::RustcAutodiff; use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; @@ -372,7 +372,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( crate::typetree::add_tt(&bx, fn_to_diff, fnc_tree); } - let call = bx.call(enzyme_ty, None, None, ad_fn, None, &args, None, None); + let call = bx.call(enzyme_ty, None, None, ad_fn, ReturnSlot::Direct, &args, None, None); let fn_ret_ty = bx.cx.val_ty(call); if fn_ret_ty == bx.cx.type_void() || fn_ret_ty == bx.cx.type_struct(&[], false) { diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index f64c8579a58c6..07c61ad34339d 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -6,7 +6,7 @@ use rustc_abi::Align; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_middle::bug; use rustc_middle::ty::offload_meta::{MappingFlags, OffloadMetadata, OffloadSize}; @@ -109,9 +109,18 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { // } let bb = Builder::append_block(cx, desc_reg_fn, "entry"); let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, register_lib, None, &[omp_descriptor], None, None); - a.call(init_ty, None, None, init_rtls, None, &[], None, None); - a.call(atexit, None, None, atexit_fn, None, &[desc_unreg_fn], None, None); + a.call( + reg_lib_decl, + None, + None, + register_lib, + ReturnSlot::Direct, + &[omp_descriptor], + None, + None, + ); + a.call(init_ty, None, None, init_rtls, ReturnSlot::Direct, &[], None, None); + a.call(atexit, None, None, atexit_fn, ReturnSlot::Direct, &[desc_unreg_fn], None, None); a.ret_void(); // define internal void @.omp_offloading.descriptor_unreg() section ".text.startup" { @@ -121,7 +130,16 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { // } let bb = Builder::append_block(cx, desc_unreg_fn, "entry"); let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, unregister_lib, None, &[omp_descriptor], None, None); + a.call( + reg_lib_decl, + None, + None, + unregister_lib, + ReturnSlot::Direct, + &[omp_descriptor], + None, + None, + ); a.ret_void(); // @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }] @@ -755,7 +773,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, None, None, fn_to_call, None, &args, None, None); + builder.call(fn_ty, None, None, fn_to_call, ReturnSlot::Direct, &args, None, None); } // Step 2) @@ -792,7 +810,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let device_id = builder.sext(device_id, cx.type_i64()); let args = vec![s_ident_t, device_id, num_workgroups, threads_per_block, region_id, a5]; - builder.call(tgt_target_kernel_ty, None, None, tgt_decl, None, &args, None, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, ReturnSlot::Direct, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4109181253556..898c28d7c2627 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1348,16 +1348,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( ) -> &'ll Value { if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.call( - try_func_ty, - None, - None, - try_func, - None, /* this function can't return indirectly */ - &[data], - None, - None, - ); + bx.call(try_func_ty, None, None, try_func, ReturnSlot::Direct, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. bx.const_bool(false) @@ -1460,7 +1451,7 @@ fn codegen_msvc_try<'ll, 'tcx>( None, None, try_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data], normal, catchswitch, @@ -1519,7 +1510,7 @@ fn codegen_msvc_try<'ll, 'tcx>( None, None, catch_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data, ptr], Some(&funclet), None, @@ -1536,7 +1527,7 @@ fn codegen_msvc_try<'ll, 'tcx>( None, None, catch_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data, null], Some(&funclet), None, @@ -1554,7 +1545,7 @@ fn codegen_msvc_try<'ll, 'tcx>( None, None, llfn, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[try_func, data, catch_func], None, None, @@ -1610,7 +1601,7 @@ fn codegen_wasm_try<'ll, 'tcx>( None, None, try_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data], normal, catchswitch, @@ -1637,7 +1628,7 @@ fn codegen_wasm_try<'ll, 'tcx>( None, None, catch_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data, ptr], Some(&funclet), None, @@ -1655,7 +1646,7 @@ fn codegen_wasm_try<'ll, 'tcx>( None, None, llfn, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[try_func, data, catch_func], None, None, @@ -1705,7 +1696,7 @@ fn codegen_gnu_try<'ll, 'tcx>( None, None, try_func, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[data], then, catch, @@ -1729,7 +1720,7 @@ fn codegen_gnu_try<'ll, 'tcx>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, None, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); }); @@ -1740,7 +1731,7 @@ fn codegen_gnu_try<'ll, 'tcx>( None, None, llfn, - None, /* this function can't return indirectly */ + ReturnSlot::Direct, &[try_func, data, catch_func], None, None, diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 5511db2dec646..da8181bb3de96 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -198,13 +198,21 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { args.push(llvm::get_param(alias_lldecl, index)); } + // For an indirect return, the alias's own first parameter is the + // caller-provided return slot: forward it to the aliasee as such. + let (return_slot, args) = if fn_abi.ret.is_indirect() { + let (sret_ptr, rest) = args.split_first().unwrap(); + (ReturnSlot::Indirect(*sret_ptr), rest) + } else { + (ReturnSlot::Direct, &args[..]) + }; let call = start_bx.call( fn_ty, Some(attrs), Some(fn_abi), aliasee, - None, - &args, + return_slot, + args, None, Some(aliasee_instance), ); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 9ba3803667275..bab073cafe218 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -592,10 +592,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ) }; - let result = bx.call( - start_ty, None, None, start_fn, None, /*The entry fn does not return indirectly.*/ - &args, None, instance, - ); + let result = + bx.call(start_ty, None, None, start_fn, ReturnSlot::Direct, &args, None, instance); if cx.sess().target.os == Os::Uefi { bx.ret(result); } else { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 306144844d919..9161f6e42e986 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -165,8 +165,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional /// return destination `destination` and the unwind action `unwind`. - /// The `indirect_return_pointer` is specified for functions returning - /// via `PassMode::indirect`, and points to a buffer, where the return value + /// The `return_slot` is [`ReturnSlot::Indirect`] for functions returning + /// via `PassMode::Indirect`, and points to a buffer where the return value /// shall be stored. fn do_call>( &self, @@ -174,7 +174,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, - indirect_return_pointer: Option, + return_slot: ReturnSlot, llargs: &[Bx::Value], destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, mut unwind: mir::UnwindAction, @@ -248,8 +248,23 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } }; + debug_assert_eq!( + return_slot.is_indirect(), + fn_abi.ret.is_indirect(), + "a return slot must be provided if and only if the return is `PassMode::Indirect`", + ); + if kind == CallKind::Tail { - bx.tail_call(fn_ty, caller_attrs, fn_abi, fn_ptr, llargs, self.funclet(fx), instance); + bx.tail_call( + fn_ty, + caller_attrs, + fn_abi, + fn_ptr, + return_slot, + llargs, + self.funclet(fx), + instance, + ); return MergingSucc::False; } @@ -264,7 +279,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, - indirect_return_pointer, + return_slot, llargs, ret_llbb, unwind_block, @@ -296,7 +311,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, - indirect_return_pointer, + return_slot, llargs, self.funclet(fx), instance, @@ -724,7 +739,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, drop_fn, - None, /*Drops always return nothing, so they can't have an indirect return.*/ + ReturnSlot::Direct, args, Some((ReturnDest::Nothing, target)), unwind, @@ -829,7 +844,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, - None, /*The assert functions don't return anything, so they can't return indirectly*/ + ReturnSlot::Direct, &args, None, unwind, @@ -861,7 +876,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, - None /*The codegen terminator does not return anything - so it can't have indirect reutns*/, + ReturnSlot::Direct, &[], None, mir::UnwindAction::Unreachable, @@ -931,7 +946,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, - None, /*Panics return nothing, so they can't have an indirect return.*/ + ReturnSlot::Direct, &[msg.0, msg.1], target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), unwind, @@ -1212,22 +1227,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // We still need to call `make_return_dest` even if there's no `target`, since // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, // and `make_return_dest` adds the return-place indirect pointer to `llargs`. - let (destination, indirect_return) = match kind { + let (destination, return_slot) = match kind { CallKind::Normal => { - let (return_dest, indirect_return) = + let (return_dest, return_slot) = self.make_return_dest(bx, destination, &fn_abi.ret); - (target.map(|target| (return_dest, target)), indirect_return) + (target.map(|target| (return_dest, target)), return_slot) } CallKind::Tail => { - if fn_abi.ret.is_indirect() { + let return_slot = if fn_abi.ret.is_indirect() { match self.make_return_dest(bx, destination, &fn_abi.ret) { - (ReturnDest::Nothing, _) => {} + (ReturnDest::Nothing, return_slot) => return_slot, _ => bug!( "tail calls to functions with indirect returns cannot store into a destination" ), } - } - (None, None) + } else { + ReturnSlot::Direct + }; + (None, return_slot) } }; @@ -1452,7 +1469,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, fn_ptr, - indirect_return, + return_slot, &llargs, destination, unwind, @@ -2376,7 +2393,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { None, Some(fn_abi), fn_ptr, - None, /*This function does not return indirectly.*/ + ReturnSlot::Direct, &[], funclet.as_ref(), None, @@ -2416,10 +2433,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, dest: mir::Place<'tcx>, fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, - ) -> (ReturnDest<'tcx, Bx::Value>, Option) { + ) -> (ReturnDest<'tcx, Bx::Value>, ReturnSlot) { // If the return is ignored, we can just return a do-nothing `ReturnDest`. if fn_ret.is_ignore() { - return (ReturnDest::Nothing, None); + return (ReturnDest::Nothing, ReturnSlot::Direct); } let dest = if let Some(index) = dest.as_local() { match self.locals[index] { @@ -2433,9 +2450,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // but the calling convention has an indirect return. let tmp = PlaceRef::alloca(bx, fn_ret.layout); tmp.storage_live(bx); - (ReturnDest::IndirectOperand(tmp, index), Some(tmp.val.llval)) + ( + ReturnDest::IndirectOperand(tmp, index), + ReturnSlot::Indirect(tmp.val.llval), + ) } else { - (ReturnDest::DirectOperand(index), None) + (ReturnDest::DirectOperand(index), ReturnSlot::Direct) }; } LocalRef::Operand(_) => { @@ -2455,9 +2475,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // to create a temporary. span_bug!(self.mir.span, "can't directly store to unaligned value"); } - (ReturnDest::Nothing, Some(dest.val.llval)) + (ReturnDest::Nothing, ReturnSlot::Indirect(dest.val.llval)) } else { - (ReturnDest::Store(dest), None) + (ReturnDest::Store(dest), ReturnSlot::Direct) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 2b1d57d9b0151..6278020eabecd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -779,7 +779,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_attrs.as_deref(), Some(fn_abi), fn_ptr, - None, /*The TLS shim does not return indirectly*/ + ReturnSlot::Direct, &[], None, Some(instance), diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 296f1cb1f8b60..e286681c293ee 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -78,7 +78,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /* fn_attrs */ None, Some(fn_abi), llfn, - None, // we know the ABI here + ReturnSlot::Direct, // we know the ABI here &[msg.0, msg.1], None, None, diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 092cd6f98562a..eb67f8dcbb1b0 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -34,6 +34,20 @@ pub enum OverflowOp { Mul, } +/// The location of the return value for the call. +#[derive(Copy, Clone, Debug)] +pub enum ReturnSlot { + Direct, + /// The return value will be passed via sret (e.g. `PassMode::Indirect`). + Indirect(V), +} + +impl ReturnSlot { + pub fn is_indirect(&self) -> bool { + matches!(self, ReturnSlot::Indirect(_)) + } +} + pub trait BuilderMethods<'a, 'tcx>: Sized + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> @@ -135,7 +149,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: Self::Value, - indirect_return_pointer: Option, + return_slot: ReturnSlot, args: &[Self::Value], then: Self::BasicBlock, catch: Self::BasicBlock, @@ -643,17 +657,20 @@ pub trait BuilderMethods<'a, 'tcx>: /// The typical case that they are None is during the codegen of intrinsics and lang-items, /// as those are "fake functions" with only a trivial ABI if any, et cetera. /// + /// `return_slot` must be `ReturnSlot::Indirect` if an argument uses `PassMode::Indirect`. + /// /// ## Return /// - /// Must return the value the function will return so it can be written to the destination, - /// assuming the function does not explicitly pass the destination as a pointer in `args`. + /// Must return the value the function will return so it can be written to the destination. + /// For calls with an indirect return, the returned value is meaningless and must not be + /// used: the return value lives in the return slot. fn call( &mut self, llty: Self::FunctionSignature, caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, fn_val: Self::Value, - indirect_return_pointer: Option, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, @@ -665,6 +682,7 @@ pub trait BuilderMethods<'a, 'tcx>: caller_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index f46d07ea5008e..1f013cb122070 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -36,7 +36,7 @@ pub use self::asm::{ AsmBuilderMethods, AsmCodegenMethods, GlobalAsmOperandRef, InlineAsmOperandRef, }; pub use self::backend::{BackendTypes, CodegenBackend, ExtraBackendMethods}; -pub use self::builder::{BuilderMethods, OverflowOp}; +pub use self::builder::{BuilderMethods, OverflowOp, ReturnSlot}; pub use self::consts::ConstCodegenMethods; pub use self::coverageinfo::CoverageInfoBuilderMethods; pub use self::debuginfo::{DebugInfoBuilderMethods, DebugInfoCodegenMethods}; From 47c07d871bb0b3b5b59876267a815acc07c2ea50 Mon Sep 17 00:00:00 2001 From: Antoni Boucher Date: Mon, 27 Jul 2026 17:19:37 -0400 Subject: [PATCH 6/7] Address review --- compiler/rustc_codegen_llvm/src/builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index fe1c7251b1e8f..a0685a0b5d58a 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -461,7 +461,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { funclet: Option<&Funclet<'ll>>, instance: Option>, ) -> &'ll Value { - // If this function returns indirectly(`PassMode::Indirect`), + // If this function returns indirectly (`PassMode::Indirect`), // the `return_slot` should be the first argument. let args = match return_slot { ReturnSlot::Direct => args.to_vec(), @@ -1478,7 +1478,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { funclet: Option<&Funclet<'ll>>, callee_instance: Option>, ) -> &'ll Value { - // If this function returns indirectly(`PassMode::Indirect`), + // If this function returns indirectly (`PassMode::Indirect`), // the `return_slot` should be the first argument. let args = match return_slot { ReturnSlot::Direct => args.to_vec(), From 1c68965650ddaa914c68061a46f8ae4c84910b7e Mon Sep 17 00:00:00 2001 From: Antoni Boucher Date: Fri, 21 Aug 2026 08:13:30 -0400 Subject: [PATCH 7/7] Add missing direct return arguments --- compiler/rustc_codegen_llvm/src/builder.rs | 1 + compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index a0685a0b5d58a..2ff11315fe884 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2114,6 +2114,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { None, None, ubsan_handler, + ReturnSlot::Direct, &[diag_data, function_address, self.const_usize(0)], None, None, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 898c28d7c2627..428216d7b78af 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -244,7 +244,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::offload_get_num_devices => { let (fn_decl, fn_ty) = declare_omp_get_num_devices(self.cx); - let llval = self.call(fn_ty, None, None, fn_decl, &[], None, None); + let llval = self.call(fn_ty, None, None, fn_decl, ReturnSlot::Direct, &[], None, None); return IntrinsicResult::Operand(OperandValue::Immediate(llval)); },