Rollup of 7 pull requests

compiler/rustc_ast/src/tokenstream.rs

@@ -249,14 +249,14 @@ impl LazyAttrTokenStreamInner {
                 break_last_token,
                 node_replacements,
             } => {
-                // The token produced by the final call to `{,inlined_}next` was not
+                // The token produced by the final call to `{,inlined_}next_and_bump` was not
                 // actually consumed by the callback. The combination of chaining the
                 // initial token and using `take` produces the desired result - we
                 // produce an empty `TokenStream` if no calls were made, and omit the
                 // final token otherwise.
                 let mut cursor_snapshot = cursor_snapshot.clone();
                 let tokens = iter::once(FlatToken::Token(*start_token))
-                    .chain(iter::repeat_with(|| FlatToken::Token(cursor_snapshot.next())))
+                    .chain(iter::repeat_with(|| FlatToken::Token(cursor_snapshot.next_and_bump())))
                     .take(*num_calls as usize);
 
                 if node_replacements.is_empty() {
@@ -883,36 +883,48 @@ impl<'t> Iterator for TokenStreamIter<'t> {
 #[derive(Clone, Debug)]
 struct TokenTreeCursor {
     stream: TokenStream,
-    /// Points to the current token tree in the stream. In `TokenCursor::curr`,
-    /// this can be any token tree. In `TokenCursor::stack`, this is always a
-    /// `TokenTree::Delimited`.
-    index: usize,
+    /// Points to the next token tree (or one past the end of the stream).
+    next_idx: usize,
 }
 
 impl TokenTreeCursor {
     #[inline]
     fn new(stream: TokenStream) -> Self {
-        TokenTreeCursor { stream, index: 0 }
+        TokenTreeCursor { stream, next_idx: 0 }
     }
 
+    /// Gets the current token tree within this cursor. In a debug build it panics on a cursor that
+    /// hasn't been bumped; in a release build it will return `None`.
     #[inline]
     fn curr(&self) -> Option<&TokenTree> {
-        self.stream.get(self.index)
+        debug_assert!(self.next_idx > 0);
+        self.stream.get(self.next_idx - 1)
     }
 
+    /// Gets the next token tree without advancing.
+    #[inline]
+    fn next(&self) -> Option<&TokenTree> {
+        self.stream.get(self.next_idx)
+    }
+
+    /// Gets the token tree `n` ahead. `look_ahead(1)` is equivalent to `next()`. `look_ahead(0)`
+    /// isn't allowed and will panic.
+    #[inline]
     fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
-        self.stream.get(self.index + n)
+        assert_ne!(n, 0);
+        self.stream.get(self.next_idx + (n - 1))
     }
 
+    /// Move the cursor to the next token tree.
     #[inline]
     fn bump(&mut self) {
-        self.index += 1;
+        self.next_idx += 1;
     }
 
-    // For skipping ahead in rare circumstances.
+    /// For skipping ahead in rare circumstances.
     #[inline]
     fn bump_to_end(&mut self) {
-        self.index = self.stream.len();
+        self.next_idx = self.stream.len();
     }
 }
 
@@ -922,15 +934,16 @@ impl TokenTreeCursor {
 /// what the parser expects, for the most part.
 #[derive(Clone, Debug)]
 pub struct TokenCursor {
-    // Cursor for the current (innermost) token stream. The index within the
+    // Cursor for the current (innermost) token stream. The `next_idx` within the
     // cursor can point to any token tree in the stream (or one past the end).
-    // The delimiters for this token stream are found in `self.stack.last()`;
-    // if that is `None` we are in the outermost token stream which never has
-    // delimiters.
+    // The delimiters for this token stream are found in the current token tree
+    // in `self.stack.last()`; if that is `None` we are in the outermost token
+    // stream which never has delimiters.
     curr: TokenTreeCursor,
 
-    // Token streams surrounding the current one. The index within each cursor
-    // always points to a `TokenTree::Delimited`.
+    // Token streams surrounding the current one. The `next_idx` within each cursor
+    // is always greater than zero and always points one past the current
+    // `TokenTree::Delimited`.
     stack: Vec<TokenTreeCursor>,
 }
 
@@ -940,12 +953,13 @@ impl TokenCursor {
         TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] }
     }
 
-    pub fn next(&mut self) -> (Token, Spacing) {
-        self.inlined_next()
+    /// Gets the next token and advances the cursor by one.
+    pub fn next_and_bump(&mut self) -> (Token, Spacing) {
+        self.inlined_next_and_bump()
     }
 
-    /// An `n` of zero is the next token tree in the current token stream; won't look outside the
-    /// current token stream.
+    /// An `n` of 1 is the next token tree in the current token stream; won't look outside the
+    /// current token stream. `look_ahead(0)` isn't allowed and will panic.
     #[inline]
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
         self.curr.look_ahead(n)
@@ -955,7 +969,7 @@ impl TokenCursor {
     /// delimited sequence. Panics if we are not within a delimited sequence.
     #[inline]
     pub fn look_ahead_past_close_delim(&self) -> Option<&TokenTree> {
-        self.stack.last().unwrap().look_ahead(1)
+        self.stack.last().unwrap().next()
     }
 
     /// Clones the `TokenTree::Delimited` that we are currently within. Panics if we are not within
@@ -991,12 +1005,12 @@ impl TokenCursor {
 
     /// This always-inlined version should only be used on hot code paths.
     #[inline(always)]
-    pub fn inlined_next(&mut self) -> (Token, Spacing) {
+    pub fn inlined_next_and_bump(&mut self) -> (Token, Spacing) {
         loop {
             // FIXME: we currently don't return `Delimiter::Invisible` open/close delims. To fix
             // #67062 we will need to, whereupon the `delim != Delimiter::Invisible` conditions
             // below can be removed.
-            if let Some(tree) = self.curr.curr() {
+            if let Some(tree) = self.curr.next() {
                 match tree {
                     &TokenTree::Token(token, spacing) => {
                         debug_assert!(!token.kind.is_delim());
@@ -1006,6 +1020,7 @@ impl TokenCursor {
                     }
                     &TokenTree::Delimited(sp, spacing, delim, ref tts) => {
                         let trees = TokenTreeCursor::new(tts.clone());
+                        self.curr.bump(); // move past the `Delimited`
                         self.stack.push(mem::replace(&mut self.curr, trees));
                         if !delim.skip() {
                             return (Token::new(delim.as_open_token_kind(), sp.open), spacing.open);
@@ -1019,7 +1034,6 @@ impl TokenCursor {
                     panic!("parent should be Delimited")
                 };
                 self.curr = parent;
-                self.curr.bump(); // move past the `Delimited`
                 if !delim.skip() {
                     return (Token::new(delim.as_close_token_kind(), span.close), spacing.close);
                 }

compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

@@ -122,8 +122,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 }
                 _ => unreachable!("{:?}", self.layout.backend_repr),
             },
-            PassMode::Cast { ref cast, pad_i32 } => {
-                assert!(!pad_i32, "padding support not yet implemented");
+            PassMode::Cast { ref cast, pad_i32_count } => {
+                assert_eq!(pad_i32_count, 0, "padding support not yet implemented");
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
             }
             PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {

compiler/rustc_codegen_gcc/src/abi.rs

@@ -168,11 +168,13 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     ));
                     continue;
                 }
-                PassMode::Cast { ref cast, pad_i32 } => {
-                    // add padding
-                    if pad_i32 {
-                        argument_tys.push(Reg::i32().gcc_type(cx));
-                    }
+                PassMode::Cast { ref cast, pad_i32_count } => {
+                    // Add padding.
+                    argument_tys.extend(std::iter::repeat_n(
+                        Reg::i32().gcc_type(cx),
+                        usize::from(pad_i32_count),
+                    ));
+
                     let ty = cast.gcc_type(cx);
                     apply_attrs(ty, &cast.attrs, argument_tys.len())
                 }

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -346,8 +346,8 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         fn_abi.ptr_to_gcc_type(self)
     }
 
-    fn reg_backend_type(&self, _ty: &Reg) -> Type<'gcc> {
-        unimplemented!();
+    fn reg_backend_type(&self, ty: &Reg) -> Type<'gcc> {
+        ty.gcc_type(self)
     }
 
     fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {

compiler/rustc_codegen_llvm/src/abi.rs

@@ -249,7 +249,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
-            PassMode::Cast { cast, pad_i32: _ } => {
+            PassMode::Cast { cast, pad_i32_count: _ } => {
                 // The ABI mandates that the value is passed as a different struct representation.
                 // Spill and reload it from the stack to convert from the ABI representation to
                 // the Rust representation.
@@ -366,7 +366,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         let llreturn_ty = match &self.ret.mode {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
-            PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
+            PassMode::Cast { cast, pad_i32_count: _ } => cast.llvm_type(cx),
             PassMode::Indirect { .. } => {
                 llargument_tys.push(cx.type_ptr());
                 cx.type_void()
@@ -405,11 +405,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     continue;
                 }
                 PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
-                PassMode::Cast { cast, pad_i32 } => {
-                    // add padding
-                    if *pad_i32 {
-                        llargument_tys.push(Reg::i32().llvm_type(cx));
-                    }
+                PassMode::Cast { cast, pad_i32_count } => {
+                    // Add padding.
+                    llargument_tys.extend(std::iter::repeat_n(
+                        Reg::i32().llvm_type(cx),
+                        usize::from(*pad_i32_count),
+                    ));
+
                     // Compute the LLVM type we use for this function from the cast type.
                     // We assume here that ABI-compatible Rust types have the same cast type.
                     cast.llvm_type(cx)
@@ -511,7 +513,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     );
                 }
             }
-            PassMode::Cast { cast, pad_i32: _ } => {
+            PassMode::Cast { cast, pad_i32_count: _ } => {
                 cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
             }
             _ => {}
@@ -580,8 +582,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b);
                     }
                 }
-                PassMode::Cast { cast, pad_i32 } => {
-                    if *pad_i32 {
+                PassMode::Cast { cast, pad_i32_count } => {
+                    for _ in 0..*pad_i32_count {
                         apply(&ArgAttributes::new());
                     }
                     apply(&cast.attrs);
@@ -630,7 +632,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 );
                 attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]);
             }
-            PassMode::Cast { cast, pad_i32: _ } => {
+            PassMode::Cast { cast, pad_i32_count: _ } => {
                 cast.attrs.apply_attrs_to_callsite(
                     llvm::AttributePlace::ReturnValue,
                     bx.cx,
@@ -666,8 +668,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     apply(bx.cx, a);
                     apply(bx.cx, b);
                 }
-                PassMode::Cast { cast, pad_i32 } => {
-                    if *pad_i32 {
+                PassMode::Cast { cast, pad_i32_count } => {
+                    for _ in 0..*pad_i32_count {
                         apply(bx.cx, &ArgAttributes::new());
                     }
                     apply(bx.cx, &cast.attrs);

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -588,7 +588,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            PassMode::Cast { cast: cast_ty, pad_i32: _ } => {
+            PassMode::Cast { cast: cast_ty, pad_i32_count: _ } => {
                 let op = match self.locals[mir::RETURN_PLACE] {
                     LocalRef::Operand(op) => op,
                     LocalRef::PendingOperand => bug!("use of return before def"),
@@ -1936,9 +1936,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) {
         match arg.mode {
             PassMode::Ignore => return,
-            PassMode::Cast { pad_i32: true, .. } => {
+            PassMode::Cast { pad_i32_count, .. } => {
                 // Fill padding with undef value, where applicable.
-                llargs.push(bx.const_undef(bx.reg_backend_type(&Reg::i32())));
+                let undef = bx.const_undef(bx.reg_backend_type(&Reg::i32()));
+                llargs.extend(std::iter::repeat_n(undef, usize::from(pad_i32_count)));
             }
             PassMode::Pair(..) => match op.val {
                 Pair(a, b) => {
@@ -2025,7 +2026,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         if by_ref && !arg.is_indirect() {
             // Have to load the argument, maybe while casting it.
-            if let PassMode::Cast { cast, pad_i32: _ } = &arg.mode {
+            if let PassMode::Cast { cast, pad_i32_count: _ } = &arg.mode {
                 // The ABI mandates that the value is passed as a different struct representation.
                 // Spill and reload it from the stack to convert from the Rust representation to
                 // the ABI representation.

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -501,8 +501,8 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 for i in 0..tupled_arg_tys.len() {
                     let arg = &fx.fn_abi.args[idx];
                     idx += 1;
-                    if let PassMode::Cast { pad_i32: true, .. } = arg.mode {
-                        llarg_idx += 1;
+                    if let PassMode::Cast { pad_i32_count, .. } = arg.mode {
+                        llarg_idx += usize::from(pad_i32_count);
                     }
                     let pr_field = place.project_field(bx, i);
                     bx.store_fn_arg(arg, &mut llarg_idx, pr_field);
@@ -529,8 +529,8 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             let arg = &fx.fn_abi.args[idx];
             idx += 1;
-            if let PassMode::Cast { pad_i32: true, .. } = arg.mode {
-                llarg_idx += 1;
+            if let PassMode::Cast { pad_i32_count, .. } = arg.mode {
+                llarg_idx += usize::from(pad_i32_count);
             }
 
             if !memory_locals.contains(local) {

compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

@@ -472,9 +472,9 @@ fn wasm_type<'tcx>(signature: &mut String, arg_abi: &ArgAbi<'_, Ty<'tcx>>, ptr_t
             }
             other => unreachable!("{other:?}"),
         },
-        PassMode::Cast { pad_i32, ref cast } => {
+        PassMode::Cast { pad_i32_count, ref cast } => {
             // For wasm, Cast is used for single-field primitive wrappers like `struct Wrapper(i64);`
-            assert!(!pad_i32, "not currently used by wasm calling convention");
+            assert_eq!(pad_i32_count, 0, "not currently used by wasm calling convention");
             assert!(cast.prefix.is_empty(), "no prefix");
             assert_eq!(cast.rest.total, arg_abi.layout.size, "single item");
 

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -24,7 +24,7 @@ enum UsesVectorRegisters {
 fn passes_vectors_by_value(mode: &PassMode, repr: &BackendRepr) -> UsesVectorRegisters {
     match mode {
         PassMode::Ignore | PassMode::Indirect { .. } => UsesVectorRegisters::No,
-        PassMode::Cast { pad_i32: _, cast }
+        PassMode::Cast { pad_i32_count: _, cast }
             if cast.prefix.iter().any(|x| matches!(x.kind, RegKind::Vector { .. }))
                 || matches!(cast.rest.unit.kind, RegKind::Vector { .. }) =>
         {