Never create an temporary PassMode::Direct when it is not a valid pass mode for a type

Author: bjorn3

Diff for: compiler/rustc_middle/src/ty/layout.rs

@@ -2745,7 +2745,11 @@ where
                 cx.layout_of(ty)
             };
 
-            let mut arg = ArgAbi::new(layout);
+            let mut arg = ArgAbi::new(cx, layout, |layout, scalar, offset| {
+                let mut attrs = ArgAttributes::new();
+                adjust_for_rust_scalar(&mut attrs, scalar, *layout, offset, is_return);
+                attrs
+            });
 
             if arg.layout.is_zst() {
                 // For some forsaken reason, x86_64-pc-windows-gnu
@@ -2762,27 +2766,6 @@ where
                 }
             }
 
-            if let Abi::ScalarPair(ref a, ref b) = arg.layout.abi {
-                let mut a_attrs = ArgAttributes::new();
-                let mut b_attrs = ArgAttributes::new();
-                adjust_for_rust_scalar(&mut a_attrs, a, arg.layout, Size::ZERO, is_return);
-                adjust_for_rust_scalar(
-                    &mut b_attrs,
-                    b,
-                    arg.layout,
-                    a.value.size(cx).align_to(b.value.align(cx).abi),
-                    is_return,
-                );
-                arg.mode = PassMode::Pair(a_attrs, b_attrs);
-                return arg;
-            }
-
-            if let Abi::Scalar(ref scalar) = arg.layout.abi {
-                if let PassMode::Direct(ref mut attrs) = arg.mode {
-                    adjust_for_rust_scalar(attrs, scalar, arg.layout, Size::ZERO, is_return);
-                }
-            }
-
             arg
         };
 
@@ -2859,9 +2842,12 @@ where
                 let max_by_val_size = Pointer.size(cx) * 2;
                 let size = arg.layout.size;
 
-                if arg.layout.is_unsized() || size > max_by_val_size {
-                    arg.make_indirect();
-                } else {
+                assert!(
+                    matches!(arg.mode, PassMode::Indirect { on_stack: false, .. }),
+                    "{:?}",
+                    arg
+                );
+                if !arg.layout.is_unsized() && size <= max_by_val_size {
                     // We want to pass small aggregates as immediates, but using
                     // a LLVM aggregate type for this leads to bad optimizations,
                     // so we pick an appropriately sized integer type instead.
@@ -2881,15 +2867,11 @@ where
     }
 }
 
-fn make_thin_self_ptr<'tcx, C>(
-    cx: &C,
-    mut layout: TyAndLayout<'tcx>,
-) -> TyAndLayout<'tcx>
-where C: LayoutOf<Ty = Ty<'tcx>, TyAndLayout = TyAndLayout<'tcx>>
-        //+ HasDataLayout
-        //+ HasTargetSpec
+fn make_thin_self_ptr<'tcx, C>(cx: &C, mut layout: TyAndLayout<'tcx>) -> TyAndLayout<'tcx>
+where
+    C: LayoutOf<Ty = Ty<'tcx>, TyAndLayout = TyAndLayout<'tcx>>
         + HasTyCtxt<'tcx>
-        + HasParamEnv<'tcx>
+        + HasParamEnv<'tcx>,
 {
     let fat_pointer_ty = if layout.is_unsized() {
         // unsized `self` is passed as a pointer to `self`

Diff for: compiler/rustc_target/src/abi/call/mod.rs

@@ -434,31 +434,49 @@ pub struct ArgAbi<'a, Ty> {
 }
 
 impl<'a, Ty> ArgAbi<'a, Ty> {
-    pub fn new(layout: TyAndLayout<'a, Ty>) -> Self {
-        ArgAbi { layout, pad: None, mode: PassMode::Direct(ArgAttributes::new()) }
+    pub fn new(
+        cx: &impl HasDataLayout,
+        layout: TyAndLayout<'a, Ty>,
+        scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, &abi::Scalar, Size) -> ArgAttributes,
+    ) -> Self {
+        let mode = match &layout.abi {
+            Abi::Uninhabited => PassMode::Ignore,
+            Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)),
+            Abi::ScalarPair(a, b) => PassMode::Pair(
+                scalar_attrs(&layout, a, Size::ZERO),
+                scalar_attrs(&layout, b, a.value.size(cx).align_to(b.value.align(cx).abi)),
+            ),
+            Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()),
+            Abi::Aggregate { .. } => Self::indirect_pass_mode(&layout),
+        };
+        ArgAbi { layout, pad: None, mode }
     }
 
-    pub fn make_indirect(&mut self) {
-        match self.mode {
-            PassMode::Direct(_) | PassMode::Pair(_, _) => {}
-            _ => panic!("Tried to make {:?} indirect", self.mode),
-        }
-
-        // Start with fresh attributes for the pointer.
+    fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode {
         let mut attrs = ArgAttributes::new();
 
         // For non-immediate arguments the callee gets its own copy of
         // the value on the stack, so there are no aliases. It's also
         // program-invisible so can't possibly capture
         attrs.set(ArgAttribute::NoAlias).set(ArgAttribute::NoCapture).set(ArgAttribute::NonNull);
-        attrs.pointee_size = self.layout.size;
+        attrs.pointee_size = layout.size;
         // FIXME(eddyb) We should be doing this, but at least on
         // i686-pc-windows-msvc, it results in wrong stack offsets.
-        // attrs.pointee_align = Some(self.layout.align.abi);
+        // attrs.pointee_align = Some(layout.align.abi);
 
-        let extra_attrs = self.layout.is_unsized().then_some(ArgAttributes::new());
+        let extra_attrs = layout.is_unsized().then_some(ArgAttributes::new());
 
-        self.mode = PassMode::Indirect { attrs, extra_attrs, on_stack: false };
+        PassMode::Indirect { attrs, extra_attrs, on_stack: false }
+    }
+
+    pub fn make_indirect(&mut self) {
+        match self.mode {
+            PassMode::Direct(_) | PassMode::Pair(_, _) => {}
+            PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: false } => return,
+            _ => panic!("Tried to make {:?} indirect", self.mode),
+        }
+
+        self.mode = Self::indirect_pass_mode(&self.layout);
     }
 
     pub fn make_indirect_byval(&mut self) {
@@ -489,10 +507,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     }
 
     pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) {
-        match self.mode {
-            PassMode::Direct(_) | PassMode::Pair(_, _) => {}
-            _ => panic!("Tried to cast {:?} to {:?}", self.mode, target.into()),
-        }
         self.mode = PassMode::Cast(target.into());
     }