Auto merge of #36151 - camlorn:struct_layout_optimization, r=eddyb

refactor to remove trans::adt and make rustc::ty::layout authoritative

I asked on IRC about optimizing struct layout by reordering fields from most-aligned to least-aligned and somehow ended up getting talked into doing this.  The goal here is to make `layout` authoritative and to remove `adt`.  The former has been accomplished by reimplementing `represent_type_uncached` and the latter is in progress.  @eddyb thought I should make the PR now.

My plan is to reserve the actual optimization for a second PR, as this work is useful by itself.

Author: bors

Diff for: src/librustc/ty/context.rs

@@ -489,6 +489,9 @@ pub struct GlobalCtxt<'tcx> {
     /// Cache for layouts computed from types.
     pub layout_cache: RefCell<FnvHashMap<Ty<'tcx>, &'tcx Layout>>,
 
+    /// Used to prevent layout from recursing too deeply.
+    pub layout_depth: Cell<usize>,
+
     /// Map from function to the `#[derive]` mode that it's defining. Only used
     /// by `rustc-macro` crates.
     pub derive_macros: RefCell<NodeMap<token::InternedString>>,
@@ -760,6 +763,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
             crate_name: token::intern_and_get_ident(crate_name),
             data_layout: data_layout,
             layout_cache: RefCell::new(FnvHashMap()),
+            layout_depth: Cell::new(0),
             derive_macros: RefCell::new(NodeMap()),
        }, f)
     }

Diff for: src/librustc/ty/layout.rs

@@ -328,6 +328,42 @@ pub enum Integer {
 }
 
 impl Integer {
+    pub fn size(&self) -> Size {
+        match *self {
+            I1 => Size::from_bits(1),
+            I8 => Size::from_bytes(1),
+            I16 => Size::from_bytes(2),
+            I32 => Size::from_bytes(4),
+            I64  => Size::from_bytes(8),
+        }
+    }
+
+    pub fn align(&self, dl: &TargetDataLayout)-> Align {
+        match *self {
+            I1 => dl.i1_align,
+            I8 => dl.i8_align,
+            I16 => dl.i16_align,
+            I32 => dl.i32_align,
+            I64 => dl.i64_align,
+        }
+    }
+
+    pub fn to_ty<'a, 'tcx>(&self, tcx: &ty::TyCtxt<'a, 'tcx, 'tcx>,
+                           signed: bool) -> Ty<'tcx> {
+        match (*self, signed) {
+            (I1, false) => tcx.types.u8,
+            (I8, false) => tcx.types.u8,
+            (I16, false) => tcx.types.u16,
+            (I32, false) => tcx.types.u32,
+            (I64, false) => tcx.types.u64,
+            (I1, true) => tcx.types.i8,
+            (I8, true) => tcx.types.i8,
+            (I16, true) => tcx.types.i16,
+            (I32, true) => tcx.types.i32,
+            (I64, true) => tcx.types.i64,
+        }
+    }
+
     /// Find the smallest Integer type which can represent the signed value.
     pub fn fit_signed(x: i64) -> Integer {
         match x {
@@ -350,6 +386,18 @@ impl Integer {
         }
     }
 
+    /// Find the smallest integer with the given alignment.
+    pub fn for_abi_align(dl: &TargetDataLayout, align: Align) -> Option<Integer> {
+        let wanted = align.abi();
+        for &candidate in &[I8, I16, I32, I64] {
+            let ty = Int(candidate);
+            if wanted == ty.align(dl).abi() && wanted == ty.size(dl).bytes() {
+                return Some(candidate);
+            }
+        }
+        None
+    }
+
     /// Get the Integer type from an attr::IntType.
     pub fn from_attr(dl: &TargetDataLayout, ity: attr::IntType) -> Integer {
         match ity {
@@ -623,6 +671,15 @@ impl<'a, 'gcx, 'tcx> Struct {
         }
         Ok(None)
     }
+
+    pub fn offset_of_field(&self, index: usize) -> Size {
+        assert!(index < self.offset_after_field.len());
+        if index == 0 {
+            Size::from_bytes(0)
+        } else {
+            self.offset_after_field[index-1]
+        }
+    }
 }
 
 /// An untagged union.
@@ -912,7 +969,7 @@ impl<'a, 'gcx, 'tcx> Layout {
                 Univariant { variant: unit, non_zero: false }
             }
 
-            // Tuples.
+            // Tuples and closures.
             ty::TyClosure(_, ty::ClosureSubsts { upvar_tys: tys, .. }) |
             ty::TyTuple(tys) => {
                 let mut st = Struct::new(dl, false);
@@ -972,10 +1029,10 @@ impl<'a, 'gcx, 'tcx> Layout {
                     });
                 }
 
-                if def.variants.len() == 1 {
+                if !def.is_enum() || def.variants.len() == 1 && hint == attr::ReprAny {
                     // Struct, or union, or univariant enum equivalent to a struct.
                     // (Typechecking will reject discriminant-sizing attrs.)
-                    assert!(!def.is_enum() || hint == attr::ReprAny);
+
                     let fields = def.variants[0].fields.iter().map(|field| {
                         field.ty(tcx, substs).layout(infcx)
                     });
@@ -1103,20 +1160,7 @@ impl<'a, 'gcx, 'tcx> Layout {
                 // won't be so conservative.
 
                 // Use the initial field alignment
-                let wanted = start_align.abi();
-                let mut ity = min_ity;
-                for &candidate in &[I16, I32, I64] {
-                    let ty = Int(candidate);
-                    if wanted == ty.align(dl).abi() && wanted == ty.size(dl).bytes() {
-                        ity = candidate;
-                        break;
-                    }
-                }
-
-                // FIXME(eddyb) conservative only to avoid diverging from trans::adt.
-                if align.abi() != start_align.abi() {
-                    ity = min_ity;
-                }
+                let mut ity = Integer::for_abi_align(dl, start_align).unwrap_or(min_ity);
 
                 // If the alignment is not larger than the chosen discriminant size,
                 // don't use the alignment as the final size.

Diff for: src/librustc/ty/util.rs

@@ -608,10 +608,19 @@ impl<'a, 'tcx> ty::TyS<'tcx> {
             }
         }
 
+        let rec_limit = tcx.sess.recursion_limit.get();
+        let depth = tcx.layout_depth.get();
+        if depth > rec_limit {
+            tcx.sess.fatal(
+                &format!("overflow representing the type `{}`", self));
+        }
+
+        tcx.layout_depth.set(depth+1);
         let layout = Layout::compute_uncached(self, infcx)?;
         if can_cache {
             tcx.layout_cache.borrow_mut().insert(self, layout);
         }
+        tcx.layout_depth.set(depth);
         Ok(layout)
     }
 

Diff for: src/librustc_trans/abi.rs

@@ -24,7 +24,7 @@ use cabi_s390x;
 use cabi_mips;
 use cabi_mips64;
 use cabi_asmjs;
-use machine::{llalign_of_min, llsize_of, llsize_of_real, llsize_of_store};
+use machine::{llalign_of_min, llsize_of, llsize_of_alloc};
 use type_::Type;
 use type_of;
 
@@ -102,7 +102,7 @@ impl ArgType {
         // Wipe old attributes, likely not valid through indirection.
         self.attrs = llvm::Attributes::default();
 
-        let llarg_sz = llsize_of_real(ccx, self.ty);
+        let llarg_sz = llsize_of_alloc(ccx, self.ty);
 
         // For non-immediate arguments the callee gets its own copy of
         // the value on the stack, so there are no aliases. It's also
@@ -200,7 +200,7 @@ impl ArgType {
                 base::call_memcpy(bcx,
                                   bcx.pointercast(dst, Type::i8p(ccx)),
                                   bcx.pointercast(llscratch, Type::i8p(ccx)),
-                                  C_uint(ccx, llsize_of_store(ccx, self.ty)),
+                                  C_uint(ccx, llsize_of_alloc(ccx, self.ty)),
                                   cmp::min(llalign_of_min(ccx, self.ty),
                                            llalign_of_min(ccx, ty)) as u32);
 
@@ -327,7 +327,7 @@ impl FnType {
                 if let Layout::CEnum { signed, .. } = *ccx.layout_of(ty) {
                     arg.signedness = Some(signed);
                 }
-                if llsize_of_real(ccx, arg.ty) == 0 {
+                if llsize_of_alloc(ccx, arg.ty) == 0 {
                     // For some forsaken reason, x86_64-pc-windows-gnu
                     // doesn't ignore zero-sized struct arguments.
                     // The same is true for s390x-unknown-linux-gnu.
@@ -358,7 +358,7 @@ impl FnType {
                 ty::TyRef(_, ty::TypeAndMut { ty, .. }) |
                 ty::TyBox(ty) => {
                     let llty = type_of::sizing_type_of(ccx, ty);
-                    let llsz = llsize_of_real(ccx, llty);
+                    let llsz = llsize_of_alloc(ccx, llty);
                     ret.attrs.set_dereferenceable(llsz);
                 }
                 _ => {}
@@ -427,7 +427,7 @@ impl FnType {
             } else {
                 if let Some(inner) = rust_ptr_attrs(ty, &mut arg) {
                     let llty = type_of::sizing_type_of(ccx, inner);
-                    let llsz = llsize_of_real(ccx, llty);
+                    let llsz = llsize_of_alloc(ccx, llty);
                     arg.attrs.set_dereferenceable(llsz);
                 }
                 args.push(arg);
@@ -469,8 +469,8 @@ impl FnType {
                     return;
                 }
 
-                let size = llsize_of_real(ccx, llty);
-                if size > llsize_of_real(ccx, ccx.int_type()) {
+                let size = llsize_of_alloc(ccx, llty);
+                if size > llsize_of_alloc(ccx, ccx.int_type()) {
                     arg.make_indirect(ccx);
                 } else if size > 0 {
                     // We want to pass small aggregates as immediates, but using