Auto merge of #86291 - crlf0710:trait_vtbl_refactor, r=bjorn3

Refactor vtable codegen

This refactor the codegen of vtables of miri interpreter, llvm, cranelift codegen backends.

This is preparation for the implementation of trait upcasting feature. cc #65991

Note that aside from code reorganization, there's an internal behavior change here that now InstanceDef::Virtual's index now include the three metadata slots, and now the first method is with index 3.

cc  `@RalfJung` `@bjorn3`

Author: bors

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -4,10 +4,7 @@
 // FIXME dedup this logic between miri, cg_llvm and cg_clif
 
 use crate::prelude::*;
-
-const DROP_FN_INDEX: usize = 0;
-const SIZE_INDEX: usize = 1;
-const ALIGN_INDEX: usize = 2;
+use ty::VtblEntry;
 
 fn vtable_memflags() -> MemFlags {
     let mut flags = MemFlags::trusted(); // A vtable access is always aligned and will never trap.
@@ -21,7 +18,7 @@ pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) ->
         pointer_ty(fx.tcx),
         vtable_memflags(),
         vtable,
-        (DROP_FN_INDEX * usize_size) as i32,
+        (ty::COMMON_VTABLE_ENTRIES_DROPINPLACE * usize_size) as i32,
     )
 }
 
@@ -31,7 +28,7 @@ pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Val
         pointer_ty(fx.tcx),
         vtable_memflags(),
         vtable,
-        (SIZE_INDEX * usize_size) as i32,
+        (ty::COMMON_VTABLE_ENTRIES_SIZE * usize_size) as i32,
     )
 }
 
@@ -41,7 +38,7 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -
         pointer_ty(fx.tcx),
         vtable_memflags(),
         vtable,
-        (ALIGN_INDEX * usize_size) as i32,
+        (ty::COMMON_VTABLE_ENTRIES_SIZE * usize_size) as i32,
     )
 }
 
@@ -62,7 +59,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
         pointer_ty(fx.tcx),
         vtable_memflags(),
         vtable,
-        ((idx + 3) * usize_size as usize) as i32,
+        (idx * usize_size as usize) as i32,
     );
     (ptr, func_ref)
 }
@@ -98,42 +95,49 @@ fn build_vtable<'tcx>(
         Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx),
     );
 
-    let mut components: Vec<_> = vec![Some(drop_in_place_fn), None, None];
-
-    let methods_root;
-    let methods = if let Some(trait_ref) = trait_ref {
-        methods_root = tcx.vtable_methods(trait_ref.with_self_ty(tcx, layout.ty));
-        methods_root.iter()
+    let vtable_entries = if let Some(trait_ref) = trait_ref {
+        tcx.vtable_entries(trait_ref.with_self_ty(tcx, layout.ty))
     } else {
-        (&[]).iter()
+        ty::COMMON_VTABLE_ENTRIES
     };
-    let methods = methods.cloned().map(|opt_mth| {
-        opt_mth.map(|(def_id, substs)| {
-            import_function(
-                tcx,
-                fx.module,
-                Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs)
-                    .unwrap()
-                    .polymorphize(fx.tcx),
-            )
-        })
-    });
-    components.extend(methods);
 
     let mut data_ctx = DataContext::new();
     let mut data = ::std::iter::repeat(0u8)
-        .take(components.len() * usize_size)
+        .take(vtable_entries.len() * usize_size)
         .collect::<Vec<u8>>()
         .into_boxed_slice();
 
-    write_usize(fx.tcx, &mut data, SIZE_INDEX, layout.size.bytes());
-    write_usize(fx.tcx, &mut data, ALIGN_INDEX, layout.align.abi.bytes());
+    for (idx, entry) in vtable_entries.iter().enumerate() {
+        match entry {
+            VtblEntry::MetadataSize => {
+                write_usize(fx.tcx, &mut data, idx, layout.size.bytes());
+            }
+            VtblEntry::MetadataAlign => {
+                write_usize(fx.tcx, &mut data, idx, layout.align.abi.bytes());
+            }
+            VtblEntry::MetadataDropInPlace | VtblEntry::Vacant | VtblEntry::Method(_, _) => {}
+        }
+    }
     data_ctx.define(data);
 
-    for (i, component) in components.into_iter().enumerate() {
-        if let Some(func_id) = component {
-            let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx);
-            data_ctx.write_function_addr((i * usize_size) as u32, func_ref);
+    for (idx, entry) in vtable_entries.iter().enumerate() {
+        match entry {
+            VtblEntry::MetadataDropInPlace => {
+                let func_ref = fx.module.declare_func_in_data(drop_in_place_fn, &mut data_ctx);
+                data_ctx.write_function_addr((idx * usize_size) as u32, func_ref);
+            }
+            VtblEntry::Method(def_id, substs) => {
+                let func_id = import_function(
+                    tcx,
+                    fx.module,
+                    Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), *def_id, substs)
+                        .unwrap()
+                        .polymorphize(fx.tcx),
+                );
+                let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx);
+                data_ctx.write_function_addr((idx * usize_size) as u32, func_ref);
+            }
+            VtblEntry::MetadataSize | VtblEntry::MetadataAlign | VtblEntry::Vacant => {}
         }
     }
 

compiler/rustc_codegen_ssa/src/glue.rs

@@ -23,7 +23,12 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         ty::Dynamic(..) => {
             // load size/align from vtable
             let vtable = info.unwrap();
-            (meth::SIZE.get_usize(bx, vtable), meth::ALIGN.get_usize(bx, vtable))
+            (
+                meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_SIZE)
+                    .get_usize(bx, vtable),
+                meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN)
+                    .get_usize(bx, vtable),
+            )
         }
         ty::Slice(_) | ty::Str => {
             let unit = layout.field(bx, 0);

compiler/rustc_codegen_ssa/src/meth.rs

@@ -1,18 +1,14 @@
 use crate::traits::*;
 
-use rustc_middle::ty::{self, Instance, Ty};
+use rustc_middle::ty::{self, Instance, Ty, VtblEntry, COMMON_VTABLE_ENTRIES};
 use rustc_target::abi::call::FnAbi;
 
 #[derive(Copy, Clone, Debug)]
 pub struct VirtualIndex(u64);
 
-pub const DESTRUCTOR: VirtualIndex = VirtualIndex(0);
-pub const SIZE: VirtualIndex = VirtualIndex(1);
-pub const ALIGN: VirtualIndex = VirtualIndex(2);
-
 impl<'a, 'tcx> VirtualIndex {
     pub fn from_index(index: usize) -> Self {
-        VirtualIndex(index as u64 + 3)
+        VirtualIndex(index as u64)
     }
 
     pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
@@ -77,43 +73,38 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
     // Not in the cache; build it.
     let nullptr = cx.const_null(cx.type_i8p_ext(cx.data_layout().instruction_address_space));
 
-    let methods_root;
-    let methods = if let Some(trait_ref) = trait_ref {
-        methods_root = tcx.vtable_methods(trait_ref.with_self_ty(tcx, ty));
-        methods_root.iter()
+    let vtable_entries = if let Some(trait_ref) = trait_ref {
+        tcx.vtable_entries(trait_ref.with_self_ty(tcx, ty))
     } else {
-        (&[]).iter()
+        COMMON_VTABLE_ENTRIES
     };
 
-    let methods = methods.cloned().map(|opt_mth| {
-        opt_mth.map_or(nullptr, |(def_id, substs)| {
-            cx.get_fn_addr(
+    let layout = cx.layout_of(ty);
+    // /////////////////////////////////////////////////////////////////////////////////////////////
+    // If you touch this code, be sure to also make the corresponding changes to
+    // `get_vtable` in `rust_mir/interpret/traits.rs`.
+    // /////////////////////////////////////////////////////////////////////////////////////////////
+    let components: Vec<_> = vtable_entries
+        .iter()
+        .map(|entry| match entry {
+            VtblEntry::MetadataDropInPlace => {
+                cx.get_fn_addr(Instance::resolve_drop_in_place(cx.tcx(), ty))
+            }
+            VtblEntry::MetadataSize => cx.const_usize(layout.size.bytes()),
+            VtblEntry::MetadataAlign => cx.const_usize(layout.align.abi.bytes()),
+            VtblEntry::Vacant => nullptr,
+            VtblEntry::Method(def_id, substs) => cx.get_fn_addr(
                 ty::Instance::resolve_for_vtable(
                     cx.tcx(),
                     ty::ParamEnv::reveal_all(),
-                    def_id,
+                    *def_id,
                     substs,
                 )
                 .unwrap()
                 .polymorphize(cx.tcx()),
-            )
+            ),
         })
-    });
-
-    let layout = cx.layout_of(ty);
-    // /////////////////////////////////////////////////////////////////////////////////////////////
-    // If you touch this code, be sure to also make the corresponding changes to
-    // `get_vtable` in `rust_mir/interpret/traits.rs`.
-    // /////////////////////////////////////////////////////////////////////////////////////////////
-    let components: Vec<_> = [
-        cx.get_fn_addr(Instance::resolve_drop_in_place(cx.tcx(), ty)),
-        cx.const_usize(layout.size.bytes()),
-        cx.const_usize(layout.align.abi.bytes()),
-    ]
-    .iter()
-    .cloned()
-    .chain(methods)
-    .collect();
+        .collect();
 
     let vtable_const = cx.const_struct(&components, false);
     let align = cx.data_layout().pointer_align.abi;

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -332,7 +332,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let fn_abi = FnAbi::of_instance(&bx, virtual_drop, &[]);
                 let vtable = args[1];
                 args = &args[..1];
-                (meth::DESTRUCTOR.get_fn(&mut bx, vtable, &fn_abi), fn_abi)
+                (
+                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                        .get_fn(&mut bx, vtable, &fn_abi),
+                    fn_abi,
+                )
             }
             _ => (bx.get_fn_addr(drop_fn), FnAbi::of_instance(&bx, drop_fn, &[])),
         };

compiler/rustc_middle/src/query/mod.rs

@@ -959,9 +959,9 @@ rustc_queries! {
         desc { |tcx| "checking if item has mir available: `{}`", tcx.def_path_str(key) }
     }
 
-    query vtable_methods(key: ty::PolyTraitRef<'tcx>)
-                        -> &'tcx [Option<(DefId, SubstsRef<'tcx>)>] {
-        desc { |tcx| "finding all methods for trait {}", tcx.def_path_str(key.def_id()) }
+    query vtable_entries(key: ty::PolyTraitRef<'tcx>)
+                        -> &'tcx [ty::VtblEntry<'tcx>] {
+        desc { |tcx| "finding all vtable entries for trait {}", tcx.def_path_str(key.def_id()) }
     }
 
     query codegen_fulfill_obligation(

compiler/rustc_middle/src/ty/mod.rs

@@ -2009,3 +2009,19 @@ impl<'tcx> fmt::Debug for SymbolName<'tcx> {
         fmt::Display::fmt(&self.name, fmt)
     }
 }
+
+#[derive(Clone, Copy, Debug, PartialEq, HashStable)]
+pub enum VtblEntry<'tcx> {
+    MetadataDropInPlace,
+    MetadataSize,
+    MetadataAlign,
+    Vacant,
+    Method(DefId, SubstsRef<'tcx>),
+}
+
+pub const COMMON_VTABLE_ENTRIES: &[VtblEntry<'_>] =
+    &[VtblEntry::MetadataDropInPlace, VtblEntry::MetadataSize, VtblEntry::MetadataAlign];
+
+pub const COMMON_VTABLE_ENTRIES_DROPINPLACE: usize = 0;
+pub const COMMON_VTABLE_ENTRIES_SIZE: usize = 1;
+pub const COMMON_VTABLE_ENTRIES_ALIGN: usize = 2;