diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index 3ec9f3ca3b8c2..279a95d7136fc 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -2,8 +2,8 @@ use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr, Memory use crate::interpret::eval_nullary_intrinsic; use crate::interpret::{ intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId, - Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, Scalar, - ScalarMaybeUninit, StackPopCleanup, + Immediate, ImmediateOrMPlace, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, + RefTracking, Scalar, ScalarMaybeUninit, StackPopCleanup, }; use rustc_errors::ErrorReported; @@ -132,15 +132,17 @@ pub(super) fn op_to_const<'tcx>( }, _ => false, }; - let immediate = if try_as_immediate { - Err(ecx.read_immediate(op).expect("normalization works on validated constants")) + let imm_or_mplace = if try_as_immediate { + ImmediateOrMPlace::Imm( + ecx.read_immediate(op).expect("normalization works on validated constants"), + ) } else { // It is guaranteed that any non-slice scalar pair is actually ByRef here. // When we come back from raw const eval, we are always by-ref. The only way our op here is // by-val is if we are in destructure_const, i.e., if this is (a field of) something that we // "tried to make immediate" before. We wouldn't do that for non-slice scalar pairs or // structs containing such. - op.try_as_mplace() + op.as_mplace_or_imm() }; // We know `offset` is relative to the allocation, so we can use `into_parts`. @@ -160,10 +162,10 @@ pub(super) fn op_to_const<'tcx>( ConstValue::Scalar(Scalar::ZST) } }; - match immediate { - Ok(ref mplace) => to_const_value(mplace), + match imm_or_mplace { + ImmediateOrMPlace::MPlace(ref mplace) => to_const_value(mplace), // see comment on `let try_as_immediate` above - Err(imm) => match *imm { + ImmediateOrMPlace::Imm(imm) => match *imm { Immediate::Scalar(x) => match x { ScalarMaybeUninit::Scalar(s) => ConstValue::Scalar(s), ScalarMaybeUninit::Uninit => to_const_value(&op.assert_mem_place()), diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 4aa3c83cc0243..c0b690a699507 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -1142,11 +1142,13 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { /// Machine pointer introspection. impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { pub fn scalar_to_ptr(&self, scalar: Scalar) -> Pointer> { + use rustc_middle::mir::interpret::BitsOrPtr::*; + // We use `to_bits_or_ptr_internal` since we are just implementing the method people need to // call to force getting out a pointer. match scalar.to_bits_or_ptr_internal(self.pointer_size()) { - Err(ptr) => ptr.into(), - Ok(bits) => { + Ptr(ptr) => ptr.into(), + Bits(bits) => { let addr = u64::try_from(bits).unwrap(); let ptr = M::ptr_from_addr(&self, addr); if addr == 0 { diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs index 2b9fe56599715..890274b34048b 100644 --- a/compiler/rustc_const_eval/src/interpret/mod.rs +++ b/compiler/rustc_const_eval/src/interpret/mod.rs @@ -25,7 +25,7 @@ pub use self::intern::{intern_const_alloc_recursive, InternKind}; pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump}; pub use self::memory::{AllocCheck, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind}; pub use self::operand::{ImmTy, Immediate, OpTy, Operand}; -pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy}; +pub use self::place::{ImmediateOrMPlace, MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy}; pub use self::validity::{CtfeValidationMode, RefTracking}; pub use self::visitor::{MutValueVisitor, ValueVisitor}; diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index e9c94c0cc434c..e888b09ee04f2 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -16,8 +16,8 @@ use rustc_target::abi::{VariantIdx, Variants}; use super::{ alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, GlobalId, - InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, Place, PlaceTy, Pointer, Provenance, - Scalar, ScalarMaybeUninit, + ImmediateOrMPlace, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, Place, PlaceTy, + Pointer, Provenance, Scalar, ScalarMaybeUninit, }; /// An `Immediate` represents a single immediate self-contained Rust value. @@ -290,25 +290,26 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - /// Try returning an immediate for the operand. + /// Return an immediate or mplace for the operand. /// If the layout does not permit loading this as an immediate, return where in memory /// we can find the data. - /// Note that for a given layout, this operation will either always fail or always - /// succeed! Whether it succeeds depends on whether the layout can be represented + /// Whether we return an immediate depends on whether the layout can be represented /// in an `Immediate`, not on which data is stored there currently. - pub fn try_read_immediate( + pub fn read_immediate_or_mplace( &self, src: &OpTy<'tcx, M::PointerTag>, - ) -> InterpResult<'tcx, Result, MPlaceTy<'tcx, M::PointerTag>>> { - Ok(match src.try_as_mplace() { - Ok(ref mplace) => { + ) -> InterpResult<'tcx, ImmediateOrMPlace<'tcx, M::PointerTag>> { + use ImmediateOrMPlace::*; + + Ok(match src.as_mplace_or_imm() { + MPlace(ref mplace) => { if let Some(val) = self.try_read_immediate_from_mplace(mplace)? { - Ok(val) + Imm(val) } else { - Err(*mplace) + MPlace(*mplace) } } - Err(val) => Ok(val), + Imm(val) => Imm(val), }) } @@ -318,7 +319,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { &self, op: &OpTy<'tcx, M::PointerTag>, ) -> InterpResult<'tcx, ImmTy<'tcx, M::PointerTag>> { - if let Ok(imm) = self.try_read_immediate(op)? { + if let ImmediateOrMPlace::Imm(imm) = self.read_immediate_or_mplace(op)? { Ok(imm) } else { span_bug!(self.cur_span(), "primitive read failed for type: {:?}", op.layout.ty); @@ -355,13 +356,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { op: &OpTy<'tcx, M::PointerTag>, field: usize, ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> { - let base = match op.try_as_mplace() { - Ok(ref mplace) => { + let base = match op.as_mplace_or_imm() { + ImmediateOrMPlace::MPlace(ref mplace) => { // We can reuse the mplace field computation logic for indirect operands. let field = self.mplace_field(mplace, field)?; return Ok(field.into()); } - Err(value) => value, + ImmediateOrMPlace::Imm(value) => value, }; let field_layout = op.layout.field(self, field); @@ -409,9 +410,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { variant: VariantIdx, ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> { // Downcasts only change the layout - Ok(match op.try_as_mplace() { - Ok(ref mplace) => self.mplace_downcast(mplace, variant)?.into(), - Err(..) => { + Ok(match op.as_mplace_or_imm() { + ImmediateOrMPlace::MPlace(ref mplace) => self.mplace_downcast(mplace, variant)?.into(), + ImmediateOrMPlace::Imm(..) => { let layout = op.layout.for_variant(self, variant); OpTy { layout, ..*op } } diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 818b95b7fc4f3..2379cfde944e2 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -97,6 +97,20 @@ impl<'tcx, Tag: Provenance> std::ops::Deref for PlaceTy<'tcx, Tag> { } } +pub enum ImmediateOrMPlace<'tcx, Tag: Provenance> { + Imm(ImmTy<'tcx, Tag>), + MPlace(MPlaceTy<'tcx, Tag>), +} + +impl<'tcx, Tag: Provenance> ImmediateOrMPlace<'tcx, Tag> { + pub fn get_mplace(self) -> MPlaceTy<'tcx, Tag> { + match self { + ImmediateOrMPlace::MPlace(p) => p, + ImmediateOrMPlace::Imm(_) => bug!("expected MPlace"), + } + } +} + /// A MemPlace with its layout. Constructing it is only possible in this module. #[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] pub struct MPlaceTy<'tcx, Tag: Provenance = AllocId> { @@ -222,11 +236,17 @@ impl<'tcx, Tag: Provenance> OpTy<'tcx, Tag> { #[inline(always)] /// Note: do not call `as_ref` on the resulting place. This function should only be used to /// read from the resulting mplace, not to get its address back. - pub fn try_as_mplace(&self) -> Result, ImmTy<'tcx, Tag>> { + pub fn as_mplace_or_imm(&self) -> ImmediateOrMPlace<'tcx, Tag> { match **self { - Operand::Indirect(mplace) => Ok(MPlaceTy { mplace, layout: self.layout }), - Operand::Immediate(_) if self.layout.is_zst() => Ok(MPlaceTy::dangling(self.layout)), - Operand::Immediate(imm) => Err(ImmTy::from_immediate(imm, self.layout)), + Operand::Indirect(mplace) => { + ImmediateOrMPlace::MPlace(MPlaceTy { mplace, layout: self.layout }) + } + Operand::Immediate(_) if self.layout.is_zst() => { + ImmediateOrMPlace::MPlace(MPlaceTy::dangling(self.layout)) + } + Operand::Immediate(imm) => { + ImmediateOrMPlace::Imm(ImmTy::from_immediate(imm, self.layout)) + } } } @@ -234,7 +254,7 @@ impl<'tcx, Tag: Provenance> OpTy<'tcx, Tag> { /// Note: do not call `as_ref` on the resulting place. This function should only be used to /// read from the resulting mplace, not to get its address back. pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Tag> { - self.try_as_mplace().unwrap() + self.as_mplace_or_imm().get_mplace() } } @@ -708,7 +728,7 @@ where } trace!("write_immediate: {:?} <- {:?}: {}", *dest, src, dest.layout.ty); - // See if we can avoid an allocation. This is the counterpart to `try_read_immediate`, + // See if we can avoid an allocation. This is the counterpart to `read_immediate_or_mplace`, // but not factored as a separate function. let mplace = match dest.place { Place::Local { frame, local } => { @@ -831,15 +851,15 @@ where } // Let us see if the layout is simple so we take a shortcut, avoid force_allocation. - let src = match self.try_read_immediate(src)? { - Ok(src_val) => { + let src = match self.read_immediate_or_mplace(src)? { + ImmediateOrMPlace::Imm(src_val) => { assert!(!src.layout.is_unsized(), "cannot have unsized immediates"); // Yay, we got a value that we can write directly. // FIXME: Add a check to make sure that if `src` is indirect, // it does not overlap with `dest`. return self.write_immediate_no_validate(*src_val, dest); } - Err(mplace) => mplace, + ImmediateOrMPlace::MPlace(mplace) => mplace, }; // Slow path, this does not fit into an immediate. Just memcpy. trace!("copy_op: {:?} <- {:?}: {}", *dest, src, dest.layout.ty); diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs index a36c9b6ed7304..16ca7198db9a8 100644 --- a/compiler/rustc_middle/src/mir/interpret/allocation.rs +++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs @@ -12,9 +12,9 @@ use rustc_span::DUMMY_SP; use rustc_target::abi::{Align, HasDataLayout, Size}; use super::{ - read_target_uint, write_target_uint, AllocId, InterpError, InterpResult, Pointer, Provenance, - ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, UndefinedBehaviorInfo, UninitBytesAccess, - UnsupportedOpInfo, + read_target_uint, write_target_uint, AllocId, BitsOrPtr, InterpError, InterpResult, Pointer, + Provenance, ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, UndefinedBehaviorInfo, + UninitBytesAccess, UnsupportedOpInfo, }; use crate::ty; @@ -396,11 +396,11 @@ impl Allocation { // `to_bits_or_ptr_internal` is the right method because we just want to store this data // as-is into memory. let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size) { - Err(val) => { + BitsOrPtr::Ptr(val) => { let (provenance, offset) = val.into_parts(); (u128::from(offset.bytes()), Some(provenance)) } - Ok(data) => (data, None), + BitsOrPtr::Bits(data) => (data, None), }; let endian = cx.data_layout().endian; diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs index b762a10da847a..f7dc3e21dee35 100644 --- a/compiler/rustc_middle/src/mir/interpret/mod.rs +++ b/compiler/rustc_middle/src/mir/interpret/mod.rs @@ -123,7 +123,9 @@ pub use self::error::{ ResourceExhaustionInfo, UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo, }; -pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit}; +pub use self::value::{ + get_slice_bytes, BitsOrPtr, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit, +}; pub use self::allocation::{ alloc_range, AllocRange, Allocation, InitChunk, InitChunkIter, InitMask, Relocations, diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index cc31d8c2c1879..2db67aba0945c 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -112,6 +112,11 @@ impl<'tcx> ConstValue<'tcx> { } } +pub enum BitsOrPtr { + Bits(u128), + Ptr(Pointer), +} + /// A `Scalar` represents an immediate, primitive value existing outside of a /// `memory::Allocation`. It is in many ways like a small chunk of an `Allocation`, up to 16 bytes in /// size. Like a range of bytes in an `Allocation`, a `Scalar` can either represent the raw bytes @@ -292,13 +297,13 @@ impl Scalar { /// This method only exists for the benefit of low-level operations that truly need to treat the /// scalar in whatever form it is. #[inline] - pub fn to_bits_or_ptr_internal(self, target_size: Size) -> Result> { + pub fn to_bits_or_ptr_internal(self, target_size: Size) -> BitsOrPtr { assert_ne!(target_size.bytes(), 0, "you should never look at the bits of a ZST"); match self { - Scalar::Int(int) => Ok(int.assert_bits(target_size)), + Scalar::Int(int) => BitsOrPtr::Bits(int.assert_bits(target_size)), Scalar::Ptr(ptr, sz) => { assert_eq!(target_size.bytes(), u64::from(sz)); - Err(ptr) + BitsOrPtr::Ptr(ptr) } } } diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs index 801c8778bff04..d212b409f153e 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs @@ -52,6 +52,7 @@ use rustc_data_structures::captures::Captures; use rustc_index::vec::Idx; use rustc_hir::{HirId, RangeEnd}; +use rustc_middle::mir::interpret::BitsOrPtr; use rustc_middle::mir::Field; use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange}; use rustc_middle::ty::layout::IntegerExt; @@ -146,7 +147,7 @@ impl IntRange { // straight to the result, after doing a bit of checking. (We // could remove this branch and just fall through, which // is more general but much slower.) - if let Ok(bits) = scalar.to_bits_or_ptr_internal(target_size) { + if let BitsOrPtr::Bits(bits) = scalar.to_bits_or_ptr_internal(target_size) { return Some(bits); } } diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 98de64cd97c9c..176669c60f8a0 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -32,8 +32,9 @@ use crate::MirPass; use rustc_const_eval::const_eval::ConstEvalErr; use rustc_const_eval::interpret::{ self, compile_time_machine, AllocId, Allocation, ConstValue, CtfeValidationMode, Frame, ImmTy, - Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, MemoryKind, OpTy, - Operand as InterpOperand, PlaceTy, Scalar, ScalarMaybeUninit, StackPopCleanup, StackPopUnwind, + Immediate, ImmediateOrMPlace, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, + MemoryKind, OpTy, Operand as InterpOperand, PlaceTy, Scalar, ScalarMaybeUninit, + StackPopCleanup, StackPopUnwind, }; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -431,8 +432,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Try to read the local as an immediate so that if it is representable as a scalar, we can // handle it as such, but otherwise, just return the value as is. - Some(match self.ecx.try_read_immediate(&op) { - Ok(Ok(imm)) => imm.into(), + Some(match self.ecx.read_immediate_or_mplace(&op) { + Ok(ImmediateOrMPlace::Imm(imm)) => imm.into(), _ => op, }) } @@ -821,10 +822,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { return; } - // FIXME> figure out what to do when try_read_immediate fails - let imm = self.use_ecx(|this| this.ecx.try_read_immediate(value)); + // FIXME> figure out what to do when read_immediate_or_mplace fails + let imm = self.use_ecx(|this| this.ecx.read_immediate_or_mplace(value)); - if let Some(Ok(imm)) = imm { + if let Some(ImmediateOrMPlace::Imm(imm)) = imm { match *imm { interpret::Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar)) => { *rval = Rvalue::Use(self.operand_from_scalar(