Skip to content

Commit 25fd504

Browse files
JohnTitorJohnTitor
authored
Rollup merge of #77511 - JulianKnodt:st_kind_cpy, r=oli-obk
Add StatementKind::CopyNonOverlapping Implements rust-lang/compiler-team#348 r? `@nagisa`
2 parents 3a5d45f + 4bceb29 commit 25fd504

File tree

26 files changed

+276
-91
lines changed

26 files changed

+276
-91
lines changed

‎compiler/rustc_codegen_cranelift/src/base.rs

+21
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,27 @@ fn codegen_stmt<'tcx>(
832832
}
833833
}
834834
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
835+
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
836+
src,
837+
dst,
838+
count,
839+
}) => {
840+
let dst = codegen_operand(fx, dst);
841+
let pointee = dst
842+
.layout()
843+
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
844+
.expect("Expected pointer");
845+
let dst = dst.load_scalar(fx);
846+
let src = codegen_operand(fx, src).load_scalar(fx);
847+
let count = codegen_operand(fx, count).load_scalar(fx);
848+
let elem_size: u64 = pointee.size.bytes();
849+
let bytes = if elem_size != 1 {
850+
fx.bcx.ins().imul_imm(count, elem_size as i64)
851+
} else {
852+
count
853+
};
854+
fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
855+
}
835856
}
836857
}
837858

‎compiler/rustc_codegen_ssa/src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#![feature(or_patterns)]
1010
#![feature(associated_type_bounds)]
1111
#![recursion_limit = "256"]
12+
#![feature(box_syntax)]
1213

1314
//! This crate contains codegen code that is used by all codegen backends (LLVM and others).
1415
//! The backend-agnostic functions of this crate use functions defined in various traits that

‎compiler/rustc_codegen_ssa/src/mir/block.rs

+60-54
Original file line numberDiff line numberDiff line change
@@ -641,67 +641,73 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
641641
return;
642642
}
643643

644-
if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) {
645-
let intrinsic = intrinsic.unwrap();
646-
let dest = match ret_dest {
647-
_ if fn_abi.ret.is_indirect() => llargs[0],
648-
ReturnDest::Nothing => {
649-
bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
650-
}
651-
ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
652-
ReturnDest::DirectOperand(_) => {
653-
bug!("Cannot use direct operand with an intrinsic call")
654-
}
655-
};
644+
match intrinsic {
645+
None | Some(sym::drop_in_place) => {}
646+
Some(sym::copy_nonoverlapping) => unreachable!(),
647+
Some(intrinsic) => {
648+
let dest = match ret_dest {
649+
_ if fn_abi.ret.is_indirect() => llargs[0],
650+
ReturnDest::Nothing => {
651+
bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
652+
}
653+
ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
654+
ReturnDest::DirectOperand(_) => {
655+
bug!("Cannot use direct operand with an intrinsic call")
656+
}
657+
};
656658

657-
let args: Vec<_> = args
658-
.iter()
659-
.enumerate()
660-
.map(|(i, arg)| {
661-
// The indices passed to simd_shuffle* in the
662-
// third argument must be constant. This is
663-
// checked by const-qualification, which also
664-
// promotes any complex rvalues to constants.
665-
if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
666-
if let mir::Operand::Constant(constant) = arg {
667-
let c = self.eval_mir_constant(constant);
668-
let (llval, ty) = self.simd_shuffle_indices(
669-
&bx,
670-
constant.span,
671-
constant.literal.ty,
672-
c,
673-
);
674-
return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty) };
675-
} else {
676-
span_bug!(span, "shuffle indices must be constant");
659+
let args: Vec<_> = args
660+
.iter()
661+
.enumerate()
662+
.map(|(i, arg)| {
663+
// The indices passed to simd_shuffle* in the
664+
// third argument must be constant. This is
665+
// checked by const-qualification, which also
666+
// promotes any complex rvalues to constants.
667+
if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
668+
if let mir::Operand::Constant(constant) = arg {
669+
let c = self.eval_mir_constant(constant);
670+
let (llval, ty) = self.simd_shuffle_indices(
671+
&bx,
672+
constant.span,
673+
constant.literal.ty,
674+
c,
675+
);
676+
return OperandRef {
677+
val: Immediate(llval),
678+
layout: bx.layout_of(ty),
679+
};
680+
} else {
681+
span_bug!(span, "shuffle indices must be constant");
682+
}
677683
}
678-
}
679684

680-
self.codegen_operand(&mut bx, arg)
681-
})
682-
.collect();
685+
self.codegen_operand(&mut bx, arg)
686+
})
687+
.collect();
688+
689+
Self::codegen_intrinsic_call(
690+
&mut bx,
691+
*instance.as_ref().unwrap(),
692+
&fn_abi,
693+
&args,
694+
dest,
695+
span,
696+
);
683697

684-
Self::codegen_intrinsic_call(
685-
&mut bx,
686-
*instance.as_ref().unwrap(),
687-
&fn_abi,
688-
&args,
689-
dest,
690-
span,
691-
);
698+
if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
699+
self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
700+
}
692701

693-
if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
694-
self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
695-
}
702+
if let Some((_, target)) = *destination {
703+
helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
704+
helper.funclet_br(self, &mut bx, target);
705+
} else {
706+
bx.unreachable();
707+
}
696708

697-
if let Some((_, target)) = *destination {
698-
helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
699-
helper.funclet_br(self, &mut bx, target);
700-
} else {
701-
bx.unreachable();
709+
return;
702710
}
703-
704-
return;
705711
}
706712

707713
// Split the rust-call tupled arguments off.

‎compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

-13
Original file line numberDiff line numberDiff line change
@@ -125,19 +125,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
125125
let offset = args[1].immediate();
126126
bx.gep(ptr, &[offset])
127127
}
128-
129-
sym::copy_nonoverlapping => {
130-
copy_intrinsic(
131-
bx,
132-
false,
133-
false,
134-
substs.type_at(0),
135-
args[1].immediate(),
136-
args[0].immediate(),
137-
args[2].immediate(),
138-
);
139-
return;
140-
}
141128
sym::copy => {
142129
copy_intrinsic(
143130
bx,

‎compiler/rustc_codegen_ssa/src/mir/statement.rs

+20
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
115115
self.codegen_coverage(&mut bx, coverage.clone());
116116
bx
117117
}
118+
mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
119+
ref src,
120+
ref dst,
121+
ref count,
122+
}) => {
123+
let dst_val = self.codegen_operand(&mut bx, dst);
124+
let src_val = self.codegen_operand(&mut bx, src);
125+
let count = self.codegen_operand(&mut bx, count).immediate();
126+
let pointee_layout = dst_val
127+
.layout
128+
.pointee_info_at(&mut bx, rustc_target::abi::Size::ZERO)
129+
.expect("Expected pointer");
130+
let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes()));
131+
132+
let align = pointee_layout.align;
133+
let dst = dst_val.immediate();
134+
let src = src_val.immediate();
135+
bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
136+
bx
137+
}
118138
mir::StatementKind::FakeRead(..)
119139
| mir::StatementKind::Retag { .. }
120140
| mir::StatementKind::AscribeUserType(..)

‎compiler/rustc_middle/src/mir/mod.rs

+20
Original file line numberDiff line numberDiff line change
@@ -1541,6 +1541,11 @@ pub enum StatementKind<'tcx> {
15411541
/// counter varible at runtime, each time the code region is executed.
15421542
Coverage(Box<Coverage>),
15431543

1544+
/// Denotes a call to the intrinsic function copy_overlapping, where `src_dst` denotes the
1545+
/// memory being read from and written to(one field to save memory), and size
1546+
/// indicates how many bytes are being copied over.
1547+
CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
1548+
15441549
/// No-op. Useful for deleting instructions without affecting statement indices.
15451550
Nop,
15461551
}
@@ -1659,6 +1664,13 @@ impl Debug for Statement<'_> {
16591664
write!(fmt, "Coverage::{:?}", coverage.kind)
16601665
}
16611666
}
1667+
CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
1668+
ref src,
1669+
ref dst,
1670+
ref count,
1671+
}) => {
1672+
write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
1673+
}
16621674
Nop => write!(fmt, "nop"),
16631675
}
16641676
}
@@ -1670,6 +1682,14 @@ pub struct Coverage {
16701682
pub code_region: Option<CodeRegion>,
16711683
}
16721684

1685+
#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable)]
1686+
pub struct CopyNonOverlapping<'tcx> {
1687+
pub src: Operand<'tcx>,
1688+
pub dst: Operand<'tcx>,
1689+
/// Number of elements to copy from src to dest, not bytes.
1690+
pub count: Operand<'tcx>,
1691+
}
1692+
16731693
///////////////////////////////////////////////////////////////////////////
16741694
// Places
16751695

‎compiler/rustc_middle/src/mir/visit.rs

+9
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,15 @@ macro_rules! make_mir_visitor {
436436
location
437437
)
438438
}
439+
StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{
440+
ref $($mutability)? src,
441+
ref $($mutability)? dst,
442+
ref $($mutability)? count,
443+
}) => {
444+
self.visit_operand(src, location);
445+
self.visit_operand(dst, location);
446+
self.visit_operand(count, location)
447+
}
439448
StatementKind::Nop => {}
440449
}
441450
}

‎compiler/rustc_mir/src/borrow_check/invalidation.rs

+9
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,15 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
9292
self.consume_operand(location, input);
9393
}
9494
}
95+
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
96+
ref src,
97+
ref dst,
98+
ref count,
99+
}) => {
100+
self.consume_operand(location, src);
101+
self.consume_operand(location, dst);
102+
self.consume_operand(location, count);
103+
}
95104
StatementKind::Nop
96105
| StatementKind::Coverage(..)
97106
| StatementKind::AscribeUserType(..)

‎compiler/rustc_mir/src/borrow_check/mod.rs

+9
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,15 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
626626
self.consume_operand(location, (input, span), flow_state);
627627
}
628628
}
629+
630+
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
631+
..
632+
}) => {
633+
span_bug!(
634+
span,
635+
"Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
636+
)
637+
}
629638
StatementKind::Nop
630639
| StatementKind::Coverage(..)
631640
| StatementKind::AscribeUserType(..)

‎compiler/rustc_mir/src/borrow_check/type_check/mod.rs

+6
Original file line numberDiff line numberDiff line change
@@ -1520,6 +1520,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
15201520
);
15211521
}
15221522
}
1523+
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
1524+
..
1525+
}) => span_bug!(
1526+
stmt.source_info.span,
1527+
"Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
1528+
),
15231529
StatementKind::FakeRead(..)
15241530
| StatementKind::StorageLive(..)
15251531
| StatementKind::StorageDead(..)

‎compiler/rustc_mir/src/dataflow/impls/borrows.rs

+1
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,7 @@ impl<'tcx> dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
305305
| mir::StatementKind::Retag { .. }
306306
| mir::StatementKind::AscribeUserType(..)
307307
| mir::StatementKind::Coverage(..)
308+
| mir::StatementKind::CopyNonOverlapping(..)
308309
| mir::StatementKind::Nop => {}
309310
}
310311
}

‎compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs

+1
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ impl<'mir, 'tcx> dataflow::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir,
149149
| StatementKind::FakeRead(..)
150150
| StatementKind::Nop
151151
| StatementKind::Retag(..)
152+
| StatementKind::CopyNonOverlapping(..)
152153
| StatementKind::StorageLive(..) => {}
153154
}
154155
}

‎compiler/rustc_mir/src/dataflow/move_paths/builder.rs

+1
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
318318
StatementKind::Retag { .. }
319319
| StatementKind::AscribeUserType(..)
320320
| StatementKind::Coverage(..)
321+
| StatementKind::CopyNonOverlapping(..)
321322
| StatementKind::Nop => {}
322323
}
323324
}

‎compiler/rustc_mir/src/interpret/intrinsics.rs

+2-22
Original file line numberDiff line numberDiff line change
@@ -323,28 +323,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
323323
let result = Scalar::from_uint(truncated_bits, layout.size);
324324
self.write_scalar(result, dest)?;
325325
}
326-
sym::copy | sym::copy_nonoverlapping => {
327-
let elem_ty = instance.substs.type_at(0);
328-
let elem_layout = self.layout_of(elem_ty)?;
329-
let count = self.read_scalar(&args[2])?.to_machine_usize(self)?;
330-
let elem_align = elem_layout.align.abi;
331-
332-
let size = elem_layout.size.checked_mul(count, self).ok_or_else(|| {
333-
err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
334-
})?;
335-
let src = self.read_scalar(&args[0])?.check_init()?;
336-
let src = self.memory.check_ptr_access(src, size, elem_align)?;
337-
let dest = self.read_scalar(&args[1])?.check_init()?;
338-
let dest = self.memory.check_ptr_access(dest, size, elem_align)?;
339-
340-
if let (Some(src), Some(dest)) = (src, dest) {
341-
self.memory.copy(
342-
src,
343-
dest,
344-
size,
345-
intrinsic_name == sym::copy_nonoverlapping,
346-
)?;
347-
}
326+
sym::copy => {
327+
self.copy(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
348328
}
349329
sym::offset => {
350330
let ptr = self.read_scalar(&args[0])?.check_init()?;

0 commit comments

Comments
 (0)