Skip to content

Commit b6cd5cf

Browse files
borsbors
committed
Auto merge of #122926 - scottmcm:assume-lengths, r=<try>
Add `assume`s to slice length calls Since `.len()` on slices is safe, let's see how this impacts things vs what we could do with #121965 and LLVM 19
2 parents c3b05c6 + eed1ccd commit b6cd5cf

8 files changed

+103
-19
lines changed

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+13-1
Original file line numberDiff line numberDiff line change
@@ -748,7 +748,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
748748
}
749749
// use common size calculation for non zero-sized types
750750
let cg_value = self.codegen_place(bx, place.as_ref());
751-
cg_value.len(bx.cx())
751+
let length = cg_value.len(bx.cx());
752+
753+
if bx.sess().opts.optimize != OptLevel::No {
754+
let elem_ty = cg_value.layout.field(bx, 0);
755+
if let Some(elem_bytes) = std::num::NonZeroU64::new(elem_ty.size.bytes()) {
756+
let isize_max = (1_u64 << (bx.sess().target.pointer_width - 1)) - 1;
757+
let len_max = isize_max / elem_bytes;
758+
let limit = bx.icmp(IntPredicate::IntULE, length, bx.const_usize(len_max));
759+
bx.assume(limit);
760+
}
761+
}
762+
763+
length
752764
}
753765

754766
/// Codegen an `Rvalue::AddressOf` or `Rvalue::Ref`

Diff for: ‎tests/codegen/issues/issue-98294-get-mut-copy-from-slice-opt.rs

+4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
pub fn test(a: &mut [u8], offset: usize, bytes: &[u8]) {
99
// CHECK-LABEL: @test(
1010
// CHECK-NOT: call
11+
// CHECK: call void @llvm.assume
12+
// CHECK-NOT: call
13+
// CHECK: call void @llvm.assume
14+
// CHECK-NOT: call
1115
// CHECK: call void @llvm.memcpy
1216
// CHECK-NOT: call
1317
// CHECK: }

Diff for: ‎tests/codegen/slice-as_chunks.rs

+6-5
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,23 @@
77
// CHECK-LABEL: @chunks4
88
#[no_mangle]
99
pub fn chunks4(x: &[u8]) -> &[[u8; 4]] {
10-
// CHECK-NEXT: start:
11-
// CHECK-NEXT: lshr i64 %x.1, 2
1210
// CHECK-NOT: shl
1311
// CHECK-NOT: mul
1412
// CHECK-NOT: udiv
1513
// CHECK-NOT: urem
16-
// CHECK: ret
14+
// CHECK: %[[NEWLEN:.+]] = lshr i64 %x.1, 2
15+
// CHECK: %[[A:.+]] = insertvalue { ptr, i64 } poison, ptr %x.0, 0
16+
// CHECK: %[[B:.+]] = insertvalue { ptr, i64 } %[[A]], i64 %[[NEWLEN]], 1
17+
// CHECK: ret { ptr, i64 } %[[B]]
1718
x.as_chunks().0
1819
}
1920

2021
// CHECK-LABEL: @chunks4_with_remainder
2122
#[no_mangle]
2223
pub fn chunks4_with_remainder(x: &[u8]) -> (&[[u8; 4]], &[u8]) {
23-
// CHECK-DAG: and i64 %x.1, -4
24+
// CHECK-DAG: and i64 %x.1, [[#0x7FFFFFFFFFFFFFFC]]
2425
// CHECK-DAG: and i64 %x.1, 3
25-
// CHECK-DAG: lshr
26+
// CHECK-DAG: lshr i64 %x.1, 2
2627
// CHECK-NOT: mul
2728
// CHECK-NOT: udiv
2829
// CHECK-NOT: urem

Diff for: ‎tests/codegen/slice-iter-fold.rs

+6-4
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@
22
#![crate_type = "lib"]
33

44
// CHECK-LABEL: @slice_fold_to_last
5+
// CHECK-SAME: %slice.0, [[USIZE:i[0-9]+]] noundef %slice.1)
56
#[no_mangle]
67
pub fn slice_fold_to_last(slice: &[i32]) -> Option<&i32> {
7-
// CHECK-NOT: loop
8-
// CHECK-NOT: br
9-
// CHECK-NOT: call
10-
// CHECK: ret
8+
// CHECK: %[[END:.+]] = getelementptr inbounds i32, ptr %slice.0, [[USIZE]] %slice.1
9+
// CHECK: %[[EMPTY:.+]] = icmp eq [[USIZE]] %slice.1, 0
10+
// CHECK: %[[LAST:.+]] = getelementptr i32, ptr %[[END]], i64 -1
11+
// CHECK: %[[R:.+]] = select i1 %[[EMPTY]], ptr null, ptr %[[LAST]]
12+
// CHECK: ret ptr %[[R]]
1113
slice.iter().fold(None, |_, i| Some(i))
1214
}

Diff for: ‎tests/codegen/slice-iter-nonnull.rs

+12-4
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,13 @@ pub fn slice_iter_next_back<'a>(it: &mut std::slice::Iter<'a, u32>) -> Option<&'
5252
// CHECK-LABEL: @slice_iter_new
5353
// CHECK-SAME: (ptr noalias noundef nonnull {{.+}} %slice.0, {{.+}} noundef %slice.1)
5454
#[no_mangle]
55-
pub fn slice_iter_new(slice: &[u32]) -> std::slice::Iter<'_, u32> {
55+
pub fn slice_iter_new(slice: &[u8]) -> std::slice::Iter<'_, u8> {
5656
// CHECK-NOT: slice
57-
// CHECK: %[[END:.+]] = getelementptr inbounds i32{{.+}} %slice.0{{.+}} %slice.1
57+
// CHECK: %[[NONNEG:.+]] = icmp sgt i64 %slice.1, -1
58+
// CHECK-NOT: slice
59+
// CHECK: call void @llvm.assume(i1 %[[NONNEG]])
60+
// CHECK-NOT: slice
61+
// CHECK: %[[END:.+]] = getelementptr inbounds i8{{.+}} %slice.0{{.+}} %slice.1
5862
// CHECK-NOT: slice
5963
// CHECK: insertvalue {{.+}} ptr %slice.0, 0
6064
// CHECK-NOT: slice
@@ -67,9 +71,13 @@ pub fn slice_iter_new(slice: &[u32]) -> std::slice::Iter<'_, u32> {
6771
// CHECK-LABEL: @slice_iter_mut_new
6872
// CHECK-SAME: (ptr noalias noundef nonnull {{.+}} %slice.0, {{.+}} noundef %slice.1)
6973
#[no_mangle]
70-
pub fn slice_iter_mut_new(slice: &mut [u32]) -> std::slice::IterMut<'_, u32> {
74+
pub fn slice_iter_mut_new(slice: &mut [u8]) -> std::slice::IterMut<'_, u8> {
75+
// CHECK-NOT: slice
76+
// CHECK: %[[NONNEG:.+]] = icmp sgt i64 %slice.1, -1
77+
// CHECK-NOT: slice
78+
// CHECK: call void @llvm.assume(i1 %[[NONNEG]])
7179
// CHECK-NOT: slice
72-
// CHECK: %[[END:.+]] = getelementptr inbounds i32{{.+}} %slice.0{{.+}} %slice.1
80+
// CHECK: %[[END:.+]] = getelementptr inbounds i8{{.+}} %slice.0{{.+}} %slice.1
7381
// CHECK-NOT: slice
7482
// CHECK: insertvalue {{.+}} ptr %slice.0, 0
7583
// CHECK-NOT: slice

Diff for: ‎tests/codegen/slice-length-limits.rs

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
//@ revisions: 32BIT 64BIT
2+
//@ compile-flags: -O
3+
//@ min-llvm-version: 18.0
4+
//@ [32BIT] only-32bit
5+
//@ [64BIT] only-64bit
6+
7+
#![crate_type = "lib"]
8+
9+
// Confirm that the `assume` calls from the length allows LLVM to know that some
10+
// math on the indices can be done without overflow risk.
11+
12+
// CHECK-LABEL: @slice_length_demo
13+
#[no_mangle]
14+
pub unsafe fn slice_length_demo(x: &[u16]) {
15+
// 32BIT: %[[LIMIT:.+]] = icmp ult [[USIZE:i32]] %x.1, [[#0x40000000]]
16+
// 64BIT: %[[LIMIT:.+]] = icmp ult [[USIZE:i64]] %x.1, [[#0x4000000000000000]]
17+
// CHECK: tail call void @llvm.assume(i1 %[[LIMIT]])
18+
19+
// CHECK: %[[Y:.+]] = phi [[USIZE]]
20+
// CHECK-SAME: [ 0, %start ]
21+
// CHECK: %[[PLUS_ONE:.+]] = add nuw nsw [[USIZE]] %[[Y]], 1
22+
// CHECK: call void @do_something([[USIZE]] noundef %[[PLUS_ONE]])
23+
// CHECK: %[[TIMES_TWO:.+]] = shl nuw nsw [[USIZE]] %[[Y]], 1
24+
// CHECK: call void @do_something([[USIZE]] noundef %[[TIMES_TWO]])
25+
for y in 0..x.len() {
26+
do_something(y + 1);
27+
do_something(y * 2);
28+
}
29+
}
30+
31+
// CHECK-LABEL: @nested_slice_length
32+
#[no_mangle]
33+
pub unsafe fn nested_slice_length(x: &[f32], y: &[f32]) {
34+
// 32BIT: %[[LIMIT:.+]] = icmp ult [[USIZE:i32]] %x.1, [[#0x20000000]]
35+
// 64BIT: %[[LIMIT:.+]] = icmp ult [[USIZE:i64]] %x.1, [[#0x2000000000000000]]
36+
// CHECK: tail call void @llvm.assume(i1 %[[LIMIT]])
37+
// 32BIT: %[[LIMIT:.+]] = icmp ult [[USIZE]] %y.1, [[#0x20000000]]
38+
// 64BIT: %[[LIMIT:.+]] = icmp ult [[USIZE]] %y.1, [[#0x2000000000000000]]
39+
// CHECK: tail call void @llvm.assume(i1 %[[LIMIT]])
40+
41+
// CHECK: %[[J:.+]] = phi [[USIZE]]
42+
// CHECK: %[[I:.+]] = phi [[USIZE]]
43+
// CHECK-NOT: phi
44+
// CHECK: %[[T:.+]] = add nuw nsw [[USIZE]] %[[I]], %[[J]]
45+
// CHECK: call void @do_something([[USIZE]] noundef %[[T]])
46+
for i in 0..x.len() {
47+
for j in 0..y.len() {
48+
do_something(i + j);
49+
}
50+
}
51+
}
52+
53+
extern {
54+
fn do_something(x: usize);
55+
}

Diff for: ‎tests/codegen/slice-ref-equality.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use std::num::NonZero;
1515
// CHECK-LABEL: @is_zero_slice_long
1616
#[no_mangle]
1717
pub fn is_zero_slice_long(data: &[u8; 456]) -> bool {
18-
// CHECK: %[[BCMP:.+]] = tail call i32 @{{bcmp|memcmp}}({{.+}})
18+
// CHECK: %[[BCMP:.+]] = tail call i32 @{{bcmp|memcmp}}({{.+}} 456)
1919
// CHECK-NEXT: %[[EQ:.+]] = icmp eq i32 %[[BCMP]], 0
2020
// CHECK-NEXT: ret i1 %[[EQ]]
2121
&data[..] == [0; 456]
@@ -48,7 +48,7 @@ pub fn is_zero_array(data: &[u8; 4]) -> bool {
4848
#[no_mangle]
4949
fn eq_slice_of_nested_u8(x: &[[u8; 3]], y: &[[u8; 3]]) -> bool {
5050
// CHECK: icmp eq [[USIZE]] %1, %3
51-
// CHECK: %[[BYTES:.+]] = mul nsw [[USIZE]] %1, 3
51+
// CHECK: %[[BYTES:.+]] = mul nuw nsw [[USIZE]] %1, 3
5252
// CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr
5353
// CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]])
5454
x == y
@@ -60,7 +60,7 @@ fn eq_slice_of_nested_u8(x: &[[u8; 3]], y: &[[u8; 3]]) -> bool {
6060
#[no_mangle]
6161
fn eq_slice_of_i32(x: &[i32], y: &[i32]) -> bool {
6262
// CHECK: icmp eq [[USIZE]] %1, %3
63-
// CHECK: %[[BYTES:.+]] = shl nsw [[USIZE]] %1, 2
63+
// CHECK: %[[BYTES:.+]] = shl nuw nsw [[USIZE]] %1, 2
6464
// CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr
6565
// CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]])
6666
x == y
@@ -72,7 +72,7 @@ fn eq_slice_of_i32(x: &[i32], y: &[i32]) -> bool {
7272
#[no_mangle]
7373
fn eq_slice_of_nonzero(x: &[NonZero<i32>], y: &[NonZero<i32>]) -> bool {
7474
// CHECK: icmp eq [[USIZE]] %1, %3
75-
// CHECK: %[[BYTES:.+]] = shl nsw [[USIZE]] %1, 2
75+
// CHECK: %[[BYTES:.+]] = shl nuw nsw [[USIZE]] %1, 2
7676
// CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr
7777
// CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]])
7878
x == y
@@ -84,7 +84,7 @@ fn eq_slice_of_nonzero(x: &[NonZero<i32>], y: &[NonZero<i32>]) -> bool {
8484
#[no_mangle]
8585
fn eq_slice_of_option_of_nonzero(x: &[Option<NonZero<i16>>], y: &[Option<NonZero<i16>>]) -> bool {
8686
// CHECK: icmp eq [[USIZE]] %1, %3
87-
// CHECK: %[[BYTES:.+]] = shl nsw [[USIZE]] %1, 1
87+
// CHECK: %[[BYTES:.+]] = shl nuw nsw [[USIZE]] %1, 1
8888
// CHECK: tail call{{( noundef)?}} i32 @{{bcmp|memcmp}}(ptr
8989
// CHECK-SAME: , [[USIZE]]{{( noundef)?}} %[[BYTES]])
9090
x == y

Diff for: ‎tests/codegen/vecdeque-drain.rs

+2
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ pub fn truncate(v: &mut VecDeque<i32>, n: usize) {
4343
// CHECK-NOT: call
4444
// CHECK: br
4545
// CHECK-NOT: call
46+
// CHECK: call void @llvm.assume
47+
// CHECK-NOT: call
4648
// CHECK: br
4749
// CHECK-NOT: call
4850
// CHECK: br

0 commit comments

Comments
 (0)