Skip to content

Commit 94105e8

Browse files
committed
wait for two short reads before uncapping the max read size
for disk IO: 1st short read = probably at end of file 2nd short read = confirming that it's indeed EOF
1 parent 4969ea8 commit 94105e8

File tree

1 file changed

+13
-3
lines changed

1 file changed

+13
-3
lines changed

std/src/io/mod.rs

+13-3
Original file line numberDiff line numberDiff line change
@@ -398,8 +398,7 @@ where
398398
// - avoid passing large buffers to readers that always initialize the free capacity if they perform short reads (#23815, #23820)
399399
// - pass large buffers to readers that do not initialize the spare capacity. this can amortize per-call overheads
400400
// - and finally pass not-too-small and not-too-large buffers to Windows read APIs because they manage to suffer from both problems
401-
// at the same time, i.e. small reads suffer from syscall overhead, all reads incur initialization cost
402-
// proportional to buffer size (#110650)
401+
// at the same time, i.e. small reads suffer from syscall overhead, all reads incur costs proportional to buffer size (#110650)
403402
//
404403
pub(crate) fn default_read_to_end<R: Read + ?Sized>(
405404
r: &mut R,
@@ -444,6 +443,8 @@ pub(crate) fn default_read_to_end<R: Read + ?Sized>(
444443
}
445444
}
446445

446+
let mut consecutive_short_reads = 0;
447+
447448
loop {
448449
if buf.len() == buf.capacity() && buf.capacity() == start_cap {
449450
// The buffer might be an exact fit. Let's read into a probe buffer
@@ -489,6 +490,12 @@ pub(crate) fn default_read_to_end<R: Read + ?Sized>(
489490
return Ok(buf.len() - start_len);
490491
}
491492

493+
if bytes_read < buf_len {
494+
consecutive_short_reads += 1;
495+
} else {
496+
consecutive_short_reads = 0;
497+
}
498+
492499
// store how much was initialized but not filled
493500
initialized = unfilled_but_initialized;
494501

@@ -503,7 +510,10 @@ pub(crate) fn default_read_to_end<R: Read + ?Sized>(
503510
// The reader is returning short reads but it doesn't call ensure_init().
504511
// In that case we no longer need to restrict read sizes to avoid
505512
// initialization costs.
506-
if !was_fully_initialized {
513+
// When reading from disk we usually don't get any short reads except at EOF.
514+
// So we wait for at least 2 short reads before uncapping the read buffer;
515+
// this helps with the Windows issue.
516+
if !was_fully_initialized && consecutive_short_reads > 1 {
507517
max_read_size = usize::MAX;
508518
}
509519

0 commit comments

Comments
 (0)