From 63dcb19927feb1edb39b9bb05786dd4c45c5ca23 Mon Sep 17 00:00:00 2001 From: vlad20012 Date: Sat, 20 May 2023 22:27:58 +0200 Subject: [PATCH] Inline non-local `#[inline(always)]` function in incremental mode --- compiler/rustc_mir_transform/src/inline.rs | 103 +++++++++++++++--- compiler/rustc_mir_transform/src/lib.rs | 2 +- .../inlining-from-extern-crate.rs | 1 + .../incremental/hashes/function_interfaces.rs | 2 +- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 12f955d46bd31..8c3eefdeb825a 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -42,19 +42,7 @@ struct CallSite<'tcx> { impl<'tcx> MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - if let Some(enabled) = sess.opts.unstable_opts.inline_mir { - return enabled; - } - - match sess.mir_opt_level() { - 0 | 1 => false, - 2 => { - (sess.opts.optimize == OptLevel::Default - || sess.opts.optimize == OptLevel::Aggressive) - && sess.opts.incremental == None - } - _ => true, - } + InlinerConfig::new(sess).is_ok() } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -69,6 +57,67 @@ impl<'tcx> MirPass<'tcx> for Inline { } } +impl Inline { + pub fn is_enabled_and_needs_mir_inliner_callees(sess: &rustc_session::Session) -> bool { + match InlinerConfig::new(sess) { + Ok(config) => config.inline_local_fns, + Err(_) => false, + } + } +} + +struct InlinerConfig { + /// Inline functions with `#[inline(always)]` attribute + inline_fns_with_inline_always_hint: bool, + /// Inline functions with `#[inline]` attribute + inline_fns_with_inline_hint: bool, + /// Inline functions without `#[inline]` attribute + /// Inline functions with `#[inline(always)]` attribute + inline_fns_without_hint: bool, + /// Inline function from current crate (much heavier during incremental compilation) + inline_local_fns: bool, +} + +impl InlinerConfig { + fn new(sess: &rustc_session::Session) -> Result { + match sess.opts.unstable_opts.inline_mir { + Some(true) => return Ok(InlinerConfig::full()), + Some(false) => return Err(InliningIsDisabled), + None => {} + } + match sess.mir_opt_level() { + 0 | 1 => Err(InliningIsDisabled), + 2 => { + let optimize = sess.opts.optimize; + if optimize == OptLevel::Default || optimize == OptLevel::Aggressive { + let is_non_incremental = sess.opts.incremental == None; + Ok(InlinerConfig { + inline_fns_with_inline_always_hint: true, + inline_fns_with_inline_hint: is_non_incremental, + inline_fns_without_hint: is_non_incremental, + inline_local_fns: is_non_incremental, + }) + } else { + Err(InliningIsDisabled) + } + } + _ => Ok(InlinerConfig::full()), + } + } + + fn full() -> InlinerConfig { + InlinerConfig { + inline_fns_with_inline_always_hint: true, + inline_fns_with_inline_hint: true, + inline_fns_without_hint: true, + inline_local_fns: true, + } + } +} + +#[derive(Debug)] +struct InliningIsDisabled; + fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { let def_id = body.source.def_id().expect_local(); @@ -91,6 +140,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { let mut this = Inliner { tcx, param_env, + config: InlinerConfig::new(tcx.sess).unwrap_or_else(|_| InlinerConfig::full()), codegen_fn_attrs: tcx.codegen_fn_attrs(def_id), history: Vec::new(), changed: false, @@ -103,6 +153,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { struct Inliner<'tcx> { tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, + config: InlinerConfig, /// Caller codegen attributes. codegen_fn_attrs: &'tcx CodegenFnAttrs, /// Stack of inlined instances. @@ -352,11 +403,35 @@ impl<'tcx> Inliner<'tcx> { if let InlineAttr::Never = callee_attrs.inline { return Err("never inline hint"); } + match callee_attrs.inline { + InlineAttr::Never => return Err("never inline hint"), + InlineAttr::Always => { + if !self.config.inline_fns_with_inline_always_hint { + return Err("inliner is configured to ignore #[inline(always)] functions"); + } + } + InlineAttr::Hint => { + if !self.config.inline_fns_with_inline_hint { + return Err("inliner is configured to ignore #[inline] functions"); + } + } + _ => { + if !self.config.inline_fns_without_hint { + return Err("inliner is configured to ignore functions without #[inline]"); + } + } + } + + let callee_is_local = callsite.callee.def_id().is_local(); + + if callee_is_local && !self.config.inline_local_fns { + return Err("inliner is configured to ignore local functions"); + } // Only inline local functions if they would be eligible for cross-crate // inlining. This is to ensure that the final crate doesn't have MIR that // reference unexported symbols - if callsite.callee.def_id().is_local() { + if callee_is_local { let is_generic = callsite.callee.substs.non_erasable_generics().next().is_some(); if !is_generic && !callee_attrs.requests_inline() { return Err("not exported"); diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 65864dc016f4f..c76d8e9edf16b 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -388,7 +388,7 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { // Do not compute the mir call graph without said call graph actually being used. - if inline::Inline.is_enabled(&tcx.sess) { + if inline::Inline::is_enabled_and_needs_mir_inliner_callees(&tcx.sess) { tcx.ensure_with_value().mir_inliner_callees(ty::InstanceDef::Item(def.to_def_id())); } } diff --git a/tests/codegen-units/partitioning/inlining-from-extern-crate.rs b/tests/codegen-units/partitioning/inlining-from-extern-crate.rs index 1cc21632e4818..6ebe8aee6b7e9 100644 --- a/tests/codegen-units/partitioning/inlining-from-extern-crate.rs +++ b/tests/codegen-units/partitioning/inlining-from-extern-crate.rs @@ -4,6 +4,7 @@ // incremental // compile-flags:-Zprint-mono-items=lazy // compile-flags:-Zinline-in-all-cgus +// compile-flags:-Zinline_mir=false #![crate_type="lib"] diff --git a/tests/incremental/hashes/function_interfaces.rs b/tests/incremental/hashes/function_interfaces.rs index 182ca7d926c6b..23b81705f9aa0 100644 --- a/tests/incremental/hashes/function_interfaces.rs +++ b/tests/incremental/hashes/function_interfaces.rs @@ -302,7 +302,7 @@ pub fn return_impl_trait() -> i32 { } #[cfg(not(any(cfail1,cfail4)))] -#[rustc_clean(cfg = "cfail2", except = "hir_owner, hir_owner_nodes, typeck, fn_sig, optimized_mir")] +#[rustc_clean(cfg = "cfail2", except = "hir_owner, hir_owner_nodes, typeck, fn_sig")] #[rustc_clean(cfg = "cfail3")] #[rustc_clean(cfg = "cfail5", except = "hir_owner, hir_owner_nodes, typeck, fn_sig, optimized_mir")] #[rustc_clean(cfg = "cfail6")]