force_collect improvements

compiler/rustc_builtin_macros/src/cmdline_attrs.rs

@@ -4,6 +4,7 @@ use crate::errors;
 use rustc_ast::attr::mk_attr;
 use rustc_ast::token;
 use rustc_ast::{self as ast, AttrItem, AttrStyle};
+use rustc_parse::parser::ForceCollect;
 use rustc_parse::{new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_session::parse::ParseSess;
 use rustc_span::FileName;
@@ -17,13 +18,14 @@ pub fn inject(krate: &mut ast::Crate, psess: &ParseSess, attrs: &[String]) {
         ));
 
         let start_span = parser.token.span;
-        let AttrItem { unsafety, path, args, tokens: _ } = match parser.parse_attr_item(false) {
-            Ok(ai) => ai,
-            Err(err) => {
-                err.emit();
-                continue;
-            }
-        };
+        let AttrItem { unsafety, path, args, tokens: _ } =
+            match parser.parse_attr_item(ForceCollect::No) {
+                Ok(ai) => ai,
+                Err(err) => {
+                    err.emit();
+                    continue;
+                }
+            };
         let end_span = parser.token.span;
         if parser.token != token::Eof {
             psess.dcx().emit_err(errors::InvalidCrateAttr { span: start_span.to(end_span) });

compiler/rustc_parse/src/parser/attr.rs

@@ -124,7 +124,7 @@ impl<'a> Parser<'a> {
                 if this.eat(&token::Not) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer };
 
             this.expect(&token::OpenDelim(Delimiter::Bracket))?;
-            let item = this.parse_attr_item(false)?;
+            let item = this.parse_attr_item(ForceCollect::No)?;
             this.expect(&token::CloseDelim(Delimiter::Bracket))?;
             let attr_sp = lo.to(this.prev_token.span);
 
@@ -248,16 +248,15 @@ impl<'a> Parser<'a> {
     ///     PATH
     ///     PATH `=` UNSUFFIXED_LIT
     /// The delimiters or `=` are still put into the resulting token stream.
-    pub fn parse_attr_item(&mut self, capture_tokens: bool) -> PResult<'a, ast::AttrItem> {
+    pub fn parse_attr_item(&mut self, force_collect: ForceCollect) -> PResult<'a, ast::AttrItem> {
         maybe_whole!(self, NtMeta, |attr| attr.into_inner());
 
-        let do_parse = |this: &mut Self| {
+        let do_parse = |this: &mut Self, _empty_attrs| {
             let is_unsafe = this.eat_keyword(kw::Unsafe);
             let unsafety = if is_unsafe {
                 let unsafe_span = this.prev_token.span;
                 this.psess.gated_spans.gate(sym::unsafe_attributes, unsafe_span);
                 this.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
-
                 ast::Safety::Unsafe(unsafe_span)
             } else {
                 ast::Safety::Default
@@ -268,10 +267,10 @@ impl<'a> Parser<'a> {
             if is_unsafe {
                 this.expect(&token::CloseDelim(Delimiter::Parenthesis))?;
             }
-            Ok(ast::AttrItem { unsafety, path, args, tokens: None })
+            Ok((ast::AttrItem { unsafety, path, args, tokens: None }, false))
         };
-        // Attr items don't have attributes
-        if capture_tokens { self.collect_tokens_no_attrs(do_parse) } else { do_parse(self) }
+        // Attr items don't have attributes.
+        self.collect_tokens_trailing_token(AttrWrapper::empty(), force_collect, do_parse)
     }
 
     /// Parses attributes that appear after the opening of an item. These should
@@ -344,7 +343,7 @@ impl<'a> Parser<'a> {
         let mut expanded_attrs = Vec::with_capacity(1);
         while self.token.kind != token::Eof {
             let lo = self.token.span;
-            let item = self.parse_attr_item(true)?;
+            let item = self.parse_attr_item(ForceCollect::Yes)?;
             expanded_attrs.push((item, lo.to(self.prev_token.span)));
             if !self.eat(&token::Comma) {
                 break;

compiler/rustc_parse/src/parser/mod.rs

@@ -942,11 +942,10 @@ impl<'a> Parser<'a> {
         let initial_semicolon = self.token.span;
 
         while self.eat(&TokenKind::Semi) {
-            let _ =
-                self.parse_stmt_without_recovery(false, ForceCollect::Yes).unwrap_or_else(|e| {
-                    e.cancel();
-                    None
-                });
+            let _ = self.parse_stmt_without_recovery(false, ForceCollect::No).unwrap_or_else(|e| {
+                e.cancel();
+                None
+            });
         }
 
         expect_err

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -171,7 +171,7 @@ impl<'a> Parser<'a> {
             NonterminalKind::Path => {
                 NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?))
             }
-            NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(true)?)),
+            NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(ForceCollect::Yes)?)),
             NonterminalKind::Vis => {
                 NtVis(P(self
                     .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))

compiler/rustc_parse/src/parser/stmt.rs

@@ -72,20 +72,23 @@ impl<'a> Parser<'a> {
                 lo,
                 attrs,
                 errors::InvalidVariableDeclarationSub::MissingLet,
+                force_collect,
             )?
         } else if self.is_kw_followed_by_ident(kw::Auto) && self.may_recover() {
             self.bump(); // `auto`
             self.recover_stmt_local_after_let(
                 lo,
                 attrs,
                 errors::InvalidVariableDeclarationSub::UseLetNotAuto,
+                force_collect,
             )?
         } else if self.is_kw_followed_by_ident(sym::var) && self.may_recover() {
             self.bump(); // `var`
             self.recover_stmt_local_after_let(
                 lo,
                 attrs,
                 errors::InvalidVariableDeclarationSub::UseLetNotVar,
+                force_collect,
             )?
         } else if self.check_path()
             && !self.token.is_qpath_start()
@@ -96,17 +99,17 @@ impl<'a> Parser<'a> {
             // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
             // that starts like a path (1 token), but it fact not a path.
             // Also, we avoid stealing syntax from `parse_item_`.
-            match force_collect {
-                ForceCollect::Yes => {
-                    self.collect_tokens_no_attrs(|this| this.parse_stmt_path_start(lo, attrs))?
+            let stmt = self.collect_tokens_trailing_token(
+                AttrWrapper::empty(),
+                force_collect,
+                |this, _empty_attrs| Ok((this.parse_stmt_path_start(lo, attrs)?, false)),
+            );
+            match stmt {
+                Ok(stmt) => stmt,
+                Err(mut err) => {
+                    self.suggest_add_missing_let_for_stmt(&mut err);
+                    return Err(err);
                 }
-                ForceCollect::No => match self.parse_stmt_path_start(lo, attrs) {
-                    Ok(stmt) => stmt,
-                    Err(mut err) => {
-                        self.suggest_add_missing_let_for_stmt(&mut err);
-                        return Err(err);
-                    }
-                },
             }
         } else if let Some(item) = self.parse_item_common(
             attrs.clone(),
@@ -123,12 +126,13 @@ impl<'a> Parser<'a> {
             self.mk_stmt(lo, StmtKind::Empty)
         } else if self.token != token::CloseDelim(Delimiter::Brace) {
             // Remainder are line-expr stmts.
-            let e = match force_collect {
-                ForceCollect::Yes => self.collect_tokens_no_attrs(|this| {
-                    this.parse_expr_res(Restrictions::STMT_EXPR, attrs)
-                })?,
-                ForceCollect::No => self.parse_expr_res(Restrictions::STMT_EXPR, attrs)?,
-            };
+            let e = self.collect_tokens_trailing_token(
+                AttrWrapper::empty(),
+                force_collect,
+                |this, _empty_attrs| {
+                    Ok((this.parse_expr_res(Restrictions::STMT_EXPR, attrs)?, false))
+                },
+            )?;
             if matches!(e.kind, ExprKind::Assign(..)) && self.eat_keyword(kw::Else) {
                 let bl = self.parse_block()?;
                 // Destructuring assignment ... else.
@@ -231,13 +235,13 @@ impl<'a> Parser<'a> {
         lo: Span,
         attrs: AttrWrapper,
         subdiagnostic: fn(Span) -> errors::InvalidVariableDeclarationSub,
+        force_collect: ForceCollect,
     ) -> PResult<'a, Stmt> {
-        let stmt =
-            self.collect_tokens_trailing_token(attrs, ForceCollect::Yes, |this, attrs| {
-                let local = this.parse_local(attrs)?;
-                // FIXME - maybe capture semicolon in recovery?
-                Ok((this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), false))
-            })?;
+        let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
+            let local = this.parse_local(attrs)?;
+            // FIXME - maybe capture semicolon in recovery?
+            Ok((this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), false))
+        })?;
         self.dcx()
             .emit_err(errors::InvalidVariableDeclaration { span: lo, sub: subdiagnostic(lo) });
         Ok(stmt)