Skip to content

Commit 96229ed

Browse files
nnethercotennethercote
committed
Remove NtItem and NtStmt.
This involves replacing `nt_pretty_printing_compatibility_hack` with `stream_pretty_printing_compatibility_hack`. The handling of statements in `transcribe` is slightly different to other nonterminal kinds, due to the lack of `from_ast` implementation for empty statements. Notable test changes: - `tests/ui/proc-macro/expand-to-derive.rs`: the diff looks large but the only difference is the insertion of a single invisible-delimited group around a metavar.
1 parent 372af98 commit 96229ed

File tree

18 files changed

+194
-140
lines changed

18 files changed

+194
-140
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,6 @@ impl HasTokens for Attribute {
199199
impl HasTokens for Nonterminal {
200200
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
201201
match self {
202-
Nonterminal::NtItem(item) => item.tokens(),
203-
Nonterminal::NtStmt(stmt) => stmt.tokens(),
204202
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
205203
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
206204
Nonterminal::NtPath(path) => path.tokens(),
@@ -209,8 +207,6 @@ impl HasTokens for Nonterminal {
209207
}
210208
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
211209
match self {
212-
Nonterminal::NtItem(item) => item.tokens_mut(),
213-
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
214210
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
215211
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
216212
Nonterminal::NtPath(path) => path.tokens_mut(),

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -892,19 +892,7 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
892892
// multiple items there....
893893
fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
894894
match nt {
895-
token::NtItem(item) => visit_clobber(item, |item| {
896-
// This is probably okay, because the only visitors likely to
897-
// peek inside interpolated nodes will be renamings/markings,
898-
// which map single items to single items.
899-
vis.flat_map_item(item).expect_one("expected visitor to produce exactly one item")
900-
}),
901895
token::NtBlock(block) => vis.visit_block(block),
902-
token::NtStmt(stmt) => visit_clobber(stmt, |stmt| {
903-
// See reasoning above.
904-
stmt.map(|stmt| {
905-
vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item")
906-
})
907-
}),
908896
token::NtExpr(expr) => vis.visit_expr(expr),
909897
token::NtLiteral(expr) => vis.visit_expr(expr),
910898
token::NtMeta(item) => {

‎compiler/rustc_ast/src/token.rs

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,9 +1071,7 @@ pub enum NtExprKind {
10711071
#[derive(Clone, Encodable, Decodable)]
10721072
/// For interpolation during macro expansion.
10731073
pub enum Nonterminal {
1074-
NtItem(P<ast::Item>),
10751074
NtBlock(P<ast::Block>),
1076-
NtStmt(P<ast::Stmt>),
10771075
NtExpr(P<ast::Expr>),
10781076
NtLiteral(P<ast::Expr>),
10791077
/// Stuff inside brackets for attributes
@@ -1167,9 +1165,7 @@ impl fmt::Display for NonterminalKind {
11671165
impl Nonterminal {
11681166
pub fn use_span(&self) -> Span {
11691167
match self {
1170-
NtItem(item) => item.span,
11711168
NtBlock(block) => block.span,
1172-
NtStmt(stmt) => stmt.span,
11731169
NtExpr(expr) | NtLiteral(expr) => expr.span,
11741170
NtMeta(attr_item) => attr_item.span(),
11751171
NtPath(path) => path.span,
@@ -1178,9 +1174,7 @@ impl Nonterminal {
11781174

11791175
pub fn descr(&self) -> &'static str {
11801176
match self {
1181-
NtItem(..) => "item",
11821177
NtBlock(..) => "block",
1183-
NtStmt(..) => "statement",
11841178
NtExpr(..) => "expression",
11851179
NtLiteral(..) => "literal",
11861180
NtMeta(..) => "attribute",
@@ -1202,9 +1196,7 @@ impl PartialEq for Nonterminal {
12021196
impl fmt::Debug for Nonterminal {
12031197
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
12041198
match *self {
1205-
NtItem(..) => f.pad("NtItem(..)"),
12061199
NtBlock(..) => f.pad("NtBlock(..)"),
1207-
NtStmt(..) => f.pad("NtStmt(..)"),
12081200
NtExpr(..) => f.pad("NtExpr(..)"),
12091201
NtLiteral(..) => f.pad("NtLiteral(..)"),
12101202
NtMeta(..) => f.pad("NtMeta(..)"),

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use rustc_macros::{Decodable, Encodable, HashStable_Generic};
2323
use rustc_serialize::{Decodable, Encodable};
2424
use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
2525

26-
use crate::ast::{AttrStyle, StmtKind};
26+
use crate::ast::AttrStyle;
2727
use crate::ast_traits::{HasAttrs, HasTokens};
2828
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
2929
use crate::{AttrVec, Attribute};
@@ -461,13 +461,7 @@ impl TokenStream {
461461

462462
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
463463
match nt {
464-
Nonterminal::NtItem(item) => TokenStream::from_ast(item),
465464
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
466-
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
467-
// FIXME: Properly collect tokens for empty statements.
468-
TokenStream::token_alone(token::Semi, stmt.span)
469-
}
470-
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
471465
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
472466
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
473467
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),

‎compiler/rustc_builtin_macros/src/cfg_eval.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,9 @@ impl CfgEval<'_> {
140140
Annotatable::ForeignItem(self.flat_map_foreign_item(item).pop().unwrap())
141141
}
142142
Annotatable::Stmt(_) => {
143-
let stmt =
144-
parser.parse_stmt_without_recovery(false, ForceCollect::Yes)?.unwrap();
143+
let stmt = parser
144+
.parse_stmt_without_recovery(false, ForceCollect::Yes, false)?
145+
.unwrap();
145146
Annotatable::Stmt(P(self.flat_map_stmt(stmt).pop().unwrap()))
146147
}
147148
Annotatable::Expr(_) => {

‎compiler/rustc_expand/src/base.rs

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use std::sync::Arc;
77

88
use rustc_ast::attr::{AttributeExt, MarkedAttrs};
99
use rustc_ast::ptr::P;
10-
use rustc_ast::token::Nonterminal;
10+
use rustc_ast::token::MetaVarKind;
1111
use rustc_ast::tokenstream::TokenStream;
1212
use rustc_ast::visit::{AssocCtxt, Visitor};
1313
use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
@@ -18,7 +18,7 @@ use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed, PResult};
1818
use rustc_feature::Features;
1919
use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools};
2020
use rustc_parse::MACRO_ARGUMENTS;
21-
use rustc_parse::parser::Parser;
21+
use rustc_parse::parser::{ForceCollect, Parser};
2222
use rustc_session::config::CollapseMacroDebuginfo;
2323
use rustc_session::parse::ParseSess;
2424
use rustc_session::{Limit, Session};
@@ -1382,13 +1382,13 @@ pub fn parse_macro_name_and_helper_attrs(
13821382
/// If this item looks like a specific enums from `rental`, emit a fatal error.
13831383
/// See #73345 and #83125 for more details.
13841384
/// FIXME(#73933): Remove this eventually.
1385-
fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
1385+
fn pretty_printing_compatibility_hack(item: &Item, psess: &ParseSess) {
13861386
let name = item.ident.name;
13871387
if name == sym::ProceduralMasqueradeDummyType
13881388
&& let ast::ItemKind::Enum(enum_def, _) = &item.kind
13891389
&& let [variant] = &*enum_def.variants
13901390
&& variant.ident.name == sym::Input
1391-
&& let FileName::Real(real) = sess.source_map().span_to_filename(item.ident.span)
1391+
&& let FileName::Real(real) = psess.source_map().span_to_filename(item.ident.span)
13921392
&& let Some(c) = real
13931393
.local_path()
13941394
.unwrap_or(Path::new(""))
@@ -1406,15 +1406,15 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
14061406
};
14071407

14081408
if crate_matches {
1409-
sess.dcx().emit_fatal(errors::ProcMacroBackCompat {
1409+
psess.dcx().emit_fatal(errors::ProcMacroBackCompat {
14101410
crate_name: "rental".to_string(),
14111411
fixed_version: "0.5.6".to_string(),
14121412
});
14131413
}
14141414
}
14151415
}
14161416

1417-
pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &Session) {
1417+
pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, psess: &ParseSess) {
14181418
let item = match ann {
14191419
Annotatable::Item(item) => item,
14201420
Annotatable::Stmt(stmt) => match &stmt.kind {
@@ -1423,17 +1423,36 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &S
14231423
},
14241424
_ => return,
14251425
};
1426-
pretty_printing_compatibility_hack(item, sess)
1426+
pretty_printing_compatibility_hack(item, psess)
14271427
}
14281428

1429-
pub(crate) fn nt_pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &Session) {
1430-
let item = match nt {
1431-
Nonterminal::NtItem(item) => item,
1432-
Nonterminal::NtStmt(stmt) => match &stmt.kind {
1433-
ast::StmtKind::Item(item) => item,
1434-
_ => return,
1435-
},
1429+
pub(crate) fn stream_pretty_printing_compatibility_hack(
1430+
kind: MetaVarKind,
1431+
stream: &TokenStream,
1432+
psess: &ParseSess,
1433+
) {
1434+
let item = match kind {
1435+
MetaVarKind::Item => {
1436+
let mut parser = Parser::new(psess, stream.clone(), None);
1437+
// No need to collect tokens for this simple check.
1438+
parser
1439+
.parse_item(ForceCollect::No)
1440+
.expect("failed to reparse item")
1441+
.expect("an actual item")
1442+
}
1443+
MetaVarKind::Stmt => {
1444+
let mut parser = Parser::new(psess, stream.clone(), None);
1445+
// No need to collect tokens for this simple check.
1446+
let stmt = parser
1447+
.parse_stmt(ForceCollect::No)
1448+
.expect("failed to reparse")
1449+
.expect("an actual stmt");
1450+
match &stmt.kind {
1451+
ast::StmtKind::Item(item) => item.clone(),
1452+
_ => return,
1453+
}
1454+
}
14361455
_ => return,
14371456
};
1438-
pretty_printing_compatibility_hack(item, sess)
1457+
pretty_printing_compatibility_hack(&item, psess)
14391458
}

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use rustc_ast::token::{
77
TokenKind,
88
};
99
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
10-
use rustc_ast::{ExprKind, TyKind};
10+
use rustc_ast::{ExprKind, StmtKind, TyKind};
1111
use rustc_data_structures::fx::FxHashMap;
1212
use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
1313
use rustc_parse::lexer::nfc_normalize;
@@ -323,6 +323,18 @@ pub(super) fn transcribe<'a>(
323323
let kind = token::NtLifetime(*ident, *is_raw);
324324
TokenTree::token_alone(kind, sp)
325325
}
326+
MatchedSingle(ParseNtResult::Item(item)) => {
327+
mk_delimited(item.span, MetaVarKind::Item, TokenStream::from_ast(item))
328+
}
329+
MatchedSingle(ParseNtResult::Stmt(stmt)) => {
330+
let stream = if let StmtKind::Empty = stmt.kind {
331+
// FIXME: Properly collect tokens for empty statements.
332+
TokenStream::token_alone(token::Semi, stmt.span)
333+
} else {
334+
TokenStream::from_ast(stmt)
335+
};
336+
mk_delimited(stmt.span, MetaVarKind::Stmt, stream)
337+
}
326338
MatchedSingle(ParseNtResult::Pat(pat, pat_kind)) => mk_delimited(
327339
pat.span,
328340
MetaVarKind::Pat(*pat_kind),

‎compiler/rustc_expand/src/proc_macro.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ impl MultiItemModifier for DeriveProcMacro {
122122
// We had a lint for a long time, but now we just emit a hard error.
123123
// Eventually we might remove the special case hard error check
124124
// altogether. See #73345.
125-
crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess);
125+
crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess);
126126
let input = item.to_tokens();
127127
let stream = {
128128
let _timer =

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -115,11 +115,25 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
115115

116116
while let Some(tree) = iter.next() {
117117
let (Token { kind, span }, joint) = match tree.clone() {
118-
tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
119-
let delimiter = pm::Delimiter::from_internal(delim);
118+
tokenstream::TokenTree::Delimited(span, _, delim, stream) => {
119+
// We used to have an alternative behaviour for crates that
120+
// needed it: a hack used to pass AST fragments to
121+
// attribute and derive macros as a single nonterminal
122+
// token instead of a token stream. Such token needs to be
123+
// "unwrapped" and not represented as a delimited group. We
124+
// had a lint for a long time, but now we just emit a hard
125+
// error. Eventually we might remove the special case hard
126+
// error check altogether. See #73345.
127+
if let Delimiter::Invisible(InvisibleOrigin::MetaVar(kind)) = delim {
128+
crate::base::stream_pretty_printing_compatibility_hack(
129+
kind,
130+
&stream,
131+
rustc.psess(),
132+
);
133+
}
120134
trees.push(TokenTree::Group(Group {
121-
delimiter,
122-
stream: Some(tts),
135+
delimiter: pm::Delimiter::from_internal(delim),
136+
stream: Some(stream),
123137
span: DelimSpan {
124138
open: span.open,
125139
close: span.close,
@@ -279,15 +293,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
279293

280294
Interpolated(nt) => {
281295
let stream = TokenStream::from_nonterminal_ast(&nt);
282-
// We used to have an alternative behaviour for crates that
283-
// needed it: a hack used to pass AST fragments to
284-
// attribute and derive macros as a single nonterminal
285-
// token instead of a token stream. Such token needs to be
286-
// "unwrapped" and not represented as a delimited group. We
287-
// had a lint for a long time, but now we just emit a hard
288-
// error. Eventually we might remove the special case hard
289-
// error check altogether. See #73345.
290-
crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.ecx.sess);
291296
trees.push(TokenTree::Group(Group {
292297
delimiter: pm::Delimiter::None,
293298
stream: Some(stream),

‎compiler/rustc_parse/src/parser/expr.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3084,7 +3084,7 @@ impl<'a> Parser<'a> {
30843084
}
30853085

30863086
self.restore_snapshot(pre_pat_snapshot);
3087-
match self.parse_stmt_without_recovery(true, ForceCollect::No) {
3087+
match self.parse_stmt_without_recovery(true, ForceCollect::No, false) {
30883088
// Consume statements for as long as possible.
30893089
Ok(Some(stmt)) => {
30903090
stmts.push(stmt);

‎compiler/rustc_parse/src/parser/item.rs

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ use super::diagnostics::{ConsumeClosingDelim, dummy_arg};
2121
use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
2222
use super::{
2323
AttrWrapper, ExpKeywordPair, ExpTokenPair, FollowedByType, ForceCollect, Parser, PathStyle,
24-
Trailing, UsePreAttrPos,
24+
Recovered, Trailing, UsePreAttrPos,
2525
};
2626
use crate::errors::{self, MacroExpandsToAdtField};
27-
use crate::{exp, fluent_generated as fluent, maybe_whole};
27+
use crate::{exp, fluent_generated as fluent};
2828

2929
impl<'a> Parser<'a> {
3030
/// Parses a source module as a crate. This is the main entry point for the parser.
@@ -142,10 +142,13 @@ impl<'a> Parser<'a> {
142142
fn_parse_mode: FnParseMode,
143143
force_collect: ForceCollect,
144144
) -> PResult<'a, Option<Item>> {
145-
maybe_whole!(self, NtItem, |item| {
145+
if let Some(item) =
146+
self.eat_metavar_seq(MetaVarKind::Item, |this| this.parse_item(ForceCollect::Yes))
147+
{
148+
let mut item = item.expect("an actual item");
146149
attrs.prepend_to_nt_inner(&mut item.attrs);
147-
Some(item.into_inner())
148-
});
150+
return Ok(Some(item.into_inner()));
151+
}
149152

150153
self.collect_tokens(None, attrs, force_collect, |this, mut attrs| {
151154
let lo = this.token.span;

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,10 +1076,12 @@ impl<'a> Parser<'a> {
10761076
let initial_semicolon = self.token.span;
10771077

10781078
while self.eat(exp!(Semi)) {
1079-
let _ = self.parse_stmt_without_recovery(false, ForceCollect::No).unwrap_or_else(|e| {
1080-
e.cancel();
1081-
None
1082-
});
1079+
let _ = self
1080+
.parse_stmt_without_recovery(false, ForceCollect::No, false)
1081+
.unwrap_or_else(|e| {
1082+
e.cancel();
1083+
None
1084+
});
10831085
}
10841086

10851087
expect_err
@@ -1746,6 +1748,8 @@ pub enum ParseNtResult {
17461748
Tt(TokenTree),
17471749
Ident(Ident, IdentIsRaw),
17481750
Lifetime(Ident, IdentIsRaw),
1751+
Item(P<ast::Item>),
1752+
Stmt(P<ast::Stmt>),
17491753
Pat(P<ast::Pat>, NtPatKind),
17501754
Ty(P<ast::Ty>),
17511755
Vis(P<ast::Visibility>),

0 commit comments

Comments
 (0)