Skip to content

Commit 5f66c2a

Browse files
borsbors
committed
Auto merge of #69801 - petrochenkov:nonorm, r=<try>
rustc_parse: Remove `Parser::normalized(_prev)_token` Perform the "normalization" (renamed to "uninterpolation") on the fly when necessary. The final part of #69579 #69384 #69376 #69211 #69034 #69006. r? @Centril
2 parents f943349 + bfc7502 commit 5f66c2a

File tree

14 files changed

+192
-188
lines changed

14 files changed

+192
-188
lines changed

Diff for: ‎src/librustc_ast/attr/mod.rs

+6-3
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,10 @@ impl MetaItem {
286286
}
287287

288288
impl AttrItem {
289+
pub fn span(&self) -> Span {
290+
self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
291+
}
292+
289293
pub fn meta(&self, span: Span) -> Option<MetaItem> {
290294
Some(MetaItem {
291295
path: self.path.clone(),
@@ -437,7 +441,7 @@ impl MetaItem {
437441
I: Iterator<Item = TokenTree>,
438442
{
439443
// FIXME: Share code with `parse_path`.
440-
let path = match tokens.next() {
444+
let path = match tokens.next().map(TokenTree::uninterpolate) {
441445
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span }))
442446
| Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
443447
let mut segments = if let token::Ident(name, _) = kind {
@@ -453,7 +457,7 @@ impl MetaItem {
453457
};
454458
loop {
455459
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
456-
tokens.next()
460+
tokens.next().map(TokenTree::uninterpolate)
457461
{
458462
segments.push(PathSegment::from_ident(Ident::new(name, span)));
459463
} else {
@@ -470,7 +474,6 @@ impl MetaItem {
470474
Path { span, segments }
471475
}
472476
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
473-
token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
474477
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
475478
token::Nonterminal::NtPath(ref path) => path.clone(),
476479
_ => return None,

Diff for: ‎src/librustc_ast/token.rs

+59-23
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ use rustc_macros::HashStable_Generic;
1414
use rustc_span::symbol::kw;
1515
use rustc_span::symbol::Symbol;
1616
use rustc_span::{self, Span, DUMMY_SP};
17-
use std::fmt;
18-
use std::mem;
17+
use std::borrow::Cow;
18+
use std::{fmt, mem};
1919

2020
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
2121
#[derive(HashStable_Generic)]
@@ -328,6 +328,18 @@ impl Token {
328328
mem::replace(self, Token::dummy())
329329
}
330330

331+
/// For interpolated tokens returns a span of the fragment to which the interpolated
332+
/// token refers, for all other tokens this is just a regular span.
333+
/// It is particularly important to use this for identifiers and lifetimes
334+
/// for which spans affect name resolution. This also includes edition checks
335+
/// for edition-specific keyword identifiers.
336+
pub fn uninterpolated_span(&self) -> Span {
337+
match &self.kind {
338+
Interpolated(nt) => nt.span(),
339+
_ => self.span,
340+
}
341+
}
342+
331343
pub fn is_op(&self) -> bool {
332344
match self.kind {
333345
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
@@ -345,7 +357,7 @@ impl Token {
345357

346358
/// Returns `true` if the token can appear at the start of an expression.
347359
pub fn can_begin_expr(&self) -> bool {
348-
match self.kind {
360+
match self.uninterpolate().kind {
349361
Ident(name, is_raw) =>
350362
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
351363
OpenDelim(..) | // tuple, array or block
@@ -363,12 +375,10 @@ impl Token {
363375
Lifetime(..) | // labeled loop
364376
Pound => true, // expression attributes
365377
Interpolated(ref nt) => match **nt {
366-
NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
367378
NtLiteral(..) |
368379
NtExpr(..) |
369380
NtBlock(..) |
370-
NtPath(..) |
371-
NtLifetime(..) => true,
381+
NtPath(..) => true,
372382
_ => false,
373383
},
374384
_ => false,
@@ -377,7 +387,7 @@ impl Token {
377387

378388
/// Returns `true` if the token can appear at the start of a type.
379389
pub fn can_begin_type(&self) -> bool {
380-
match self.kind {
390+
match self.uninterpolate().kind {
381391
Ident(name, is_raw) =>
382392
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
383393
OpenDelim(Paren) | // tuple
@@ -391,8 +401,7 @@ impl Token {
391401
Lt | BinOp(Shl) | // associated path
392402
ModSep => true, // global path
393403
Interpolated(ref nt) => match **nt {
394-
NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
395-
NtTy(..) | NtPath(..) | NtLifetime(..) => true,
404+
NtTy(..) | NtPath(..) => true,
396405
_ => false,
397406
},
398407
_ => false,
@@ -433,38 +442,47 @@ impl Token {
433442
///
434443
/// Keep this in sync with `Lit::from_token`.
435444
pub fn can_begin_literal_or_bool(&self) -> bool {
436-
match self.kind {
445+
match self.uninterpolate().kind {
437446
Literal(..) | BinOp(Minus) => true,
438447
Ident(name, false) if name.is_bool_lit() => true,
439448
Interpolated(ref nt) => match &**nt {
440-
NtIdent(ident, false) if ident.name.is_bool_lit() => true,
441449
NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
442450
_ => false,
443451
},
444452
_ => false,
445453
}
446454
}
447455

456+
// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
457+
// into the regular identifier or lifetime token it refers to,
458+
// otherwise returns the original token.
459+
pub fn uninterpolate(&self) -> Cow<'_, Token> {
460+
match &self.kind {
461+
Interpolated(nt) => match **nt {
462+
NtIdent(ident, is_raw) => {
463+
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
464+
}
465+
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
466+
_ => Cow::Borrowed(self),
467+
},
468+
_ => Cow::Borrowed(self),
469+
}
470+
}
471+
448472
/// Returns an identifier if this token is an identifier.
449473
pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
450-
match self.kind {
451-
Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
452-
Interpolated(ref nt) => match **nt {
453-
NtIdent(ident, is_raw) => Some((ident, is_raw)),
454-
_ => None,
455-
},
474+
let token = self.uninterpolate();
475+
match token.kind {
476+
Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)),
456477
_ => None,
457478
}
458479
}
459480

460481
/// Returns a lifetime identifier if this token is a lifetime.
461482
pub fn lifetime(&self) -> Option<ast::Ident> {
462-
match self.kind {
463-
Lifetime(name) => Some(ast::Ident::new(name, self.span)),
464-
Interpolated(ref nt) => match **nt {
465-
NtLifetime(ident) => Some(ident),
466-
_ => None,
467-
},
483+
let token = self.uninterpolate();
484+
match token.kind {
485+
Lifetime(name) => Some(ast::Ident::new(name, token.span)),
468486
_ => None,
469487
}
470488
}
@@ -714,6 +732,24 @@ pub enum Nonterminal {
714732
#[cfg(target_arch = "x86_64")]
715733
rustc_data_structures::static_assert_size!(Nonterminal, 40);
716734

735+
impl Nonterminal {
736+
fn span(&self) -> Span {
737+
match self {
738+
NtItem(item) => item.span,
739+
NtBlock(block) => block.span,
740+
NtStmt(stmt) => stmt.span,
741+
NtPat(pat) => pat.span,
742+
NtExpr(expr) | NtLiteral(expr) => expr.span,
743+
NtTy(ty) => ty.span,
744+
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
745+
NtMeta(attr_item) => attr_item.span(),
746+
NtPath(path) => path.span,
747+
NtVis(vis) => vis.span,
748+
NtTT(tt) => tt.span(),
749+
}
750+
}
751+
}
752+
717753
impl PartialEq for Nonterminal {
718754
fn eq(&self, rhs: &Self) -> bool {
719755
match (self, rhs) {

Diff for: ‎src/librustc_ast/tokenstream.rs

+7
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,13 @@ impl TokenTree {
116116
pub fn close_tt(span: DelimSpan, delim: DelimToken) -> TokenTree {
117117
TokenTree::token(token::CloseDelim(delim), span.close)
118118
}
119+
120+
pub fn uninterpolate(self) -> TokenTree {
121+
match self {
122+
TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
123+
tt => tt,
124+
}
125+
}
119126
}
120127

121128
impl<CTX> HashStable<CTX> for TokenStream

Diff for: ‎src/librustc_ast/util/literal.rs

+4-11
Original file line numberDiff line numberDiff line change
@@ -191,23 +191,16 @@ impl Lit {
191191
///
192192
/// Keep this in sync with `Token::can_begin_literal_or_bool`.
193193
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
194-
let lit = match token.kind {
194+
let lit = match token.uninterpolate().kind {
195195
token::Ident(name, false) if name.is_bool_lit() => {
196196
token::Lit::new(token::Bool, name, None)
197197
}
198198
token::Literal(lit) => lit,
199199
token::Interpolated(ref nt) => {
200-
match &**nt {
201-
token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
202-
let lit = token::Lit::new(token::Bool, ident.name, None);
203-
return Lit::from_lit_token(lit, ident.span);
200+
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
201+
if let ast::ExprKind::Lit(lit) = &expr.kind {
202+
return Ok(lit.clone());
204203
}
205-
token::NtExpr(expr) | token::NtLiteral(expr) => {
206-
if let ast::ExprKind::Lit(lit) = &expr.kind {
207-
return Ok(lit.clone());
208-
}
209-
}
210-
_ => {}
211204
}
212205
return Err(LitError::NotLiteral);
213206
}

Diff for: ‎src/librustc_builtin_macros/format.rs

+32-33
Original file line numberDiff line numberDiff line change
@@ -156,44 +156,43 @@ fn parse_args<'a>(
156156
if p.token == token::Eof {
157157
break;
158158
} // accept trailing commas
159-
if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
160-
named = true;
161-
let name = if let token::Ident(name, _) = p.normalized_token.kind {
159+
match p.token.ident() {
160+
Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => {
161+
named = true;
162162
p.bump();
163-
name
164-
} else {
165-
unreachable!();
166-
};
163+
p.expect(&token::Eq)?;
164+
let e = p.parse_expr()?;
165+
if let Some(prev) = names.get(&ident.name) {
166+
ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", ident))
167+
.span_label(args[*prev].span, "previously here")
168+
.span_label(e.span, "duplicate argument")
169+
.emit();
170+
continue;
171+
}
167172

168-
p.expect(&token::Eq)?;
169-
let e = p.parse_expr()?;
170-
if let Some(prev) = names.get(&name) {
171-
ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name))
172-
.span_label(args[*prev].span, "previously here")
173-
.span_label(e.span, "duplicate argument")
174-
.emit();
175-
continue;
173+
// Resolve names into slots early.
174+
// Since all the positional args are already seen at this point
175+
// if the input is valid, we can simply append to the positional
176+
// args. And remember the names.
177+
let slot = args.len();
178+
names.insert(ident.name, slot);
179+
args.push(e);
176180
}
177-
178-
// Resolve names into slots early.
179-
// Since all the positional args are already seen at this point
180-
// if the input is valid, we can simply append to the positional
181-
// args. And remember the names.
182-
let slot = args.len();
183-
names.insert(name, slot);
184-
args.push(e);
185-
} else {
186-
let e = p.parse_expr()?;
187-
if named {
188-
let mut err = ecx
189-
.struct_span_err(e.span, "positional arguments cannot follow named arguments");
190-
err.span_label(e.span, "positional arguments must be before named arguments");
191-
for pos in names.values() {
192-
err.span_label(args[*pos].span, "named argument");
181+
_ => {
182+
let e = p.parse_expr()?;
183+
if named {
184+
let mut err = ecx.struct_span_err(
185+
e.span,
186+
"positional arguments cannot follow named arguments",
187+
);
188+
err.span_label(e.span, "positional arguments must be before named arguments");
189+
for pos in names.values() {
190+
err.span_label(args[*pos].span, "named argument");
191+
}
192+
err.emit();
193193
}
194-
err.emit();
194+
args.push(e);
195195
}
196-
args.push(e);
197196
}
198197
}
199198
Ok((fmtstr, args, names))

Diff for: ‎src/librustc_expand/mbe/macro_parser.rs

+6-12
Original file line numberDiff line numberDiff line change
@@ -750,15 +750,9 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
750750

751751
/// The token is an identifier, but not `_`.
752752
/// We prohibit passing `_` to macros expecting `ident` for now.
753-
fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
754-
match token.kind {
755-
token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
756-
token::Interpolated(ref nt) => match **nt {
757-
token::NtIdent(ident, is_raw) if ident.name != kw::Underscore => {
758-
Some((ident.name, is_raw))
759-
}
760-
_ => None,
761-
},
753+
fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
754+
match token.ident() {
755+
Some((ident, is_raw)) if ident.name != kw::Underscore => Some((ident, is_raw)),
762756
_ => None,
763757
}
764758
}
@@ -783,7 +777,7 @@ fn may_begin_with(token: &Token, name: Name) -> bool {
783777
&& !token.is_keyword(kw::Let)
784778
}
785779
sym::ty => token.can_begin_type(),
786-
sym::ident => get_macro_name(token).is_some(),
780+
sym::ident => get_macro_ident(token).is_some(),
787781
sym::literal => token.can_begin_literal_or_bool(),
788782
sym::vis => match token.kind {
789783
// The follow-set of :vis + "priv" keyword + interpolated
@@ -888,9 +882,9 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
888882
sym::ty => token::NtTy(p.parse_ty()?),
889883
// this could be handled like a token, since it is one
890884
sym::ident => {
891-
if let Some((name, is_raw)) = get_macro_name(&p.token) {
885+
if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
892886
p.bump();
893-
token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
887+
token::NtIdent(ident, is_raw)
894888
} else {
895889
let token_str = pprust::token_to_string(&p.token);
896890
let msg = &format!("expected ident, found {}", &token_str);

Diff for: ‎src/librustc_parse/lib.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#![feature(crate_visibility_modifier)]
55

66
use rustc_ast::ast;
7-
use rustc_ast::token::{self, Nonterminal, Token};
7+
use rustc_ast::token::{self, Nonterminal};
88
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
99
use rustc_ast_pretty::pprust;
1010
use rustc_data_structures::sync::Lrc;
@@ -171,8 +171,7 @@ fn maybe_source_file_to_parser(
171171
let mut parser = stream_to_parser(sess, stream, None);
172172
parser.unclosed_delims = unclosed_delims;
173173
if parser.token == token::Eof {
174-
let span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
175-
parser.set_token(Token::new(token::Eof, span));
174+
parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
176175
}
177176

178177
Ok(parser)

Diff for: ‎src/librustc_parse/parser/diagnostics.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -192,17 +192,19 @@ impl<'a> Parser<'a> {
192192
TokenKind::CloseDelim(token::DelimToken::Brace),
193193
TokenKind::CloseDelim(token::DelimToken::Paren),
194194
];
195-
if let token::Ident(name, false) = self.normalized_token.kind {
196-
if Ident::new(name, self.normalized_token.span).is_raw_guess()
197-
&& self.look_ahead(1, |t| valid_follow.contains(&t.kind))
195+
match self.token.ident() {
196+
Some((ident, false))
197+
if ident.is_raw_guess()
198+
&& self.look_ahead(1, |t| valid_follow.contains(&t.kind)) =>
198199
{
199200
err.span_suggestion(
200-
self.normalized_token.span,
201+
ident.span,
201202
"you can escape reserved keywords to use them as identifiers",
202-
format!("r#{}", name),
203+
format!("r#{}", ident.name),
203204
Applicability::MaybeIncorrect,
204205
);
205206
}
207+
_ => {}
206208
}
207209
if let Some(token_descr) = super::token_descr_opt(&self.token) {
208210
err.span_label(self.token.span, format!("expected identifier, found {}", token_descr));

0 commit comments

Comments
 (0)