Skip to content

Commit

Permalink
Rollup merge of rust-lang#58476 - nnethercote:rm-LazyTokenStream, r=p…
Browse files Browse the repository at this point in the history 
…etrochenkov

Remove `LazyTokenStream`.

`LazyTokenStream` was added in rust-lang#40939. Perhaps it was an effective optimization then, but no longer. This PR removes it, making the code both simpler and faster.

r? @alexcrichton
  • Loading branch information
@Centril
Centril authored Feb 23, 2019
2 parents 27cc893 + 895a794 commit 117747d
Show file tree
Hide file tree
Showing 16 changed files with 168 additions and 226 deletions.
12 changes: 6 additions & 6 deletions src/librustc/hir/lowering.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1124,19 +1124,19 @@ impl<'a> LoweringContext<'a> {
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
span,
delim,
self.lower_token_stream(tts.into()).into(),
self.lower_token_stream(tts),
).into(),
}
}

fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
match token {
Token::Interpolated(_) => {}
other => return TokenTree::Token(span, other).into(),
Token::Interpolated(nt) => {
let tts = nt.to_tokenstream(&self.sess.parse_sess, span);
self.lower_token_stream(tts)
}
other => TokenTree::Token(span, other).into(),
}

let tts = token.interpolated_to_tokenstream(&self.sess.parse_sess, span);
self.lower_token_stream(tts)
}

fn lower_arm(&mut self, arm: &Arm) -> hir::Arm {
Expand Down
2 changes: 1 addition & 1 deletion src/librustc/hir/map/def_collector.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {

fn visit_token(&mut self, t: Token) {
if let Token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = nt.0 {
if let token::NtExpr(ref expr) = *nt {
if let ExprKind::Mac(..) = expr.node {
self.visit_macro_invoc(expr.id);
}
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_resolve/build_reduced_graph.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1025,7 +1025,7 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {

fn visit_token(&mut self, t: Token) {
if let Token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = nt.0 {
if let token::NtExpr(ref expr) = *nt {
if let ast::ExprKind::Mac(..) = expr.node {
self.visit_invoc(expr.id);
}
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/attr/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ impl MetaItem {
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
Path { span, segments }
}
Some(TokenTree::Token(_, Token::Interpolated(ref nt))) => match nt.0 {
Some(TokenTree::Token(_, Token::Interpolated(nt))) => match *nt {
token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()),
token::Nonterminal::NtPath(ref path) => path.clone(),
Expand Down Expand Up @@ -682,7 +682,7 @@ impl LitKind {
match token {
Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)),
Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)),
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(nt) => match *nt {
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
ExprKind::Lit(ref lit) => Some(lit.node.clone()),
_ => None,
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ext/base.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ impl<F> TTMacroExpander for F
impl MutVisitor for AvoidInterpolatedIdents {
fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) {
if let tokenstream::TokenTree::Token(_, token::Interpolated(nt)) = tt {
if let token::NtIdent(ident, is_raw) = nt.0 {
if let token::NtIdent(ident, is_raw) = **nt {
*tt = tokenstream::TokenTree::Token(ident.span,
token::Ident(ident, is_raw));
}
Expand Down
5 changes: 3 additions & 2 deletions src/libsyntax/ext/expand.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use syntax_pos::{Span, DUMMY_SP, FileName};
use syntax_pos::hygiene::ExpnFormat;

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
use std::fs;
use std::io::ErrorKind;
use std::{iter, mem};
Expand Down Expand Up @@ -584,14 +585,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
}
AttrProcMacro(ref mac, ..) => {
self.gate_proc_macro_attr_item(attr.span, &item);
let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
let item_tok = TokenTree::Token(DUMMY_SP, Token::Interpolated(Lrc::new(match item {
Annotatable::Item(item) => token::NtItem(item),
Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
Annotatable::Expr(expr) => token::NtExpr(expr),
})).into();
}))).into();
let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span);
let tok_result = mac.expand(self.cx, attr.span, input, item_tok);
let res = self.parse_ast_fragment(tok_result, invoc.fragment_kind,
Expand Down
29 changes: 15 additions & 14 deletions src/libsyntax/ext/tt/macro_parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ use smallvec::{smallvec, SmallVec};
use syntax_pos::Span;

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::mem;
use std::ops::{Deref, DerefMut};
Expand Down Expand Up @@ -179,7 +180,7 @@ struct MatcherPos<'root, 'tt: 'root> {
/// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
/// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
/// wants the shared `matches`, one should use `up.matches`.
matches: Box<[Rc<NamedMatchVec>]>,
matches: Box<[Lrc<NamedMatchVec>]>,
/// The position in `matches` corresponding to the first metavar in this matcher's sequence of
/// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
/// to `matches[match_lo]`.
Expand Down Expand Up @@ -218,7 +219,7 @@ struct MatcherPos<'root, 'tt: 'root> {
impl<'root, 'tt> MatcherPos<'root, 'tt> {
/// Adds `m` as a named match for the `idx`-th metavar.
fn push_match(&mut self, idx: usize, m: NamedMatch) {
let matches = Rc::make_mut(&mut self.matches[idx]);
let matches = Lrc::make_mut(&mut self.matches[idx]);
matches.push(m);
}
}
Expand Down Expand Up @@ -295,11 +296,11 @@ pub fn count_names(ms: &[TokenTree]) -> usize {
}

/// `len` `Vec`s (initially shared and empty) that will store matches of metavars.
fn create_matches(len: usize) -> Box<[Rc<NamedMatchVec>]> {
fn create_matches(len: usize) -> Box<[Lrc<NamedMatchVec>]> {
if len == 0 {
vec![]
} else {
let empty_matches = Rc::new(SmallVec::new());
let empty_matches = Lrc::new(SmallVec::new());
vec![empty_matches; len]
}.into_boxed_slice()
}
Expand Down Expand Up @@ -353,8 +354,8 @@ fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherP
/// token tree it was derived from.
#[derive(Debug, Clone)]
pub enum NamedMatch {
MatchedSeq(Rc<NamedMatchVec>, DelimSpan),
MatchedNonterminal(Rc<Nonterminal>),
MatchedSeq(Lrc<NamedMatchVec>, DelimSpan),
MatchedNonterminal(Lrc<Nonterminal>),
}

/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
Expand Down Expand Up @@ -561,7 +562,7 @@ fn inner_parse_loop<'root, 'tt>(
new_item.match_cur += seq.num_captures;
new_item.idx += 1;
for idx in item.match_cur..item.match_cur + seq.num_captures {
new_item.push_match(idx, MatchedSeq(Rc::new(smallvec![]), sp));
new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp));
}
cur_items.push(new_item);
}
Expand Down Expand Up @@ -707,7 +708,7 @@ pub fn parse(
let matches = eof_items[0]
.matches
.iter_mut()
.map(|dv| Rc::make_mut(dv).pop().unwrap());
.map(|dv| Lrc::make_mut(dv).pop().unwrap());
return nameize(sess, ms, matches);
} else if eof_items.len() > 1 {
return Error(
Expand Down Expand Up @@ -780,7 +781,7 @@ pub fn parse(
let match_cur = item.match_cur;
item.push_match(
match_cur,
MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.as_str()))),
MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, &ident.as_str()))),
);
item.idx += 1;
item.match_cur += 1;
Expand Down Expand Up @@ -829,7 +830,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
},
"block" => match *token {
Token::OpenDelim(token::Brace) => true,
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(ref nt) => match **nt {
token::NtItem(_)
| token::NtPat(_)
| token::NtTy(_)
Expand All @@ -843,9 +844,9 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
},
"path" | "meta" => match *token {
Token::ModSep | Token::Ident(..) => true,
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(ref nt) => match **nt {
token::NtPath(_) | token::NtMeta(_) => true,
_ => may_be_ident(&nt.0),
_ => may_be_ident(&nt),
},
_ => false,
},
Expand All @@ -862,12 +863,12 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
Token::ModSep | // path
Token::Lt | // path (UFCS constant)
Token::BinOp(token::Shl) => true, // path (double UFCS)
Token::Interpolated(ref nt) => may_be_ident(&nt.0),
Token::Interpolated(ref nt) => may_be_ident(nt),
_ => false,
},
"lifetime" => match *token {
Token::Lifetime(_) => true,
Token::Interpolated(ref nt) => match nt.0 {
Token::Interpolated(ref nt) => match **nt {
token::NtLifetime(_) | token::NtTT(_) => true,
_ => false,
},
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ext/tt/transcribe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ pub fn transcribe(cx: &ExtCtxt<'_>,
result.push(tt.clone().into());
} else {
sp = sp.apply_mark(cx.current_expansion.mark);
let token = TokenTree::Token(sp, Token::interpolated((**nt).clone()));
let token = TokenTree::Token(sp, Token::Interpolated(nt.clone()));
result.push(token.into());
}
} else {
Expand Down
5 changes: 2 additions & 3 deletions src/libsyntax/mut_visit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,9 +581,8 @@ pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
token::Ident(id, _is_raw) => vis.visit_ident(id),
token::Lifetime(id) => vis.visit_ident(id),
token::Interpolated(nt) => {
let nt = Lrc::make_mut(nt);
vis.visit_interpolated(&mut nt.0);
nt.1 = token::LazyTokenStream::new();
let mut nt = Lrc::make_mut(nt);
vis.visit_interpolated(&mut nt);
}
_ => {}
}
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/parse/attr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ impl<'a> Parser<'a> {
/// The delimiters or `=` are still put into the resulting token stream.
crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
let meta = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
_ => None,
},
Expand Down Expand Up @@ -227,7 +227,7 @@ impl<'a> Parser<'a> {
/// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
let nt_meta = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtMeta(ref e) => Some(e.clone()),
_ => None,
},
Expand Down
16 changes: 8 additions & 8 deletions src/libsyntax/parse/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ enum BlockMode {
macro_rules! maybe_whole_expr {
($p:expr) => {
if let token::Interpolated(nt) = $p.token.clone() {
match nt.0 {
match *nt {
token::NtExpr(ref e) | token::NtLiteral(ref e) => {
$p.bump();
return Ok((*e).clone());
Expand All @@ -146,7 +146,7 @@ macro_rules! maybe_whole_expr {
macro_rules! maybe_whole {
($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
if let token::Interpolated(nt) = $p.token.clone() {
if let token::$constructor($x) = nt.0.clone() {
if let token::$constructor($x) = (*nt).clone() {
$p.bump();
return Ok($e);
}
Expand Down Expand Up @@ -1570,7 +1570,7 @@ impl<'a> Parser<'a> {
Some(body)
}
token::Interpolated(ref nt) => {
match &nt.0 {
match **nt {
token::NtBlock(..) => {
*at_end = true;
let (inner_attrs, body) = self.parse_inner_attrs_and_block()?;
Expand Down Expand Up @@ -1913,7 +1913,7 @@ impl<'a> Parser<'a> {

fn is_named_argument(&mut self) -> bool {
let offset = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
_ => 0,
}
Expand Down Expand Up @@ -2099,7 +2099,7 @@ impl<'a> Parser<'a> {
/// Matches `token_lit = LIT_INTEGER | ...`.
fn parse_lit_token(&mut self) -> PResult<'a, LitKind> {
let out = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
ExprKind::Lit(ref lit) => { lit.node.clone() }
_ => { return self.unexpected_last(&self.token); }
Expand Down Expand Up @@ -2299,7 +2299,7 @@ impl<'a> Parser<'a> {
/// attributes.
pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> {
let meta_ident = match self.token {
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtMeta(ref meta) => match meta.node {
ast::MetaItemKind::Word => Some(meta.ident.clone()),
_ => None,
Expand Down Expand Up @@ -3271,7 +3271,7 @@ impl<'a> Parser<'a> {
self.meta_var_span = Some(self.span);
// Interpolated identifier and lifetime tokens are replaced with usual identifier
// and lifetime tokens, so the former are never encountered during normal parsing.
match nt.0 {
match **nt {
token::NtIdent(ident, is_raw) => (token::Ident(ident, is_raw), ident.span),
token::NtLifetime(ident) => (token::Lifetime(ident), ident.span),
_ => return,
Expand Down Expand Up @@ -3403,7 +3403,7 @@ impl<'a> Parser<'a> {
// can't continue an expression after an ident
token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw),
token::Literal(..) | token::Pound => true,
token::Interpolated(ref nt) => match nt.0 {
token::Interpolated(ref nt) => match **nt {
token::NtIdent(..) | token::NtExpr(..) |
token::NtBlock(..) | token::NtPath(..) => true,
_ => false,
Expand Down
Loading

0 comments on commit 117747d

Please sign in to comment.