Rollup merge of rust-lang#39118 - jseyfried:token_tree_based_parser, …

…r=nrc

Refactor the parser to consume token trees

This is groundwork for efficiently parsing attribute proc macro invocations, bang macro invocations, and `TokenStream`-based attributes and fragment matchers.

This improves parsing performance by 8-15% and expansion performance by 0-5% on a sampling of the compiler's crates.

r? @nrc

src/librustc/hir/print.rs

@@ -13,6 +13,7 @@ pub use self::AnnNode::*;
 use syntax::abi::Abi;
 use syntax::ast;
 use syntax::codemap::{CodeMap, Spanned};
+use syntax::parse::ParseSess;
 use syntax::parse::lexer::comments;
 use syntax::print::pp::{self, break_offset, word, space, hardbreak};
 use syntax::print::pp::{Breaks, eof};
@@ -21,7 +22,6 @@ use syntax::print::pprust::{self as ast_pp, PrintState};
 use syntax::ptr::P;
 use syntax::symbol::keywords;
 use syntax_pos::{self, BytePos};
-use errors;
 
 use hir;
 use hir::{PatKind, RegionTyParamBound, TraitTyParamBound, TraitBoundModifier};
@@ -116,16 +116,15 @@ pub const default_columns: usize = 78;
 /// it can scan the input text for comments and literals to
 /// copy forward.
 pub fn print_crate<'a>(cm: &'a CodeMap,
-                       span_diagnostic: &errors::Handler,
+                       sess: &ParseSess,
                        krate: &hir::Crate,
                        filename: String,
                        input: &mut Read,
                        out: Box<Write + 'a>,
                        ann: &'a PpAnn,
                        is_expanded: bool)
                        -> io::Result<()> {
-    let mut s = State::new_from_input(cm, span_diagnostic, filename, input,
-                                      out, ann, is_expanded);
+    let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
 
     // When printing the AST, we sometimes need to inject `#[no_std]` here.
     // Since you can't compile the HIR, it's not necessary.
@@ -137,16 +136,14 @@ pub fn print_crate<'a>(cm: &'a CodeMap,
 
 impl<'a> State<'a> {
     pub fn new_from_input(cm: &'a CodeMap,
-                          span_diagnostic: &errors::Handler,
+                          sess: &ParseSess,
                           filename: String,
                           input: &mut Read,
                           out: Box<Write + 'a>,
                           ann: &'a PpAnn,
                           is_expanded: bool)
                           -> State<'a> {
-        let (cmnts, lits) = comments::gather_comments_and_literals(span_diagnostic,
-                                                                   filename,
-                                                                   input);
+        let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
 
         State::new(cm,
                    out,

src/librustc/session/config.rs

@@ -25,6 +25,7 @@ use lint;
 use middle::cstore;
 
 use syntax::ast::{self, IntTy, UintTy};
+use syntax::parse::token;
 use syntax::parse;
 use syntax::symbol::Symbol;
 use syntax::feature_gate::UnstableFeatures;
@@ -1259,7 +1260,7 @@ pub fn parse_cfgspecs(cfgspecs: Vec<String> ) -> ast::CrateConfig {
 
         let meta_item = panictry!(parser.parse_meta_item());
 
-        if !parser.reader.is_eof() {
+        if parser.token != token::Eof {
             early_error(ErrorOutputType::default(), &format!("invalid --cfg argument: {}", s))
         } else if meta_item.is_meta_item_list() {
             let msg =

src/librustc_driver/pretty.rs

@@ -838,7 +838,7 @@ pub fn print_after_parsing(sess: &Session,
                 debug!("pretty printing source code {:?}", s);
                 let sess = annotation.sess();
                 pprust::print_crate(sess.codemap(),
-                                    sess.diagnostic(),
+                                    &sess.parse_sess,
                                     krate,
                                     src_name.to_string(),
                                     &mut rdr,
@@ -896,7 +896,7 @@ pub fn print_after_hir_lowering<'tcx, 'a: 'tcx>(sess: &'a Session,
                     debug!("pretty printing source code {:?}", s);
                     let sess = annotation.sess();
                     pprust::print_crate(sess.codemap(),
-                                        sess.diagnostic(),
+                                        &sess.parse_sess,
                                         krate,
                                         src_name.to_string(),
                                         &mut rdr,
@@ -920,7 +920,7 @@ pub fn print_after_hir_lowering<'tcx, 'a: 'tcx>(sess: &'a Session,
                     debug!("pretty printing source code {:?}", s);
                     let sess = annotation.sess();
                     pprust_hir::print_crate(sess.codemap(),
-                                            sess.diagnostic(),
+                                            &sess.parse_sess,
                                             krate,
                                             src_name.to_string(),
                                             &mut rdr,
@@ -945,7 +945,7 @@ pub fn print_after_hir_lowering<'tcx, 'a: 'tcx>(sess: &'a Session,
                     let sess = annotation.sess();
                     let ast_map = annotation.ast_map().expect("--unpretty missing HIR map");
                     let mut pp_state = pprust_hir::State::new_from_input(sess.codemap(),
-                                                                         sess.diagnostic(),
+                                                                         &sess.parse_sess,
                                                                          src_name.to_string(),
                                                                          &mut rdr,
                                                                          box out,

src/librustc_metadata/cstore_impl.rs

@@ -29,7 +29,7 @@ use rustc_back::PanicStrategy;
 
 use syntax::ast;
 use syntax::attr;
-use syntax::parse::new_parser_from_source_str;
+use syntax::parse::filemap_to_tts;
 use syntax::symbol::Symbol;
 use syntax_pos::{mk_sp, Span};
 use rustc::hir::svh::Svh;
@@ -399,19 +399,9 @@ impl<'tcx> CrateStore<'tcx> for cstore::CStore {
         let (name, def) = data.get_macro(id.index);
         let source_name = format!("<{} macros>", name);
 
-        // NB: Don't use parse_tts_from_source_str because it parses with quote_depth > 0.
-        let mut parser = new_parser_from_source_str(&sess.parse_sess, source_name, def.body);
-
-        let lo = parser.span.lo;
-        let body = match parser.parse_all_token_trees() {
-            Ok(body) => body,
-            Err(mut err) => {
-                err.emit();
-                sess.abort_if_errors();
-                unreachable!();
-            }
-        };
-        let local_span = mk_sp(lo, parser.prev_span.hi);
+        let filemap = sess.parse_sess.codemap().new_filemap(source_name, None, def.body);
+        let local_span = mk_sp(filemap.start_pos, filemap.end_pos);
+        let body = filemap_to_tts(&sess.parse_sess, filemap);
 
         // Mark the attrs as used
         let attrs = data.get_item_attrs(id.index);

src/librustc_save_analysis/span_utils.rs

@@ -17,9 +17,9 @@ use std::env;
 use std::path::Path;
 
 use syntax::ast;
-use syntax::parse::lexer::{self, Reader, StringReader};
+use syntax::parse::filemap_to_tts;
+use syntax::parse::lexer::{self, StringReader};
 use syntax::parse::token::{self, Token};
-use syntax::parse::parser::Parser;
 use syntax::symbol::keywords;
 use syntax::tokenstream::TokenTree;
 use syntax_pos::*;
@@ -85,14 +85,13 @@ impl<'a> SpanUtils<'a> {
         let filemap = self.sess
                           .codemap()
                           .new_filemap(String::from("<anon-dxr>"), None, self.snippet(span));
-        let s = self.sess;
-        lexer::StringReader::new(s.diagnostic(), filemap)
+        lexer::StringReader::new(&self.sess.parse_sess, filemap)
     }
 
     fn span_to_tts(&self, span: Span) -> Vec<TokenTree> {
-        let srdr = self.retokenise_span(span);
-        let mut p = Parser::new(&self.sess.parse_sess, Box::new(srdr), None, false);
-        p.parse_all_token_trees().expect("Couldn't re-parse span")
+        let filename = String::from("<anon-dxr>");
+        let filemap = self.sess.codemap().new_filemap(filename, None, self.snippet(span));
+        filemap_to_tts(&self.sess.parse_sess, filemap)
     }
 
     // Re-parses a path and returns the span for the last identifier in the path

src/librustdoc/html/highlight.rs

@@ -27,7 +27,7 @@ use std::io;
 use std::io::prelude::*;
 
 use syntax::codemap::CodeMap;
-use syntax::parse::lexer::{self, Reader, TokenAndSpan};
+use syntax::parse::lexer::{self, TokenAndSpan};
 use syntax::parse::token;
 use syntax::parse;
 use syntax_pos::Span;
@@ -42,8 +42,7 @@ pub fn render_with_highlighting(src: &str, class: Option<&str>, id: Option<&str>
     let mut out = Vec::new();
     write_header(class, id, &mut out).unwrap();
 
-    let mut classifier = Classifier::new(lexer::StringReader::new(&sess.span_diagnostic, fm),
-                                         sess.codemap());
+    let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm), sess.codemap());
     if let Err(_) = classifier.write_source(&mut out) {
         return format!("<pre>{}</pre>", src);
     }
@@ -63,8 +62,7 @@ pub fn render_inner_with_highlighting(src: &str) -> io::Result<String> {
     let fm = sess.codemap().new_filemap("<stdin>".to_string(), None, src.to_string());
 
     let mut out = Vec::new();
-    let mut classifier = Classifier::new(lexer::StringReader::new(&sess.span_diagnostic, fm),
-                                         sess.codemap());
+    let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm), sess.codemap());
     classifier.write_source(&mut out)?;
 
     Ok(String::from_utf8_lossy(&out).into_owned())
@@ -185,10 +183,10 @@ impl<'a> Classifier<'a> {
                 Ok(tas) => tas,
                 Err(_) => {
                     self.lexer.emit_fatal_errors();
-                    self.lexer.span_diagnostic.struct_warn("Backing out of syntax highlighting")
-                                              .note("You probably did not intend to render this \
-                                                     as a rust code-block")
-                                              .emit();
+                    self.lexer.sess.span_diagnostic
+                        .struct_warn("Backing out of syntax highlighting")
+                        .note("You probably did not intend to render this as a rust code-block")
+                        .emit();
                     return Err(io::Error::new(io::ErrorKind::Other, ""));
                 }
             };

src/libsyntax/ext/base.rs

@@ -615,9 +615,7 @@ impl<'a> ExtCtxt<'a> {
 
     pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree])
         -> parser::Parser<'a> {
-        let mut parser = parse::tts_to_parser(self.parse_sess, tts.to_vec());
-        parser.allow_interpolated_tts = false; // FIXME(jseyfried) `quote!` can't handle these yet
-        parser
+        parse::tts_to_parser(self.parse_sess, tts.to_vec())
     }
     pub fn codemap(&self) -> &'a CodeMap { self.parse_sess.codemap() }
     pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }

src/libsyntax/ext/expand.rs

@@ -21,7 +21,7 @@ use ext::base::*;
 use feature_gate::{self, Features};
 use fold;
 use fold::*;
-use parse::{ParseSess, DirectoryOwnership, PResult, lexer};
+use parse::{ParseSess, DirectoryOwnership, PResult, filemap_to_tts};
 use parse::parser::Parser;
 use parse::token;
 use print::pprust;
@@ -669,12 +669,8 @@ fn tts_for_attr_args(attr: &ast::Attribute, parse_sess: &ParseSess) -> Vec<Token
 }
 
 fn string_to_tts(text: String, parse_sess: &ParseSess) -> Vec<TokenTree> {
-    let filemap = parse_sess.codemap()
-                            .new_filemap(String::from("<macro expansion>"), None, text);
-
-    let lexer = lexer::StringReader::new(&parse_sess.span_diagnostic, filemap);
-    let mut parser = Parser::new(parse_sess, Box::new(lexer), None, false);
-    panictry!(parser.parse_all_token_trees())
+    let filename = String::from("<macro expansion>");
+    filemap_to_tts(parse_sess, parse_sess.codemap().new_filemap(filename, None, text))
 }
 
 impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {

src/libsyntax/ext/tt/macro_parser.rs

@@ -82,7 +82,6 @@ use ast::Ident;
 use syntax_pos::{self, BytePos, mk_sp, Span};
 use codemap::Spanned;
 use errors::FatalError;
-use parse::lexer::*; //resolve bug?
 use parse::{Directory, ParseSess};
 use parse::parser::{PathStyle, Parser};
 use parse::token::{DocComment, MatchNt, SubstNt};
@@ -407,9 +406,9 @@ fn inner_parse_loop(cur_eis: &mut SmallVector<Box<MatcherPos>>,
     Success(())
 }
 
-pub fn parse(sess: &ParseSess, rdr: TtReader, ms: &[TokenTree], directory: Option<Directory>)
+pub fn parse(sess: &ParseSess, tts: Vec<TokenTree>, ms: &[TokenTree], directory: Option<Directory>)
              -> NamedParseResult {
-    let mut parser = Parser::new(sess, Box::new(rdr), directory, true);
+    let mut parser = Parser::new(sess, tts, directory, true);
     let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo));
     let mut next_eis = Vec::new(); // or proceed normally
 
@@ -481,23 +480,8 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
     match name {
         "tt" => {
             p.quote_depth += 1; //but in theory, non-quoted tts might be useful
-            let mut tt = panictry!(p.parse_token_tree());
+            let tt = panictry!(p.parse_token_tree());
             p.quote_depth -= 1;
-            while let TokenTree::Token(sp, token::Interpolated(nt)) = tt {
-                if let token::NtTT(..) = *nt {
-                    match Rc::try_unwrap(nt) {
-                        Ok(token::NtTT(sub_tt)) => tt = sub_tt,
-                        Ok(_) => unreachable!(),
-                        Err(nt_rc) => match *nt_rc {
-                            token::NtTT(ref sub_tt) => tt = sub_tt.clone(),
-                            _ => unreachable!(),
-                        },
-                    }
-                } else {
-                    tt = TokenTree::Token(sp, token::Interpolated(nt.clone()));
-                    break
-                }
-            }
             return token::NtTT(tt);
         }
         _ => {}
@@ -527,7 +511,7 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
         "ident" => match p.token {
             token::Ident(sn) => {
                 p.bump();
-                token::NtIdent(Spanned::<Ident>{node: sn, span: p.span})
+                token::NtIdent(Spanned::<Ident>{node: sn, span: p.prev_span})
             }
             _ => {
                 let token_str = pprust::token_to_string(&p.token);

src/libsyntax/ext/tt/macro_rules.rs

@@ -16,8 +16,8 @@ use ext::expand::{Expansion, ExpansionKind};
 use ext::tt::macro_parser::{Success, Error, Failure};
 use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
 use ext::tt::macro_parser::{parse, parse_failure_msg};
+use ext::tt::transcribe::transcribe;
 use parse::{Directory, ParseSess};
-use parse::lexer::new_tt_reader;
 use parse::parser::Parser;
 use parse::token::{self, NtTT, Token};
 use parse::token::Token::*;
@@ -113,13 +113,12 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                     _ => cx.span_bug(sp, "malformed macro rhs"),
                 };
                 // rhs has holes ( `$id` and `$(...)` that need filled)
-                let trncbr =
-                    new_tt_reader(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs);
+                let tts = transcribe(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs);
                 let directory = Directory {
                     path: cx.current_expansion.module.directory.clone(),
                     ownership: cx.current_expansion.directory_ownership,
                 };
-                let mut p = Parser::new(cx.parse_sess(), Box::new(trncbr), Some(directory), false);
+                let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), false);
                 p.root_module_name = cx.current_expansion.module.mod_path.last()
                     .map(|id| (*id.name.as_str()).to_owned());
 
@@ -187,10 +186,8 @@ pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension {
         })),
     ];
 
-    // Parse the macro_rules! invocation (`none` is for no interpolations):
-    let arg_reader = new_tt_reader(&sess.span_diagnostic, None, def.body.clone());
-
-    let argument_map = match parse(sess, arg_reader, &argument_gram, None) {
+    // Parse the macro_rules! invocation
+    let argument_map = match parse(sess, def.body.clone(), &argument_gram, None) {
         Success(m) => m,
         Failure(sp, tok) => {
             let s = parse_failure_msg(tok);