Skip to content

Commit

Permalink
syntax: Get rid of token::IdentStyle
Browse files Browse the repository at this point in the history
  • Loading branch information
@petrochenkov
petrochenkov committed Apr 24, 2016
1 parent 8dbf8f5 commit 546c052
Show file tree
Hide file tree
Showing 16 changed files with 111 additions and 183 deletions.
2 changes: 1 addition & 1 deletion src/librustdoc/html/highlight.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ fn write_source(sess: &parse::ParseSess,
}

// keywords are also included in the identifier set
token::Ident(ident, _is_mod_sep) => {
token::Ident(ident) => {
match &*ident.name.as_str() {
"ref" | "mut" => "kw-2",

Expand Down
12 changes: 5 additions & 7 deletions src/libsyntax/ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1206,8 +1206,7 @@ impl TokenTree {
TokenTree::Delimited(sp, Rc::new(Delimited {
delim: token::Bracket,
open_span: sp,
tts: vec![TokenTree::Token(sp, token::Ident(token::str_to_ident("doc"),
token::Plain)),
tts: vec![TokenTree::Token(sp, token::Ident(token::str_to_ident("doc"))),
TokenTree::Token(sp, token::Eq),
TokenTree::Token(sp, token::Literal(
token::StrRaw(token::intern(&stripped), num_of_hashes), None))],
Expand All @@ -1225,14 +1224,13 @@ impl TokenTree {
}
(&TokenTree::Token(sp, token::SpecialVarNt(var)), _) => {
let v = [TokenTree::Token(sp, token::Dollar),
TokenTree::Token(sp, token::Ident(token::str_to_ident(var.as_str()),
token::Plain))];
TokenTree::Token(sp, token::Ident(token::str_to_ident(var.as_str())))];
v[index].clone()
}
(&TokenTree::Token(sp, token::MatchNt(name, kind, name_st, kind_st)), _) => {
let v = [TokenTree::Token(sp, token::SubstNt(name, name_st)),
(&TokenTree::Token(sp, token::MatchNt(name, kind)), _) => {
let v = [TokenTree::Token(sp, token::SubstNt(name)),
TokenTree::Token(sp, token::Colon),
TokenTree::Token(sp, token::Ident(kind, kind_st))];
TokenTree::Token(sp, token::Ident(kind))];
v[index].clone()
}
(&TokenTree::Sequence(_, ref seq), _) => {
Expand Down
10 changes: 5 additions & 5 deletions src/libsyntax/diagnostics/plugin.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt,
token_tree: &[TokenTree])
-> Box<MacResult+'cx> {
let code = match (token_tree.len(), token_tree.get(0)) {
(1, Some(&TokenTree::Token(_, token::Ident(code, _)))) => code,
(1, Some(&TokenTree::Token(_, token::Ident(code)))) => code,
_ => unreachable!()
};

Expand Down Expand Up @@ -92,10 +92,10 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
token_tree.get(1),
token_tree.get(2)
) {
(1, Some(&TokenTree::Token(_, token::Ident(ref code, _))), None, None) => {
(1, Some(&TokenTree::Token(_, token::Ident(ref code))), None, None) => {
(code, None)
},
(3, Some(&TokenTree::Token(_, token::Ident(ref code, _))),
(3, Some(&TokenTree::Token(_, token::Ident(ref code))),
Some(&TokenTree::Token(_, token::Comma)),
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => {
(code, Some(description))
Expand Down Expand Up @@ -160,9 +160,9 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt,
let (crate_name, name) = match (&token_tree[0], &token_tree[2]) {
(
// Crate name.
&TokenTree::Token(_, token::Ident(ref crate_name, _)),
&TokenTree::Token(_, token::Ident(ref crate_name)),
// DIAGNOSTICS ident.
&TokenTree::Token(_, token::Ident(ref name, _))
&TokenTree::Token(_, token::Ident(ref name))
) => (*&crate_name, name),
_ => unreachable!()
};
Expand Down
25 changes: 6 additions & 19 deletions src/libsyntax/ext/quote.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ pub mod rt {

impl ToTokens for ast::Ident {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP, token::Ident(*self, token::Plain))]
vec![TokenTree::Token(DUMMY_SP, token::Ident(*self))]
}
}

Expand Down Expand Up @@ -646,14 +646,10 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
cx.expr_usize(sp, n))
}

token::Ident(ident, style) => {
token::Ident(ident) => {
return cx.expr_call(sp,
mk_token_path(cx, sp, "Ident"),
vec![mk_ident(cx, sp, ident),
match style {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
}]);
vec![mk_ident(cx, sp, ident)]);
}

token::Lifetime(ident) => {
Expand All @@ -668,19 +664,10 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
vec!(mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident))));
}

token::MatchNt(name, kind, namep, kindp) => {
token::MatchNt(name, kind) => {
return cx.expr_call(sp,
mk_token_path(cx, sp, "MatchNt"),
vec!(mk_ident(cx, sp, name),
mk_ident(cx, sp, kind),
match namep {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
},
match kindp {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
}));
vec![mk_ident(cx, sp, name), mk_ident(cx, sp, kind)]);
}

token::Interpolated(_) => panic!("quote! with interpolated token"),
Expand Down Expand Up @@ -722,7 +709,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {

fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<ast::Stmt> {
match *tt {
TokenTree::Token(sp, SubstNt(ident, _)) => {
TokenTree::Token(sp, SubstNt(ident)) => {
// tt.extend($ident.to_tokens(ext_cx))

let e_to_toks =
Expand Down
12 changes: 6 additions & 6 deletions src/libsyntax/ext/tt/macro_parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
n_rec(p_s, next_m, res, ret_val, idx)?;
}
}
TokenTree::Token(sp, MatchNt(bind_name, _, _, _)) => {
TokenTree::Token(sp, MatchNt(bind_name, _)) => {
match ret_val.entry(bind_name.name) {
Vacant(spot) => {
spot.insert(res[*idx].clone());
Expand Down Expand Up @@ -263,7 +263,7 @@ pub type PositionalParseResult = ParseResult<Vec<Rc<NamedMatch>>>;
/// unhygienic comparison)
pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
match (t1,t2) {
(&token::Ident(id1,_),&token::Ident(id2,_))
(&token::Ident(id1),&token::Ident(id2))
| (&token::Lifetime(id1),&token::Lifetime(id2)) =>
id1.name == id2.name,
_ => *t1 == *t2
Expand Down Expand Up @@ -451,7 +451,7 @@ pub fn parse(sess: &ParseSess,
if (!bb_eis.is_empty() && !next_eis.is_empty())
|| bb_eis.len() > 1 {
let nts = bb_eis.iter().map(|ei| match ei.top_elts.get_tt(ei.idx) {
TokenTree::Token(_, MatchNt(bind, name, _, _)) => {
TokenTree::Token(_, MatchNt(bind, name)) => {
format!("{} ('{}')", name, bind)
}
_ => panic!()
Expand Down Expand Up @@ -479,7 +479,7 @@ pub fn parse(sess: &ParseSess,

let mut ei = bb_eis.pop().unwrap();
match ei.top_elts.get_tt(ei.idx) {
TokenTree::Token(span, MatchNt(_, ident, _, _)) => {
TokenTree::Token(span, MatchNt(_, ident)) => {
let match_cur = ei.match_cur;
(&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
parse_nt(&mut rust_parser, span, &ident.name.as_str()))));
Expand Down Expand Up @@ -534,9 +534,9 @@ pub fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
"ty" => token::NtTy(panictry!(p.parse_ty())),
// this could be handled like a token, since it is one
"ident" => match p.token {
token::Ident(sn,b) => {
token::Ident(sn) => {
p.bump();
token::NtIdent(Box::new(Spanned::<Ident>{node: sn, span: p.span}),b)
token::NtIdent(Box::new(Spanned::<Ident>{node: sn, span: p.span}))
}
_ => {
let token_str = pprust::token_to_string(&p.token);
Expand Down
22 changes: 10 additions & 12 deletions src/libsyntax/ext/tt/macro_rules.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
// $( $lhs:tt => $rhs:tt );+
// ...quasiquoting this would be nice.
// These spans won't matter, anyways
let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain);
let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain);
let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt);
let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt);
let argument_gram = vec!(
TokenTree::Sequence(DUMMY_SP,
Rc::new(ast::SequenceRepetition {
Expand Down Expand Up @@ -415,7 +415,7 @@ fn check_matcher_old<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token, on_fai
let mut tokens = matcher.peekable();
while let Some(token) = tokens.next() {
last = match *token {
TokenTree::Token(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
TokenTree::Token(sp, MatchNt(ref name, ref frag_spec)) => {
// ii. If T is a simple NT, look ahead to the next token T' in
// M. If T' is in the set FOLLOW(NT), continue. Else; reject.
if can_be_followed_by_any(&frag_spec.name.as_str()) {
Expand Down Expand Up @@ -881,7 +881,7 @@ fn check_matcher_core(cx: &mut ExtCtxt,
// Now `last` holds the complete set of NT tokens that could
// end the sequence before SUFFIX. Check that every one works with `suffix`.
'each_last: for &(_sp, ref t) in &last.tokens {
if let MatchNt(ref name, ref frag_spec, _, _) = *t {
if let MatchNt(ref name, ref frag_spec) = *t {
for &(sp, ref next_token) in &suffix_first.tokens {
match is_in_follow(cx, next_token, &frag_spec.name.as_str()) {
Err(msg) => {
Expand Down Expand Up @@ -917,9 +917,8 @@ fn check_matcher_core(cx: &mut ExtCtxt,
last
}


fn token_can_be_followed_by_any(tok: &Token) -> bool {
if let &MatchNt(_, ref frag_spec, _, _) = tok {
if let &MatchNt(_, ref frag_spec) = tok {
frag_can_be_followed_by_any(&frag_spec.name.as_str())
} else {
// (Non NT's can always be followed by anthing in matchers.)
Expand Down Expand Up @@ -1005,18 +1004,17 @@ fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {
"pat" => {
match *tok {
FatArrow | Comma | Eq | BinOp(token::Or) => Ok(true),
Ident(i, _) if (i.name.as_str() == "if" ||
i.name.as_str() == "in") => Ok(true),
Ident(i) if (i.name.as_str() == "if" ||
i.name.as_str() == "in") => Ok(true),
_ => Ok(false)
}
},
"path" | "ty" => {
match *tok {
OpenDelim(token::DelimToken::Brace) | OpenDelim(token::DelimToken::Bracket) |
Comma | FatArrow | Colon | Eq | Gt | Semi | BinOp(token::Or) => Ok(true),
MatchNt(_, ref frag, _, _) if frag.name.as_str() == "block" => Ok(true),
Ident(i, _) if (i.name.as_str() == "as" ||
i.name.as_str() == "where") => Ok(true),
MatchNt(_, ref frag) if frag.name.as_str() == "block" => Ok(true),
Ident(i) if i.name.as_str() == "as" || i.name.as_str() == "where" => Ok(true),
_ => Ok(false)
}
},
Expand All @@ -1036,7 +1034,7 @@ fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {

fn has_legal_fragment_specifier(tok: &Token) -> Result<(), String> {
debug!("has_legal_fragment_specifier({:?})", tok);
if let &MatchNt(_, ref frag_spec, _, _) = tok {
if let &MatchNt(_, ref frag_spec) = tok {
let s = &frag_spec.name.as_str();
if !is_legal_fragment_specifier(s) {
return Err(s.to_string());
Expand Down
12 changes: 6 additions & 6 deletions src/libsyntax/ext/tt/transcribe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
size + lockstep_iter_size(tt, r)
})
},
TokenTree::Token(_, SubstNt(name, _)) | TokenTree::Token(_, MatchNt(name, _, _, _)) =>
TokenTree::Token(_, SubstNt(name)) | TokenTree::Token(_, MatchNt(name, _)) =>
match lookup_cur_matched(r, name) {
Some(matched) => match *matched {
MatchedNonterminal(_) => LisUnconstrained,
Expand All @@ -186,7 +186,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
None => (),
Some(sp) => {
r.cur_span = sp;
r.cur_tok = token::Ident(r.imported_from.unwrap(), token::Plain);
r.cur_tok = token::Ident(r.imported_from.unwrap());
return ret_val;
},
}
Expand Down Expand Up @@ -278,12 +278,12 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
}
}
// FIXME #2887: think about span stuff here
TokenTree::Token(sp, SubstNt(ident, namep)) => {
TokenTree::Token(sp, SubstNt(ident)) => {
r.stack.last_mut().unwrap().idx += 1;
match lookup_cur_matched(r, ident) {
None => {
r.cur_span = sp;
r.cur_tok = SubstNt(ident, namep);
r.cur_tok = SubstNt(ident);
return ret_val;
// this can't be 0 length, just like TokenTree::Delimited
}
Expand All @@ -292,9 +292,9 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
// sidestep the interpolation tricks for ident because
// (a) idents can be in lots of places, so it'd be a pain
// (b) we actually can, since it's a token.
MatchedNonterminal(NtIdent(ref sn, b)) => {
MatchedNonterminal(NtIdent(ref sn)) => {
r.cur_span = sn.span;
r.cur_tok = token::Ident(sn.node, b);
r.cur_tok = token::Ident(sn.node);
return ret_val;
}
MatchedNonterminal(ref other_whole_nt) => {
Expand Down
17 changes: 5 additions & 12 deletions src/libsyntax/fold.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,17 +610,11 @@ pub fn noop_fold_tts<T: Folder>(tts: &[TokenTree], fld: &mut T) -> Vec<TokenTree
// apply ident folder if it's an ident, apply other folds to interpolated nodes
pub fn noop_fold_token<T: Folder>(t: token::Token, fld: &mut T) -> token::Token {
match t {
token::Ident(id, followed_by_colons) => {
token::Ident(fld.fold_ident(id), followed_by_colons)
}
token::Ident(id) => token::Ident(fld.fold_ident(id)),
token::Lifetime(id) => token::Lifetime(fld.fold_ident(id)),
token::Interpolated(nt) => token::Interpolated(fld.fold_interpolated(nt)),
token::SubstNt(ident, namep) => {
token::SubstNt(fld.fold_ident(ident), namep)
}
token::MatchNt(name, kind, namep, kindp) => {
token::MatchNt(fld.fold_ident(name), fld.fold_ident(kind), namep, kindp)
}
token::SubstNt(ident) => token::SubstNt(fld.fold_ident(ident)),
token::MatchNt(name, kind) => token::MatchNt(fld.fold_ident(name), fld.fold_ident(kind)),
_ => t
}
}
Expand Down Expand Up @@ -664,9 +658,8 @@ pub fn noop_fold_interpolated<T: Folder>(nt: token::Nonterminal, fld: &mut T)
token::NtPat(pat) => token::NtPat(fld.fold_pat(pat)),
token::NtExpr(expr) => token::NtExpr(fld.fold_expr(expr)),
token::NtTy(ty) => token::NtTy(fld.fold_ty(ty)),
token::NtIdent(id, is_mod_name) =>
token::NtIdent(Box::new(Spanned::<Ident>{node: fld.fold_ident(id.node), .. *id}),
is_mod_name),
token::NtIdent(id) =>
token::NtIdent(Box::new(Spanned::<Ident>{node: fld.fold_ident(id.node), ..*id})),
token::NtMeta(meta_item) => token::NtMeta(fld.fold_meta_item(meta_item)),
token::NtPath(path) => token::NtPath(Box::new(fld.fold_path(*path))),
token::NtTT(tt) => token::NtTT(P(fld.fold_tt(&tt))),
Expand Down
Loading

0 comments on commit 546c052

Please sign in to comment.