Auto merge of rust-lang#56647 - petrochenkov:dcrate2, r=alexcrichton

Rework treatment of `$crate` in procedural macros

Important clarification: `$crate` below means "processed `$crate`" or "output `$crate`". In the input of a decl macro `$crate` is just two separate tokens, but in the *output of a decl macro* `$crate` is a single keyword identifier (rust-lang#55640 (comment)).

First of all, this PR removes the `eliminate_crate_var` hack.
`$crate::foo` is no longer replaced with `::foo` or `::crate_name::foo` in the input of derive proc macros, it's passed to the macro instead with its precise span and hygiene data, and can be treated as any other path segment keyword (like `crate` or `self`) after that. (Note: `eliminate_crate_var` was never used for non-derive proc macros.)

This creates an annoying problem - derive macros still may stringify their input before processing and expect `$crate` survive that stringification and refer to the same crate (the Rust 1.15-1.29 way of doing things).
Moreover, the input of proc macro attributes and derives (but not fn-like proc macros) also effectively survives stringification before being passed to the macro (also for legacy implementation reasons).

So we kind of resurrect the `eliminate_crate_var` hack in reduced form, but apply it only to AST pretty-printing.
If an AST fragment is pretty-printed, the resulting *text* will have `$crate` replaced with `crate` or `::crate_name`. This should be enough to keep all the legacy cases working.

Closes rust-lang#55640
Closes rust-lang#56622
r? @ghost

src/libproc_macro/lib.rs

@@ -729,11 +729,6 @@ impl Punct {
     /// which can be further configured with the `set_span` method below.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn new(ch: char, spacing: Spacing) -> Punct {
-        const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^',
-                                       '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\''];
-        if !LEGAL_CHARS.contains(&ch) {
-            panic!("unsupported character `{:?}`", ch)
-        }
         Punct(bridge::client::Punct::new(ch, spacing))
     }
 
@@ -800,16 +795,6 @@ impl fmt::Debug for Punct {
 pub struct Ident(bridge::client::Ident);
 
 impl Ident {
-    fn is_valid(string: &str) -> bool {
-        let mut chars = string.chars();
-        if let Some(start) = chars.next() {
-            (start == '_' || start.is_xid_start())
-                && chars.all(|cont| cont == '_' || cont.is_xid_continue())
-        } else {
-            false
-        }
-    }
-
     /// Creates a new `Ident` with the given `string` as well as the specified
     /// `span`.
     /// The `string` argument must be a valid identifier permitted by the
@@ -831,18 +816,12 @@ impl Ident {
     /// tokens, requires a `Span` to be specified at construction.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
     pub fn new(string: &str, span: Span) -> Ident {
-        if !Ident::is_valid(string) {
-            panic!("`{:?}` is not a valid identifier", string)
-        }
         Ident(bridge::client::Ident::new(string, span.0, false))
     }
 
     /// Same as `Ident::new`, but creates a raw identifier (`r#ident`).
     #[unstable(feature = "proc_macro_raw_ident", issue = "54723")]
     pub fn new_raw(string: &str, span: Span) -> Ident {
-        if !Ident::is_valid(string) {
-            panic!("`{:?}` is not a valid identifier", string)
-        }
         Ident(bridge::client::Ident::new(string, span.0, true))
     }
 

src/librustc/hir/print.rs

@@ -1622,8 +1622,7 @@ impl<'a> State<'a> {
             if i > 0 {
                 self.s.word("::")?
             }
-            if segment.ident.name != keywords::PathRoot.name() &&
-               segment.ident.name != keywords::DollarCrate.name() {
+            if segment.ident.name != keywords::PathRoot.name() {
                self.print_ident(segment.ident)?;
                segment.with_generic_args(|generic_args| {
                    self.print_generic_args(generic_args, segment.infer_types,
@@ -1636,8 +1635,7 @@ impl<'a> State<'a> {
     }
 
     pub fn print_path_segment(&mut self, segment: &hir::PathSegment) -> io::Result<()> {
-        if segment.ident.name != keywords::PathRoot.name() &&
-           segment.ident.name != keywords::DollarCrate.name() {
+        if segment.ident.name != keywords::PathRoot.name() {
            self.print_ident(segment.ident)?;
            segment.with_generic_args(|generic_args| {
                self.print_generic_args(generic_args, segment.infer_types, false)
@@ -1664,8 +1662,7 @@ impl<'a> State<'a> {
                     if i > 0 {
                         self.s.word("::")?
                     }
-                    if segment.ident.name != keywords::PathRoot.name() &&
-                       segment.ident.name != keywords::DollarCrate.name() {
+                    if segment.ident.name != keywords::PathRoot.name() {
                         self.print_ident(segment.ident)?;
                         segment.with_generic_args(|generic_args| {
                             self.print_generic_args(generic_args,

src/librustc_resolve/build_reduced_graph.rs

@@ -1035,4 +1035,15 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {
         }
         visit::walk_attribute(self, attr);
     }
+
+    fn visit_ident(&mut self, ident: Ident) {
+        if ident.name == keywords::DollarCrate.name() {
+            let name = match self.resolver.resolve_crate_root(ident).kind {
+                ModuleKind::Def(_, name) if name != keywords::Invalid.name() => name,
+                _ => keywords::Crate.name(),
+            };
+            ident.span.ctxt().set_dollar_crate_name(name);
+        }
+        visit::walk_ident(self, ident);
+    }
 }

src/librustc_resolve/lib.rs

@@ -1173,10 +1173,6 @@ impl<'a> ModuleData<'a> {
         }
     }
 
-    fn is_local(&self) -> bool {
-        self.normal_ancestor_id.is_local()
-    }
-
     fn nearest_item_scope(&'a self) -> Module<'a> {
         if self.is_trait() { self.parent.unwrap() } else { self }
     }

src/librustc_resolve/macros.rs

@@ -10,7 +10,7 @@
 
 use {AmbiguityError, AmbiguityKind, AmbiguityErrorMisc};
 use {CrateLint, Resolver, ResolutionError, ScopeSet, Weak};
-use {Module, ModuleKind, NameBinding, NameBindingKind, PathResult, Segment, ToNameBinding};
+use {Module, NameBinding, NameBindingKind, PathResult, Segment, ToNameBinding};
 use {is_known_tool, resolve_error};
 use ModuleOrUniformRoot;
 use Namespace::*;
@@ -30,8 +30,6 @@ use syntax::ext::expand::{AstFragment, Invocation, InvocationKind};
 use syntax::ext::hygiene::{self, Mark};
 use syntax::ext::tt::macro_rules;
 use syntax::feature_gate::{feature_err, is_builtin_attr_name, GateIssue};
-use syntax::fold::{self, Folder};
-use syntax::ptr::P;
 use syntax::symbol::{Symbol, keywords};
 use syntax::util::lev_distance::find_best_match_for_name;
 use syntax_pos::{Span, DUMMY_SP};
@@ -138,58 +136,6 @@ impl<'a> base::Resolver for Resolver<'a> {
         mark
     }
 
-    fn eliminate_crate_var(&mut self, item: P<ast::Item>) -> P<ast::Item> {
-        struct EliminateCrateVar<'b, 'a: 'b>(
-            &'b mut Resolver<'a>, Span
-        );
-
-        impl<'a, 'b> Folder for EliminateCrateVar<'a, 'b> {
-            fn fold_path(&mut self, path: ast::Path) -> ast::Path {
-                match self.fold_qpath(None, path) {
-                    (None, path) => path,
-                    _ => unreachable!(),
-                }
-            }
-
-            fn fold_qpath(&mut self, mut qself: Option<ast::QSelf>, mut path: ast::Path)
-                          -> (Option<ast::QSelf>, ast::Path) {
-                qself = qself.map(|ast::QSelf { ty, path_span, position }| {
-                    ast::QSelf {
-                        ty: self.fold_ty(ty),
-                        path_span: self.new_span(path_span),
-                        position,
-                    }
-                });
-
-                if path.segments[0].ident.name == keywords::DollarCrate.name() {
-                    let module = self.0.resolve_crate_root(path.segments[0].ident);
-                    path.segments[0].ident.name = keywords::PathRoot.name();
-                    if !module.is_local() {
-                        let span = path.segments[0].ident.span;
-                        path.segments.insert(1, match module.kind {
-                            ModuleKind::Def(_, name) => ast::PathSegment::from_ident(
-                                ast::Ident::with_empty_ctxt(name).with_span_pos(span)
-                            ),
-                            _ => unreachable!(),
-                        });
-                        if let Some(qself) = &mut qself {
-                            qself.position += 1;
-                        }
-                    }
-                }
-                (qself, path)
-            }
-
-            fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
-                fold::noop_fold_mac(mac, self)
-            }
-        }
-
-        let ret = EliminateCrateVar(self, item.span).fold_item(item);
-        assert!(ret.len() == 1);
-        ret.into_iter().next().unwrap()
-    }
-
     fn visit_ast_fragment_with_placeholders(&mut self, mark: Mark, fragment: &AstFragment,
                                             derives: &[Mark]) {
         let invocation = self.invocations[&mark];
@@ -259,7 +205,6 @@ impl<'a> base::Resolver for Resolver<'a> {
             self.definitions.add_parent_module_of_macro_def(invoc.expansion_data.mark,
                                                             normal_module_def_id);
             invoc.expansion_data.mark.set_default_transparency(ext.default_transparency());
-            invoc.expansion_data.mark.set_is_builtin(def_id.krate == CrateNum::BuiltinMacros);
         }
 
         Ok(Some(ext))

src/libsyntax/ext/base.rs

@@ -732,7 +732,6 @@ pub type NamedSyntaxExtension = (Name, SyntaxExtension);
 pub trait Resolver {
     fn next_node_id(&mut self) -> ast::NodeId;
     fn get_module_scope(&mut self, id: ast::NodeId) -> Mark;
-    fn eliminate_crate_var(&mut self, item: P<ast::Item>) -> P<ast::Item>;
 
     fn visit_ast_fragment_with_placeholders(&mut self, mark: Mark, fragment: &AstFragment,
                                             derives: &[Mark]);
@@ -766,7 +765,6 @@ pub struct DummyResolver;
 impl Resolver for DummyResolver {
     fn next_node_id(&mut self) -> ast::NodeId { ast::DUMMY_NODE_ID }
     fn get_module_scope(&mut self, _id: ast::NodeId) -> Mark { Mark::root() }
-    fn eliminate_crate_var(&mut self, item: P<ast::Item>) -> P<ast::Item> { item }
 
     fn visit_ast_fragment_with_placeholders(&mut self, _invoc: Mark, _fragment: &AstFragment,
                                             _derives: &[Mark]) {}

src/libsyntax/ext/expand.rs

@@ -203,10 +203,7 @@ fn macro_bang_format(path: &ast::Path) -> ExpnFormat {
         if i != 0 {
             path_str.push_str("::");
         }
-
-        if segment.ident.name != keywords::PathRoot.name() &&
-            segment.ident.name != keywords::DollarCrate.name()
-        {
+        if segment.ident.name != keywords::PathRoot.name() {
             path_str.push_str(&segment.ident.as_str())
         }
     }

src/libsyntax/parse/token.rs

@@ -633,7 +633,9 @@ impl Token {
             (&Shebang(a), &Shebang(b)) => a == b,
 
             (&Lifetime(a), &Lifetime(b)) => a.name == b.name,
-            (&Ident(a, b), &Ident(c, d)) => a.name == c.name && b == d,
+            (&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name ||
+                                                       a.name == keywords::DollarCrate.name() ||
+                                                       c.name == keywords::DollarCrate.name()),
 
             (&Literal(ref a, b), &Literal(ref c, d)) => {
                 b == d && a.probably_equal_for_proc_macro(c)

src/libsyntax/print/pprust.rs

@@ -16,7 +16,6 @@ use util::parser::{self, AssocOp, Fixity};
 use attr;
 use source_map::{self, SourceMap, Spanned};
 use syntax_pos::{self, BytePos};
-use syntax_pos::hygiene::{Mark, SyntaxContext};
 use parse::token::{self, BinOpToken, Token};
 use parse::lexer::comments;
 use parse::{self, ParseSess};
@@ -724,12 +723,12 @@ pub trait PrintState<'a> {
             if i > 0 {
                 self.writer().word("::")?
             }
-            if segment.ident.name != keywords::PathRoot.name() &&
-               segment.ident.name != keywords::DollarCrate.name()
-            {
-                self.writer().word(segment.ident.as_str().get())?;
-            } else if segment.ident.name == keywords::DollarCrate.name() {
-                self.print_dollar_crate(segment.ident.span.ctxt())?;
+            if segment.ident.name != keywords::PathRoot.name() {
+                if segment.ident.name == keywords::DollarCrate.name() {
+                    self.print_dollar_crate(segment.ident)?;
+                } else {
+                    self.writer().word(segment.ident.as_str().get())?;
+                }
             }
         }
         Ok(())
@@ -843,17 +842,19 @@ pub trait PrintState<'a> {
 
     fn nbsp(&mut self) -> io::Result<()> { self.writer().word(" ") }
 
-    fn print_dollar_crate(&mut self, mut ctxt: SyntaxContext) -> io::Result<()> {
-        if let Some(mark) = ctxt.adjust(Mark::root()) {
-            // Make a best effort to print something that complies
-            if mark.is_builtin() {
-                if let Some(name) = std_inject::injected_crate_name() {
-                    self.writer().word("::")?;
-                    self.writer().word(name)?;
-                }
-            }
+    // AST pretty-printer is used as a fallback for turning AST structures into token streams for
+    // proc macros. Additionally, proc macros may stringify their input and expect it survive the
+    // stringification (especially true for proc macro derives written between Rust 1.15 and 1.30).
+    // So we need to somehow pretty-print `$crate` in paths in a way preserving at least some of
+    // its hygiene data, most importantly name of the crate it refers to.
+    // As a result we print `$crate` as `crate` if it refers to the local crate
+    // and as `::other_crate_name` if it refers to some other crate.
+    fn print_dollar_crate(&mut self, ident: ast::Ident) -> io::Result<()> {
+        let name = ident.span.ctxt().dollar_crate_name();
+        if !ast::Ident::with_empty_ctxt(name).is_path_segment_keyword() {
+            self.writer().word("::")?;
         }
-        Ok(())
+        self.writer().word(name.as_str().get())
     }
 }
 
@@ -2463,14 +2464,15 @@ impl<'a> State<'a> {
                           colons_before_params: bool)
                           -> io::Result<()>
     {
-        if segment.ident.name != keywords::PathRoot.name() &&
-           segment.ident.name != keywords::DollarCrate.name() {
-            self.print_ident(segment.ident)?;
+        if segment.ident.name != keywords::PathRoot.name() {
+            if segment.ident.name == keywords::DollarCrate.name() {
+                self.print_dollar_crate(segment.ident)?;
+            } else {
+                self.print_ident(segment.ident)?;
+            }
             if let Some(ref args) = segment.args {
                 self.print_generic_args(args, colons_before_params)?;
             }
-        } else if segment.ident.name == keywords::DollarCrate.name() {
-            self.print_dollar_crate(segment.ident.span.ctxt())?;
         }
         Ok(())
     }

src/libsyntax/tokenstream.rs

@@ -348,7 +348,9 @@ impl TokenStream {
                 | TokenTree::Token(_, Token::Semi)
                 // The pretty printer collapses whitespace arbitrarily and can
                 // introduce whitespace from `NoDelim`.
-                | TokenTree::Token(_, Token::Whitespace) => false,
+                | TokenTree::Token(_, Token::Whitespace)
+                // The pretty printer can turn `$crate` into `::crate_name`
+                | TokenTree::Token(_, Token::ModSep) => false,
                 _ => true
             }
         }

src/libsyntax_ext/deriving/custom.rs

@@ -74,7 +74,6 @@ impl MultiItemModifier for ProcMacroDerive {
         // Mark attributes as known, and used.
         MarkAttrs(&self.attrs).visit_item(&item);
 
-        let item = ecx.resolver.eliminate_crate_var(item);
         let token = Token::interpolated(token::NtItem(item));
         let input = tokenstream::TokenTree::Token(DUMMY_SP, token).into();