Rollup merge of rust-lang#62476 - petrochenkov:expref, r=matthewjasper

Continue refactoring macro expansion and resolution

This PR continues the work started in rust-lang#62042.
It contains a set of more or less related refactorings with the general goal of making things simpler and more orthogonal.
Along the way most of the issues uncovered in rust-lang#62086 are fixed.

The PR is better read in per-commit fashion with whitespace changes ignored.
I tried to leave some more detailed commit messages describing the motivation behind the individual changes.

Fixes rust-lang#44692
Fixes rust-lang#52363
Unblocks rust-lang#62086
r? @matthewjasper

src/doc/unstable-book/src/language-features/on-unimplemented.md

@@ -98,7 +98,8 @@ application of these fields based on a variety of attributes when using
    `crate_local`) or matching against a particular method. Currently used
    for `try`.
  - `from_desugaring`: usable both as boolean (whether the flag is present)
-   or matching against a particular desugaring.
+   or matching against a particular desugaring. The desugaring is identified
+   with its variant name in the `DesugaringKind` enum.
 
 For example, the `Iterator` trait can be annotated in the following way:
 

src/libcore/macros.rs

@@ -1244,19 +1244,22 @@ mod builtin {
 
     /// Attribute macro applied to a function to turn it into a unit test.
     #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(test, rustc_attrs)]
     #[rustc_builtin_macro]
     #[rustc_macro_transparency = "semitransparent"]
     pub macro test($item:item) { /* compiler built-in */ }
 
     /// Attribute macro applied to a function to turn it into a benchmark test.
     #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(test, rustc_attrs)]
     #[rustc_builtin_macro]
     #[rustc_macro_transparency = "semitransparent"]
     pub macro bench($item:item) { /* compiler built-in */ }
 
     /// An implementation detail of the `#[test]` and `#[bench]` macros.
     #[unstable(feature = "custom_test_frameworks", issue = "50297",
                reason = "custom test frameworks are an unstable feature")]
+    #[allow_internal_unstable(test, rustc_attrs)]
     #[rustc_builtin_macro]
     #[rustc_macro_transparency = "semitransparent"]
     pub macro test_case($item:item) { /* compiler built-in */ }

src/libcore/ops/try.rs

@@ -8,12 +8,12 @@
 #[rustc_on_unimplemented(
    on(all(
        any(from_method="from_error", from_method="from_ok"),
-       from_desugaring="?"),
+       from_desugaring="QuestionMark"),
       message="the `?` operator can only be used in a \
                function that returns `Result` or `Option` \
                (or another type that implements `{Try}`)",
       label="cannot use the `?` operator in a function that returns `{Self}`"),
-   on(all(from_method="into_result", from_desugaring="?"),
+   on(all(from_method="into_result", from_desugaring="QuestionMark"),
       message="the `?` operator can only be applied to values \
                that implement `{Try}`",
       label="the `?` operator cannot be applied to type `{Self}`")

src/librustc/hir/lowering.rs

@@ -60,10 +60,9 @@ use syntax::attr;
 use syntax::ast;
 use syntax::ast::*;
 use syntax::errors;
-use syntax::ext::hygiene::{Mark, SyntaxContext};
+use syntax::ext::hygiene::Mark;
 use syntax::print::pprust;
-use syntax::source_map::{self, respan, ExpnInfo, CompilerDesugaringKind, Spanned};
-use syntax::source_map::CompilerDesugaringKind::CondTemporary;
+use syntax::source_map::{respan, ExpnInfo, ExpnKind, DesugaringKind, Spanned};
 use syntax::std_inject;
 use syntax::symbol::{kw, sym, Symbol};
 use syntax::tokenstream::{TokenStream, TokenTree};
@@ -872,17 +871,15 @@ impl<'a> LoweringContext<'a> {
     /// allowed inside this span.
     fn mark_span_with_reason(
         &self,
-        reason: CompilerDesugaringKind,
+        reason: DesugaringKind,
         span: Span,
         allow_internal_unstable: Option<Lrc<[Symbol]>>,
     ) -> Span {
-        let mark = Mark::fresh(Mark::root());
-        mark.set_expn_info(ExpnInfo {
-            def_site: Some(span),
+        span.fresh_expansion(Mark::root(), ExpnInfo {
+            def_site: span,
             allow_internal_unstable,
-            ..ExpnInfo::default(source_map::CompilerDesugaring(reason), span, self.sess.edition())
-        });
-        span.with_ctxt(SyntaxContext::empty().apply_mark(mark))
+            ..ExpnInfo::default(ExpnKind::Desugaring(reason), span, self.sess.edition())
+        })
     }
 
     fn with_anonymous_lifetime_mode<R>(
@@ -1188,7 +1185,7 @@ impl<'a> LoweringContext<'a> {
         };
 
         let unstable_span = self.mark_span_with_reason(
-            CompilerDesugaringKind::Async,
+            DesugaringKind::Async,
             span,
             self.allow_gen_future.clone(),
         );
@@ -1733,7 +1730,7 @@ impl<'a> LoweringContext<'a> {
         // Not tracking it makes lints in rustc and clippy very fragile, as
         // frequently opened issues show.
         let exist_ty_span = self.mark_span_with_reason(
-            CompilerDesugaringKind::ExistentialType,
+            DesugaringKind::ExistentialType,
             span,
             None,
         );
@@ -2603,7 +2600,7 @@ impl<'a> LoweringContext<'a> {
         let span = output.span();
 
         let exist_ty_span = self.mark_span_with_reason(
-            CompilerDesugaringKind::Async,
+            DesugaringKind::Async,
             span,
             None,
         );
@@ -3275,7 +3272,7 @@ impl<'a> LoweringContext<'a> {
                 };
 
                 let desugared_span =
-                    this.mark_span_with_reason(CompilerDesugaringKind::Async, span, None);
+                    this.mark_span_with_reason(DesugaringKind::Async, span, None);
 
                 // Construct an argument representing `__argN: <ty>` to replace the argument of the
                 // async function.
@@ -4410,7 +4407,9 @@ impl<'a> LoweringContext<'a> {
                     _ => {
                         // Lower condition:
                         let cond = self.lower_expr(cond);
-                        let span_block = self.mark_span_with_reason(CondTemporary, cond.span, None);
+                        let span_block = self.mark_span_with_reason(
+                            DesugaringKind::CondTemporary, cond.span, None
+                        );
                         // Wrap in a construct equivalent to `{ let _t = $cond; _t }`
                         // to preserve drop semantics since `if cond { ... }` does not
                         // let temporaries live outside of `cond`.
@@ -4469,7 +4468,9 @@ impl<'a> LoweringContext<'a> {
 
                         // Lower condition:
                         let cond = this.with_loop_condition_scope(|this| this.lower_expr(cond));
-                        let span_block = this.mark_span_with_reason(CondTemporary, cond.span, None);
+                        let span_block = this.mark_span_with_reason(
+                            DesugaringKind::CondTemporary, cond.span, None
+                        );
                         // Wrap in a construct equivalent to `{ let _t = $cond; _t }`
                         // to preserve drop semantics since `while cond { ... }` does not
                         // let temporaries live outside of `cond`.
@@ -4508,7 +4509,7 @@ impl<'a> LoweringContext<'a> {
             ExprKind::TryBlock(ref body) => {
                 self.with_catch_scope(body.id, |this| {
                     let unstable_span = this.mark_span_with_reason(
-                        CompilerDesugaringKind::TryBlock,
+                        DesugaringKind::TryBlock,
                         body.span,
                         this.allow_try_trait.clone(),
                     );
@@ -4836,7 +4837,7 @@ impl<'a> LoweringContext<'a> {
                 let mut head = self.lower_expr(head);
                 let head_sp = head.span;
                 let desugared_span = self.mark_span_with_reason(
-                    CompilerDesugaringKind::ForLoop,
+                    DesugaringKind::ForLoop,
                     head_sp,
                     None,
                 );
@@ -4990,13 +4991,13 @@ impl<'a> LoweringContext<'a> {
                 // }
 
                 let unstable_span = self.mark_span_with_reason(
-                    CompilerDesugaringKind::QuestionMark,
+                    DesugaringKind::QuestionMark,
                     e.span,
                     self.allow_try_trait.clone(),
                 );
                 let try_span = self.sess.source_map().end_point(e.span);
                 let try_span = self.mark_span_with_reason(
-                    CompilerDesugaringKind::QuestionMark,
+                    DesugaringKind::QuestionMark,
                     try_span,
                     self.allow_try_trait.clone(),
                 );
@@ -5811,12 +5812,12 @@ impl<'a> LoweringContext<'a> {
             }
         }
         let span = self.mark_span_with_reason(
-            CompilerDesugaringKind::Await,
+            DesugaringKind::Await,
             await_span,
             None,
         );
         let gen_future_span = self.mark_span_with_reason(
-            CompilerDesugaringKind::Await,
+            DesugaringKind::Await,
             await_span,
             self.allow_gen_future.clone(),
         );

src/librustc/hir/map/def_collector.rs

@@ -1,6 +1,5 @@
 use crate::hir::map::definitions::*;
-use crate::hir::def_id::{CRATE_DEF_INDEX, DefIndex};
-use crate::session::CrateDisambiguator;
+use crate::hir::def_id::DefIndex;
 
 use syntax::ast::*;
 use syntax::ext::hygiene::Mark;
@@ -12,51 +11,30 @@ use syntax_pos::Span;
 /// Creates `DefId`s for nodes in the AST.
 pub struct DefCollector<'a> {
     definitions: &'a mut Definitions,
-    parent_def: Option<DefIndex>,
+    parent_def: DefIndex,
     expansion: Mark,
-    pub visit_macro_invoc: Option<&'a mut dyn FnMut(MacroInvocationData)>,
-}
-
-pub struct MacroInvocationData {
-    pub mark: Mark,
-    pub def_index: DefIndex,
 }
 
 impl<'a> DefCollector<'a> {
     pub fn new(definitions: &'a mut Definitions, expansion: Mark) -> Self {
-        DefCollector {
-            definitions,
-            expansion,
-            parent_def: None,
-            visit_macro_invoc: None,
-        }
-    }
-
-    pub fn collect_root(&mut self,
-                        crate_name: &str,
-                        crate_disambiguator: CrateDisambiguator) {
-        let root = self.definitions.create_root_def(crate_name,
-                                                    crate_disambiguator);
-        assert_eq!(root, CRATE_DEF_INDEX);
-        self.parent_def = Some(root);
+        let parent_def = definitions.invocation_parent(expansion);
+        DefCollector { definitions, parent_def, expansion }
     }
 
     fn create_def(&mut self,
                   node_id: NodeId,
                   data: DefPathData,
                   span: Span)
                   -> DefIndex {
-        let parent_def = self.parent_def.unwrap();
+        let parent_def = self.parent_def;
         debug!("create_def(node_id={:?}, data={:?}, parent_def={:?})", node_id, data, parent_def);
-        self.definitions
-            .create_def_with_parent(parent_def, node_id, data, self.expansion, span)
+        self.definitions.create_def_with_parent(parent_def, node_id, data, self.expansion, span)
     }
 
     pub fn with_parent<F: FnOnce(&mut Self)>(&mut self, parent_def: DefIndex, f: F) {
-        let parent = self.parent_def;
-        self.parent_def = Some(parent_def);
+        let orig_parent_def = std::mem::replace(&mut self.parent_def, parent_def);
         f(self);
-        self.parent_def = parent;
+        self.parent_def = orig_parent_def;
     }
 
     fn visit_async_fn(
@@ -97,12 +75,7 @@ impl<'a> DefCollector<'a> {
     }
 
     fn visit_macro_invoc(&mut self, id: NodeId) {
-        if let Some(ref mut visit) = self.visit_macro_invoc {
-            visit(MacroInvocationData {
-                mark: id.placeholder_to_mark(),
-                def_index: self.parent_def.unwrap(),
-            })
-        }
+        self.definitions.set_invocation_parent(id.placeholder_to_mark(), self.parent_def);
     }
 }
 
@@ -275,36 +248,24 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {
     }
 
     fn visit_expr(&mut self, expr: &'a Expr) {
-        let parent_def = self.parent_def;
-
-        match expr.node {
+        let parent_def = match expr.node {
             ExprKind::Mac(..) => return self.visit_macro_invoc(expr.id),
             ExprKind::Closure(_, asyncness, ..) => {
-                let closure_def = self.create_def(expr.id,
-                                          DefPathData::ClosureExpr,
-                                          expr.span);
-                self.parent_def = Some(closure_def);
-
                 // Async closures desugar to closures inside of closures, so
                 // we must create two defs.
-                if let IsAsync::Async { closure_id, .. } = asyncness {
-                    let async_def = self.create_def(closure_id,
-                                                    DefPathData::ClosureExpr,
-                                                    expr.span);
-                    self.parent_def = Some(async_def);
+                let closure_def = self.create_def(expr.id, DefPathData::ClosureExpr, expr.span);
+                match asyncness {
+                    IsAsync::Async { closure_id, .. } =>
+                        self.create_def(closure_id, DefPathData::ClosureExpr, expr.span),
+                    IsAsync::NotAsync => closure_def,
                 }
             }
-            ExprKind::Async(_, async_id, _) => {
-                let async_def = self.create_def(async_id,
-                                                DefPathData::ClosureExpr,
-                                                expr.span);
-                self.parent_def = Some(async_def);
-            }
-            _ => {}
+            ExprKind::Async(_, async_id, _) =>
+                self.create_def(async_id, DefPathData::ClosureExpr, expr.span),
+            _ => self.parent_def,
         };
 
-        visit::walk_expr(self, expr);
-        self.parent_def = parent_def;
+        self.with_parent(parent_def, |this| visit::walk_expr(this, expr));
     }
 
     fn visit_ty(&mut self, ty: &'a Ty) {

src/librustc/hir/map/definitions.rs

@@ -100,6 +100,9 @@ pub struct Definitions {
     expansions_that_defined: FxHashMap<DefIndex, Mark>,
     next_disambiguator: FxHashMap<(DefIndex, DefPathData), u32>,
     def_index_to_span: FxHashMap<DefIndex, Span>,
+    /// When collecting definitions from an AST fragment produced by a macro invocation `Mark`
+    /// we know what parent node that fragment should be attached to thanks to this table.
+    invocation_parents: FxHashMap<Mark, DefIndex>,
 }
 
 /// A unique identifier that we can use to lookup a definition
@@ -434,6 +437,7 @@ impl Definitions {
         assert!(self.def_index_to_node.is_empty());
         self.def_index_to_node.push(ast::CRATE_NODE_ID);
         self.node_to_def_index.insert(ast::CRATE_NODE_ID, root_index);
+        self.set_invocation_parent(Mark::root(), root_index);
 
         // Allocate some other DefIndices that always must exist.
         GlobalMetaDataKind::allocate_def_indices(self);
@@ -526,6 +530,15 @@ impl Definitions {
     pub fn add_parent_module_of_macro_def(&mut self, mark: Mark, module: DefId) {
         self.parent_modules_of_macro_defs.insert(mark, module);
     }
+
+    pub fn invocation_parent(&self, invoc_id: Mark) -> DefIndex {
+        self.invocation_parents[&invoc_id]
+    }
+
+    pub fn set_invocation_parent(&mut self, invoc_id: Mark, parent: DefIndex) {
+        let old_parent = self.invocation_parents.insert(invoc_id, parent);
+        assert!(old_parent.is_none(), "parent def-index is reset for an invocation");
+    }
 }
 
 impl DefPathData {

src/librustc/hir/map/mod.rs

@@ -1,5 +1,5 @@
 use self::collector::NodeCollector;
-pub use self::def_collector::{DefCollector, MacroInvocationData};
+pub use self::def_collector::DefCollector;
 pub use self::definitions::{
     Definitions, DefKey, DefPath, DefPathData, DisambiguatedDefPathData, DefPathHash
 };

src/librustc/ich/impls_syntax.rs

@@ -89,7 +89,6 @@ impl_stable_hash_for!(enum ::syntax::ext::base::MacroKind {
     Bang,
     Attr,
     Derive,
-    ProcMacroStub,
 });
 
 
@@ -399,7 +398,7 @@ impl_stable_hash_for!(enum ::syntax_pos::hygiene::Transparency {
 
 impl_stable_hash_for!(struct ::syntax_pos::hygiene::ExpnInfo {
     call_site,
-    format,
+    kind,
     def_site,
     default_transparency,
     allow_internal_unstable,
@@ -408,13 +407,13 @@ impl_stable_hash_for!(struct ::syntax_pos::hygiene::ExpnInfo {
     edition
 });
 
-impl_stable_hash_for!(enum ::syntax_pos::hygiene::ExpnFormat {
-    MacroAttribute(sym),
-    MacroBang(sym),
-    CompilerDesugaring(kind)
+impl_stable_hash_for!(enum ::syntax_pos::hygiene::ExpnKind {
+    Root,
+    Macro(kind, descr),
+    Desugaring(kind)
 });
 
-impl_stable_hash_for!(enum ::syntax_pos::hygiene::CompilerDesugaringKind {
+impl_stable_hash_for!(enum ::syntax_pos::hygiene::DesugaringKind {
     CondTemporary,
     Async,
     Await,