Auto merge of rust-lang#78677 - Aaron1011:fix/capture-inner-attrs, r=…

…petrochenkov

Use reparsed `TokenStream` if we captured any inner attributes

Fixes rust-lang#78675

We now bail out of `prepend_attrs` if we ended up capturing any inner
attributes (which can happen in several places, due to token capturing
for `macro_rules!` arguments.

compiler/rustc_parse/src/lib.rs

@@ -611,11 +611,11 @@ fn prepend_attrs(
     }
     let mut builder = tokenstream::TokenStreamBuilder::new();
     for attr in attrs {
-        assert_eq!(
-            attr.style,
-            ast::AttrStyle::Outer,
-            "inner attributes should prevent cached tokens from existing"
-        );
+        // FIXME: Correctly handle tokens for inner attributes.
+        // For now, we fall back to reparsing the original AST node
+        if attr.style == ast::AttrStyle::Inner {
+            return None;
+        }
         builder.push(
             attr.tokens
                 .as_ref()

compiler/rustc_parse/src/parser/item.rs

@@ -7,7 +7,7 @@ use crate::maybe_whole;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
-use rustc_ast::{self as ast, AttrStyle, AttrVec, Attribute, DUMMY_NODE_ID};
+use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
 use rustc_ast::{AssocItem, AssocItemKind, ForeignItemKind, Item, ItemKind, Mod};
 use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
 use rustc_ast::{BindingMode, Block, FnDecl, FnSig, Param, SelfKind};
@@ -127,34 +127,19 @@ impl<'a> Parser<'a> {
 
         let (mut item, tokens) = if needs_tokens {
             let (item, tokens) = self.collect_tokens(parse_item)?;
-            (item, Some(tokens))
+            (item, tokens)
         } else {
             (parse_item(self)?, None)
         };
-
-        self.unclosed_delims.append(&mut unclosed_delims);
-
-        // Once we've parsed an item and recorded the tokens we got while
-        // parsing we may want to store `tokens` into the item we're about to
-        // return. Note, though, that we specifically didn't capture tokens
-        // related to outer attributes. The `tokens` field here may later be
-        // used with procedural macros to convert this item back into a token
-        // stream, but during expansion we may be removing attributes as we go
-        // along.
-        //
-        // If we've got inner attributes then the `tokens` we've got above holds
-        // these inner attributes. If an inner attribute is expanded we won't
-        // actually remove it from the token stream, so we'll just keep yielding
-        // it (bad!). To work around this case for now we just avoid recording
-        // `tokens` if we detect any inner attributes. This should help keep
-        // expansion correct, but we should fix this bug one day!
-        if let Some(tokens) = tokens {
-            if let Some(item) = &mut item {
-                if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
-                    item.tokens = tokens;
-                }
+        if let Some(item) = &mut item {
+            // If we captured tokens during parsing (due to encountering an `NtItem`),
+            // use those instead
+            if item.tokens.is_none() {
+                item.tokens = tokens;
             }
         }
+
+        self.unclosed_delims.append(&mut unclosed_delims);
         Ok(item)
     }
 

src/test/ui/proc-macro/issue-78675-captured-inner-attrs.rs

@@ -0,0 +1,32 @@
+// check-pass
+// edition:2018
+// compile-flags: -Z span-debug
+// aux-build:test-macros.rs
+
+#![no_std] // Don't load unnecessary hygiene information from std
+extern crate std;
+
+#[macro_use] extern crate test_macros;
+
+macro_rules! foo {(
+    #[fake_attr]
+    $item:item
+) => (
+    $item
+)}
+
+macro_rules! outer {($item:item) => (
+    print_bang! { // Identity proc-macro
+        foo! {
+            #[fake_attr]
+            $item
+        }
+    }
+)}
+outer! {
+    mod bar {
+        //! Foo
+    }
+}
+
+fn main() {}

src/test/ui/proc-macro/issue-78675-captured-inner-attrs.stdout

@@ -0,0 +1,86 @@
+PRINT-BANG INPUT (DISPLAY): foo ! { #[fake_attr] mod bar {
+    #![doc = r" Foo"]
+} }
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Ident {
+        ident: "foo",
+        span: $DIR/issue-78675-captured-inner-attrs.rs:20:9: 20:12 (#4),
+    },
+    Punct {
+        ch: '!',
+        spacing: Alone,
+        span: $DIR/issue-78675-captured-inner-attrs.rs:20:12: 20:13 (#4),
+    },
+    Group {
+        delimiter: Brace,
+        stream: TokenStream [
+            Punct {
+                ch: '#',
+                spacing: Alone,
+                span: $DIR/issue-78675-captured-inner-attrs.rs:21:13: 21:14 (#4),
+            },
+            Group {
+                delimiter: Bracket,
+                stream: TokenStream [
+                    Ident {
+                        ident: "fake_attr",
+                        span: $DIR/issue-78675-captured-inner-attrs.rs:21:15: 21:24 (#4),
+                    },
+                ],
+                span: $DIR/issue-78675-captured-inner-attrs.rs:21:14: 21:25 (#4),
+            },
+            Group {
+                delimiter: None,
+                stream: TokenStream [
+                    Ident {
+                        ident: "mod",
+                        span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                    },
+                    Ident {
+                        ident: "bar",
+                        span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                    },
+                    Group {
+                        delimiter: Brace,
+                        stream: TokenStream [
+                            Punct {
+                                ch: '#',
+                                spacing: Joint,
+                                span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                            },
+                            Punct {
+                                ch: '!',
+                                spacing: Alone,
+                                span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                            },
+                            Group {
+                                delimiter: Bracket,
+                                stream: TokenStream [
+                                    Ident {
+                                        ident: "doc",
+                                        span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                                    },
+                                    Punct {
+                                        ch: '=',
+                                        spacing: Alone,
+                                        span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                                    },
+                                    Literal {
+                                        kind: StrRaw(0),
+                                        symbol: " Foo",
+                                        suffix: None,
+                                        span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                                    },
+                                ],
+                                span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                            },
+                        ],
+                        span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+                    },
+                ],
+                span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
+            },
+        ],
+        span: $DIR/issue-78675-captured-inner-attrs.rs:20:14: 23:10 (#4),
+    },
+]