Merge pull request #28215 from ProvableHQ/doc-annotation-lex-parse

Clarify ABNF vs. lexer/parser annotations.

compiler/parser/src/parser/file.rs

@@ -327,7 +327,7 @@ impl<N: Network> ParserContext<'_, N> {
         let span = start + identifier.span;
 
         // TODO: Verify that this check is sound.
-        // Check that there is no whitespace in between the `@` symbol and identifier.
+        // Check that there is no whitespace or comments in between the `@` symbol and identifier.
         match identifier.span.hi.0 - start.lo.0 > 1 + identifier.name.to_string().len() as u32 {
             true => Err(ParserError::space_in_annotation(span).into()),
             false => Ok(Annotation { identifier, span, id: self.node_builder.next_id() }),

compiler/parser/src/tokenizer/token.rs

@@ -102,10 +102,18 @@ pub enum Token {
     Arrow,
     BigArrow,
     Underscore,
-    At,
+    At, // @ is not a symbol token in the ABNF grammar (see explanation about annotations below)
     // There is no symbol for `)group` here (unlike the ABNF grammar),
     // because we handle that differently in the lexer.
 
+    // The ABNF grammar has annotations as tokens,
+    // defined as @ immediately followed by an identifier.
+    // Here instead we regard the @ sign alone as a token (see `At` above),
+    // and we lex it separately from the identifier that is supposed to follow it in an annotation.
+    // When parsing annotations, we check that there is no whitespace or comments
+    // between the @ and the identifier, thus eventually complying to the ABNF grammar.
+    // See the parse_annotation function.
+
     // Type keywords
     Address,
     Bool,