Allow symbols to be escaped in DocText (#3375)

Related to #3374, this is an implementation of escaping characters in
doc comments, such as the `@` symbol.

---------

Co-authored-by: Timothee Guerin <[email protected]>
Co-authored-by: Timothee Guerin <[email protected]>

.chronus/changes/bug-escape-characters-in-docblock-2024-4-23-16-0-21.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: feature
+packages:
+  - "@typespec/compiler"
+---
+
+Allow `@` to be escaped in doc comment with `\`

packages/compiler/src/core/parser.ts

@@ -2850,6 +2850,12 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa
           }
           nextToken();
           break;
+        case Token.DocText:
+          parts.push(source.substring(start, tokenPos()));
+          parts.push(tokenValue());
+          nextToken();
+          start = tokenPos();
+          break;
         default:
           nextToken();
           break;

packages/compiler/src/core/scanner.ts

@@ -459,6 +459,8 @@ export function createScanner(
         return getStringTokenValue(token, tokenFlags);
       case Token.Identifier:
         return getIdentifierTokenValue();
+      case Token.DocText:
+        return getDocTextValue();
       default:
         return getTokenText();
     }
@@ -656,6 +658,10 @@ export function createScanner(
         case CharCode.LineFeed:
           return next(Token.NewLine);
 
+        case CharCode.Backslash:
+          tokenFlags |= TokenFlags.Escaped;
+          return position === endPosition - 1 ? next(Token.DocText) : next(Token.DocText, 2);
+
         case CharCode.Space:
         case CharCode.Tab:
         case CharCode.VerticalTab:
@@ -1036,6 +1042,44 @@ export function createScanner(
     return text;
   }
 
+  function getDocTextValue(): string {
+    if (tokenFlags & TokenFlags.Escaped) {
+      let start = tokenPosition;
+      const end = position;
+
+      let result = "";
+      let pos = start;
+
+      while (pos < end) {
+        const ch = input.charCodeAt(pos);
+        if (ch !== CharCode.Backslash) {
+          pos++;
+          continue;
+        }
+
+        if (pos === end - 1) {
+          break;
+        }
+
+        result += input.substring(start, pos);
+        switch (input.charCodeAt(pos + 1)) {
+          case CharCode.At:
+            result += "@";
+            break;
+          default:
+            result += input.substring(pos, pos + 2);
+        }
+        pos += 2;
+        start = pos;
+      }
+
+      result += input.substring(start, end);
+      return result;
+    } else {
+      return input.substring(tokenPosition, position);
+    }
+  }
+
   function findTripleQuotedStringIndent(start: number, end: number): [number, number] {
     end = end - 3; // Remove the """
     // remove whitespace before closing delimiter and record it as required
@@ -1231,6 +1275,8 @@ export function createScanner(
         return "\\";
       case CharCode.$:
         return "$";
+      case CharCode.At:
+        return "@";
       case CharCode.Backtick:
         return "`";
       default:

packages/compiler/test/parser.test.ts

@@ -1007,6 +1007,19 @@ describe("compiler: parser", () => {
             strictEqual(docs[0].tags.length, 0);
           },
         ],
+        [
+          `
+          /** Escape at the end \\*/
+          model M {}
+          `,
+          (script) => {
+            const docs = script.statements[0].docs;
+            strictEqual(docs?.length, 1);
+            strictEqual(docs[0].content.length, 1);
+            strictEqual(docs[0].content[0].text, "Escape at the end \\");
+            strictEqual(docs[0].tags.length, 0);
+          },
+        ],
         [
           `
           /**
@@ -1024,6 +1037,8 @@ describe("compiler: parser", () => {
            *\`\`\`
            *
            * \`This is not a @tag either because we're in a code span\`.
+           * 
+           * This is not a \\@tag because it is escaped.
            *
            * @param x the param
            * that continues on another line
@@ -1052,7 +1067,9 @@ describe("compiler: parser", () => {
                 "This code fence is glued\n" +
                 "to the stars\n" +
                 "```\n\n" +
-                "`This is not a @tag either because we're in a code span`."
+                "`This is not a @tag either because we're in a code span`.\n" +
+                "\n" +
+                "This is not a @tag because it is escaped."
             );
             strictEqual(docs[0].tags.length, 6);
             const [xParam, yParam, tTemplate, uTemplate, returns, pretend] = docs[0].tags;