wip

JetBrains · Apr 3, 2023 · 1000b12 · 1000b12
1 parent d7e2c79
commit 1000b12
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 130 deletions.
diff --git a/...sharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/FSharpDummyParser.kt b/...sharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/FSharpDummyParser.kt
@@ -12,134 +12,120 @@ import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.tryParse
 import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.whileMakingProgress
 
 class FSharpDummyParser : PsiParser {
-  override fun parse(root: IElementType, builder: PsiBuilder): ASTNode {
-    builder.parseFile(root)
-    return builder.treeBuilt
-  }
-
-  private fun PsiBuilder.parseFile(fileElementType: IElementType) {
-    parse(fileElementType) {
-      whileMakingProgress {
-        if (!parseDummyExpression()) advanceLexerWithNewLineCounting()
-        true
-      }
+    override fun parse(root: IElementType, builder: PsiBuilder): ASTNode {
+        builder.setWhitespaceSkippedCallback { elementType, _, _ ->
+            if (elementType == FSharpTokenType.NEW_LINE) ++currentLineIndex
+        }
+        builder.parseFile(root)
+        return builder.treeBuilt
     }
-  }
 
-  private fun PsiBuilder.getCurrentIndentation() =
-    currentOffset - currentLineIndent
+    private fun PsiBuilder.parseFile(fileElementType: IElementType) {
+        parse(fileElementType) {
+            whileMakingProgress {
+                if (!parseDummyExpression()) advanceLexerWithNewLineCounting()
+                true
+            }
+        }
+    }
 
-  private fun PsiBuilder.parseDummyExpression() = parseConcatenation()
+    private fun PsiBuilder.getCurrentIndentation() =
+        currentOffset - currentLineIndent
 
-  private fun PsiBuilder.parseConcatenation() =
-    tryParse(FSharpElementTypes.DUMMY_EXPRESSION) {
-      val currentIndent = getCurrentIndentation()
+    private fun PsiBuilder.parseDummyExpression() = parseConcatenation()
 
-      if (!parseStringExpression()) false
-      else if (!scanOrRollback { tryParseConcatenationPartAhead(currentIndent) }) false
-      else {
-        whileMakingProgress {
-          scanOrRollback { tryParseConcatenationPartAhead(currentIndent) }
+    private fun PsiBuilder.parseConcatenation() =
+        tryParse(FSharpElementTypes.DUMMY_EXPRESSION) {
+            val currentIndent = getCurrentIndentation()
+
+            if (!parseStringExpression()) false
+            else if (!scanOrRollback { tryParseConcatenationPartAhead(currentIndent) }) false
+            else {
+                whileMakingProgress {
+                    scanOrRollback { tryParseConcatenationPartAhead(currentIndent) }
+                }
+                true
+            }
         }
-        true
-      }
-    }
 
-  private fun PsiBuilder.tryParseConcatenationPartAhead(requiredStringIndent: Int): Boolean {
-    val oldLineIndex = currentLineIndex
-    val hasSpaceBeforePlus = eatFilteredTokens()
-    val movedToNewLine = currentLineIndex != oldLineIndex
-
-    if (tokenType != FSharpTokenType.PLUS) return false
-    advanceLexer() // eat plus token
-    val afterPlusTokenIndent = getCurrentIndentation()
-
-    // since "123" +"123" is not allowed
-    if (hasSpaceBeforePlus && !eatFilteredTokens()) return false
-
-    // since
-    //    "123"
-    // more than one space after plus
-    // +  "123"
-    // is not allowed
-    val secondStringOperandIndent = getCurrentIndentation()
-    if (movedToNewLine &&
-      afterPlusTokenIndent < requiredStringIndent &&
-      secondStringOperandIndent - afterPlusTokenIndent != 1
-    ) return false
-
-    // since
-    //    "123"
-    // + "123"
-    // is not allowed
-    return secondStringOperandIndent >= requiredStringIndent && parseStringExpression()
-  }
-
-  private fun PsiBuilder.parseStringExpression() =
-    parseInterpolatedStringExpression() || parseAnyStringExpression()
-
-  private fun PsiBuilder.parseAnyStringExpression() =
-    if (tokenType !in FSharpTokenType.ALL_STRINGS) false
-    else parse {
-      val interpolated = tokenType in FSharpTokenType.INTERPOLATED_STRINGS
-      advanceLexerWithNewLineCounting()
-      if (interpolated) FSharpElementTypes.INTERPOLATED_STRING_LITERAL_EXPRESSION_PART
-      else FSharpElementTypes.STRING_LITERAL_EXPRESSION
+    private fun PsiBuilder.tryParseConcatenationPartAhead(requiredStringIndent: Int): Boolean {
+        val oldLineIndex = currentLineIndex
+        val hasSpaceBeforePlus = eatFilteredTokens()
+        val movedToNewLine = currentLineIndex != oldLineIndex
+
+        if (tokenType != FSharpTokenType.PLUS) return false
+        advanceLexer() // eat plus token
+        val afterPlusTokenIndent = getCurrentIndentation()
+
+        // since "123" +"123" is not allowed
+        if (hasSpaceBeforePlus && !eatFilteredTokens()) return false
+
+        // since
+        //    "123"
+        // more than one space after plus
+        // +  "123"
+        // is not allowed
+        val secondStringOperandIndent = getCurrentIndentation()
+        if (movedToNewLine &&
+            afterPlusTokenIndent < requiredStringIndent &&
+            secondStringOperandIndent - afterPlusTokenIndent != 1
+        ) return false
+
+        // since
+        //    "123"
+        // + "123"
+        // is not allowed
+        return secondStringOperandIndent >= requiredStringIndent && parseStringExpression()
     }
 
-  private fun PsiBuilder.parseInterpolatedStringExpression() =
-    if (!FSharpTokenType.INTERPOLATED_STRINGS.contains(tokenType)) false
-    else parse(FSharpElementTypes.INTERPOLATED_STRING_LITERAL_EXPRESSION) {
-      var nestingDepth = 0
-      whileMakingProgress {
-        if (tokenType in FSharpTokenType.INTERPOLATED_STRING_STARTS) nestingDepth += 1
-        if (tokenType in FSharpTokenType.INTERPOLATED_STRING_ENDS) nestingDepth -= 1
-        if (!parseAnyStringExpression()) advanceLexerWithNewLineCounting()
-        nestingDepth != 0
-      }
-    }
+    private fun PsiBuilder.parseStringExpression() =
+        parseInterpolatedStringExpression() || parseAnyStringExpression()
+
+    private fun PsiBuilder.parseAnyStringExpression() =
+        if (tokenType !in FSharpTokenType.ALL_STRINGS) false
+        else parse {
+            val interpolated = tokenType in FSharpTokenType.INTERPOLATED_STRINGS
+            advanceLexerWithNewLineCounting()
+            if (interpolated) FSharpElementTypes.INTERPOLATED_STRING_LITERAL_EXPRESSION_PART
+            else FSharpElementTypes.STRING_LITERAL_EXPRESSION
+        }
 
-  private fun PsiBuilder.eatFilteredTokens() =
-    tryEatAllTokens(
-      FSharpTokenType.WHITESPACE,
-      FSharpTokenType.NEW_LINE,
-      FSharpTokenType.LINE_COMMENT,
-      FSharpTokenType.BLOCK_COMMENT
-    )
-
-  private fun PsiBuilder.advanceLexerWithNewLineCounting() {
-    if (eof()) return
-
-    when (tokenType) {
-      FSharpTokenType.NEW_LINE -> {
-        advanceLexer()
-        currentLineIndent = currentOffset
-        ++currentLineIndex
-      }
-
-      in FSharpTokenType.STRINGS -> {
-        val lastEndOfLineIndex = tokenText!!.lastIndexOf('\n')
-        val stringStartIndent = currentOffset
-        advanceLexer()
-        if (lastEndOfLineIndex != -1) {
-          currentLineIndent = stringStartIndent + lastEndOfLineIndex + 1
-          ++currentLineIndex
+    private fun PsiBuilder.parseInterpolatedStringExpression() =
+        if (tokenType !in FSharpTokenType.INTERPOLATED_STRINGS) false
+        else parse(FSharpElementTypes.INTERPOLATED_STRING_LITERAL_EXPRESSION) {
+            var nestingDepth = 0
+            whileMakingProgress {
+                if (tokenType in FSharpTokenType.INTERPOLATED_STRING_STARTS) nestingDepth += 1
+                if (tokenType in FSharpTokenType.INTERPOLATED_STRING_ENDS) nestingDepth -= 1
+                if (!parseAnyStringExpression()) advanceLexerWithNewLineCounting()
+                nestingDepth != 0
+            }
         }
-      }
 
-      else -> advanceLexer()
+    private fun PsiBuilder.eatFilteredTokens(): Boolean {
+        val offset = currentOffset
+        eof()
+        return currentOffset > offset
     }
-  }
 
-  private fun PsiBuilder.tryEatAllTokens(vararg tokens: IElementType): Boolean {
-    var count = 0
-    while (tokenType in tokens) {
-      advanceLexerWithNewLineCounting()
-      count++
+    private fun PsiBuilder.advanceLexerWithNewLineCounting() {
+        if (eof()) return
+
+        when (tokenType) {
+            in FSharpTokenType.STRINGS -> {
+                val lastEndOfLineIndex = tokenText!!.lastIndexOf('\n')
+                val stringStartIndent = currentOffset
+                advanceLexer()
+                if (lastEndOfLineIndex != -1) {
+                    currentLineIndent = stringStartIndent + lastEndOfLineIndex + 1
+                    ++currentLineIndex
+                }
+            }
+
+            else -> advanceLexer()
+        }
     }
-    return count > 0
-  }
 
-  private var currentLineIndent = 0
-  private var currentLineIndex = 0
+    private var currentLineIndent = 0
+    private var currentLineIndex = 0
 }
diff --git a/.../src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/FSharpParserDefinition.kt b/.../src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/FSharpParserDefinition.kt
@@ -6,6 +6,7 @@ import com.intellij.lang.PsiParser
 import com.intellij.openapi.project.Project
 import com.intellij.psi.FileViewProvider
 import com.intellij.psi.PsiElement
+import com.intellij.psi.TokenType
 import com.intellij.psi.tree.IFileElementType
 import com.intellij.psi.tree.TokenSet
 import com.intellij.psi.util.PsiUtilCore
@@ -16,20 +17,22 @@ import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.impl.FSharpElementTy
 import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.impl.FSharpFileImpl
 
 class FSharpParserDefinition : ParserDefinition {
-  private val logger = getLogger<FSharpParserDefinition>()
-  override fun createLexer(project: Project?) = FSharpLexer()
-  override fun createParser(project: Project): PsiParser = FSharpDummyParser()
-  override fun getCommentTokens(): TokenSet = FSharpTokenType.COMMENTS
-  override fun getStringLiteralElements(): TokenSet = FSharpTokenType.ALL_STRINGS
-  override fun createElement(node: ASTNode): PsiElement {
-    if (node is PsiElement) {
-      logger.error("Dummy blocks should be lazy and not parsed like this")
-      return node
+    private val logger = getLogger<FSharpParserDefinition>()
+    override fun getWhitespaceTokens(): TokenSet = TokenSet.create(FSharpTokenType.NEW_LINE, TokenType.WHITE_SPACE)
+    override fun createLexer(project: Project?) = FSharpLexer()
+    override fun createParser(project: Project): PsiParser = FSharpDummyParser()
+    override fun getCommentTokens(): TokenSet = FSharpTokenType.COMMENTS
+    override fun getStringLiteralElements(): TokenSet = FSharpTokenType.ALL_STRINGS
+    override fun createElement(node: ASTNode): PsiElement {
+        if (node is PsiElement) {
+            logger.error("Dummy blocks should be lazy and not parsed like this")
+            return node
+        }
+
+        logger.error("An attempt to parse unexpected element")
+        return PsiUtilCore.NULL_PSI_ELEMENT
     }
 
-    logger.error("An attempt to parse unexpected element")
-    return PsiUtilCore.NULL_PSI_ELEMENT
-  }
-  override fun createFile(viewProvider: FileViewProvider) = FSharpFileImpl(viewProvider)
-  override fun getFileNodeType(): IFileElementType = FSharpElementTypes.FILE
+    override fun createFile(viewProvider: FileViewProvider) = FSharpFileImpl(viewProvider)
+    override fun getFileNodeType(): IFileElementType = FSharpElementTypes.FILE
 }
diff --git a/...src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/lexer/FSharpTokenType.java b/...src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/lexer/FSharpTokenType.java
@@ -1,5 +1,6 @@
 package com.jetbrains.rider.ideaInterop.fileTypes.fsharp.lexer;
 
+import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.NotNull;
@@ -99,7 +100,7 @@ public interface FSharpTokenType {
   IElementType VERBATIM_INTERPOLATED_STRING_MIDDLE = createToken("VERBATIM_INTERPOLATED_STRING_MIDDLE");
   IElementType VERBATIM_INTERPOLATED_STRING_END = createToken("VERBATIM_INTERPOLATED_STRING_END");
   IElementType NEW_LINE = createToken("NEW_LINE");
-  IElementType WHITESPACE = createToken("WHITESPACE");
+  IElementType WHITESPACE = TokenType.WHITE_SPACE;
   IElementType KEYWORD_STRING_SOURCE_DIRECTORY = createToken("KEYWORD_STRING_SOURCE_DIRECTORY");
   IElementType KEYWORD_STRING_SOURCE_FILE = createToken("KEYWORD_STRING_SOURCE_FILE");
   IElementType KEYWORD_STRING_LINE = createToken("KEYWORD_STRING_LINE");

diff --git a/...com/jetbrains/rider/ideaInterop/fileTypes/fsharp/psi/FSharpStringLiteralExpressionUtil.kt b/...com/jetbrains/rider/ideaInterop/fileTypes/fsharp/psi/FSharpStringLiteralExpressionUtil.kt
@@ -68,7 +68,7 @@ fun createLiteralTextEscaper(expression: FSharpStringLiteralExpression): Literal
       isInterpolated = false
     )
 
-    FSharpStringLiteralType.ByteArray -> TODO("Error")
+    FSharpStringLiteralType.ByteArray -> TODO("Unexpected escaping call on ByteArray string")
   }
 }
-Original file line number
+Diff line change
@@ Expand Up @@
           isInterpolated = false
         )
-        FSharpStringLiteralType.ByteArray -> TODO("Error")
+        FSharpStringLiteralType.ByteArray -> TODO("Unexpected escaping call on ByteArray string")
       }
     }
@@ Expand Down @@