wip

rider-fsharp/build.gradle.kts

@@ -16,6 +16,12 @@ plugins {
   kotlin("jvm") version "1.8.0"
 }
 
+dependencies {
+  testImplementation("junit:junit:4.13.2")
+  testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.9.2")
+  testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.9.2")
+}
+
 apply {
   plugin("kotlin")
 }
@@ -303,11 +309,18 @@ tasks {
     dependsOn(generateFSharpLexer, rdgen)
   }
 
-  withType<Test> {
+  val parserTest by register<Test>("parserTest") {
+    useJUnitPlatform()
+  }
+
+  named<Test>("test") {
+    dependsOn(parserTest)
     useTestNG {
       groupByInstances = true
     }
+  }
 
+  withType<Test> {
     testLogging {
       showStandardStreams = true
       exceptionFormat = TestExceptionFormat.FULL

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/FSharpDummyParser.kt

@@ -0,0 +1,115 @@
+package com.jetbrains.rider.ideaInterop.fileTypes.fsharp
+
+import com.intellij.lang.ASTNode
+import com.intellij.lang.PsiBuilder
+import com.intellij.lang.PsiParser
+import com.intellij.psi.tree.IElementType
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.lexer.FSharpTokenType
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.impl.FSharpElementTypes
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.parse
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.scanOrRollback
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.whileMakingProgress
+
+class FSharpDummyParser : PsiParser {
+  override fun parse(root: IElementType, builder: PsiBuilder): ASTNode {
+    builder.setWhitespaceSkippedCallback { elementType, _, end ->
+      if (elementType == FSharpTokenType.NEW_LINE) currentLineStart = end
+    }
+    builder.parseFile(root)
+    return builder.treeBuilt
+  }
+
+  private fun PsiBuilder.parseFile(fileElementType: IElementType) {
+    parse(fileElementType) {
+      whileMakingProgress {
+        if (!parseDummyExpression()) advanceLexerWithNewLineCounting()
+        true
+      }
+    }
+  }
+
+  private fun PsiBuilder.getCurrentLineOffset() = currentOffset - currentLineStart
+  private fun PsiBuilder.parseDummyExpression() = parseConcatenation()
+
+  private fun PsiBuilder.parseConcatenation() =
+    parse {
+      val currentIndent = getCurrentLineOffset()
+
+      if (!parseStringExpression()) null
+      else if (!scanOrRollback { tryParseConcatenationPartAhead(currentIndent) }) null
+      else {
+        whileMakingProgress {
+          scanOrRollback { tryParseConcatenationPartAhead(currentIndent) }
+        }
+        FSharpElementTypes.DUMMY_EXPRESSION
+      }
+    }
+
+  private fun PsiBuilder.tryParseConcatenationPartAhead(requiredStringIndent: Int): Boolean {
+    val hasSpaceBeforePlus = rawLookup(-1)?.let { isWhitespaceOrComment(it) } ?: false
+
+    if (tokenType != FSharpTokenType.PLUS) return false
+    val afterPlusTokenIndent = getCurrentLineOffset() + 1
+    advanceLexer() // eat plus token
+
+    // since "123" +"123" is not allowed
+    if (hasSpaceBeforePlus && currentOffset - afterPlusTokenIndent == 0) return false
+
+    val secondStringOperandIndent = getCurrentLineOffset()
+    // since
+    //    "123"
+    // + "123"
+    // is not allowed
+    if (secondStringOperandIndent < requiredStringIndent) return false
+
+    // since
+    //    "123"
+    // more than one space after plus
+    // +  "123"
+    // is not allowed
+    if (secondStringOperandIndent == requiredStringIndent &&
+      secondStringOperandIndent - afterPlusTokenIndent > 1
+    ) return false
+
+    return parseStringExpression()
+  }
+
+  private fun PsiBuilder.parseStringExpression() =
+    parseInterpolatedStringExpression() || parseAnyStringExpression()
+
+  private fun PsiBuilder.parseAnyStringExpression() =
+    if (tokenType !in FSharpTokenType.ALL_STRINGS) false
+    else parse {
+      val interpolated = tokenType in FSharpTokenType.INTERPOLATED_STRINGS
+      advanceLexerWithNewLineCounting()
+      if (interpolated) FSharpElementTypes.INTERPOLATED_STRING_LITERAL_EXPRESSION_PART
+      else FSharpElementTypes.STRING_LITERAL_EXPRESSION
+    }
+
+  private fun PsiBuilder.parseInterpolatedStringExpression() =
+    if (tokenType !in FSharpTokenType.INTERPOLATED_STRINGS) false
+    else parse(FSharpElementTypes.INTERPOLATED_STRING_LITERAL_EXPRESSION) {
+      var nestingDepth = 0
+      whileMakingProgress {
+        if (tokenType in FSharpTokenType.INTERPOLATED_STRING_STARTS) nestingDepth += 1
+        if (tokenType in FSharpTokenType.INTERPOLATED_STRING_ENDS) nestingDepth -= 1
+        if (!parseAnyStringExpression()) advanceLexerWithNewLineCounting()
+        nestingDepth != 0
+      }
+    }
+
+  private fun PsiBuilder.advanceLexerWithNewLineCounting() {
+    when (tokenType) {
+      in FSharpTokenType.STRINGS -> {
+        val lastEndOfLineIndex = tokenText!!.lastIndexOf('\n')
+        val stringStartIndent = currentOffset
+        advanceLexer()
+        if (lastEndOfLineIndex != -1) currentLineStart = stringStartIndent + lastEndOfLineIndex + 1
+      }
+
+      else -> advanceLexer()
+    }
+  }
+
+  private var currentLineStart = 0
+}

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/FSharpParserDefinition.kt

@@ -1,19 +1,37 @@
 package com.jetbrains.rider.ideaInterop.fileTypes.fsharp
 
-import com.intellij.lexer.Lexer
+import com.intellij.lang.ASTNode
+import com.intellij.lang.ParserDefinition
+import com.intellij.lang.PsiParser
 import com.intellij.openapi.project.Project
-import com.intellij.psi.tree.IElementType
+import com.intellij.psi.FileViewProvider
+import com.intellij.psi.PsiElement
 import com.intellij.psi.tree.IFileElementType
-import com.jetbrains.rider.ideaInterop.fileTypes.RiderFileElementType
-import com.jetbrains.rider.ideaInterop.fileTypes.RiderParserDefinitionBase
+import com.intellij.psi.tree.TokenSet
+import com.intellij.psi.util.PsiUtilCore
+import com.jetbrains.rd.platform.util.getLogger
 import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.lexer.FSharpLexer
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.lexer.FSharpTokenType
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.impl.FSharpElementTypes
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi.impl.FSharpFileImpl
 
-class FSharpParserDefinition : RiderParserDefinitionBase(FSharpFileElementType, FSharpFileType) {
-  companion object {
-    val FSharpElementType = IElementType("RIDER_FSHARP", FSharpLanguage)
-    val FSharpFileElementType = RiderFileElementType("RIDER_FSHARP_FILE", FSharpLanguage, FSharpElementType)
-  }
+class FSharpParserDefinition : ParserDefinition {
+    private val logger = getLogger<FSharpParserDefinition>()
+    override fun getWhitespaceTokens() = TokenSet.create(FSharpTokenType.NEW_LINE, FSharpTokenType.WHITESPACE)
+    override fun createLexer(project: Project?) = FSharpLexer()
+    override fun createParser(project: Project): PsiParser = FSharpDummyParser()
+    override fun getCommentTokens(): TokenSet = FSharpTokenType.COMMENTS
+    override fun getStringLiteralElements(): TokenSet = FSharpTokenType.ALL_STRINGS
+    override fun createElement(node: ASTNode): PsiElement {
+        if (node is PsiElement) {
+            logger.error("Dummy blocks should be lazy and not parsed like this")
+            return node
+        }
 
-  override fun createLexer(project: Project?): Lexer = FSharpLexer()
-  override fun getFileNodeType(): IFileElementType = FSharpFileElementType
+        logger.error("An attempt to parse unexpected element")
+        return PsiUtilCore.NULL_PSI_ELEMENT
+    }
+
+    override fun createFile(viewProvider: FileViewProvider) = FSharpFileImpl(viewProvider)
+    override fun getFileNodeType(): IFileElementType = FSharpElementTypes.FILE
 }

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/highlighting/FSharpSyntaxHighlighterFactory.kt

@@ -14,15 +14,15 @@ class FSharpSyntaxHighlighter : SyntaxHighlighterBase() {
     private val pp_keywords = PP_KEYWORDS.types.map { it to FSharpTextAttributeKeys.PREPROCESSOR_KEYWORD }
     private val strings = STRINGS.types.map { it to FSharpTextAttributeKeys.STRING }
     private val interpolated_strings = INTERPOLATED_STRINGS.types.map { it to FSharpTextAttributeKeys.STRING }
-    private val comments = COMMENTS.types.map { it to FSharpTextAttributeKeys.BLOCK_COMMENT }
+    private val block_comments = BLOCK_COMMENTS.types.map { it to FSharpTextAttributeKeys.BLOCK_COMMENT }
     private val numbers = NUMBERS.types.map { it to FSharpTextAttributeKeys.NUMBER }
 
     private val ourKeys = mapOf(
       CHARACTER_LITERAL to FSharpTextAttributeKeys.STRING,
       BYTECHAR to FSharpTextAttributeKeys.STRING,
       LINE_COMMENT to FSharpTextAttributeKeys.COMMENT,
       TokenType.BAD_CHARACTER to HighlighterColors.BAD_CHARACTER
-    ) + keywords + pp_keywords + comments + strings + interpolated_strings + numbers
+    ) + keywords + pp_keywords + block_comments + strings + interpolated_strings + numbers
   }
 
   override fun getHighlightingLexer() = FSharpLexer()

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/lexer/FSharpTokenType.java

@@ -1,5 +1,6 @@
 package com.jetbrains.rider.ideaInterop.fileTypes.fsharp.lexer;
 
+import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.NotNull;
@@ -99,7 +100,7 @@ public interface FSharpTokenType {
   IElementType VERBATIM_INTERPOLATED_STRING_MIDDLE = createToken("VERBATIM_INTERPOLATED_STRING_MIDDLE");
   IElementType VERBATIM_INTERPOLATED_STRING_END = createToken("VERBATIM_INTERPOLATED_STRING_END");
   IElementType NEW_LINE = createToken("NEW_LINE");
-  IElementType WHITESPACE = createToken("WHITESPACE");
+  IElementType WHITESPACE = TokenType.WHITE_SPACE;
   IElementType KEYWORD_STRING_SOURCE_DIRECTORY = createToken("KEYWORD_STRING_SOURCE_DIRECTORY");
   IElementType KEYWORD_STRING_SOURCE_FILE = createToken("KEYWORD_STRING_SOURCE_FILE");
   IElementType KEYWORD_STRING_LINE = createToken("KEYWORD_STRING_LINE");
@@ -260,17 +261,31 @@ public interface FSharpTokenType {
     UNFINISHED_TRIPLE_QUOTE_INTERPOLATED_STRING
   );
 
+  TokenSet ALL_STRINGS = TokenSet.orSet(STRINGS, INTERPOLATED_STRINGS);
+
+  TokenSet INTERPOLATED_STRING_STARTS = TokenSet.create(
+    REGULAR_INTERPOLATED_STRING_START,
+    VERBATIM_INTERPOLATED_STRING_START,
+    TRIPLE_QUOTE_INTERPOLATED_STRING_START
+  );
+
   TokenSet INTERPOLATED_STRING_ENDS = TokenSet.create(
     REGULAR_INTERPOLATED_STRING_END,
     VERBATIM_INTERPOLATED_STRING_END,
     TRIPLE_QUOTE_INTERPOLATED_STRING_END
   );
 
-  TokenSet COMMENTS = TokenSet.create(
+  TokenSet BLOCK_COMMENTS = TokenSet.create(
     SHEBANG,
     BLOCK_COMMENT
   );
 
+  TokenSet COMMENTS = TokenSet.create(
+    SHEBANG,
+    BLOCK_COMMENT,
+    LINE_COMMENT
+  );
+
   TokenSet IDENT_KEYWORDS = TokenSet.create(
     ABSTRACT,
     AND,

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/psi/FSharpElementTypes.kt

@@ -0,0 +1,17 @@
+package com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi
+
+import com.intellij.lang.ASTNode
+import com.intellij.psi.tree.ICompositeElementType
+import com.intellij.psi.tree.IElementType
+import com.intellij.psi.tree.IFileElementType
+import com.jetbrains.rider.ideaInterop.fileTypes.fsharp.FSharpLanguage
+
+class FSharpFileElementType : IFileElementType("FSharpFile", FSharpLanguage)
+
+open class FSharpElementType(debugName: String, val text: String = debugName) : IElementType(debugName, FSharpLanguage)
+abstract class FSharpCompositeElementType(debugName: String) : FSharpElementType(debugName), ICompositeElementType
+
+inline fun createCompositeElementType(debugName: String, crossinline elementFactory: (FSharpElementType) -> ASTNode) =
+  object : FSharpCompositeElementType(debugName) {
+    override fun createCompositeNode() = elementFactory(this)
+  }

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/psi/FSharpElements.kt

@@ -0,0 +1,16 @@
+package com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi
+
+import com.intellij.psi.PsiElement
+import com.intellij.psi.PsiFile
+
+interface FSharpElement : PsiElement
+
+interface FSharpFile : FSharpElement, PsiFile
+
+interface FSharpExpression : FSharpElement
+
+interface FSharpStringLiteralExpression : FSharpElement
+
+interface FSharpInterpolatedStringLiteralExpressionPart : FSharpElement
+
+interface FSharpInterpolatedStringLiteralExpression : FSharpStringLiteralExpression

rider-fsharp/src/main/java/com/jetbrains/rider/ideaInterop/fileTypes/fsharp/psi/FSharpStringLiteralTypes.kt

@@ -0,0 +1,24 @@
+package com.jetbrains.rider.ideaInterop.fileTypes.fsharp.psi
+
+enum class FSharpStringLiteralType {
+  /** "{string representation}" */
+  RegularString,
+
+  /** @"{string representation}" */
+  VerbatimString,
+
+  /** """{string representation}""" */
+  TripleQuoteString,
+
+  /** $"{string representation}" */
+  RegularInterpolatedString,
+
+  /** $@"{string representation}" */
+  VerbatimInterpolatedString,
+
+  /** $"""{string representation}""" */
+  TripleQuoteInterpolatedString,
+
+  /** "{string representation}"B */
+  ByteArray
+}