WIP custom token lexer analysis.

paket.dependencies

@@ -82,4 +82,4 @@ group compiler
     nuget FSharp.Core 6.0.1 content: none
     nuget System.Memory
     nuget System.Runtime
-    nuget FsLexYacc 11.0.0-beta1
+    nuget FsLexYacc 10.2.0

paket.lock

@@ -1441,11 +1441,11 @@ RESTRICTION: == netstandard2.0
 NUGET
   remote: https://api.nuget.org/v3/index.json
     FSharp.Core (6.0.1) - content: none
-    FsLexYacc (11.0.0-beta1)
-      FSharp.Core (>= 4.6.2)
-      FsLexYacc.Runtime (>= 11.0.0-beta1)
-    FsLexYacc.Runtime (11.0.0-beta1)
-      FSharp.Core (>= 4.6.2)
+    FsLexYacc (10.2)
+      FSharp.Core (>= 4.5.2)
+      FsLexYacc.Runtime (>= 10.2 < 10.3)
+    FsLexYacc.Runtime (10.2)
+      FSharp.Core (>= 4.5.2)
     Microsoft.NETCore.Platforms (6.0.1)
     Microsoft.NETCore.Targets (5.0)
     System.Buffers (4.5.1)

src/Fantomas.Benchmarks/Runners.fs

@@ -2,11 +2,8 @@ module Fantomas.Benchmarks.Runners
 
 open System.IO
 open BenchmarkDotNet.Attributes
-open FSharp.Compiler.CodeAnalysis
 open Fantomas
-open Fantomas.Extras
 
-let sharedChecker = lazy (FSharpChecker.Create())
 let config = FormatConfig.FormatConfig.Default
 
 [<MemoryDiagnoser>]
@@ -20,13 +17,12 @@ type CodePrinterTest() =
 
         let fileName = Path.GetFileName(path)
 
-        let parsingOptions = CodeFormatterImpl.createParsingOptionsFromFile fileName
 
         let content =
             File.ReadAllText(path)
             |> SourceOrigin.SourceString
 
-        CodeFormatter.FormatDocumentAsync(fileName, content, config, parsingOptions, sharedChecker.Value)
+        CodeFormatter.FormatDocumentAsync(fileName, content, config)
         |> Async.RunSynchronously
         |> ignore
 

src/Fantomas.CoreGlobalTool/Daemon.fs

@@ -59,9 +59,7 @@ type FantomasDaemon(sender: Stream, reader: Stream) as this =
                         CodeFormatter.FormatDocumentAsync(
                             request.FilePath,
                             SourceString request.SourceCode,
-                            config,
-                            CodeFormatterImpl.createParsingOptionsFromFile request.FilePath,
-                            CodeFormatterImpl.sharedChecker.Value
+                            config
                         )
 
                     if formatted = request.SourceCode then
@@ -76,31 +74,32 @@ type FantomasDaemon(sender: Stream, reader: Stream) as this =
     [<JsonRpcMethod(Methods.FormatSelection, UseSingleObjectParameterDeserialization = true)>]
     member _.FormatSelectionAsync(request: FormatSelectionRequest) : Task<FormatSelectionResponse> =
         async {
-            let config =
-                match request.Config with
-                | Some configProperties ->
-                    let config = readConfiguration request.FilePath
-                    parseOptionsFromEditorConfig config configProperties
-                | None -> readConfiguration request.FilePath
-
-            let range =
-                let r = request.Range
-                mkRange request.FilePath (mkPos r.StartLine r.StartColumn) (mkPos r.EndLine r.EndColumn)
-
-            try
-                let! formatted =
-                    CodeFormatter.FormatSelectionAsync(
-                        request.FilePath,
-                        range,
-                        SourceString request.SourceCode,
-                        config,
-                        CodeFormatterImpl.createParsingOptionsFromFile request.FilePath,
-                        CodeFormatterImpl.sharedChecker.Value
-                    )
-
-                return FormatSelectionResponse.Formatted(request.FilePath, formatted)
-            with
-            | ex -> return FormatSelectionResponse.Error(request.FilePath, ex.Message)
+            return FormatSelectionResponse.Error(request.FilePath, "Format selection is no longer supported in Fantomas 5.")
+//            let config =
+//                match request.Config with
+//                | Some configProperties ->
+//                    let config = readConfiguration request.FilePath
+//                    parseOptionsFromEditorConfig config configProperties
+//                | None -> readConfiguration request.FilePath
+//
+//            let range =
+//                let r = request.Range
+//                mkRange request.FilePath (mkPos r.StartLine r.StartColumn) (mkPos r.EndLine r.EndColumn)
+//
+//            try
+//                let! formatted =
+//                    CodeFormatter.FormatSelectionAsync(
+//                        request.FilePath,
+//                        range,
+//                        SourceString request.SourceCode,
+//                        config,
+//                        CodeFormatterImpl.createParsingOptionsFromFile request.FilePath,
+//                        CodeFormatterImpl.sharedChecker.Value
+//                    )
+//
+//                return FormatSelectionResponse.Formatted(request.FilePath, formatted)
+//            with
+//            | ex -> return FormatSelectionResponse.Error(request.FilePath, ex.Message)
         }
         |> Async.StartAsTask
 

src/Fantomas.Extras/FakeHelpers.fs

@@ -2,7 +2,7 @@ module Fantomas.Extras.FakeHelpers
 
 open System
 open System.IO
-open FSharp.Compiler.CodeAnalysis
+//open FSharp.Compiler.CodeAnalysis
 open Fantomas
 open Fantomas.FormatConfig
 open Fantomas.Extras
@@ -49,20 +49,9 @@ let private formatContentInternalAsync
     else
         async {
             try
-                let fileName =
-                    if Path.GetExtension(file) = ".fsi" then
-                        "tmp.fsi"
-                    else
-                        "tmp.fsx"
+                let isSignatureFile = Path.GetExtension(file) = ".fsi"
 
-                let! formattedContent =
-                    CodeFormatter.FormatDocumentAsync(
-                        fileName,
-                        SourceOrigin.SourceString originalContent,
-                        config,
-                        CodeFormatterImpl.createParsingOptionsFromFile fileName,
-                        CodeFormatterImpl.sharedChecker.Value
-                    )
+                let! formattedContent = CodeFormatter.FormatDocumentAsync(isSignatureFile, originalContent, config)
 
                 let contentChanged =
                     if compareWithoutLineEndings then
@@ -75,13 +64,7 @@ let private formatContentInternalAsync
                         originalContent <> formattedContent
 
                 if contentChanged then
-                    let! isValid =
-                        CodeFormatter.IsValidFSharpCodeAsync(
-                            fileName,
-                            (SourceOrigin.SourceString(formattedContent)),
-                            CodeFormatterImpl.createParsingOptionsFromFile fileName,
-                            CodeFormatterImpl.sharedChecker.Value
-                        )
+                    let! isValid = CodeFormatter.IsValidFSharpCodeAsync(isSignatureFile, formattedContent)
 
                     if not isValid then
                         raise

src/Fantomas.FCS/AssemblyInfo.fs

@@ -0,0 +1,11 @@
+namespace System
+
+open System.Runtime.CompilerServices
+
+[<assembly: InternalsVisibleTo("Fantomas")>]
+
+do ()
+
+module internal AssemblyVersionInformation =
+    [<Literal>]
+    let InternalsVisibleTo = "Fantomas"

src/Fantomas.FCS/Fantomas.FCS.fsproj

@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <None Include="paket.references" />
+    <Compile Include="AssemblyInfo.fs" />
     <EmbeddedText Include="..\..\..\fsharp\src\fsharp\FSComp.txt">
       <Link>FSComp.txt</Link>
     </EmbeddedText>

src/Fantomas.FCS/Lex.fs

@@ -17,10 +17,9 @@ type FSharpLexerFlags =
     | UseLexFilter = 0x10000
 
 let lex
-    (text: ISourceText)
-    (conditionalCompilationDefines: string list)
     (onToken: Parser.token -> range -> unit)
-    (pathMap: Internal.Utilities.PathMap)
+    (conditionalCompilationDefines: string list)
+    (text: ISourceText)
     : unit =
     let errorLogger: ErrorLogger =
         { new ErrorLogger("DiscardErrorsLogger") with
@@ -30,7 +29,7 @@ let lex
     let langVersion = Features.LanguageVersion.Default
     let reportLibraryOnlyFeatures = false
     let canSkipTrivia = false //(flags &&& FSharpLexerFlags.SkipTrivia) = FSharpLexerFlags.SkipTrivia
-    let isLightSyntaxOn = false // (flags &&& FSharpLexerFlags.LightSyntaxOn) = FSharpLexerFlags.LightSyntaxOn
+    let isLightSyntaxOn = true // (flags &&& FSharpLexerFlags.LightSyntaxOn) = FSharpLexerFlags.LightSyntaxOn
     let isCompiling = false // (flags &&& FSharpLexerFlags.Compiling) = FSharpLexerFlags.Compiling
     let isCompilingFSharpCore = false // (flags &&& FSharpLexerFlags.CompilingFSharpCore) = FSharpLexerFlags.CompilingFSharpCore
     let canUseLexFilter = true // No idea...  (flags &&& FSharpLexerFlags.UseLexFilter) = FSharpLexerFlags.UseLexFilter
@@ -41,7 +40,14 @@ let lex
     let lightStatus = LightSyntaxStatus(isLightSyntaxOn, true)
 
     let lexargs =
-        mkLexargs (conditionalCompilationDefines, lightStatus, LexResourceManager(0), [], errorLogger, pathMap)
+        mkLexargs (
+            conditionalCompilationDefines,
+            lightStatus,
+            LexResourceManager(0),
+            [],
+            errorLogger,
+            Internal.Utilities.PathMap.empty
+        )
 
     let lexargs = { lexargs with applyLineDirectives = isCompiling }