diff --git a/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj b/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj index 8b04614760b..513ff0d4ec2 100644 --- a/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj +++ b/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj @@ -12,12 +12,17 @@ $(SystemValueTuplePackageVersion) + + AnyCPU + + + diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index dbf5eaecfc2..e69d6d6ab50 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -4,9 +4,80 @@ open BenchmarkDotNet.Attributes open BenchmarkDotNet.Running open Microsoft.FSharp.Compiler.ErrorLogger open Microsoft.FSharp.Compiler.SourceCodeServices +open Microsoft.FSharp.Compiler.Text open System.Text +open Microsoft.CodeAnalysis.Text -[] +module private SourceText = + + open System.Runtime.CompilerServices + + let weakTable = ConditionalWeakTable() + + let create (sourceText: SourceText) = + + let sourceText = + { new ISourceText with + + member __.Item with get index = sourceText.[index] + + member __.GetLineString(lineIndex) = + sourceText.Lines.[lineIndex].ToString() + + member __.GetLineCount() = + sourceText.Lines.Count + + member __.GetLastCharacterPosition() = + if sourceText.Lines.Count > 0 then + (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + else + (0, 0) + + member __.GetSubTextString(start, length) = + sourceText.GetSubText(TextSpan(start, length)).ToString() + + member __.SubTextEquals(target, startIndex) = + if startIndex < 0 || startIndex >= sourceText.Length then + raise (ArgumentOutOfRangeException("startIndex")) + + if String.IsNullOrEmpty(target) then + raise (ArgumentException("Target is null or empty.", "target")) + + let lastIndex = startIndex + target.Length + if lastIndex <= startIndex || lastIndex >= sourceText.Length then + raise (ArgumentException("Target is too big.", "target")) + + let mutable finished = false + let mutable didEqual = true + let mutable i = 0 + while not finished && i < target.Length do + if target.[i] <> sourceText.[startIndex + i] then + didEqual <- false + finished <- true // bail out early + else + i <- i + 1 + + didEqual + + member __.ContentEquals(sourceText) = + match sourceText with + | :? SourceText as sourceText -> sourceText.ContentEquals(sourceText) + | _ -> false + + member __.Length = sourceText.Length + + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + sourceText.CopyTo(sourceIndex, destination, destinationIndex, count) + } + + sourceText + +type SourceText with + + member this.ToFSharpSourceText() = + SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create)) + +[] type CompilerServiceParsing() = let mutable checkerOpt = None @@ -32,8 +103,7 @@ type CompilerServiceParsing() = match sourceOpt with | None -> - let source = File.ReadAllText("""..\..\..\..\..\src\fsharp\TypeChecker.fs""") - sourceOpt <- Some(source) + sourceOpt <- Some <| SourceText.From(File.OpenRead("""..\..\..\..\..\src\fsharp\TypeChecker.fs"""), Encoding.Default, SourceHashAlgorithm.Sha1, true) | _ -> () [] @@ -43,7 +113,7 @@ type CompilerServiceParsing() = | Some(checker) -> checker.InvalidateAll() checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - checker.ParseFile("dummy.fs", "dummy", parsingOptions) |> Async.RunSynchronously |> ignore + checker.ParseFile("dummy.fs", SourceText.ofString "dummy", parsingOptions) |> Async.RunSynchronously |> ignore [] member __.Parsing() = @@ -51,10 +121,10 @@ type CompilerServiceParsing() = | None, _ -> failwith "no checker" | _, None -> failwith "no source" | Some(checker), Some(source) -> - let results = checker.ParseFile("TypeChecker.fs", source, parsingOptions) |> Async.RunSynchronously + let results = checker.ParseFile("TypeChecker.fs", source.ToFSharpSourceText(), parsingOptions) |> Async.RunSynchronously if results.ParseHadErrors then failwithf "parse had errors: %A" results.Errors [] let main argv = let _ = BenchmarkRunner.Run() - 0 + 0 \ No newline at end of file diff --git a/fcs/samples/EditorService/Program.fs b/fcs/samples/EditorService/Program.fs index 5a2cb624f0c..17fc133e6e3 100644 --- a/fcs/samples/EditorService/Program.fs +++ b/fcs/samples/EditorService/Program.fs @@ -8,6 +8,7 @@ open Microsoft.FSharp.Compiler.QuickParse let checker = FSharpChecker.Create() let parseWithTypeInfo (file, input) = + let input = Microsoft.FSharp.Compiler.Text.SourceText.ofString input let checkOptions, _errors = checker.GetProjectOptionsFromScript(file, input) |> Async.RunSynchronously let parsingOptions, _errors = checker.GetParsingOptionsFromProjectOptions(checkOptions) let untypedRes = checker.ParseFile(file, input, parsingOptions) |> Async.RunSynchronously diff --git a/fcs/samples/UntypedTree/Program.fs b/fcs/samples/UntypedTree/Program.fs index 3334b093bad..8d4986125e7 100644 --- a/fcs/samples/UntypedTree/Program.fs +++ b/fcs/samples/UntypedTree/Program.fs @@ -11,7 +11,7 @@ let checker = FSharpChecker.Create() // Get untyped tree for a specified input let getUntypedTree (file, input) = let parsingOptions = { FSharpParsingOptions.Default with SourceFiles = [| file |] } - let untypedRes = checker.ParseFile(file, input, parsingOptions) |> Async.RunSynchronously + let untypedRes = checker.ParseFile(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, parsingOptions) |> Async.RunSynchronously match untypedRes.ParseTree with | Some tree -> tree | None -> failwith "Something went wrong during parsing!" diff --git a/src/fsharp/CheckFormatStrings.fs b/src/fsharp/CheckFormatStrings.fs index 9d7fd72d7a0..37295ac987c 100644 --- a/src/fsharp/CheckFormatStrings.fs +++ b/src/fsharp/CheckFormatStrings.fs @@ -54,15 +54,17 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe let (offset, fmt) = match context with | Some context -> - let length = context.Source.Length - if m.EndLine < context.LineStartPositions.Length then - let startIndex = context.LineStartPositions.[m.StartLine-1] + m.StartColumn - let endIndex = context.LineStartPositions.[m.EndLine-1] + m.EndColumn - 1 - if startIndex < length-3 && context.Source.[startIndex..startIndex+2] = "\"\"\"" then - (3, context.Source.[startIndex+3..endIndex-3]) - elif startIndex < length-2 && context.Source.[startIndex..startIndex+1] = "@\"" then - (2, context.Source.[startIndex+2..endIndex-1]) - else (1, context.Source.[startIndex+1..endIndex-1]) + let sourceText = context.SourceText + let lineStartPositions = context.LineStartPositions + let length = sourceText.Length + if m.EndLine < lineStartPositions.Length then + let startIndex = lineStartPositions.[m.StartLine-1] + m.StartColumn + let endIndex = lineStartPositions.[m.EndLine-1] + m.EndColumn - 1 + if startIndex < length-3 && sourceText.SubTextEquals("\"\"\"", startIndex) then + (3, sourceText.GetSubTextString(startIndex + 3, endIndex - startIndex)) + elif startIndex < length-2 && sourceText.SubTextEquals("@\"", startIndex) then + (2, sourceText.GetSubTextString(startIndex + 2, endIndex + 1 - startIndex)) + else (1, sourceText.GetSubTextString(startIndex + 1, endIndex - startIndex)) else (1, fmt) | None -> (1, fmt) diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index 11e9e9a74fa..240afb5a2d2 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -22,7 +22,8 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Microsoft.FSharp.Compiler.AbstractIL.Extensions.ILX open Microsoft.FSharp.Compiler.AbstractIL.Diagnostics -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.AttributeChecking open Microsoft.FSharp.Compiler.ConstraintSolver @@ -5037,7 +5038,7 @@ module private ScriptPreprocessClosure = open Internal.Utilities.Text.Lexing /// Represents an input to the closure finding process - type ClosureSource = ClosureSource of filename: string * referenceRange: range * sourceText: string * parseRequired: bool + type ClosureSource = ClosureSource of filename: string * referenceRange: range * sourceText: ISourceText * parseRequired: bool /// Represents an output of the closure finding process type ClosureFile = ClosureFile of string * range * ParsedInput option * (PhasedDiagnostic * bool) list * (PhasedDiagnostic * bool) list * (string * range) list // filename, range, errors, warnings, nowarns @@ -5052,7 +5053,7 @@ module private ScriptPreprocessClosure = seen.ContainsKey(check) /// Parse a script from source. - let ParseScriptText(filename:string, source:string, tcConfig:TcConfig, codeContext, lexResourceManager:Lexhelp.LexResourceManager, errorLogger:ErrorLogger) = + let ParseScriptText(filename:string, sourceText:ISourceText, tcConfig:TcConfig, codeContext, lexResourceManager:Lexhelp.LexResourceManager, errorLogger:ErrorLogger) = // fsc.exe -- COMPILED\!INTERACTIVE // fsi.exe -- !COMPILED\INTERACTIVE @@ -5064,7 +5065,7 @@ module private ScriptPreprocessClosure = | CodeContext.CompilationAndEvaluation -> ["INTERACTIVE"] | CodeContext.Compilation -> ["COMPILED"] | CodeContext.Editing -> "EDITING" :: (if IsScript filename then ["INTERACTIVE"] else ["COMPILED"]) - let lexbuf = UnicodeLexing.StringAsLexbuf source + let lexbuf = UnicodeLexing.SourceTextAsLexbuf(sourceText) let isLastCompiland = (IsScript filename), tcConfig.target.IsExe // The root compiland is last in the list of compilands. ParseOneInputLexbuf (tcConfig, lexResourceManager, defines, lexbuf, filename, isLastCompiland, errorLogger) @@ -5101,7 +5102,7 @@ module private ScriptPreprocessClosure = | None -> new StreamReader(stream, true) | Some (n: int) -> new StreamReader(stream, Encoding.GetEncoding(n)) let source = reader.ReadToEnd() - [ClosureSource(filename, m, source, parseRequired)] + [ClosureSource(filename, m, SourceText.ofString source, parseRequired)] with e -> errorRecovery e m [] @@ -5129,7 +5130,7 @@ module private ScriptPreprocessClosure = let tcConfig = ref tcConfig let observedSources = Observed() - let rec loop (ClosureSource(filename, m, source, parseRequired)) = + let rec loop (ClosureSource(filename, m, sourceText, parseRequired)) = [ if not (observedSources.HaveSeen(filename)) then observedSources.SetSeen(filename) //printfn "visiting %s" filename @@ -5137,7 +5138,7 @@ module private ScriptPreprocessClosure = let parseResult, parseDiagnostics = let errorLogger = CapturingErrorLogger("FindClosureParse") use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _ -> errorLogger) - let result = ParseScriptText (filename, source, !tcConfig, codeContext, lexResourceManager, errorLogger) + let result = ParseScriptText (filename, sourceText, !tcConfig, codeContext, lexResourceManager, errorLogger) result, errorLogger.Diagnostics match parseResult with @@ -5237,7 +5238,7 @@ module private ScriptPreprocessClosure = result /// Given source text, find the full load closure. Used from service.fs, when editing a script file - let GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) = + let GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) = // Resolve the basic references such as FSharp.Core.dll first, before processing any #I directives in the script // // This is tries to mimic the action of running the script in F# Interactive - the initial context for scripting is created @@ -5250,7 +5251,7 @@ module private ScriptPreprocessClosure = let tcConfig = CreateScriptTextTcConfig(legacyReferenceResolver, defaultFSharpBinariesDir, filename, codeContext, useSimpleResolution, useFsiAuxLib, Some references0, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) - let closureSources = [ClosureSource(filename, range0, source, true)] + let closureSources = [ClosureSource(filename, range0, sourceText, true)] let closureFiles, tcConfig = FindClosureFiles(closureSources, tcConfig, codeContext, lexResourceManager) GetLoadClosure(ctok, filename, closureFiles, tcConfig, codeContext) @@ -5268,9 +5269,9 @@ type LoadClosure with // /// A temporary TcConfig is created along the way, is why this routine takes so many arguments. We want to be sure to use exactly the /// same arguments as the rest of the application. - static member ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename:string, source:string, codeContext, useSimpleResolution:bool, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) : LoadClosure = + static member ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename:string, sourceText:ISourceText, codeContext, useSimpleResolution:bool, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) : LoadClosure = use unwindBuildPhase = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse - ScriptPreprocessClosure.GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) + ScriptPreprocessClosure.GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) /// Analyze a set of script files and find the closure of their references. The resulting references are then added to the given TcConfig. /// Used from fsi.fs and fsc.fs, for #load and command line. diff --git a/src/fsharp/CompileOps.fsi b/src/fsharp/CompileOps.fsi index eb371fa5927..0e9f53c843e 100755 --- a/src/fsharp/CompileOps.fsi +++ b/src/fsharp/CompileOps.fsi @@ -11,6 +11,7 @@ open Microsoft.FSharp.Compiler.AbstractIL.IL open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.TypeChecker open Microsoft.FSharp.Compiler.Range open Microsoft.FSharp.Compiler.Ast @@ -796,7 +797,7 @@ type LoadClosure = // /// A temporary TcConfig is created along the way, is why this routine takes so many arguments. We want to be sure to use exactly the /// same arguments as the rest of the application. - static member ComputeClosureOfScriptText: CompilationThreadToken * legacyReferenceResolver: ReferenceResolver.Resolver * defaultFSharpBinariesDir: string * filename: string * source: string * implicitDefines:CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) * assumeDotNetFramework: bool * tryGetMetadataSnapshot: ILReaderTryGetMetadataSnapshot * reduceMemoryUsage: ReduceMemoryFlag -> LoadClosure + static member ComputeClosureOfScriptText: CompilationThreadToken * legacyReferenceResolver: ReferenceResolver.Resolver * defaultFSharpBinariesDir: string * filename: string * sourceText: ISourceText * implicitDefines:CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) * assumeDotNetFramework: bool * tryGetMetadataSnapshot: ILReaderTryGetMetadataSnapshot * reduceMemoryUsage: ReduceMemoryFlag -> LoadClosure /// Analyze a set of script files and find the closure of their references. The resulting references are then added to the given TcConfig. /// Used from fsi.fs and fsc.fs, for #load and command line. diff --git a/src/fsharp/NameResolution.fs b/src/fsharp/NameResolution.fs index 43c15418490..d3d1c7db881 100644 --- a/src/fsharp/NameResolution.fs +++ b/src/fsharp/NameResolution.fs @@ -5,7 +5,8 @@ module internal Microsoft.FSharp.Compiler.NameResolution open Internal.Utilities -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Range open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.ErrorLogger @@ -1251,7 +1252,7 @@ type OpenDeclaration = IsOwnNamespace = isOwnNamespace } type FormatStringCheckContext = - { Source: string + { SourceText: ISourceText LineStartPositions: int[] } /// An abstract type for reporting the results of name resolution and type checking. @@ -1261,7 +1262,7 @@ type ITypecheckResultsSink = abstract NotifyNameResolution : pos * Item * Item * TyparInst * ItemOccurence * Tastops.DisplayEnv * NameResolutionEnv * AccessorDomain * range * bool -> unit abstract NotifyFormatSpecifierLocation : range * int -> unit abstract NotifyOpenDeclaration : OpenDeclaration -> unit - abstract CurrentSource : string option + abstract CurrentSourceText : ISourceText option abstract FormatStringCheckContext : FormatStringCheckContext option let (|ValRefOfProp|_|) (pi : PropInfo) = pi.ArbitraryValRef @@ -1513,7 +1514,7 @@ type TcSymbolUses(g, capturedNameResolutions : ResizeArray() let capturedExprTypings = ResizeArray<_>() let capturedNameResolutions = ResizeArray<_>() @@ -1537,18 +1538,18 @@ type TcResultsSinkImpl(g, ?source: string) = let formatStringCheckContext = lazy - source |> Option.map (fun source -> + sourceText |> Option.map (fun sourceText -> let positions = [| yield 0 - for i in 1..source.Length do - let c = source.[i-1] - if c = '\r' && i < source.Length && source.[i] = '\n' then () + for i in 1..sourceText.Length do + let c = sourceText.[i-1] + if c = '\r' && i < sourceText.Length && sourceText.[i] = '\n' then () elif c = '\r' then yield i elif c = '\n' then yield i - yield source.Length + yield sourceText.Length |] - { Source = source + { SourceText = sourceText LineStartPositions = positions }) member this.GetResolutions() = @@ -1603,7 +1604,7 @@ type TcResultsSinkImpl(g, ?source: string) = member sink.NotifyOpenDeclaration(openDeclaration) = capturedOpenDeclarations.Add(openDeclaration) - member sink.CurrentSource = source + member sink.CurrentSourceText = sourceText member sink.FormatStringCheckContext = formatStringCheckContext.Value diff --git a/src/fsharp/NameResolution.fsi b/src/fsharp/NameResolution.fsi index 10fe03ff0b1..8912c8946a5 100755 --- a/src/fsharp/NameResolution.fsi +++ b/src/fsharp/NameResolution.fsi @@ -3,6 +3,7 @@ module internal Microsoft.FSharp.Compiler.NameResolution open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AccessibilityLogic open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.Infos @@ -347,7 +348,7 @@ type internal OpenDeclaration = /// Source text and an array of line end positions, used for format string parsing type FormatStringCheckContext = { /// Source text - Source: string + SourceText: ISourceText /// Array of line start positions LineStartPositions: int[] } @@ -370,7 +371,7 @@ type ITypecheckResultsSink = abstract NotifyOpenDeclaration : OpenDeclaration -> unit /// Get the current source - abstract CurrentSource : string option + abstract CurrentSourceText : ISourceText option /// Cached line-end normalized source text and an array of line end positions, used for format string parsing abstract FormatStringCheckContext : FormatStringCheckContext option @@ -379,7 +380,7 @@ type ITypecheckResultsSink = type internal TcResultsSinkImpl = /// Create a TcResultsSinkImpl - new : tcGlobals : TcGlobals * ?source:string -> TcResultsSinkImpl + new : tcGlobals : TcGlobals * ?sourceText: ISourceText -> TcResultsSinkImpl /// Get all the resolutions reported to the sink member GetResolutions : unit -> TcResolutions diff --git a/src/fsharp/TypeChecker.fs b/src/fsharp/TypeChecker.fs index 495e0d4c470..3b9c9e07647 100755 --- a/src/fsharp/TypeChecker.fs +++ b/src/fsharp/TypeChecker.fs @@ -1860,7 +1860,7 @@ let MakeAndPublishSimpleVals cenv env m names mergeNamesInOneNameresEnv = member this.NotifyExprHasType(_, _, _, _, _, _) = assert false // no expr typings in MakeSimpleVals member this.NotifyFormatSpecifierLocation(_, _) = () member this.NotifyOpenDeclaration(_) = () - member this.CurrentSource = None + member this.CurrentSourceText = None member this.FormatStringCheckContext = None } use _h = WithNewTypecheckResultsSink(sink, cenv.tcSink) diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index b6013c0342d..9dfdaf32df7 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -6,6 +6,7 @@ module internal Microsoft.FSharp.Compiler.UnicodeLexing // Functions for Unicode char-based lexing (new code). // +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Internal.Utilities open System.IO @@ -19,6 +20,9 @@ let StringAsLexbuf (s:string) : Lexbuf = let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf = LexBuffer<_>.FromFunction bufferFiller + +let SourceTextAsLexbuf sourceText = + LexBuffer.FromSourceText(sourceText) // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure // uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold diff --git a/src/fsharp/UnicodeLexing.fsi b/src/fsharp/UnicodeLexing.fsi index 58f6828c95e..a2c2c18fed6 100644 --- a/src/fsharp/UnicodeLexing.fsi +++ b/src/fsharp/UnicodeLexing.fsi @@ -2,6 +2,7 @@ module internal Microsoft.FSharp.Compiler.UnicodeLexing +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Text open Internal.Utilities.Text.Lexing @@ -9,3 +10,4 @@ type Lexbuf = LexBuffer val internal StringAsLexbuf : string -> Lexbuf val public FunctionAsLexbuf : (char [] * int * int -> int) -> Lexbuf val public UnicodeFileAsLexbuf :string * int option * (*retryLocked*) bool -> Lexbuf +val public SourceTextAsLexbuf : ISourceText -> Lexbuf diff --git a/src/fsharp/fsi/fsi.fs b/src/fsharp/fsi/fsi.fs index b2df5102032..07ddd0e395a 100644 --- a/src/fsharp/fsi/fsi.fs +++ b/src/fsharp/fsi/fsi.fs @@ -20,6 +20,7 @@ open System.Threading open System.Reflection open System.Runtime.CompilerServices open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL open Microsoft.FSharp.Compiler.AbstractIL.Diagnostics open Microsoft.FSharp.Compiler.AbstractIL.IL @@ -2350,7 +2351,7 @@ type internal FsiInteractionProcessor let tcConfig = TcConfig.Create(tcConfigB,validate=false) let fsiInteractiveChecker = FsiInteractiveChecker(legacyReferenceResolver, checker, tcConfig, istate.tcGlobals, istate.tcImports, istate.tcState) - fsiInteractiveChecker.ParseAndCheckInteraction(ctok, text) + fsiInteractiveChecker.ParseAndCheckInteraction(ctok, SourceText.ofString text) //---------------------------------------------------------------------------- diff --git a/src/fsharp/pars.fsy b/src/fsharp/pars.fsy index 9e75787d29f..de42111b414 100644 --- a/src/fsharp/pars.fsy +++ b/src/fsharp/pars.fsy @@ -3857,9 +3857,9 @@ parenExpr: arbExpr("parenExpr2rbcs", lhsm) } | LPAREN OBLOCKEND_COMING_SOON - { reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnmatchedParen()) - let lhsm = unionRangeWithPos (rhs parseState 1) (rhs parseState 2).Start - arbExpr("parenExpr2obecs", lhsm) } + { let lparenRange = (rhs parseState 1) + reportParseErrorAt lparenRange (FSComp.SR.parsUnmatchedParen()) + SynExpr.Paren(arbExpr("parenExpr2obecs", lparenRange.EndRange), lparenRange, None, lparenRange) } | LPAREN recover %prec prec_atomexpr_lparen_error { reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnmatchedParen()) diff --git a/src/fsharp/service/ServiceParamInfoLocations.fs b/src/fsharp/service/ServiceParamInfoLocations.fs index e0e9affd2d5..7295e2e08e6 100755 --- a/src/fsharp/service/ServiceParamInfoLocations.fs +++ b/src/fsharp/service/ServiceParamInfoLocations.fs @@ -83,7 +83,7 @@ module internal NoteworthyParamInfoLocationsImpl = let inner = traverseSynExpr synExpr match inner with | None -> - if AstTraversal.rangeContainsPosEdgesExclusive parenRange pos then + if AstTraversal.rangeContainsPosLeftEdgeExclusiveAndRightEdgeInclusive parenRange pos then Found (parenRange.Start, [(parenRange.End, getNamedParamName synExpr)], rpRangeOpt.IsSome), None else NotFound, None diff --git a/src/fsharp/service/ServiceParseTreeWalk.fs b/src/fsharp/service/ServiceParseTreeWalk.fs index 660528e03ee..f84f84bb993 100755 --- a/src/fsharp/service/ServiceParseTreeWalk.fs +++ b/src/fsharp/service/ServiceParseTreeWalk.fs @@ -27,6 +27,8 @@ module public AstTraversal = // treat ranges as though they are fully open: (,) let rangeContainsPosEdgesExclusive (m1:range) p = posGt p m1.Start && posGt m1.End p + let rangeContainsPosLeftEdgeExclusiveAndRightEdgeInclusive (m1:range) p = posGt p m1.Start && posGeq m1.End p + /// used to track route during traversal AST [] type TraverseStep = diff --git a/src/fsharp/service/ServiceXmlDocParser.fs b/src/fsharp/service/ServiceXmlDocParser.fs index 0cfd5ec6b7b..9191637baa9 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fs +++ b/src/fsharp/service/ServiceXmlDocParser.fs @@ -2,6 +2,8 @@ namespace Microsoft.FSharp.Compiler.SourceCodeServices +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library /// Represent an Xml documentation block in source code @@ -38,10 +40,10 @@ module XmlDocParsing = | SynPat.InstanceMember _ | SynPat.FromParseError _ -> [] - let getXmlDocablesImpl(sourceCodeLinesOfTheFile: string [], input: ParsedInput option) = + let getXmlDocablesImpl(sourceText: ISourceText, input: ParsedInput option) = let indentOf (lineNum: int) = let mutable i = 0 - let line = sourceCodeLinesOfTheFile.[lineNum-1] // -1 because lineNum reported by xmldocs are 1-based, but array is 0-based + let line = sourceText.GetLineString(lineNum-1) // -1 because lineNum reported by xmldocs are 1-based, but array is 0-based while i < line.Length && line.Chars(i) = ' ' do i <- i + 1 i @@ -184,7 +186,7 @@ module XmlDocComment = res module XmlDocParser = + /// Get the list of Xml documentation from current source code - let getXmlDocables (sourceCodeOfTheFile, input) = - let sourceCodeLinesOfTheFile = String.getLines sourceCodeOfTheFile - XmlDocParsing.getXmlDocablesImpl (sourceCodeLinesOfTheFile, input) \ No newline at end of file + let getXmlDocables (sourceText: ISourceText, input) = + XmlDocParsing.getXmlDocablesImpl (sourceText, input) \ No newline at end of file diff --git a/src/fsharp/service/ServiceXmlDocParser.fsi b/src/fsharp/service/ServiceXmlDocParser.fsi index 7ba0ae6e7ba..80c509ce8bf 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fsi +++ b/src/fsharp/service/ServiceXmlDocParser.fsi @@ -3,6 +3,7 @@ namespace Microsoft.FSharp.Compiler.SourceCodeServices open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Range open Microsoft.FSharp.Compiler.Ast @@ -16,6 +17,7 @@ module public XmlDocComment = val isBlank : string -> int option module public XmlDocParser = + /// Get the list of Xml documentation from current source code - val getXmlDocables : sourceCodeOfTheFile : string * input : Ast.ParsedInput option -> XmlDocable list + val getXmlDocables : ISourceText * input: Ast.ParsedInput option -> XmlDocable list \ No newline at end of file diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index d8eb7442b4a..7a0e9f2f69d 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -14,7 +14,8 @@ open System.Reflection open System.Text open Microsoft.FSharp.Core.Printf -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL open Microsoft.FSharp.Compiler.AbstractIL.IL open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader @@ -1469,23 +1470,14 @@ type FSharpParsingOptions = module internal Parser = - // We'll need number of lines for adjusting error messages at EOF - let GetFileInfoForLastLineErrors (source: string) = - // number of lines in the source file - let lastLine = (source |> Seq.sumBy (fun c -> if c = '\n' then 1 else 0)) + 1 - // length of the last line - let lastLineLength = source.Length - source.LastIndexOf("\n",StringComparison.Ordinal) - 1 - lastLine, lastLineLength - - /// Error handler for parsing & type checking while processing a single file - type ErrorHandler(reportErrors, mainInputFileName, errorSeverityOptions: FSharpErrorSeverityOptions, source) = + type ErrorHandler(reportErrors, mainInputFileName, errorSeverityOptions: FSharpErrorSeverityOptions, sourceText: ISourceText) = let mutable options = errorSeverityOptions let errorsAndWarningsCollector = new ResizeArray<_>() let mutable errorCount = 0 // We'll need number of lines for adjusting error messages at EOF - let fileInfo = GetFileInfoForLastLineErrors source + let fileInfo = sourceText.GetLastCharacterPosition() // This function gets called whenever an error happens during parsing or checking let diagnosticSink sev (exn: PhasedDiagnostic) = @@ -1545,12 +1537,10 @@ module internal Parser = let tokenizer = LexFilter.LexFilter(lightSyntaxStatus, options.CompilingFsLib, Lexer.token lexargs true, lexbuf) tokenizer.Lexer - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + let createLexbuf sourceText = + UnicodeLexing.SourceTextAsLexbuf(sourceText) - let matchBraces(source, fileName, options: FSharpParsingOptions, userOpName: string) = + let matchBraces(sourceText, fileName, options: FSharpParsingOptions, userOpName: string) = let delayedLogger = CapturingErrorLogger("matchBraces") use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _ -> delayedLogger) use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse @@ -1563,8 +1553,8 @@ module internal Parser = use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse let matchingBraces = new ResizeArray<_>() - Lexhelp.usingLexbufForParsing(UnicodeLexing.StringAsLexbuf(addNewLine source), fileName) (fun lexbuf -> - let errHandler = ErrorHandler(false, fileName, options.ErrorSeverityOptions, source) + Lexhelp.usingLexbufForParsing(createLexbuf sourceText, fileName) (fun lexbuf -> + let errHandler = ErrorHandler(false, fileName, options.ErrorSeverityOptions, sourceText) let lexfun = createLexerFunction fileName options lexbuf errHandler let parenTokensBalance t1 t2 = match t1, t2 with @@ -1592,14 +1582,14 @@ module internal Parser = matchBraces []) matchingBraces.ToArray() - let parseFile(source, fileName, options: FSharpParsingOptions, userOpName: string) = + let parseFile(sourceText: ISourceText, fileName, options: FSharpParsingOptions, userOpName: string) = Trace.TraceInformation("FCS: {0}.{1} ({2})", userOpName, "parseFile", fileName) - let errHandler = new ErrorHandler(true, fileName, options.ErrorSeverityOptions, source) + let errHandler = new ErrorHandler(true, fileName, options.ErrorSeverityOptions, sourceText) use unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _oldLogger -> errHandler.ErrorLogger) use unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse let parseResult = - Lexhelp.usingLexbufForParsing(UnicodeLexing.StringAsLexbuf(addNewLine source), fileName) (fun lexbuf -> + Lexhelp.usingLexbufForParsing(createLexbuf sourceText, fileName) (fun lexbuf -> let lexfun = createLexerFunction fileName options lexbuf errHandler let isLastCompiland = fileName.Equals(options.LastFileName, StringComparison.CurrentCultureIgnoreCase) || @@ -1617,7 +1607,7 @@ module internal Parser = // Type check a single file against an initial context, gleaning both errors and intellisense information. let CheckOneFile (parseResults: FSharpParseFileResults, - source: string, + sourceText: ISourceText, mainInputFileName: string, projectFileName: string, tcConfig: TcConfig, @@ -1643,7 +1633,7 @@ module internal Parser = // Run the type checker... | Some parsedMainInput -> // Initialize the error handler - let errHandler = new ErrorHandler(true, mainInputFileName, tcConfig.errorSeverityOptions, source) + let errHandler = new ErrorHandler(true, mainInputFileName, tcConfig.errorSeverityOptions, sourceText) use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _oldLogger -> errHandler.ErrorLogger) use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.TypeCheck @@ -1720,7 +1710,8 @@ module internal Parser = tcState.NiceNameGenerator.Reset() // Typecheck the real input. - let sink = TcResultsSinkImpl(tcGlobals, source = source) + let sink = TcResultsSinkImpl(tcGlobals, sourceText = sourceText) + let! ct = Async.CancellationToken let! resOpt = @@ -2197,17 +2188,17 @@ module Helpers = && FSharpProjectOptions.UseSameProject(o1,o2) /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. parsing - let AreSameForParsing((fileName1: string, source1: string, options1), (fileName2, source2, options2)) = - fileName1 = fileName2 && options1 = options2 && source1 = source2 + let AreSameForParsing((fileName1: string, source1: ISourceText, options1), (fileName2, source2, options2)) = + fileName1 = fileName2 && options1 = options2 && source1.ContentEquals(source2) let AreSimilarForParsing((fileName1, _, _), (fileName2, _, _)) = fileName1 = fileName2 /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. checking - let AreSameForChecking3((fileName1: string, source1: string, options1: FSharpProjectOptions), (fileName2, source2, options2)) = + let AreSameForChecking3((fileName1: string, source1: ISourceText, options1: FSharpProjectOptions), (fileName2, source2, options2)) = (fileName1 = fileName2) && FSharpProjectOptions.AreSameForChecking(options1,options2) - && (source1 = source2) + && source1.ContentEquals(source2) /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. resource usage let AreSubsumable3((fileName1:string,_,o1:FSharpProjectOptions),(fileName2:string,_,o2:FSharpProjectOptions)) = @@ -2324,8 +2315,7 @@ module CompileHelpers = | None -> () -type FileName = string -type Source = string +type FileName = string type FilePath = string type ProjectPath = string type FileVersion = int @@ -2470,7 +2460,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC // Also keyed on source. This can only be out of date if the antecedent is out of date let checkFileInProjectCache = - MruCache + MruCache (keepStrongly=checkFileInProjectCacheSize, areSame=AreSameForChecking3, areSimilar=AreSubsumable3) @@ -2504,7 +2494,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC | Parser.TypeCheckAborted.Yes -> FSharpCheckFileAnswer.Aborted | Parser.TypeCheckAborted.No scope -> FSharpCheckFileAnswer.Succeeded(MakeCheckFileResults(filename, options, builder, scope, dependencyFiles, creationErrors, parseErrors, tcErrors)) - member bc.RecordTypeCheckFileInProjectResults(filename,options,parsingOptions,parseResults,fileVersion,priorTimeStamp,checkAnswer,source) = + member bc.RecordTypeCheckFileInProjectResults(filename,options,parsingOptions,parseResults,fileVersion,priorTimeStamp,checkAnswer,sourceText) = match checkAnswer with | None | Some FSharpCheckFileAnswer.Aborted -> () @@ -2512,22 +2502,22 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC foregroundTypeCheckCount <- foregroundTypeCheckCount + 1 parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCachePossiblyStale.Set(ltok, (filename,options),(parseResults,typedResults,fileVersion)) - checkFileInProjectCache.Set(ltok, (filename,source,options),(parseResults,typedResults,fileVersion,priorTimeStamp)) - parseFileCache.Set(ltok, (filename, source, parsingOptions), parseResults)) + checkFileInProjectCache.Set(ltok, (filename,sourceText,options),(parseResults,typedResults,fileVersion,priorTimeStamp)) + parseFileCache.Set(ltok, (filename, sourceText, parsingOptions), parseResults)) member bc.ImplicitlyStartCheckProjectInBackground(options, userOpName) = if implicitlyStartBackgroundWork then - bc.CheckProjectInBackground(options, userOpName + ".ImplicitlyStartCheckProjectInBackground") + bc.CheckProjectInBackground(options, userOpName + ".ImplicitlyStartCheckProjectInBackground") - member bc.ParseFile(filename: string, source: string, options: FSharpParsingOptions, userOpName: string) = + member bc.ParseFile(filename: string, sourceText: ISourceText, options: FSharpParsingOptions, userOpName: string) = async { - match parseCacheLock.AcquireLock(fun ltok -> parseFileCache.TryGet(ltok, (filename, source, options))) with + match parseCacheLock.AcquireLock(fun ltok -> parseFileCache.TryGet(ltok, (filename, sourceText, options))) with | Some res -> return res | None -> foregroundParseCount <- foregroundParseCount + 1 - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile(source, filename, options, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile(sourceText, filename, options, userOpName) let res = FSharpParseFileResults(parseErrors, parseTreeOpt, anyErrors, options.SourceFiles) - parseCacheLock.AcquireLock(fun ltok -> parseFileCache.Set(ltok, (filename, source, options), res)) + parseCacheLock.AcquireLock(fun ltok -> parseFileCache.Set(ltok, (filename, sourceText, options), res)) return res } @@ -2546,9 +2536,9 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC } ) - member bc.GetCachedCheckFileResult(builder: IncrementalBuilder,filename,source,options) = + member bc.GetCachedCheckFileResult(builder: IncrementalBuilder,filename,sourceText,options) = // Check the cache. We can only use cached results when there is no work to do to bring the background builder up-to-date - let cachedResults = parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCache.TryGet(ltok, (filename,source,options))) + let cachedResults = parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCache.TryGet(ltok, (filename,sourceText,options))) match cachedResults with // | Some (parseResults, checkResults, _, _) when builder.AreCheckResultsBeforeFileInProjectReady(filename) -> @@ -2577,7 +2567,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC /// 7. Releases the file "lock". member private bc.CheckOneFileImpl (parseResults: FSharpParseFileResults, - source: string, + sourceText: ISourceText, fileName: string, options: FSharpProjectOptions, textSnapshotInfo: obj option, @@ -2593,7 +2583,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC let rec loop() = async { // results may appear while we were waiting for the lock, let's recheck if it's the case - let cachedResults = bc.GetCachedCheckFileResult(builder, fileName, source, options) + let cachedResults = bc.GetCachedCheckFileResult(builder, fileName, sourceText, options) match cachedResults with | Some (_, checkResults) -> return FSharpCheckFileAnswer.Succeeded checkResults @@ -2604,11 +2594,11 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC // For scripts, this will have been recorded by GetProjectOptionsFromScript. let loadClosure = scriptClosureCacheLock.AcquireLock (fun ltok -> scriptClosureCache.TryGet (ltok, options)) let! tcErrors, tcFileResult = - Parser.CheckOneFile(parseResults, source, fileName, options.ProjectFileName, tcPrior.TcConfig, tcPrior.TcGlobals, tcPrior.TcImports, + Parser.CheckOneFile(parseResults, sourceText, fileName, options.ProjectFileName, tcPrior.TcConfig, tcPrior.TcGlobals, tcPrior.TcImports, tcPrior.TcState, loadClosure, tcPrior.TcErrors, reactorOps, (fun () -> builder.IsAlive), textSnapshotInfo, userOpName) let parsingOptions = FSharpParsingOptions.FromTcConfig(tcPrior.TcConfig, Array.ofList builder.SourceFiles, options.UseScriptResolutionRules) let checkAnswer = MakeCheckFileAnswer(fileName, tcFileResult, options, builder, Array.ofList tcPrior.TcDependencyFiles, creationErrors, parseResults.Errors, tcErrors) - bc.RecordTypeCheckFileInProjectResults(fileName, options, parsingOptions, parseResults, fileVersion, tcPrior.TimeStamp, Some checkAnswer, source) + bc.RecordTypeCheckFileInProjectResults(fileName, options, parsingOptions, parseResults, fileVersion, tcPrior.TimeStamp, Some checkAnswer, sourceText) return checkAnswer finally let dummy = ref () @@ -2624,7 +2614,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC } /// Type-check the result obtained by parsing, but only if the antecedent type checking context is available. - member bc.CheckFileInProjectAllowingStaleCachedResults(parseResults: FSharpParseFileResults, filename, fileVersion, source, options, textSnapshotInfo: obj option, userOpName) = + member bc.CheckFileInProjectAllowingStaleCachedResults(parseResults: FSharpParseFileResults, filename, fileVersion, sourceText, options, textSnapshotInfo: obj option, userOpName) = let execWithReactorAsync action = reactor.EnqueueAndAwaitOpAsync(userOpName, "CheckFileInProjectAllowingStaleCachedResults ", filename, action) async { try @@ -2639,7 +2629,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC match incrementalBuildersCache.TryGetAny (ctok, options) with | Some (Some builder, creationErrors, _) -> - match bc.GetCachedCheckFileResult(builder, filename, source, options) with + match bc.GetCachedCheckFileResult(builder, filename, sourceText, options) with | Some (_, checkResults) -> return Some (builder, creationErrors, Some (FSharpCheckFileAnswer.Succeeded checkResults)) | _ -> return Some (builder, creationErrors, None) | _ -> return None // the builder wasn't ready @@ -2659,7 +2649,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC match tcPrior with | Some tcPrior -> - let! checkResults = bc.CheckOneFileImpl(parseResults, source, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) + let! checkResults = bc.CheckOneFileImpl(parseResults, sourceText, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) return Some checkResults | None -> return None // the incremental builder was not up to date finally @@ -2667,7 +2657,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC } /// Type-check the result obtained by parsing. Force the evaluation of the antecedent type checking context if needed. - member bc.CheckFileInProject(parseResults: FSharpParseFileResults, filename, fileVersion, source, options, textSnapshotInfo, userOpName) = + member bc.CheckFileInProject(parseResults: FSharpParseFileResults, filename, fileVersion, sourceText, options, textSnapshotInfo, userOpName) = let execWithReactorAsync action = reactor.EnqueueAndAwaitOpAsync(userOpName, "CheckFileInProject", filename, action) async { try @@ -2679,7 +2669,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC | None -> return FSharpCheckFileAnswer.Succeeded (MakeCheckFileResultsEmpty(filename, creationErrors)) | Some builder -> // Check the cache. We can only use cached results when there is no work to do to bring the background builder up-to-date - let cachedResults = bc.GetCachedCheckFileResult(builder, filename, source, options) + let cachedResults = bc.GetCachedCheckFileResult(builder, filename, sourceText, options) match cachedResults with | Some (_, checkResults) -> return FSharpCheckFileAnswer.Succeeded checkResults @@ -2688,14 +2678,14 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC let! tcPrior = execWithReactorAsync <| fun ctok -> builder.GetCheckResultsBeforeFileInProject (ctok, filename) let parseTreeOpt = parseResults.ParseTree |> Option.map builder.DeduplicateParsedInputModuleNameInProject let parseResultsAterDeDuplication = FSharpParseFileResults(parseResults.Errors, parseTreeOpt, parseResults.ParseHadErrors, parseResults.DependencyFiles) - let! checkAnswer = bc.CheckOneFileImpl(parseResultsAterDeDuplication, source, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) + let! checkAnswer = bc.CheckOneFileImpl(parseResultsAterDeDuplication, sourceText, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) return checkAnswer finally bc.ImplicitlyStartCheckProjectInBackground(options, userOpName) } /// Parses and checks the source file and returns untyped AST and check results. - member bc.ParseAndCheckFileInProject (filename:string, fileVersion, source, options:FSharpProjectOptions, textSnapshotInfo, userOpName) = + member bc.ParseAndCheckFileInProject (filename:string, fileVersion, sourceText, options:FSharpProjectOptions, textSnapshotInfo, userOpName) = let execWithReactorAsync action = reactor.EnqueueAndAwaitOpAsync(userOpName, "ParseAndCheckFileInProject", filename, action) async { try @@ -2716,7 +2706,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC return (parseResults, FSharpCheckFileAnswer.Aborted) | Some builder -> - let cachedResults = bc.GetCachedCheckFileResult(builder, filename, source, options) + let cachedResults = bc.GetCachedCheckFileResult(builder, filename, sourceText, options) match cachedResults with | Some (parseResults, checkResults) -> @@ -2730,10 +2720,10 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC // Do the parsing. let parsingOptions = FSharpParsingOptions.FromTcConfig(builder.TcConfig, Array.ofList (builder.SourceFiles), options.UseScriptResolutionRules) - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (source, filename, parsingOptions, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (sourceText, filename, parsingOptions, userOpName) let parseTreeOpt = parseTreeOpt |> Option.map builder.DeduplicateParsedInputModuleNameInProject let parseResults = FSharpParseFileResults(parseErrors, parseTreeOpt, anyErrors, builder.AllDependenciesDeprecated) - let! checkResults = bc.CheckOneFileImpl(parseResults, source, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) + let! checkResults = bc.CheckOneFileImpl(parseResults, sourceText, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) Logger.LogBlockMessageStop (filename + strGuid + "-Successful") LogCompilerFunctionId.Service_ParseAndCheckFileInProject @@ -2778,8 +2768,8 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC /// Try to get recent approximate type check results for a file. - member bc.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, source, _userOpName: string) = - match source with + member bc.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, sourceText: ISourceText option, _userOpName: string) = + match sourceText with | Some sourceText -> parseCacheLock.AcquireLock (fun ltok -> match checkFileInProjectCache.TryGet(ltok,(filename,sourceText,options)) with @@ -2829,7 +2819,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC member bc.ParseAndCheckProject(options, userOpName) = reactor.EnqueueAndAwaitOpAsync(userOpName, "ParseAndCheckProject", options.ProjectFileName, fun ctok -> bc.ParseAndCheckProjectImpl(options, ctok, userOpName)) - member bc.GetProjectOptionsFromScript(filename, source, loadedTimeStamp, otherFlags, useFsiAuxLib: bool option, assumeDotNetFramework: bool option, extraProjectInfo: obj option, optionsStamp: int64 option, userOpName) = + member bc.GetProjectOptionsFromScript(filename, sourceText, loadedTimeStamp, otherFlags, useFsiAuxLib: bool option, assumeDotNetFramework: bool option, extraProjectInfo: obj option, optionsStamp: int64 option, userOpName) = reactor.EnqueueAndAwaitOpAsync (userOpName, "GetProjectOptionsFromScript", filename, fun ctok -> cancellable { use errors = new ErrorScope() @@ -2855,7 +2845,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, - defaultFSharpBinariesDir, filename, source, + defaultFSharpBinariesDir, filename, sourceText, CodeContext.Editing, useSimpleResolution, useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=tryGetMetadataSnapshot, @@ -3018,37 +3008,36 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten member ic.ReferenceResolver = legacyReferenceResolver - member ic.MatchBraces(filename, source, options: FSharpParsingOptions, ?userOpName: string) = + member ic.MatchBraces(filename, sourceText, options: FSharpParsingOptions, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" async { - match braceMatchCache.TryGet(AssumeAnyCallerThreadWithoutEvidence(), (filename, source, options)) with + match braceMatchCache.TryGet(AssumeAnyCallerThreadWithoutEvidence(), (filename, sourceText, options)) with | Some res -> return res | None -> - let res = Parser.matchBraces(source, filename, options, userOpName) - braceMatchCache.Set(AssumeAnyCallerThreadWithoutEvidence(), (filename, source, options), res) + let res = Parser.matchBraces(sourceText, filename, options, userOpName) + braceMatchCache.Set(AssumeAnyCallerThreadWithoutEvidence(), (filename, sourceText, options), res) return res } + member ic.MatchBraces(filename, source: string, options: FSharpProjectOptions, ?userOpName: string) = + let userOpName = defaultArg userOpName "Unknown" + let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) + ic.MatchBraces(filename, SourceText.ofString source, parsingOptions, userOpName) + member ic.GetParsingOptionsFromProjectOptions(options): FSharpParsingOptions * _ = let sourceFiles = List.ofArray options.SourceFiles let argv = List.ofArray options.OtherOptions ic.GetParsingOptionsFromCommandLineArgs(sourceFiles, argv, options.UseScriptResolutionRules) - member ic.MatchBraces(filename, source, options: FSharpProjectOptions, ?userOpName: string) = - let userOpName = defaultArg userOpName "Unknown" - let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) - ic.MatchBraces(filename, source, parsingOptions, userOpName) - - member ic.ParseFile(filename, source, options, ?userOpName: string) = + member ic.ParseFile(filename, sourceText, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() - backgroundCompiler.ParseFile(filename, source, options, userOpName) + backgroundCompiler.ParseFile(filename, sourceText, options, userOpName) - - member ic.ParseFileInProject(filename, source, options, ?userOpName: string) = + member ic.ParseFileInProject(filename, source: string, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) - ic.ParseFile(filename, source, parsingOptions, userOpName) + ic.ParseFile(filename, SourceText.ofString source, parsingOptions, userOpName) member ic.GetBackgroundParseResultsForFileInProject (filename,options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3059,9 +3048,9 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten backgroundCompiler.GetBackgroundCheckResultsForFileInProject(filename,options, userOpName) /// Try to get recent approximate type check results for a file. - member ic.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, ?source, ?userOpName: string) = + member ic.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, ?sourceText, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.TryGetRecentCheckResultsForFile(filename,options,source, userOpName) + backgroundCompiler.TryGetRecentCheckResultsForFile(filename,options,sourceText, userOpName) member ic.Compile(argv: string[], ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3191,21 +3180,21 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten /// parse including the reconstructed types in the file. member ic.CheckFileInProjectAllowingStaleCachedResults(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.CheckFileInProjectAllowingStaleCachedResults(parseResults,filename,fileVersion,source,options,textSnapshotInfo, userOpName) - + backgroundCompiler.CheckFileInProjectAllowingStaleCachedResults(parseResults,filename,fileVersion,SourceText.ofString source,options,textSnapshotInfo, userOpName) + /// Typecheck a source code file, returning a handle to the results of the /// parse including the reconstructed types in the file. - member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, sourceText:ISourceText, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() - backgroundCompiler.CheckFileInProject(parseResults,filename,fileVersion,source,options,textSnapshotInfo, userOpName) - + backgroundCompiler.CheckFileInProject(parseResults,filename,fileVersion,sourceText,options,textSnapshotInfo, userOpName) + /// Typecheck a source code file, returning a handle to the results of the /// parse including the reconstructed types in the file. - member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, sourceText:ISourceText, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() - backgroundCompiler.ParseAndCheckFileInProject(filename, fileVersion, source, options, textSnapshotInfo, userOpName) + backgroundCompiler.ParseAndCheckFileInProject(filename, fileVersion, sourceText, options, textSnapshotInfo, userOpName) member ic.ParseAndCheckProject(options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3217,9 +3206,9 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten backgroundCompiler.KeepProjectAlive(options, userOpName) /// For a given script file, get the ProjectOptions implied by the #load closure - member ic.GetProjectOptionsFromScript(filename, source, ?loadedTimeStamp, ?otherFlags, ?useFsiAuxLib, ?assumeDotNetFramework, ?extraProjectInfo: obj, ?optionsStamp: int64, ?userOpName: string) = + member ic.GetProjectOptionsFromScript(filename, sourceText, ?loadedTimeStamp, ?otherFlags, ?useFsiAuxLib, ?assumeDotNetFramework, ?extraProjectInfo: obj, ?optionsStamp: int64, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.GetProjectOptionsFromScript(filename, source, loadedTimeStamp, otherFlags, useFsiAuxLib, assumeDotNetFramework, extraProjectInfo, optionsStamp, userOpName) + backgroundCompiler.GetProjectOptionsFromScript(filename, sourceText, loadedTimeStamp, otherFlags, useFsiAuxLib, assumeDotNetFramework, extraProjectInfo, optionsStamp, userOpName) member ic.GetProjectOptionsFromCommandLineArgs(projectFileName, argv, ?loadedTimeStamp, ?extraProjectInfo: obj) = let loadedTimeStamp = defaultArg loadedTimeStamp DateTime.MaxValue // Not 'now', we don't want to force reloading @@ -3312,13 +3301,13 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten type FsiInteractiveChecker(legacyReferenceResolver, reactorOps: IReactorOperations, tcConfig: TcConfig, tcGlobals, tcImports, tcState) = let keepAssemblyContents = false - member __.ParseAndCheckInteraction (ctok, source, ?userOpName: string) = + member __.ParseAndCheckInteraction (ctok, sourceText: ISourceText, ?userOpName: string) = async { let userOpName = defaultArg userOpName "Unknown" let filename = Path.Combine(tcConfig.implicitIncludeDir, "stdin.fsx") // Note: projectSourceFiles is only used to compute isLastCompiland, and is ignored if Build.IsScript(mainInputFileName) is true (which it is in this case). let parsingOptions = FSharpParsingOptions.FromTcConfig(tcConfig, [| filename |], true) - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (source, filename, parsingOptions, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (sourceText, filename, parsingOptions, userOpName) let dependencyFiles = [| |] // interactions have no dependencies let parseResults = FSharpParseFileResults(parseErrors, parseTreeOpt, parseHadErrors = anyErrors, dependencyFiles = dependencyFiles) @@ -3330,8 +3319,8 @@ type FsiInteractiveChecker(legacyReferenceResolver, reactorOps: IReactorOperatio let fsiCompilerOptions = CompileOptions.GetCoreFsiCompilerOptions tcConfigB CompileOptions.ParseCompilerOptions (ignore, fsiCompilerOptions, [ ]) - let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, CodeContext.Editing, tcConfig.useSimpleResolution, tcConfig.useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=(fun _ -> None), reduceMemoryUsage=reduceMemoryUsage) - let! tcErrors, tcFileResult = Parser.CheckOneFile(parseResults, source, filename, "project", tcConfig, tcGlobals, tcImports, tcState, Some loadClosure, backgroundDiagnostics, reactorOps, (fun () -> true), None, userOpName) + let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, CodeContext.Editing, tcConfig.useSimpleResolution, tcConfig.useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=(fun _ -> None), reduceMemoryUsage=reduceMemoryUsage) + let! tcErrors, tcFileResult = Parser.CheckOneFile(parseResults, sourceText, filename, "project", tcConfig, tcGlobals, tcImports, tcState, Some loadClosure, backgroundDiagnostics, reactorOps, (fun () -> true), None, userOpName) return match tcFileResult with diff --git a/src/fsharp/service/service.fsi b/src/fsharp/service/service.fsi index 2d07c235614..82efb201c53 100755 --- a/src/fsharp/service/service.fsi +++ b/src/fsharp/service/service.fsi @@ -14,7 +14,8 @@ open Microsoft.FSharp.Compiler.AbstractIL open Microsoft.FSharp.Compiler.AbstractIL.IL open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.Driver open Microsoft.FSharp.Compiler.ErrorLogger @@ -379,10 +380,10 @@ type public FSharpChecker = /// /// /// The filename for the file, used to help caching of results. - /// The full source for the file. + /// The full source for the file. /// Parsing options for the project or script. /// An optional string used for tracing compiler operations associated with this request. - member MatchBraces: filename: string * source: string * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> + member MatchBraces: filename: string * sourceText: ISourceText * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> /// /// Parse a source code file, returning information about brace matching in the file. @@ -402,10 +403,10 @@ type public FSharpChecker = /// /// /// The filename for the file. - /// The full source for the file. + /// The full source for the file. /// Parsing options for the project or script. /// An optional string used for tracing compiler operations associated with this request. - member ParseFile: filename: string * source: string * options: FSharpParsingOptions * ?userOpName: string -> Async + member ParseFile: filename: string * sourceText: ISourceText * options: FSharpParsingOptions * ?userOpName: string -> Async /// /// Parse a source code file, returning a handle that can be used for obtaining navigation bar information @@ -460,7 +461,7 @@ type public FSharpChecker = /// The results of ParseFile for this file. /// The name of the file in the project whose source is being checked. /// An integer that can be used to indicate the version of the file. This will be returned by TryGetRecentCheckResultsForFile when looking up the file. - /// The full source for the file. + /// The full source for the file. /// The options for the project or script. /// /// An item passed back to 'hasTextChangedSinceLastTypecheck' (from some calls made on 'FSharpCheckFileResults') to help determine if @@ -468,7 +469,7 @@ type public FSharpChecker = /// can be used to marginally increase accuracy of intellisense results in some situations. /// /// An optional string used for tracing compiler operations associated with this request. - member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * source: string * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * sourceText: ISourceText * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async /// /// @@ -492,7 +493,7 @@ type public FSharpChecker = /// can be used to marginally increase accuracy of intellisense results in some situations. /// /// An optional string used for tracing compiler operations associated with this request. - member ParseAndCheckFileInProject : filename: string * fileversion: int * source: string * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + member ParseAndCheckFileInProject : filename: string * fileversion: int * sourceText: ISourceText * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async /// /// Parse and typecheck all files in a project. @@ -523,7 +524,7 @@ type public FSharpChecker = /// so that an 'unload' and 'reload' action will cause the script to be considered as a new project, /// so that references are re-resolved. /// An optional string used for tracing compiler operations associated with this request. - member GetProjectOptionsFromScript : filename: string * source: string * ?loadedTimeStamp: DateTime * ?otherFlags: string[] * ?useFsiAuxLib: bool * ?assumeDotNetFramework: bool * ?extraProjectInfo: obj * ?optionsStamp: int64 * ?userOpName: string -> Async + member GetProjectOptionsFromScript : filename: string * sourceText: ISourceText * ?loadedTimeStamp: DateTime * ?otherFlags: string[] * ?useFsiAuxLib: bool * ?assumeDotNetFramework: bool * ?extraProjectInfo: obj * ?optionsStamp: int64 * ?userOpName: string -> Async /// /// Get the FSharpProjectOptions implied by a set of command line arguments. @@ -621,9 +622,9 @@ type public FSharpChecker = /// /// The filename for the file. /// The options for the project or script, used to determine active --define conditionals and other options relevant to parsing. - /// Optionally, specify source that must match the previous parse precisely. + /// Optionally, specify source that must match the previous parse precisely. /// An optional string used for tracing compiler operations associated with this request. - member TryGetRecentCheckResultsForFile : filename: string * options:FSharpProjectOptions * ?source: string * ?userOpName: string -> (FSharpParseFileResults * FSharpCheckFileResults * (*version*)int) option + member TryGetRecentCheckResultsForFile : filename: string * options:FSharpProjectOptions * ?sourceText: ISourceText * ?userOpName: string -> (FSharpParseFileResults * FSharpCheckFileResults * (*version*)int) option /// This function is called when the entire environment is known to have changed for reasons not encoded in the ProjectOptions of any project/compilation. member InvalidateAll : unit -> unit @@ -723,7 +724,7 @@ type internal FsiInteractiveChecker = internal new : ReferenceResolver.Resolver * ops: IReactorOperations * tcConfig: TcConfig * tcGlobals: TcGlobals * tcImports: TcImports * tcState: TcState -> FsiInteractiveChecker /// An optional string used for tracing compiler operations associated with this request. - member internal ParseAndCheckInteraction : CompilationThreadToken * source:string * ?userOpName: string -> Async + member internal ParseAndCheckInteraction : CompilationThreadToken * sourceText:ISourceText * ?userOpName: string -> Async /// Information about the compilation environment [] diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 2a49afc5799..87012685b3f 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -2,11 +2,106 @@ #nowarn "47" // recursive initialization of LexBuffer +namespace Microsoft.FSharp.Compiler.Text + +open System +open System.IO + +type ISourceText = + + abstract Item : int -> char with get + + abstract GetLineString : lineIndex: int -> string + + abstract GetLineCount : unit -> int + + abstract GetLastCharacterPosition : unit -> int * int + + abstract GetSubTextString : start: int * length: int -> string + + abstract SubTextEquals : target: string * startIndex: int -> bool + + abstract Length : int + + abstract ContentEquals : sourceText: ISourceText -> bool + + abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit + +[] +type StringText(str: string) = + + let getLines (str: string) = + use reader = new StringReader(str) + [| + let mutable line = reader.ReadLine() + while not (isNull line) do + yield line + line <- reader.ReadLine() + if str.EndsWith("\n", StringComparison.Ordinal) then + // last trailing space not returned + // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak + yield String.Empty + |] + + let getLines = + // This requires allocating and getting all the lines. + // However, likely whoever is calling it is using a different implementation of ISourceText + // So, it's ok that we do this for now. + lazy getLines str + + member __.String = str + + interface ISourceText with + + member __.Item with get index = str.[index] + + member __.GetLastCharacterPosition() = + let lines = getLines.Value + if lines.Length > 0 then + (lines.Length, lines.[lines.Length - 1].Length) + else + (0, 0) + + member __.GetLineString(lineIndex) = + getLines.Value.[lineIndex] + + member __.GetLineCount() = getLines.Value.Length + + member __.GetSubTextString(start, length) = + str.Substring(start, length) + + member __.SubTextEquals(target, startIndex) = + if startIndex < 0 || startIndex >= str.Length then + invalidArg "startIndex" "Out of range." + + if String.IsNullOrEmpty(target) then + invalidArg "target" "Is null or empty." + + let lastIndex = startIndex + target.Length + if lastIndex <= startIndex || lastIndex >= str.Length then + invalidArg "target" "Too big." + + str.IndexOf(target, startIndex, target.Length) <> -1 + + member __.Length = str.Length + + member this.ContentEquals(sourceText) = + match sourceText with + | :? StringText as sourceText when sourceText = this || sourceText.String = str -> true + | _ -> false + + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + str.CopyTo(sourceIndex, destination, destinationIndex, count) + +module SourceText = + + let ofString str = StringText(str) :> ISourceText // NOTE: the code in this file is a drop-in replacement runtime for Lexing.fs from the FsLexYacc repository namespace Internal.Utilities.Text.Lexing open Microsoft.FSharp.Core + open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Collections open System.Collections.Generic @@ -168,7 +263,23 @@ namespace Internal.Utilities.Text.Lexing LexBuffer<'Char>.FromArrayNoCopy buffer // Important: This method takes ownership of the array - static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy arr + static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy arr + + static member FromSourceText (sourceText: ISourceText) = + let mutable currentSourceIndex = 0 + LexBuffer.FromFunction(fun (chars, start, length) -> + let lengthToCopy = + if currentSourceIndex + length <= sourceText.Length then + length + else + sourceText.Length - currentSourceIndex + + if lengthToCopy <= 0 then 0 + else + sourceText.CopyTo(currentSourceIndex, chars, start, lengthToCopy) + currentSourceIndex <- currentSourceIndex + lengthToCopy + lengthToCopy + ) module GenericImplFragments = let startInterpret(lexBuffer:LexBuffer)= diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 9ee890b8d19..b968ba3605b 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -2,6 +2,33 @@ // LexBuffers are for use with automatically generated lexical analyzers, // in particular those produced by 'fslex'. + +namespace Microsoft.FSharp.Compiler.Text + +type ISourceText = + + abstract Item : int -> char with get + + abstract GetLineString : lineIndex: int -> string + + abstract GetLineCount : unit -> int + + abstract GetLastCharacterPosition : unit -> int * int + + abstract GetSubTextString : start: int * length: int -> string + + abstract SubTextEquals : target: string * startIndex: int -> bool + + abstract Length : int + + abstract ContentEquals : sourceText: ISourceText -> bool + + abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit + +module SourceText = + + val ofString : string -> ISourceText + // // NOTE: the code in this file is a drop-in replacement runtime for Lexing.fsi from the FsLexYacc repository // and is referenced by generated code for the three FsLex generated lexers in the F# compiler. @@ -9,6 +36,7 @@ namespace Internal.Utilities.Text.Lexing open System.Collections.Generic +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Core open Microsoft.FSharp.Control @@ -85,6 +113,8 @@ type internal LexBuffer<'Char> = /// Create a lex buffer that reads character or byte inputs by using the given function. static member FromFunction: ('Char[] * int * int -> int) -> LexBuffer<'Char> + /// Create a lex buffer backed by source text. + static member FromSourceText : ISourceText -> LexBuffer /// The type of tables for an unicode lexer generated by fslex.exe. [] diff --git a/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj b/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj index 7e69ab35171..bbeac7ee59f 100644 --- a/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj +++ b/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj @@ -18,6 +18,7 @@ + diff --git a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs new file mode 100644 index 00000000000..3754e75ae2d --- /dev/null +++ b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Compiler.UnitTests + +open System +open NUnit.Framework + +open Microsoft.FSharp.Compiler.Text + +[] +module SourceTextTests = + + [] + let StringText () = + let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" + let sourceText = SourceText.ofString text + + Assert.AreEqual("test", sourceText.GetLineString(0)) + Assert.AreEqual("test2", sourceText.GetLineString(1)) + Assert.AreEqual("test3", sourceText.GetLineString(2)) + Assert.AreEqual("", sourceText.GetLineString(3)) + Assert.AreEqual("test4", sourceText.GetLineString(4)) + Assert.AreEqual("test5", sourceText.GetLineString(5)) + Assert.AreEqual("test6", sourceText.GetLineString(6)) + Assert.AreEqual("", sourceText.GetLineString(7)) + Assert.AreEqual(8, sourceText.GetLineCount()) + + let (count, length) = sourceText.GetLastCharacterPosition() + Assert.AreEqual(8, count) + Assert.AreEqual(0, length) + + Assert.True(sourceText.SubTextEquals("test", 0)) + Assert.True(sourceText.SubTextEquals("test2", 5)) + Assert.True(sourceText.SubTextEquals("test3", 12)) + + Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("", 0) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals(text + text, 0) |> ignore) |> ignore + + Assert.False(sourceText.SubTextEquals("test", 1)) + Assert.False(sourceText.SubTextEquals("test", 4)) + Assert.False(sourceText.SubTextEquals("test", 11)) \ No newline at end of file diff --git a/tests/service/AssemblyContentProviderTests.fs b/tests/service/AssemblyContentProviderTests.fs index 30f11f4cdd5..b98eaf33a0c 100644 --- a/tests/service/AssemblyContentProviderTests.fs +++ b/tests/service/AssemblyContentProviderTests.fs @@ -43,7 +43,7 @@ let (=>) (source: string) (expected: string list) = // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak yield "" |] - let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, source, projectOptions) |> Async.RunSynchronously + let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, projectOptions) |> Async.RunSynchronously let checkFileResults = match checkFileAnswer with diff --git a/tests/service/AssemblyReaderShim.fs b/tests/service/AssemblyReaderShim.fs index e7fd0a48ba3..51664dd32e6 100644 --- a/tests/service/AssemblyReaderShim.fs +++ b/tests/service/AssemblyReaderShim.fs @@ -29,5 +29,5 @@ let x = 123 """ let fileName, options = Common.mkTestFileAndOptions source [| |] - Common.checker.ParseAndCheckFileInProject(fileName, 0, source, options) |> Async.RunSynchronously |> ignore + Common.checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, options) |> Async.RunSynchronously |> ignore gotRequest |> should be True diff --git a/tests/service/Common.fs b/tests/service/Common.fs index 07c2484e70a..2cfaf0c6119 100644 --- a/tests/service/Common.fs +++ b/tests/service/Common.fs @@ -57,13 +57,13 @@ type TempFile(ext, contents) = let getBackgroundParseResultsForScriptText (input) = use file = new TempFile("fsx", input) - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, input) |> Async.RunSynchronously + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) |> Async.RunSynchronously checker.GetBackgroundParseResultsForFileInProject(file.Name, checkOptions) |> Async.RunSynchronously let getBackgroundCheckResultsForScriptText (input) = use file = new TempFile("fsx", input) - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, input) |> Async.RunSynchronously + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) |> Async.RunSynchronously checker.GetBackgroundCheckResultsForFileInProject(file.Name, checkOptions) |> Async.RunSynchronously @@ -167,7 +167,7 @@ let mkTestFileAndOptions source additionalArgs = fileName, options let parseAndCheckFile fileName source options = - match checker.ParseAndCheckFileInProject(fileName, 0, source, options) |> Async.RunSynchronously with + match checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, options) |> Async.RunSynchronously with | parseResults, FSharpCheckFileAnswer.Succeeded(checkResults) -> parseResults, checkResults | _ -> failwithf "Parsing aborted unexpectedly..." @@ -181,11 +181,11 @@ let parseAndCheckScript (file, input) = let projectOptions = checker.GetProjectOptionsFromCommandLineArgs (projName, args) #else - let projectOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, input) |> Async.RunSynchronously + let projectOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) |> Async.RunSynchronously printfn "projectOptions = %A" projectOptions #endif - let parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, input, projectOptions) |> Async.RunSynchronously + let parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, projectOptions) |> Async.RunSynchronously // if parseResult.Errors.Length > 0 then // printfn "---> Parse Input = %A" input @@ -204,7 +204,7 @@ let parseSourceCode (name: string, code: string) = let dllPath = Path.Combine(location, name + ".dll") let args = mkProjectCommandLineArgs(dllPath, [filePath]) let options, errors = checker.GetParsingOptionsFromCommandLineArgs(List.ofArray args) - let parseResults = checker.ParseFile(filePath, code, options) |> Async.RunSynchronously + let parseResults = checker.ParseFile(filePath, Microsoft.FSharp.Compiler.Text.SourceText.ofString code, options) |> Async.RunSynchronously parseResults.ParseTree /// Extract range info diff --git a/tests/service/EditorTests.fs b/tests/service/EditorTests.fs index 6579a72cc1b..5fb5249e83a 100644 --- a/tests/service/EditorTests.fs +++ b/tests/service/EditorTests.fs @@ -116,8 +116,8 @@ let ``Basic cancellation test`` () = let file = "/home/user/Test.fsx" async { checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - let! checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, input) - let! parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, input, checkOptions) + let! checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) + let! parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, checkOptions) return parseResult, typedRes } |> Async.RunSynchronously |> ignore diff --git a/tests/service/InteractiveCheckerTests.fs b/tests/service/InteractiveCheckerTests.fs index fba9adcb639..f3ee91b0f12 100644 --- a/tests/service/InteractiveCheckerTests.fs +++ b/tests/service/InteractiveCheckerTests.fs @@ -45,8 +45,11 @@ let internal identsAndRanges (input: Ast.ParsedInput) = | Ast.SynModuleDecl.Attributes(_attrs, _range) -> failwith "Not implemented yet" | Ast.SynModuleDecl.HashDirective(_, _range) -> failwith "Not implemented yet" | Ast.SynModuleDecl.NamespaceFragment(moduleOrNamespace) -> extractFromModuleOrNamespace moduleOrNamespace - and extractFromModuleOrNamespace (Ast.SynModuleOrNamespace(longIdent, _, _, moduleDecls, _, _, _, range)) = - (identAndRange (longIdentToString longIdent) range) :: (moduleDecls |> List.collect extractFromModuleDecl) + and extractFromModuleOrNamespace (Ast.SynModuleOrNamespace(longIdent, _, _, moduleDecls, _, _, _, _)) = + let xs = moduleDecls |> List.collect extractFromModuleDecl + if longIdent.IsEmpty then xs + else + (identAndRange (longIdentToString longIdent) (longIdent |> List.map (fun id -> id.idRange) |> List.reduce Range.unionRanges)) :: xs match input with | Ast.ParsedInput.ImplFile(Ast.ParsedImplFileInput(_, _, _, _, _, modulesOrNamespaces, _)) -> @@ -71,7 +74,7 @@ let input = let ``Test ranges - namespace`` () = let res = parseAndExtractRanges input printfn "Test ranges - namespace, res = %A" res - res |> shouldEqual [("N", ((4, 4), (6, 0))); ("Sample", ((4, 9), (4, 15)))] + res |> shouldEqual [("N", ((2, 14), (2, 15))); ("Sample", ((4, 9), (4, 15)))] let input2 = """ @@ -84,7 +87,7 @@ let input2 = let ``Test ranges - module`` () = let res = parseAndExtractRanges input2 printfn "Test ranges - module, res = %A" res - res |> shouldEqual [("M", ((2, 4), (4, 26))); ("Sample", ((4, 9), (4, 15)))] + res |> shouldEqual [("M", ((2, 11), (2, 12))); ("Sample", ((4, 9), (4, 15)))] let input3 = """ @@ -97,4 +100,4 @@ let input3 = let ``Test ranges - global namespace`` () = let res = parseAndExtractRanges input3 printfn "Test ranges - global namespace, res = %A" res - res |> shouldEqual [("", ((4, 4), (6, 0))); ("Sample", ((4, 9), (4, 15)))] + res |> shouldEqual [("Sample", ((4, 9), (4, 15)))] diff --git a/tests/service/MultiProjectAnalysisTests.fs b/tests/service/MultiProjectAnalysisTests.fs index 0e4d38a3321..4dfe416858b 100644 --- a/tests/service/MultiProjectAnalysisTests.fs +++ b/tests/service/MultiProjectAnalysisTests.fs @@ -913,7 +913,7 @@ let ``Type provider project references should not throw exceptions`` () = //printfn "options: %A" options let fileName = __SOURCE_DIRECTORY__ + @"/data/TypeProviderConsole/Program.fs" let fileSource = File.ReadAllText(fileName) - let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, fileSource, options) |> Async.RunSynchronously + let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource, options) |> Async.RunSynchronously let fileCheckResults = match fileCheckAnswer with | FSharpCheckFileAnswer.Succeeded(res) -> res @@ -1010,7 +1010,7 @@ let ``Projects creating generated types should not utilize cross-project-referen //printfn "options: %A" options let fileName = __SOURCE_DIRECTORY__ + @"/data/TypeProvidersBug/TestConsole/Program.fs" let fileSource = File.ReadAllText(fileName) - let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, fileSource, options) |> Async.RunSynchronously + let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource, options) |> Async.RunSynchronously let fileCheckResults = match fileCheckAnswer with | FSharpCheckFileAnswer.Succeeded(res) -> res diff --git a/tests/service/PerfTests.fs b/tests/service/PerfTests.fs index 662e00c5645..ad544f39499 100644 --- a/tests/service/PerfTests.fs +++ b/tests/service/PerfTests.fs @@ -31,7 +31,7 @@ module internal Project1 = let projFileName = Path.ChangeExtension(base2, ".fsproj") let fileSources = [ for (i,f) in fileNamesI -> (f, "module M" + string i) ] for (f,text) in fileSources do File.WriteAllText(f, text) - let fileSources2 = [ for (i,f) in fileSources -> f ] + let fileSources2 = [ for (i,f) in fileSources -> Microsoft.FSharp.Compiler.Text.SourceText.ofString f ] let fileNames = [ for (_,f) in fileNamesI -> f ] let args = mkProjectCommandLineArgs (dllName, fileNames) diff --git a/tests/service/ProjectAnalysisTests.fs b/tests/service/ProjectAnalysisTests.fs index 5ea55a40505..0913c673988 100644 --- a/tests/service/ProjectAnalysisTests.fs +++ b/tests/service/ProjectAnalysisTests.fs @@ -28,7 +28,7 @@ module internal Project1 = let fileName2 = Path.ChangeExtension(base2, ".fs") let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module M type C() = @@ -39,9 +39,10 @@ let fff () = xxx + xxx type CAbbrev = C """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) - let fileSource2 = """ + let fileSource2Text = """ module N open M @@ -83,7 +84,8 @@ let mmmm1 : M.C = new M.C() // note, these don't count as uses of CA let mmmm2 : M.CAbbrev = new M.CAbbrev() // note, these don't count as uses of C """ - File.WriteAllText(fileName2, fileSource2) + let fileSource2 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource2Text + File.WriteAllText(fileName2, fileSource2Text) let fileNames = [fileName1; fileName2] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -2411,7 +2413,7 @@ module internal Project16 = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module Impl type C() = @@ -2427,9 +2429,10 @@ and F = { Field1 : int; Field2 : int } and G = Case1 | Case2 of int """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) - let sigFileSource1 = """ + let sigFileSource1Text = """ module Impl type C = @@ -2448,7 +2451,8 @@ and F = { Field1 : int; Field2 : int } and G = Case1 | Case2 of int """ - File.WriteAllText(sigFileName1, sigFileSource1) + let sigFileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString sigFileSource1Text + File.WriteAllText(sigFileName1, sigFileSource1Text) let cleanFileName a = if a = fileName1 then "file1" elif a = sigFileName1 then "sig1" else "??" let fileNames = [sigFileName1; fileName1] @@ -4506,11 +4510,12 @@ module internal Project35b = open System.IO let fileName1 = Path.ChangeExtension(Path.GetTempFileName(), ".fsx") - let fileSource1 = """ + let fileSource1Text = """ #r "System.dll" #r "notexist.dll" """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let cleanFileName a = if a = fileName1 then "file1" else "??" let fileNames = [fileName1] @@ -5153,7 +5158,7 @@ module internal ProjectBig = let projFileName = Path.ChangeExtension(base2, ".fsproj") let fileSources = [ for (i,f) in fileNamesI -> (f, "module M" + string i) ] for (f,text) in fileSources do File.WriteAllText(f, text) - let fileSources2 = [ for (i,f) in fileSources -> f ] + let fileSources2 = [ for (i,f) in fileSources -> Microsoft.FSharp.Compiler.Text.SourceText.ofString f ] let fileNames = [ for (_,f) in fileNamesI -> f ] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5286,14 +5291,14 @@ module internal ProjectLineDirectives = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module M # 10 "Test.fsy" let x = (1 = 3.0) """ - - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) let options = checker.GetProjectOptionsFromCommandLineArgs (projFileName, args) @@ -5329,11 +5334,12 @@ let ``ParseAndCheckFileResults contains ImplFile list if FSharpChecker is create let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ type A(i:int) = member x.Value = i """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5386,7 +5392,7 @@ let ``Unused opens in rec module smoke test 1``() = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module rec Module open System.Collections // unused @@ -5423,7 +5429,8 @@ type UseTheThings(i:int) = member x.UseSomeUsedModuleContainingExtensionMember() = (3).Q member x.UseSomeUsedModuleContainingUnion() = A """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5458,7 +5465,7 @@ let ``Unused opens in non rec module smoke test 1``() = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module Module open System.Collections // unused @@ -5495,7 +5502,8 @@ type UseTheThings(i:int) = member x.UseSomeUsedModuleContainingExtensionMember() = (3).Q member x.UseSomeUsedModuleContainingUnion() = A """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5530,7 +5538,7 @@ let ``Unused opens smoke test auto open``() = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ open System.Collections // unused open System.Collections.Generic // used, should not appear open FSharp.Control // unused @@ -5567,7 +5575,8 @@ type UseTheThings(i:int) = member x.UseSomeUsedModuleContainingExtensionMember() = (3).Q member x.UseSomeUsedModuleContainingUnion() = A """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) diff --git a/tests/service/ProjectOptionsTests.fs b/tests/service/ProjectOptionsTests.fs index b390f53e62b..5c5d86db3ba 100644 --- a/tests/service/ProjectOptionsTests.fs +++ b/tests/service/ProjectOptionsTests.fs @@ -508,7 +508,7 @@ let ``Test SourceFiles order for GetProjectOptionsFromScript`` () = // See #594 let scriptPath = __SOURCE_DIRECTORY__ + @"/data/ScriptProject/" + scriptName + ".fsx" let scriptSource = File.ReadAllText scriptPath let projOpts, _diagnostics = - checker.GetProjectOptionsFromScript(scriptPath, scriptSource) + checker.GetProjectOptionsFromScript(scriptPath, Microsoft.FSharp.Compiler.Text.SourceText.ofString scriptSource) |> Async.RunSynchronously projOpts.SourceFiles |> Array.map Path.GetFileNameWithoutExtension @@ -527,21 +527,21 @@ let ``Script load closure project`` () = let fileName1 = Path.GetTempPath() + Path.DirectorySeparatorChar.ToString() + "Impl.fs" let fileName2 = Path.ChangeExtension(Path.GetTempFileName(), ".fsx") - let fileSource1 = """ + let fileSource1Text = """ module ImplFile #if INTERACTIVE let x = 42 #endif """ - - let fileSource2 = """ + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + let fileSource2Text = """ #load "Impl.fs" ImplFile.x """ - - File.WriteAllText(fileName1, fileSource1) - File.WriteAllText(fileName2, fileSource2) + let fileSource2 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource2Text + File.WriteAllText(fileName1, fileSource1Text) + File.WriteAllText(fileName2, fileSource2Text) printfn "------Starting Script load closure project----" printfn "Getting project options..." diff --git a/tests/service/StructureTests.fs b/tests/service/StructureTests.fs index d87c3c18e78..9229ab6ccb5 100644 --- a/tests/service/StructureTests.fs +++ b/tests/service/StructureTests.fs @@ -60,7 +60,7 @@ let (=>) (source: string) (expectedRanges: (Range * Range) list) = reraise() [] -let ``empty file``() = "" => [ (1, 0, 2, 0), (1, 0, 2, 0) ] +let ``empty file``() = "" => [ ] [] let ``nested module``() = diff --git a/vsintegration/Utils/LanguageServiceProfiling/Program.fs b/vsintegration/Utils/LanguageServiceProfiling/Program.fs index 5413e8b9094..a131a5f65ab 100644 --- a/vsintegration/Utils/LanguageServiceProfiling/Program.fs +++ b/vsintegration/Utils/LanguageServiceProfiling/Program.fs @@ -42,6 +42,7 @@ Results look like this: *) open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.SourceCodeServices open System open System.IO @@ -89,7 +90,7 @@ let main argv = async { eprintfn "ParseAndCheckFileInProject(%s)..." options.FileToCheck let sw = Stopwatch.StartNew() - let! _, answer = checker.ParseAndCheckFileInProject(options.FileToCheck, fileVersion, File.ReadAllText options.FileToCheck, options.Options) + let! _, answer = checker.ParseAndCheckFileInProject(options.FileToCheck, fileVersion, SourceText.ofString (File.ReadAllText options.FileToCheck), options.Options) match answer with | FSharpCheckFileAnswer.Aborted -> eprintfn "Abortedin %O!" sw.Elapsed @@ -110,7 +111,7 @@ let main argv = let answers = options.FilesToCheck |> List.map (fun file -> eprintfn "doing %s" file - checker.ParseAndCheckFileInProject(file, fileVersion, File.ReadAllText file, options.Options) |> Async.RunSynchronously) + checker.ParseAndCheckFileInProject(file, fileVersion, SourceText.ofString (File.ReadAllText file), options.Options) |> Async.RunSynchronously) for _,answer in answers do match answer with | FSharpCheckFileAnswer.Aborted -> @@ -157,7 +158,7 @@ let main argv = match fileResults with | Some fileResults -> let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions(options.Options) - let! parseResult = checker.ParseFile(options.FileToCheck, getFileText(), parsingOptions) + let! parseResult = checker.ParseFile(options.FileToCheck, SourceText.ofString (getFileText()), parsingOptions) for completion in options.CompletionPositions do eprintfn "querying %A %s" completion.QualifyingNames completion.PartialName let! listInfo = diff --git a/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs b/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs index e7cd7306190..8f6d7109a2e 100644 --- a/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs +++ b/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs @@ -25,7 +25,7 @@ type internal FSharpHelpContextService static let userOpName = "ImplementInterfaceCodeFix" static member GetHelpTerm(checker: FSharpChecker, sourceText : SourceText, fileName, options, span: TextSpan, tokens: List, textVersion, perfOptions) : Async = asyncMaybe { - let! _, _, check = checker.ParseAndCheckDocument(fileName, textVersion, sourceText.ToString(), options, perfOptions, userOpName = userOpName) + let! _, _, check = checker.ParseAndCheckDocument(fileName, textVersion, sourceText, options, perfOptions, userOpName = userOpName) let textLines = sourceText.Lines let lineInfo = textLines.GetLineFromPosition(span.Start) let line = lineInfo.LineNumber @@ -34,12 +34,11 @@ type internal FSharpHelpContextService let caretColumn = textLines.GetLinePosition(span.Start).Character let shouldTryToFindSurroundingIdent (token : ClassifiedSpan) = - let span = token.TextSpan - let content = sourceText.ToString().Substring(span.Start, span.End - span.Start) + let content = sourceText.GetSubText(token.TextSpan) match token.ClassificationType with | ClassificationTypeNames.Text | ClassificationTypeNames.WhiteSpace -> true - | (ClassificationTypeNames.Operator|ClassificationTypeNames.Punctuation)when content = "." -> true + | (ClassificationTypeNames.Operator|ClassificationTypeNames.Punctuation)when content.Length > 0 && content.[0] = '.' -> true | _ -> false let tokenInformation, col = diff --git a/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs b/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs index b7850948a86..46024320ca4 100644 --- a/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs +++ b/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs @@ -68,9 +68,9 @@ type internal XmlDocCommandFilter let curLineNum = wpfTextView.Caret.Position.BufferPosition.GetContainingLine().LineNumber + 1 let! document = document.Value let! parsingOptions, _options = projectInfoManager.TryGetOptionsForEditingDocumentOrProject(document, CancellationToken.None) - let sourceText = wpfTextView.TextBuffer.CurrentSnapshot.GetText() + let! sourceText = document.GetTextAsync(CancellationToken.None) let! parsedInput = checker.ParseDocument(document, parsingOptions, sourceText, userOpName) - let xmlDocables = XmlDocParser.getXmlDocables (sourceText, Some parsedInput) + let xmlDocables = XmlDocParser.getXmlDocables (sourceText.ToFSharpSourceText(), Some parsedInput) let xmlDocablesBelowThisLine = // +1 because looking below current line for e.g. a 'member' xmlDocables |> List.filter (fun (XmlDocable(line,_indent,_paramNames)) -> line = curLineNum+1) diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index d1de7e83508..04245876425 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -6,7 +6,9 @@ module internal Microsoft.VisualStudio.FSharp.Editor.Extensions open System open System.IO open Microsoft.CodeAnalysis +open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.Host +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.SourceCodeServices @@ -40,6 +42,74 @@ type Document with languageServices.GetService<'T>() |> Some +module private SourceText = + + open System.Runtime.CompilerServices + + let weakTable = ConditionalWeakTable() + + let create (sourceText: SourceText) = + let sourceText = + { new ISourceText with + + member __.Item with get index = sourceText.[index] + + member __.GetLineString(lineIndex) = + sourceText.Lines.[lineIndex].ToString() + + member __.GetLineCount() = + sourceText.Lines.Count + + member __.GetLastCharacterPosition() = + if sourceText.Lines.Count > 0 then + (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + else + (0, 0) + + member __.GetSubTextString(start, length) = + sourceText.GetSubText(TextSpan(start, length)).ToString() + + member __.SubTextEquals(target, startIndex) = + if startIndex < 0 || startIndex >= sourceText.Length then + invalidArg "startIndex" "Out of range." + + if String.IsNullOrEmpty(target) then + invalidArg "target" "Is null or empty." + + let lastIndex = startIndex + target.Length + if lastIndex <= startIndex || lastIndex >= sourceText.Length then + invalidArg "target" "Too big." + + let mutable finished = false + let mutable didEqual = true + let mutable i = 0 + while not finished && i < target.Length do + if target.[i] <> sourceText.[startIndex + i] then + didEqual <- false + finished <- true // bail out early + else + i <- i + 1 + + didEqual + + member __.ContentEquals(sourceText) = + match sourceText with + | :? SourceText as sourceText -> sourceText.ContentEquals(sourceText) + | _ -> false + + member __.Length = sourceText.Length + + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + sourceText.CopyTo(sourceIndex, destination, destinationIndex, count) + } + + sourceText + +type SourceText with + + member this.ToFSharpSourceText() = + SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create)) + type FSharpNavigationDeclarationItem with member x.RoslynGlyph : Glyph = match x.Glyph with diff --git a/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs b/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs index 297f7c20954..0332b6f1848 100644 --- a/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs +++ b/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs @@ -105,7 +105,7 @@ type internal FSharpCompletionProvider static member ProvideCompletionsAsyncAux(checker: FSharpChecker, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, filePath: string, textVersionHash: int, getAllSymbols: FSharpCheckFileResults -> AssemblySymbol list, languageServicePerformanceOptions: LanguageServicePerformanceOptions, intellisenseOptions: IntelliSenseOptions) = asyncMaybe { - let! parseResults, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName = userOpName) + let! parseResults, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName = userOpName) let textLines = sourceText.Lines let caretLinePos = textLines.GetLinePosition(caretPosition) diff --git a/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs b/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs index 2d158e50ed2..7293b43e380 100644 --- a/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs +++ b/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs @@ -36,7 +36,7 @@ type internal FSharpSignatureHelpProvider // Unit-testable core routine static member internal ProvideMethodsAsyncAux(checker: FSharpChecker, documentationBuilder: IDocumentationBuilder, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, triggerIsTypedChar: char option, filePath: string, textVersionHash: int) = async { - let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToString(), options, userOpName = userOpName) + let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToFSharpSourceText(), options, userOpName = userOpName) match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> return None | FSharpCheckFileAnswer.Succeeded(checkFileResults) -> diff --git a/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs b/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs index c4fc0c4d4b0..08a72c54473 100644 --- a/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs +++ b/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs @@ -37,7 +37,7 @@ type internal FSharpBreakpointResolutionService else let textLineColumn = textLinePos.Character let fcsTextLineNumber = Line.fromZ textLinePos.Line // Roslyn line numbers are zero-based, FSharp.Compiler.Service line numbers are 1-based - let! parseResults = checker.ParseFile(fileName, sourceText.ToString(), parsingOptions, userOpName = userOpName) + let! parseResults = checker.ParseFile(fileName, sourceText.ToFSharpSourceText(), parsingOptions, userOpName = userOpName) return parseResults.ValidateBreakpointLocation(mkPos fcsTextLineNumber textLineColumn) } diff --git a/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs b/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs index bda8f5ca64f..9ef06d1081e 100644 --- a/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs +++ b/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs @@ -59,12 +59,13 @@ type internal FSharpDocumentDiagnosticAnalyzer() = static member GetDiagnostics(checker: FSharpChecker, filePath: string, sourceText: SourceText, textVersionHash: int, parsingOptions: FSharpParsingOptions, options: FSharpProjectOptions, diagnosticType: DiagnosticsType) = async { - let! parseResults = checker.ParseFile(filePath, sourceText.ToString(), parsingOptions, userOpName=userOpName) + let fsSourceText = sourceText.ToFSharpSourceText() + let! parseResults = checker.ParseFile(filePath, fsSourceText, parsingOptions, userOpName=userOpName) let! errors = async { match diagnosticType with | DiagnosticsType.Semantic -> - let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, sourceText.ToString(), options, userOpName=userOpName) + let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, fsSourceText, options, userOpName=userOpName) match checkResultsAnswer with | FSharpCheckFileAnswer.Aborted -> return [||] | FSharpCheckFileAnswer.Succeeded results -> diff --git a/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs b/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs index e9ea87fea5d..bc55ce999fa 100644 --- a/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs +++ b/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs @@ -59,7 +59,7 @@ type internal FSharpDocumentHighlightsService [] (checkerP let textLinePos = sourceText.Lines.GetLinePosition(position) let fcsTextLineNumber = Line.fromZ textLinePos.Line let! symbol = Tokenizer.getSymbolAtPosition(documentKey, sourceText, position, filePath, defines, SymbolLookupKind.Greedy, false) - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName = userOpName) + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName = userOpName) let! symbolUse = checkFileResults.GetSymbolUseAtLocation(fcsTextLineNumber, symbol.Ident.idRange.EndColumn, textLine.ToString(), symbol.FullIsland, userOpName=userOpName) let! symbolUses = checkFileResults.GetUsesOfSymbolInFile(symbolUse.Symbol) |> liftAsync return diff --git a/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs b/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs index 77dc1ca35e1..5a8380d9caf 100644 --- a/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs @@ -3,6 +3,7 @@ namespace Microsoft.VisualStudio.FSharp.Editor open System.ComponentModel.Composition +open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.Editor open Microsoft.FSharp.Compiler.SourceCodeServices open System.Runtime.InteropServices @@ -17,9 +18,9 @@ type internal FSharpBraceMatchingService static let defaultUserOpName = "BraceMatching" - static member GetBraceMatchingResult(checker: FSharpChecker, sourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [] forFormatting: bool) = + static member GetBraceMatchingResult(checker: FSharpChecker, sourceText: SourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [] forFormatting: bool) = async { - let! matchedBraces = checker.MatchBraces(fileName, sourceText.ToString(), parsingOptions, userOpName) + let! matchedBraces = checker.MatchBraces(fileName, sourceText.ToFSharpSourceText(), parsingOptions, userOpName) let isPositionInRange range = match RoslynHelpers.TryFSharpRangeToTextSpan(sourceText, range) with | None -> false diff --git a/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs b/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs index c191dee6fb9..1f2538ee9ee 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs @@ -10,20 +10,17 @@ open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.SourceCodeServices type FSharpChecker with - member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: string, userOpName: string) = + member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) = asyncMaybe { - let! fileParseResults = checker.ParseFile(document.FilePath, sourceText, parsingOptions, userOpName=userOpName) |> liftAsync + let! fileParseResults = checker.ParseFile(document.FilePath, sourceText.ToFSharpSourceText(), parsingOptions, userOpName=userOpName) |> liftAsync return! fileParseResults.ParseTree } - member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) = - checker.ParseDocument(document, parsingOptions, sourceText=sourceText.ToString(), userOpName=userOpName) - - member checker.ParseAndCheckDocument(filePath: string, textVersionHash: int, sourceText: string, options: FSharpProjectOptions, languageServicePerformanceOptions: LanguageServicePerformanceOptions, userOpName: string) = + member checker.ParseAndCheckDocument(filePath: string, textVersionHash: int, sourceText: SourceText, options: FSharpProjectOptions, languageServicePerformanceOptions: LanguageServicePerformanceOptions, userOpName: string) = async { let parseAndCheckFile = async { - let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText, options, userOpName=userOpName) + let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToFSharpSourceText(), options, userOpName=userOpName) return match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> @@ -82,21 +79,21 @@ type FSharpChecker with match allowStaleResults with | Some b -> { document.FSharpOptions.LanguageServicePerformance with AllowStaleCompletionResults = b } | _ -> document.FSharpOptions.LanguageServicePerformance - return! checker.ParseAndCheckDocument(document.FilePath, textVersion.GetHashCode(), sourceText.ToString(), options, perfOpts, userOpName=userOpName) + return! checker.ParseAndCheckDocument(document.FilePath, textVersion.GetHashCode(), sourceText, options, perfOpts, userOpName=userOpName) } - member checker.TryParseAndCheckFileInProject (projectOptions, fileName, source, userOpName) = async { - let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0, source,projectOptions, userOpName=userOpName) + member checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText: SourceText, userOpName) = async { + let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0,sourceText.ToFSharpSourceText(),projectOptions, userOpName=userOpName) match checkAnswer with | FSharpCheckFileAnswer.Aborted -> return None | FSharpCheckFileAnswer.Succeeded checkResults -> return Some (parseResults,checkResults) } - member checker.GetAllUsesOfAllSymbolsInSourceString (projectOptions, fileName, source: string, checkForUnusedOpens, userOpName) = async { + member checker.GetAllUsesOfAllSymbolsInSourceString (projectOptions, fileName, sourceText: SourceText, checkForUnusedOpens, userOpName) = async { - let! parseAndCheckResults = checker.TryParseAndCheckFileInProject (projectOptions, fileName, source, userOpName=userOpName) + let! parseAndCheckResults = checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText, userOpName=userOpName) match parseAndCheckResults with | None -> return [||] | Some(_parseResults,checkResults) -> diff --git a/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs b/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs index 2d9280c5474..eae4f7d1a77 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs @@ -119,9 +119,9 @@ type private FSharpProjectOptionsReactor (workspace: VisualStudioWorkspaceImpl, let rec tryComputeOptionsByFile (document: Document) cancellationToken = async { - let! text = document.GetTextAsync(cancellationToken) |> Async.AwaitTask + let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask let! fileStamp = document.GetTextVersionAsync(cancellationToken) |> Async.AwaitTask - let! scriptProjectOptions, _ = checkerProvider.Checker.GetProjectOptionsFromScript(document.FilePath, text.ToString(), DateTime.Now) + let! scriptProjectOptions, _ = checkerProvider.Checker.GetProjectOptionsFromScript(document.FilePath, sourceText.ToFSharpSourceText(), DateTime.Now) match singleFileCache.TryGetValue(document.Id) with | false, _ -> let projectOptions = diff --git a/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs b/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs index 00abd4c309c..31366d930f9 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs @@ -34,7 +34,7 @@ module internal SymbolHelpers = let defines = CompilerEnvironment.GetCompilationDefinesForEditing parsingOptions let! symbol = Tokenizer.getSymbolAtPosition(document.Id, sourceText, position, document.FilePath, defines, SymbolLookupKind.Greedy, false) let settings = document.FSharpOptions - let! _, _, checkFileResults = checker.ParseAndCheckDocument(document.FilePath, textVersionHash, sourceText.ToString(), projectOptions, settings.LanguageServicePerformance, userOpName = userOpName) + let! _, _, checkFileResults = checker.ParseAndCheckDocument(document.FilePath, textVersionHash, sourceText, projectOptions, settings.LanguageServicePerformance, userOpName = userOpName) let! symbolUse = checkFileResults.GetSymbolUseAtLocation(fcsTextLineNumber, symbol.Ident.idRange.EndColumn, textLine.ToString(), symbol.FullIsland, userOpName=userOpName) let! symbolUses = checkFileResults.GetUsesOfSymbolInFile(symbolUse.Symbol) |> liftAsync return symbolUses diff --git a/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs b/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs index 9e855e8719a..f664db8de1b 100644 --- a/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs +++ b/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs @@ -207,12 +207,12 @@ type internal GoToDefinition(checker: FSharpChecker, projectInfoManager: FSharpP /// if the symbol is defined in the given file, return its declaration location, otherwise use the targetSymbol to find the first /// instance of its presence in the provided source file. The first case is needed to return proper declaration location for /// recursive type definitions, where the first its usage may not be the declaration. - member __.FindSymbolDeclarationInFile(targetSymbolUse: FSharpSymbolUse, filePath: string, source: string, options: FSharpProjectOptions, fileVersion:int) = + member __.FindSymbolDeclarationInFile(targetSymbolUse: FSharpSymbolUse, filePath: string, sourceText: SourceText, options: FSharpProjectOptions, fileVersion:int) = asyncMaybe { match targetSymbolUse.Symbol.DeclarationLocation with | Some decl when decl.FileName = filePath -> return decl | _ -> - let! _, checkFileAnswer = checker.ParseAndCheckFileInProject (filePath, fileVersion, source, options, userOpName = userOpName) |> liftAsync + let! _, checkFileAnswer = checker.ParseAndCheckFileInProject (filePath, fileVersion, sourceText.ToFSharpSourceText(), options, userOpName = userOpName) |> liftAsync match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> return! None | FSharpCheckFileAnswer.Succeeded checkFileResults -> @@ -273,7 +273,7 @@ type internal GoToDefinition(checker: FSharpChecker, projectInfoManager: FSharpP let! implSourceText = implDocument.GetTextAsync () |> liftTaskAsync let! implVersion = implDocument.GetTextVersionAsync () |> liftTaskAsync - let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText.ToString(), projectOptions, implVersion.GetHashCode()) + let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText, projectOptions, implVersion.GetHashCode()) let! implTextSpan = RoslynHelpers.TryFSharpRangeToTextSpan (implSourceText, targetRange) let navItem = FSharpNavigableItem (implDocument, implTextSpan) @@ -312,7 +312,7 @@ type internal GoToDefinition(checker: FSharpChecker, projectInfoManager: FSharpP let! implSourceText = implDocument.GetTextAsync () |> liftTaskAsync let! _parsingOptions, projectOptions = projectInfoManager.TryGetOptionsForEditingDocumentOrProject(implDocument, CancellationToken.None) - let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText.ToString(), projectOptions, implVersion.GetHashCode()) + let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText, projectOptions, implVersion.GetHashCode()) let! implTextSpan = RoslynHelpers.TryFSharpRangeToTextSpan (implSourceText, targetRange) let navItem = FSharpNavigableItem (implDocument, implTextSpan) diff --git a/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs b/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs index eedcf57b007..bb65cb6452e 100644 --- a/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs +++ b/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs @@ -199,7 +199,7 @@ type internal FSharpNavigateToSearchService async { let! cancellationToken = Async.CancellationToken let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask - let! parseResults = checkerProvider.Checker.ParseFile(document.FilePath, sourceText.ToString(), parsingOptions) + let! parseResults = checkerProvider.Checker.ParseFile(document.FilePath, sourceText.ToFSharpSourceText(), parsingOptions) let navItems parsedInput = NavigateTo.getNavigableItems parsedInput diff --git a/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs b/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs index 5547aa177aa..6758584d51a 100644 --- a/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs +++ b/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs @@ -169,7 +169,7 @@ type internal FSharpAsyncQuickInfoSource // test helper static member ProvideQuickInfo(checker:FSharpChecker, documentId:DocumentId, sourceText:SourceText, filePath:string, position:int, parsingOptions:FSharpParsingOptions, options:FSharpProjectOptions, textVersionHash:int, languageServicePerformanceOptions: LanguageServicePerformanceOptions) = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName=FSharpQuickInfo.userOpName) + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName=FSharpQuickInfo.userOpName) let textLine = sourceText.Lines.GetLineFromPosition position let textLineNumber = textLine.LineNumber + 1 // Roslyn line numbers are zero-based let defines = CompilerEnvironment.GetCompilationDefinesForEditing parsingOptions diff --git a/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs b/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs index 42c673d90b7..ac3c054682a 100644 --- a/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs +++ b/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs @@ -12,7 +12,6 @@ open Microsoft.VisualStudio.Text open Microsoft.FSharp.Compiler.SourceCodeServices open Microsoft.VisualStudio.FSharp.LanguageService.SiteProvider - #nowarn "44" // use of obsolete CheckFileInProjectAllowingStaleCachedResults // @@ -95,7 +94,7 @@ type internal FSharpLanguageServiceBackgroundRequests_DEPRECATED lazy // This portion is executed on the language service thread let timestamp = if source=null then System.DateTime(2000,1,1) else source.OpenedTime // source is null in unit tests let checker = getInteractiveChecker() - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(fileName, sourceText, timestamp, [| |]) |> Async.RunSynchronously + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(fileName, Microsoft.FSharp.Compiler.Text.SourceText.ofString sourceText, timestamp, [| |]) |> Async.RunSynchronously let referencedProjectFileNames = [| |] let projectSite = ProjectSitesAndFiles.CreateProjectSiteForScript(fileName, referencedProjectFileNames, checkOptions) { ProjectSite = projectSite diff --git a/vsintegration/src/FSharp.LanguageService/FSharpSource.fs b/vsintegration/src/FSharp.LanguageService/FSharpSource.fs index 3146f5d0ba1..a818b6f3d1b 100644 --- a/vsintegration/src/FSharp.LanguageService/FSharpSource.fs +++ b/vsintegration/src/FSharp.LanguageService/FSharpSource.fs @@ -371,7 +371,7 @@ type internal FSharpSource_DEPRECATED(service:LanguageService_DEPRECATED, textLi Stamp = None } |> ic.GetParsingOptionsFromProjectOptions - ic.ParseFile(fileName, source.GetText(), co) |> Async.RunSynchronously + ic.ParseFile(fileName, Microsoft.FSharp.Compiler.Text.SourceText.ofString (source.GetText()), co) |> Async.RunSynchronously override source.GetCommentFormat() = let mutable info = new CommentInfo() diff --git a/vsintegration/tests/Salsa/salsa.fs b/vsintegration/tests/Salsa/salsa.fs index 8a86f834180..7306c68dfa7 100644 --- a/vsintegration/tests/Salsa/salsa.fs +++ b/vsintegration/tests/Salsa/salsa.fs @@ -1101,7 +1101,7 @@ module internal Salsa = member file.GetFileName() = filename member file.GetProjectOptionsOfScript() = - project.Solution.Vs.LanguageService.FSharpChecker.GetProjectOptionsFromScript(filename, file.CombinedLines, System.DateTime(2000,1,1), [| |]) + project.Solution.Vs.LanguageService.FSharpChecker.GetProjectOptionsFromScript(filename, Microsoft.FSharp.Compiler.Text.SourceText.ofString file.CombinedLines, System.DateTime(2000,1,1), [| |]) |> Async.RunSynchronously |> fst // drop diagnostics @@ -1113,7 +1113,7 @@ module internal Salsa = member file.OnIdle() = while file.Source.NeedsVisualRefresh do file.OnIdleTypeCheck() - member file.CombinedLines = + member file.CombinedLines : string = if combinedLines = null then combinedLines<-String.Join("\n",lines) combinedLines diff --git a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs index a2650c519e4..d442b9a8963 100644 --- a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs +++ b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs @@ -53,7 +53,7 @@ module GoToDefinitionServiceTests = let textLinePos = sourceText.Lines.GetLinePosition position let fcsTextLineNumber = Line.fromZ textLinePos.Line let! lexerSymbol = Tokenizer.getSymbolAtPosition(documentKey, sourceText, position, filePath, defines, SymbolLookupKind.Greedy, false) - let! _, _, checkFileResults = checker.ParseAndCheckDocument (filePath, textVersionHash, sourceText.ToString(), options, LanguageServicePerformanceOptions.Default, userOpName=userOpName) |> Async.RunSynchronously + let! _, _, checkFileResults = checker.ParseAndCheckDocument (filePath, textVersionHash, sourceText, options, LanguageServicePerformanceOptions.Default, userOpName=userOpName) |> Async.RunSynchronously let declarations = checkFileResults.GetDeclarationLocation (fcsTextLineNumber, lexerSymbol.Ident.idRange.EndColumn, textLine.ToString(), lexerSymbol.FullIsland, false, userOpName=userOpName) |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs b/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs index 7578f34bb9f..93c135ad8bf 100644 --- a/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs +++ b/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs @@ -39,6 +39,7 @@ let main argv = 0 // return an integer exit code " + static member private testCases: Object[][] = [| [| "123456"; None |] // Numeric literals are not interesting [| "is a string"; Some("\"This is a string\"") |] diff --git a/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs new file mode 100644 index 00000000000..23b1e6702a0 --- /dev/null +++ b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace Microsoft.VisualStudio.FSharp.Editor.Tests.Roslyn + +open System +open NUnit.Framework + +open Microsoft.VisualStudio.FSharp.Editor +open Microsoft.FSharp.Compiler.Text +open Microsoft.CodeAnalysis.Text + +[] +module RoslynSourceTextTests = + + [] + let SourceText () = + let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" + let sourceText = SourceText.From(text).ToFSharpSourceText() + + Assert.AreEqual("test", sourceText.GetLineString(0)) + Assert.AreEqual("test2", sourceText.GetLineString(1)) + Assert.AreEqual("test3", sourceText.GetLineString(2)) + Assert.AreEqual("", sourceText.GetLineString(3)) + Assert.AreEqual("test4", sourceText.GetLineString(4)) + Assert.AreEqual("test5", sourceText.GetLineString(5)) + Assert.AreEqual("test6", sourceText.GetLineString(6)) + Assert.AreEqual("", sourceText.GetLineString(7)) + Assert.AreEqual(8, sourceText.GetLineCount()) + + let (count, length) = sourceText.GetLastCharacterPosition() + Assert.AreEqual(8, count) + Assert.AreEqual(0, length) + + Assert.True(sourceText.SubTextEquals("test", 0)) + Assert.True(sourceText.SubTextEquals("test2", 5)) + Assert.True(sourceText.SubTextEquals("test3", 12)) + + Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("", 0) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals(text + text, 0) |> ignore) |> ignore + + Assert.False(sourceText.SubTextEquals("test", 1)) + Assert.False(sourceText.SubTextEquals("test", 4)) + Assert.False(sourceText.SubTextEquals("test", 11)) \ No newline at end of file diff --git a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs index 3c5b8f40147..8ba79f5aad6 100644 --- a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs @@ -30,10 +30,10 @@ type SemanticClassificationServiceTests() = let checker = FSharpChecker.Create() let perfOptions = { LanguageServicePerformanceOptions.Default with AllowStaleCompletionResults = false } - let getRanges (sourceText: string) : (Range.range * SemanticClassificationType) list = + let getRanges (source: string) : (Range.range * SemanticClassificationType) list = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, sourceText, projectOptions, perfOptions, "") + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(source), projectOptions, perfOptions, "") return checkFileResults.GetSemanticClassification(None) } |> Async.RunSynchronously @@ -41,7 +41,7 @@ type SemanticClassificationServiceTests() = |> List.collect Array.toList let verifyClassificationAtEndOfMarker(fileContents: string, marker: string, classificationType: string) = - let text = SourceText.From fileContents + let text = SourceText.From(fileContents) let ranges = getRanges fileContents let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1) let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1) @@ -50,7 +50,7 @@ type SemanticClassificationServiceTests() = | Some(_, ty) -> Assert.AreEqual(classificationType, FSharpClassificationTypes.getClassificationTypeName ty, "Classification data doesn't match for end of marker") let verifyNoClassificationDataAtEndOfMarker(fileContents: string, marker: string, classificationType: string) = - let text = SourceText.From fileContents + let text = SourceText.From(fileContents) let ranges = getRanges fileContents let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1) let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1) diff --git a/vsintegration/tests/UnitTests/UnusedOpensTests.fs b/vsintegration/tests/UnitTests/UnusedOpensTests.fs index d23f5ed4e18..64f3fa2e8a3 100644 --- a/vsintegration/tests/UnitTests/UnusedOpensTests.fs +++ b/vsintegration/tests/UnitTests/UnusedOpensTests.fs @@ -29,7 +29,7 @@ let private checker = FSharpChecker.Create() let (=>) (source: string) (expectedRanges: ((*line*)int * ((*start column*)int * (*end column*)int)) list) = let sourceLines = source.Split ([|"\r\n"; "\n"; "\r"|], StringSplitOptions.None) - let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, source, projectOptions) |> Async.RunSynchronously + let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, projectOptions) |> Async.RunSynchronously let checkFileResults = match checkFileAnswer with diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index ea8b5992d1f..997471f88ce 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -122,6 +122,7 @@ --> + Roslyn\IndentationServiceTests.fs