From 1ba6b74fa4daf7d88590072239a94060a0b7a215 Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 9 Apr 2020 19:19:45 +0100 Subject: [PATCH] string interploation implementation --- .gitignore | 2 + src/fsharp/CheckFormatStrings.fs | 117 +++--- src/fsharp/CheckFormatStrings.fsi | 5 +- src/fsharp/CompileOps.fs | 4 + src/fsharp/ErrorLogger.fs | 7 +- src/fsharp/FSComp.txt | 5 + src/fsharp/FSStrings.resx | 12 + src/fsharp/FSharp.Core/printf.fs | 171 ++++++--- src/fsharp/FSharp.Core/printf.fsi | 8 + src/fsharp/LanguageFeatures.fs | 3 + src/fsharp/LanguageFeatures.fsi | 1 + src/fsharp/LexFilter.fs | 22 +- src/fsharp/ParseHelpers.fs | 16 +- src/fsharp/PostInferenceChecks.fs | 2 +- src/fsharp/SyntaxTree.fs | 8 +- src/fsharp/SyntaxTreeOps.fs | 3 + src/fsharp/TcGlobals.fs | 6 + src/fsharp/TypeChecker.fs | 72 +++- src/fsharp/TypedTreeOps.fs | 3 + src/fsharp/TypedTreeOps.fsi | 2 + src/fsharp/lex.fsl | 353 +++++++++++------- src/fsharp/lexhelp.fs | 56 ++- src/fsharp/lexhelp.fsi | 15 +- src/fsharp/pars.fsy | 35 +- src/fsharp/service/ServiceLexing.fs | 121 ++++-- src/fsharp/service/ServiceParseTreeWalk.fs | 75 +++- src/fsharp/service/ServiceUntypedParse.fs | 3 + .../SurfaceArea.coreclr.fs | 1 + .../SurfaceArea.net40.fs | 1 + tests/fsharp/FSharpSuite.Tests.fsproj | 1 + tests/service/TokenizerTests.fs | 36 +- 31 files changed, 827 insertions(+), 339 deletions(-) diff --git a/.gitignore b/.gitignore index 7f3870c3d05..997f346844c 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,5 @@ msbuild.binlog /fcs/FSharp.Compiler.Service.netstandard/*.fsi /.ionide/ **/.DS_Store +Session.vim +coc-settings.json diff --git a/src/fsharp/CheckFormatStrings.fs b/src/fsharp/CheckFormatStrings.fs index 4b09c88665e..fb473ced98a 100644 --- a/src/fsharp/CheckFormatStrings.fs +++ b/src/fsharp/CheckFormatStrings.fs @@ -47,7 +47,7 @@ let newInfo () = addZeros = false precision = false} -let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringCheckContext option) fmt bty cty = +let parseFormatStringInternal (m:range) (g: TcGlobals) isInterp (context: FormatStringCheckContext option) fmt bty cty = // Offset is used to adjust ranges depending on whether input string is regular, verbatim or triple-quote. // We construct a new 'fmt' string since the current 'fmt' string doesn't distinguish between "\n" and escaped "\\n". let (offset, fmt) = @@ -77,7 +77,7 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe if acc |> List.forall (fun (p, _) -> p = None) then // without positional specifiers acc |> List.map snd |> List.rev else - failwithf "%s" <| FSComp.SR.forPositionalSpecifiersNotPermitted() + raise (Failure (FSComp.SR.forPositionalSpecifiersNotPermitted())) argtys elif System.Char.IsSurrogatePair(fmt,i) then parseLoop acc (i+2, relLine, relCol+2) @@ -88,65 +88,65 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe let startCol = relCol let relCol = relCol+1 let i = i+1 - if i >= len then failwithf "%s" <| FSComp.SR.forMissingFormatSpecifier() + if i >= len then raise (Failure (FSComp.SR.forMissingFormatSpecifier())) let info = newInfo() let rec flags i = - if i >= len then failwithf "%s" <| FSComp.SR.forMissingFormatSpecifier() + if i >= len then raise (Failure (FSComp.SR.forMissingFormatSpecifier())) match fmt.[i] with | '-' -> - if info.leftJustify then failwithf "%s" <| FSComp.SR.forFlagSetTwice("-") + if info.leftJustify then raise (Failure (FSComp.SR.forFlagSetTwice("-"))) info.leftJustify <- true flags(i+1) | '+' -> - if info.numPrefixIfPos <> None then failwithf "%s" <| FSComp.SR.forPrefixFlagSpacePlusSetTwice() + if info.numPrefixIfPos <> None then raise (Failure (FSComp.SR.forPrefixFlagSpacePlusSetTwice())) info.numPrefixIfPos <- Some '+' flags(i+1) | '0' -> - if info.addZeros then failwithf "%s" <| FSComp.SR.forFlagSetTwice("0") + if info.addZeros then raise (Failure (FSComp.SR.forFlagSetTwice("0"))) info.addZeros <- true flags(i+1) | ' ' -> - if info.numPrefixIfPos <> None then failwithf "%s" <| FSComp.SR.forPrefixFlagSpacePlusSetTwice() + if info.numPrefixIfPos <> None then raise (Failure (FSComp.SR.forPrefixFlagSpacePlusSetTwice())) info.numPrefixIfPos <- Some ' ' flags(i+1) - | '#' -> failwithf "%s" <| FSComp.SR.forHashSpecifierIsInvalid() + | '#' -> raise (Failure (FSComp.SR.forHashSpecifierIsInvalid() )) | _ -> i let rec digitsPrecision i = - if i >= len then failwithf "%s" <| FSComp.SR.forBadPrecision() + if i >= len then raise (Failure (FSComp.SR.forBadPrecision())) match fmt.[i] with | c when System.Char.IsDigit c -> digitsPrecision (i+1) | _ -> i let precision i = - if i >= len then failwithf "%s" <| FSComp.SR.forBadWidth() + if i >= len then raise (Failure (FSComp.SR.forBadWidth())) match fmt.[i] with | c when System.Char.IsDigit c -> info.precision <- true; false,digitsPrecision (i+1) | '*' -> info.precision <- true; true,(i+1) - | _ -> failwithf "%s" <| FSComp.SR.forPrecisionMissingAfterDot() + | _ -> raise (Failure (FSComp.SR.forPrecisionMissingAfterDot())) let optionalDotAndPrecision i = - if i >= len then failwithf "%s" <| FSComp.SR.forBadPrecision() + if i >= len then raise (Failure (FSComp.SR.forBadPrecision())) match fmt.[i] with | '.' -> precision (i+1) | _ -> false,i let rec digitsWidthAndPrecision i = - if i >= len then failwithf "%s" <| FSComp.SR.forBadPrecision() + if i >= len then raise (Failure (FSComp.SR.forBadPrecision())) match fmt.[i] with | c when System.Char.IsDigit c -> digitsWidthAndPrecision (i+1) | _ -> optionalDotAndPrecision i let widthAndPrecision i = - if i >= len then failwithf "%s" <| FSComp.SR.forBadPrecision() + if i >= len then raise (Failure (FSComp.SR.forBadPrecision())) match fmt.[i] with | c when System.Char.IsDigit c -> false,digitsWidthAndPrecision i | '*' -> true,optionalDotAndPrecision (i+1) | _ -> false,optionalDotAndPrecision i let rec digitsPosition n i = - if i >= len then failwithf "%s" <| FSComp.SR.forBadPrecision() + if i >= len then raise (Failure (FSComp.SR.forBadPrecision())) match fmt.[i] with | c when System.Char.IsDigit c -> digitsPosition (n*10 + int c - int '0') (i+1) | '$' -> Some n, i+1 @@ -171,22 +171,35 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe let widthArg,(precisionArg,i) = widthAndPrecision i let relCol = relCol + i - oldI - if i >= len then failwithf "%s" <| FSComp.SR.forBadPrecision() + if i >= len then raise (Failure (FSComp.SR.forBadPrecision())) let acc = if precisionArg then (Option.map ((+)1) posi, g.int_ty) :: acc else acc let acc = if widthArg then (Option.map ((+)1) posi, g.int_ty) :: acc else acc - let checkNoPrecision c = if info.precision then failwithf "%s" <| FSComp.SR.forFormatDoesntSupportPrecision(c.ToString()) - let checkNoZeroFlag c = if info.addZeros then failwithf "%s" <| FSComp.SR.forDoesNotSupportZeroFlag(c.ToString()) - let checkNoNumericPrefix c = if info.numPrefixIfPos <> None then - failwithf "%s" <| FSComp.SR.forDoesNotSupportPrefixFlag(c.ToString(), (Option.get info.numPrefixIfPos).ToString()) + let checkNoPrecision c = + if info.precision then raise (Failure (FSComp.SR.forFormatDoesntSupportPrecision(c.ToString()))) + + let checkNoZeroFlag c = + if info.addZeros then raise (Failure (FSComp.SR.forDoesNotSupportZeroFlag(c.ToString()))) + + let checkNoNumericPrefix c = + match info.numPrefixIfPos with + | Some n -> raise (Failure (FSComp.SR.forDoesNotSupportPrefixFlag(c.ToString(), n.ToString()))) + | None -> () let checkOtherFlags c = checkNoPrecision c checkNoZeroFlag c checkNoNumericPrefix c + let skipInterp i = + // Explicitly typed holes in interpolated strings get '%P' after them as hole place marker + if isInterp then + if i+1 < fmt.Length && fmt.[i] = '%' && fmt.[i+1] = 'P' then i + 2 + else raise (Failure (FSComp.SR.forFormatInvalidForInterpolated())) + else i + let collectSpecifierLocation relLine relCol numStdArgs = let numArgsForSpecifier = numStdArgs + (if widthArg then 1 else 0) + (if precisionArg then 1 else 0) @@ -209,55 +222,68 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe parseLoop acc (i+1, relLine, relCol+1) | ('d' | 'i' | 'o' | 'u' | 'x' | 'X') -> - if info.precision then failwithf "%s" <| FSComp.SR.forFormatDoesntSupportPrecision(ch.ToString()) + if info.precision then raise (Failure (FSComp.SR.forFormatDoesntSupportPrecision(ch.ToString()))) collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, mkFlexibleIntFormatTypar g m) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, mkFlexibleIntFormatTypar g m) :: acc) (i, relLine, relCol+1) | ('l' | 'L') -> - if info.precision then failwithf "%s" <| FSComp.SR.forFormatDoesntSupportPrecision(ch.ToString()) + if info.precision then raise (Failure (FSComp.SR.forFormatDoesntSupportPrecision(ch.ToString()))) let relCol = relCol+1 let i = i+1 // "bad format specifier ... In F# code you can use %d, %x, %o or %u instead ..." if i >= len then - failwithf "%s" <| FSComp.SR.forBadFormatSpecifier() + raise (Failure (FSComp.SR.forBadFormatSpecifier())) // Always error for %l and %Lx - failwithf "%s" <| FSComp.SR.forLIsUnnecessary() + raise (Failure (FSComp.SR.forLIsUnnecessary())) match fmt.[i] with | ('d' | 'i' | 'o' | 'u' | 'x' | 'X') -> collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, mkFlexibleIntFormatTypar g m) :: acc) (i+1, relLine, relCol+1) - | _ -> failwithf "%s" <| FSComp.SR.forBadFormatSpecifier() + let i = skipInterp (i+1) + parseLoop ((posi, mkFlexibleIntFormatTypar g m) :: acc) (i, relLine, relCol+1) + | _ -> raise (Failure (FSComp.SR.forBadFormatSpecifier())) | ('h' | 'H') -> - failwithf "%s" <| FSComp.SR.forHIsUnnecessary() + raise (Failure (FSComp.SR.forHIsUnnecessary())) | 'M' -> collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, mkFlexibleDecimalFormatTypar g m) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, mkFlexibleDecimalFormatTypar g m) :: acc) (i, relLine, relCol+1) | ('f' | 'F' | 'e' | 'E' | 'g' | 'G') -> collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, mkFlexibleFloatFormatTypar g m) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, mkFlexibleFloatFormatTypar g m) :: acc) (i, relLine, relCol+1) | 'b' -> checkOtherFlags ch collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, g.bool_ty) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, g.bool_ty) :: acc) (i, relLine, relCol+1) | 'c' -> checkOtherFlags ch collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, g.char_ty) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, g.char_ty) :: acc) (i, relLine, relCol+1) | 's' -> checkOtherFlags ch collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, g.string_ty) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, g.string_ty) :: acc) (i, relLine, relCol+1) | 'O' -> checkOtherFlags ch collectSpecifierLocation relLine relCol 1 + let i = skipInterp (i+1) + parseLoop ((posi, NewInferenceType ()) :: acc) (i, relLine, relCol+1) + + // residue of hole "...{n}..." in interpolated strings + | 'P' when isInterp -> + checkOtherFlags ch parseLoop ((posi, NewInferenceType ()) :: acc) (i+1, relLine, relCol+1) | 'A' -> @@ -265,22 +291,25 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe | None // %A has BindingFlags=Public, %+A has BindingFlags=Public | NonPublic | Some '+' -> collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, NewInferenceType ()) :: acc) (i+1, relLine, relCol+1) - | Some _ -> failwithf "%s" <| FSComp.SR.forDoesNotSupportPrefixFlag(ch.ToString(), (Option.get info.numPrefixIfPos).ToString()) + let i = skipInterp (i+1) + parseLoop ((posi, NewInferenceType ()) :: acc) (i, relLine, relCol+1) + | Some n -> raise (Failure (FSComp.SR.forDoesNotSupportPrefixFlag(ch.ToString(), n.ToString()))) | 'a' -> checkOtherFlags ch let xty = NewInferenceType () let fty = bty --> (xty --> cty) collectSpecifierLocation relLine relCol 2 - parseLoop ((Option.map ((+)1) posi, xty) :: (posi, fty) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((Option.map ((+)1) posi, xty) :: (posi, fty) :: acc) (i, relLine, relCol+1) | 't' -> checkOtherFlags ch collectSpecifierLocation relLine relCol 1 - parseLoop ((posi, bty --> cty) :: acc) (i+1, relLine, relCol+1) + let i = skipInterp (i+1) + parseLoop ((posi, bty --> cty) :: acc) (i, relLine, relCol+1) - | c -> failwithf "%s" <| FSComp.SR.forBadFormatSpecifierGeneral(String.make 1 c) + | c -> raise (Failure (FSComp.SR.forBadFormatSpecifierGeneral(String.make 1 c))) | '\n' -> parseLoop acc (i+1, relLine+1, 0) | _ -> parseLoop acc (i+1, relLine, relCol+1) @@ -288,15 +317,15 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe let results = parseLoop [] (0, 0, m.StartColumn) results, Seq.toList specifierLocations -let ParseFormatString m g formatStringCheckContext fmt bty cty dty = - let argtys, specifierLocations = parseFormatStringInternal m g formatStringCheckContext fmt bty cty +let ParseFormatString m g isInterp formatStringCheckContext fmt bty cty dty = + let argtys, specifierLocations = parseFormatStringInternal m g isInterp formatStringCheckContext fmt bty cty let aty = List.foldBack (-->) argtys dty let ety = mkRefTupledTy g argtys - (aty, ety), specifierLocations + (argtys, aty, ety), specifierLocations -let TryCountFormatStringArguments m g fmt bty cty = +let TryCountFormatStringArguments m g isInterp fmt bty cty = try - let argtys, _specifierLocations = parseFormatStringInternal m g None fmt bty cty + let argtys, _specifierLocations = parseFormatStringInternal m g isInterp None fmt bty cty Some argtys.Length with _ -> None diff --git a/src/fsharp/CheckFormatStrings.fsi b/src/fsharp/CheckFormatStrings.fsi index 91cce79b7d4..4053813e291 100644 --- a/src/fsharp/CheckFormatStrings.fsi +++ b/src/fsharp/CheckFormatStrings.fsi @@ -9,9 +9,10 @@ module internal FSharp.Compiler.CheckFormatStrings open FSharp.Compiler open FSharp.Compiler.NameResolution +open FSharp.Compiler.Range open FSharp.Compiler.TypedTree open FSharp.Compiler.TcGlobals -val ParseFormatString : Range.range -> TcGlobals -> formatStringCheckContext: FormatStringCheckContext option -> fmt: string -> bty: TType -> cty: TType -> dty: TType -> (TType * TType) * (Range.range * int) list +val ParseFormatString : m: range -> g: TcGlobals -> isInterp: bool -> formatStringCheckContext: FormatStringCheckContext option -> fmt: string -> bty: TType -> cty: TType -> dty: TType -> (TType list * TType * TType) * (range * int) list -val TryCountFormatStringArguments : m:Range.range -> g:TcGlobals -> fmt:string -> bty:TType -> cty:TType -> int option +val TryCountFormatStringArguments : m:Range.range -> g:TcGlobals -> isInterp: bool -> fmt:string -> bty:TType -> cty:TType -> int option diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index 161b6731950..29c99d9258f 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -1165,6 +1165,10 @@ let OutputPhasedErrorR (os: StringBuilder) (err: PhasedDiagnostic) (canSuggestNa | Parser.TOKEN_EOF -> getErrorString("Parser.TOKEN.EOF") | Parser.TOKEN_CONST -> getErrorString("Parser.TOKEN.CONST") | Parser.TOKEN_FIXED -> getErrorString("Parser.TOKEN.FIXED") + | Parser.TOKEN_INTERP_STRING_BEGIN_END -> getErrorString("Parser.TOKEN.INTERP.STRING.BEGIN.END") + | Parser.TOKEN_INTERP_STRING_BEGIN_PART -> getErrorString("Parser.TOKEN.INTERP.STRING.BEGIN.PART") + | Parser.TOKEN_INTERP_STRING_PART -> getErrorString("Parser.TOKEN.INTERP.STRING.PART") + | Parser.TOKEN_INTERP_STRING_END -> getErrorString("Parser.TOKEN.INTERP.STRING.END") | unknown -> Debug.Assert(false, "unknown token tag") let result = sprintf "%+A" unknown diff --git a/src/fsharp/ErrorLogger.fs b/src/fsharp/ErrorLogger.fs index 3e345ea4abf..85caca57d78 100755 --- a/src/fsharp/ErrorLogger.fs +++ b/src/fsharp/ErrorLogger.fs @@ -688,4 +688,9 @@ let internal tryLanguageFeatureError langVersion langFeature m = tryLanguageFeatureErrorAux langVersion langFeature m error let internal tryLanguageFeatureErrorRecover langVersion langFeature m = - tryLanguageFeatureErrorAux langVersion langFeature m errorR \ No newline at end of file + tryLanguageFeatureErrorAux langVersion langFeature m errorR + +let internal languageFeatureNotSupportedInLibraryError (langVersion: LanguageVersion) (langFeature: LanguageFeature) (m: range) = + let featureStr = langVersion.GetFeatureString langFeature + let suggestedVersionStr = langVersion.GetFeatureVersionString langFeature + error (Error(FSComp.SR.chkFeatureNotSupportedInLibrary(featureStr, suggestedVersionStr), m)) diff --git a/src/fsharp/FSComp.txt b/src/fsharp/FSComp.txt index bbcd42bf9f7..6b9d4975bbf 100644 --- a/src/fsharp/FSComp.txt +++ b/src/fsharp/FSComp.txt @@ -1489,6 +1489,8 @@ notAFunctionButMaybeDeclaration,"This value is not a function and cannot be appl 3350,chkFeatureNotLanguageSupported,"Feature '%s' is not available in F# %s. Please use language version %s or greater." 3351,chkFeatureNotRuntimeSupported,"Feature '%s' is not supported by target runtime." 3352,typrelInterfaceMemberNoMostSpecificImplementation,"Interface member '%s' does not have a most specific implementation." +3353,chkFeatureNotSupportedInLibrary,"Feature '%s' requires the F# library for language version %s or greater." +3360,lexByteStringMayNotBeInterpolated,"a byte string may not be interpolated" useSdkRefs,"Use reference assemblies for .NET framework references when available (Enabled by default)." fSharpBannerVersion,"%s for F# %s" optsLangVersion,"Display the allowed values for language version, specify language version such as 'latest' or 'preview'" @@ -1509,3 +1511,6 @@ featureFixedIndexSlice3d4d,"fixed-index slice 3d/4d" featureAndBang,"applicative computation expressions" featureNullableOptionalInterop,"nullable optional interop" featureDefaultInterfaceMemberConsumption,"default interface member consumption" +featureStringInterpolation,"string interpolation" +3361,tcInterpolationMixedWithPercent,"Mismatch in interpolated string. Interpolated strings may not use '%%' formats unless each is given an expression, e.g. '%%d{{1+1}}'" +forFormatInvalidForInterpolated,"Invalid interpolated string. interpolated strings may not use '%%' formats unless each is given an expression, e.g. '%%d{{1+1}}'" diff --git a/src/fsharp/FSStrings.resx b/src/fsharp/FSStrings.resx index e755b590b49..1b0f5b38379 100644 --- a/src/fsharp/FSStrings.resx +++ b/src/fsharp/FSStrings.resx @@ -441,6 +441,18 @@ keyword 'fixed' + + interpolated string + + + interpolated string (first part) + + + interpolated string (part) + + + interpolated string (final part) + keyword 'constraint' diff --git a/src/fsharp/FSharp.Core/printf.fs b/src/fsharp/FSharp.Core/printf.fs index d751ba1317f..0e008aa4fad 100644 --- a/src/fsharp/FSharp.Core/printf.fs +++ b/src/fsharp/FSharp.Core/printf.fs @@ -41,13 +41,12 @@ module internal PrintfImpl = open System.Text open System.Collections.Generic + open System.Collections.Concurrent open System.Reflection open Microsoft.FSharp.Core open Microsoft.FSharp.Core.Operators open Microsoft.FSharp.Collections open LanguagePrimitives.IntrinsicOperators - - open System.IO [] type FormatFlags = @@ -130,6 +129,9 @@ module internal PrintfImpl = let parseTypeChar (s: string) i = s.[i], (i + 1) + + let skipInterpHole isInterp (s:string) i = + if isInterp && i+1 < s.Length && s.[i] = '%' && s.[i+1] = 'P' then i+2 else i let findNextFormatSpecifier (s: string) i = let rec go i (buf: Text.StringBuilder) = @@ -143,6 +145,7 @@ module internal PrintfImpl = let w, i2 = parseWidth s i1 let p, i3 = parsePrecision s i2 let typeChar, i4 = parseTypeChar s i3 + // shortcut for the simpliest case // if typeChar is not % or it has star as width\precision - resort to long path if typeChar = '%' && not (w = StarValue || p = StarValue) then @@ -170,33 +173,43 @@ module internal PrintfImpl = static member inline Write (env: PrintfEnv<_, _, _>, a, b) = env.Write a env.Write b + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c) = Utils.Write(env, a, b) env.Write c + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d) = Utils.Write(env, a, b) Utils.Write(env, c, d) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e) = Utils.Write(env, a, b, c) Utils.Write(env, d, e) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f) = Utils.Write(env, a, b, c, d) Utils.Write(env, e, f) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f, g) = Utils.Write(env, a, b, c, d, e) Utils.Write(env, f, g) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f, g, h) = Utils.Write(env, a, b, c, d, e, f) Utils.Write(env, g, h) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f, g, h, i) = Utils.Write(env, a, b, c, d, e, f, g) Utils.Write(env, h, i) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f, g, h, i, j) = Utils.Write(env, a, b, c, d, e, f, g, h) Utils.Write(env, i, j) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f, g, h, i, j, k) = Utils.Write(env, a, b, c, d, e, f, g, h, i) Utils.Write(env, j, k) + static member inline Write (env: PrintfEnv<_, _, _>, a, b, c, d, e, f, g, h, i, j, k, l, m) = Utils.Write(env, a, b, c, d, e, f, g, h, i, j, k) Utils.Write(env, l, m) @@ -980,6 +993,7 @@ module internal PrintfImpl = System.Diagnostics.Debug.Assert((padChar = ' ')) fun (fmt: string) (w: int) v -> rightJustifyWithSpaceAsPadChar (f fmt v) true (isPositive v) w prefixForPositives + module Float = let inline noJustification f (prefixForPositives: string) = fun (fmt: string) v -> @@ -1054,16 +1068,20 @@ module internal PrintfImpl = else raise (ArgumentException()) type ObjectPrinter = + static member ObjectToString<'T>(spec: FormatSpecifier) = basicWithPadding spec (fun (v: 'T) -> match box v with null -> "" | x -> x.ToString()) + /// Convert an interpoland to a string + static member InterpolandToString<'T>(spec: FormatSpecifier) = + basicWithPadding spec (fun (v: 'T) -> match box v with null -> "" | x -> x.ToString()) + static member GenericToStringCore(v: 'T, opts: Microsoft.FSharp.Text.StructuredPrintfImpl.FormatOptions, bindingFlags) = - // printfn %0A is considered to mean 'print width zero' - match box v with - | null -> - Microsoft.FSharp.Text.StructuredPrintfImpl.Display.anyToStringForPrintf opts bindingFlags (v, typeof<'T>) - | _ -> - Microsoft.FSharp.Text.StructuredPrintfImpl.Display.anyToStringForPrintf opts bindingFlags (v, v.GetType()) + let vty = + match box v with + | null -> typeof<'T> + | _ -> v.GetType() + Microsoft.FSharp.Text.StructuredPrintfImpl.Display.anyToStringForPrintf opts bindingFlags (v, vty) static member GenericToString<'T>(spec: FormatSpecifier) = let bindingFlags = @@ -1079,6 +1097,7 @@ module internal PrintfImpl = else o if spec.IsPrecisionSpecified then { o with PrintSize = spec.Precision} else o + match spec.IsStarWidth, spec.IsStarPrecision with | true, true -> box (fun (v: 'T) (width: int) (prec: int) -> @@ -1086,21 +1105,21 @@ module internal PrintfImpl = let opts = if not useZeroWidth then { opts with PrintWidth = width} else opts ObjectPrinter.GenericToStringCore(v, opts, bindingFlags) ) + | true, false -> box (fun (v: 'T) (width: int) -> let opts = if not useZeroWidth then { opts with PrintWidth = width} else opts - ObjectPrinter.GenericToStringCore(v, opts, bindingFlags) - ) + ObjectPrinter.GenericToStringCore(v, opts, bindingFlags)) + | false, true -> box (fun (v: 'T) (prec: int) -> let opts = { opts with PrintSize = prec } - ObjectPrinter.GenericToStringCore(v, opts, bindingFlags) - ) + ObjectPrinter.GenericToStringCore(v, opts, bindingFlags) ) + | false, false -> box (fun (v: 'T) -> - ObjectPrinter.GenericToStringCore(v, opts, bindingFlags) - ) - + ObjectPrinter.GenericToStringCore(v, opts, bindingFlags)) + let basicNumberToString (ty: Type) (spec: FormatSpecifier) = System.Diagnostics.Debug.Assert(not spec.IsPrecisionSpecified, "not spec.IsPrecisionSpecified") @@ -1139,6 +1158,10 @@ module internal PrintfImpl = let private NonPublicStatics = BindingFlags.NonPublic ||| BindingFlags.Static + let mi_GenericToString = typeof.GetMethod("GenericToString", NonPublicStatics) + let mi_ObjectToString = typeof.GetMethod("ObjectToString", NonPublicStatics) + let mi_InterpolandToString = typeof.GetMethod("InterpolandToString", NonPublicStatics) + let private getValueConverter (ty: Type) (spec: FormatSpecifier) : obj = match spec.TypeChar with | 'b' -> @@ -1160,12 +1183,13 @@ module internal PrintfImpl = | 'g' | 'G' -> basicFloatToString ty spec | 'A' -> - let mi = typeof.GetMethod("GenericToString", NonPublicStatics) - let mi = mi.MakeGenericMethod ty + let mi = mi_GenericToString.MakeGenericMethod ty mi.Invoke(null, [| box spec |]) | 'O' -> - let mi = typeof.GetMethod("ObjectToString", NonPublicStatics) - let mi = mi.MakeGenericMethod ty + let mi = mi_ObjectToString.MakeGenericMethod ty + mi.Invoke(null, [| box spec |]) + | 'P' -> + let mi = mi_InterpolandToString.MakeGenericMethod ty mi.Invoke(null, [| box spec |]) | _ -> raise (ArgumentException(SR.GetString(SR.printfBadFormatSpecifier))) @@ -1242,6 +1266,15 @@ module internal PrintfImpl = System.Diagnostics.Debug.Assert(args.Count = types.Count, "args.Count = types.Count") args.Count = 0 + /// Type of element that is stored in cache + /// Pair: factory for the printer + number of text blocks that printer will produce (used to preallocate buffers) + [] + type CachedItem<'T, 'State, 'Residue, 'Result> = + { format: string + factory: PrintfFactory<'State, 'Residue, 'Result, 'T> + blockCount: int + isInterp: bool } + /// Parses format string and creates result printer function. /// First it recursively consumes format string up to the end, then during unwinding builds printer using PrintfBuilderStack as storage for arguments. /// idea of implementation is very simple: every step can either push argument to the stack (if current block of 5 format specifiers is not yet filled) @@ -1399,10 +1432,9 @@ module internal PrintfImpl = else buildPlainFinal(plainArgs, plainTypes) - let rec parseFromFormatSpecifier (prefix: string) (s: string) (funcTy: Type) i: int = + let rec parseFromFormatSpecifier isInterp (prefix: string) (s: string) (funcTy: Type) i: int = - if i >= s.Length then 0 - else + if i >= s.Length then 0 else System.Diagnostics.Debug.Assert(s.[i] = '%', "s.[i] = '%'") count <- count + 1 @@ -1411,6 +1443,9 @@ module internal PrintfImpl = let width, i = FormatString.parseWidth s i let precision, i = FormatString.parsePrecision s i let typeChar, i = FormatString.parseTypeChar s i + // Skip %P insertion points added after %d in interpolated strings + let i = FormatString.skipInterpHole isInterp s i + let spec = { TypeChar = typeChar; Precision = precision; Flags = flags; Width = width} let next, suffix = FormatString.findNextFormatSpecifier s i @@ -1431,7 +1466,7 @@ module internal PrintfImpl = let retTy = argTys.[argTys.Length - 1] - let numberOfArgs = parseFromFormatSpecifier suffix s retTy next + let numberOfArgs = parseFromFormatSpecifier isInterp suffix s retTy next if spec.TypeChar = 'a' || spec.TypeChar = 't' || spec.IsStarWidth || spec.IsStarPrecision then if numberOfArgs = ContinuationOnStack then @@ -1491,7 +1526,7 @@ module internal PrintfImpl = else numberOfArgs + 1 - let parseFormatString (s: string) (funcTy: System.Type) : obj = + let parseFormatString isInterp (s: string) (funcTy: System.Type) : obj = optimizedArgCount <- 0 let prefixPos, prefix = FormatString.findNextFormatSpecifier s 0 if prefixPos = s.Length then @@ -1501,41 +1536,55 @@ module internal PrintfImpl = env.Finish() ) else - let n = parseFromFormatSpecifier prefix s funcTy prefixPos + let n = parseFromFormatSpecifier isInterp prefix s funcTy prefixPos if n = ContinuationOnStack || n = 0 then builderStack.PopValueUnsafe() else buildPlain n prefix - member __.Build<'T>(s: string) : PrintfFactory<'S, 'Re, 'Res, 'T> * int = - parseFormatString s typeof<'T> :?> _, (2 * count + 1) - optimizedArgCount // second component is used in SprintfEnv as value for internal buffer - - /// Type of element that is stored in cache - /// Pair: factory for the printer + number of text blocks that printer will produce (used to preallocate buffers) - type CachedItem<'T, 'State, 'Residue, 'Result> = PrintfFactory<'State, 'Residue, 'Result, 'T> * int + member __.Build<'T>(s: string, isInterp: bool) = + { format = s + factory = (parseFormatString isInterp s typeof<'T> :?> PrintfFactory<'S, 'Re, 'Res, 'T>) + blockCount = (2 * count + 1) - optimizedArgCount // second component is used in SprintfEnv as value for internal buffer + isInterp = isInterp } /// 2-level cache. - /// 1st-level stores last value that was consumed by the current thread in thread-static field thus providing shortcuts for scenarios when - /// printf is called in tight loop - /// 2nd level is global dictionary that maps format string to the corresponding PrintfFactory + /// + /// We can use the same caches for both interpolated and non-interpolated strings + /// since interpolated strings contain %P and don't overlap with non-interpolation strings, and if an interpolated + /// string doesn't contain %P then the processing of the format strings is semantically identical. type Cache<'T, 'State, 'Residue, 'Result>() = - static let generate fmt = PrintfBuilder<'State, 'Residue, 'Result>().Build<'T>(fmt) - static let mutable map = System.Collections.Concurrent.ConcurrentDictionary>() - static let getOrAddFunc = Func<_, _>(generate) - static let get (key: string) = map.GetOrAdd(key, getOrAddFunc) - - [] - [] - static val mutable private last: string * CachedItem<'T, 'State, 'Residue, 'Result> + + /// 1st level cache (type-indexed). Stores last value that was consumed by the current + /// thread in thread-static field thus providing shortcuts for scenarios when printf is + /// called in tight loop. + [] + static val mutable private mostRecent: CachedItem<'T, 'State, 'Residue, 'Result> - static member Get(key: Format<'T, 'State, 'Residue, 'Result>) = - if not (Cache<'T, 'State, 'Residue, 'Result>.last === null) - && key.Value.Equals (fst Cache<'T, 'State, 'Residue, 'Result>.last) then - snd Cache<'T, 'State, 'Residue, 'Result>.last + // 2nd level cache (type-indexed). Dictionary that maps format string to the corresponding cache entry + static let mutable dict : ConcurrentDictionary> = null + + static member Get(fmt: Format<'T, 'State, 'Residue, 'Result>, isInterp) = + let cacheEntry = Cache<'T, 'State, 'Residue, 'Result>.mostRecent + let key = fmt.Value + if not (cacheEntry === null) && key.Equals cacheEntry.format then + cacheEntry else - let v = get key.Value - Cache<'T, 'State, 'Residue, 'Result>.last <- (key.Value, v) + // Initialize the 2nd level cache if necessary. Note there's a race condition but it doesn't + // matter if we initialize these values twice (and lose one entry) + if isNull dict then + dict <- ConcurrentDictionary<_,_>() + + let v = + match dict.TryGetValue(key) with + | true, res -> res + | _ -> + let entry = PrintfBuilder<'State, 'Residue, 'Result>().Build<'T>(key, isInterp) + // Note there's a race condition but it doesn't matter if lose one entry + dict.TryAdd(key, entry) |> ignore + entry + Cache<'T, 'State, 'Residue, 'Result>.mostRecent <- v v type StringPrintfEnv<'Result>(k, n) = @@ -1573,10 +1622,10 @@ module internal PrintfImpl = override __.Write(s: string) = tw.Write s override __.WriteT(()) = () - let inline doPrintf fmt f = - let formatter, n = Cache<_, _, _, _>.Get fmt - let env() = f n - formatter env + let inline doPrintf fmt isInterp f = + let formatter = Cache<_, _, _, _>.Get (fmt, isInterp) + let env() = f formatter.blockCount + formatter.factory env [] module Printf = @@ -1595,34 +1644,40 @@ module Printf = [] let ksprintf continuation (format: StringFormat<'T, 'Result>) : 'T = - doPrintf format (fun n -> + doPrintf format false (fun n -> if n <= 2 then SmallStringPrintfEnv continuation :> PrintfEnv<_, _, _> else StringPrintfEnv(continuation, n) :> PrintfEnv<_, _, _> ) - [] - let sprintf (format: StringFormat<'T>) = - doPrintf format (fun n -> + let inline sprintfAux isInterp (format: StringFormat<'T>) = + doPrintf format isInterp (fun n -> if n <= 2 then SmallStringPrintfEnv id :> PrintfEnv<_, _, _> else StringPrintfEnv(id, n) :> PrintfEnv<_, _, _> ) + [] + let sprintf (format: StringFormat<'T>) = sprintfAux false format + + [] + [] + let isprintf (format: StringFormat<'T>) = sprintfAux true format + [] let kprintf continuation format = ksprintf continuation format [] let kbprintf continuation (builder: StringBuilder) format = - doPrintf format (fun _ -> + doPrintf format false (fun _ -> StringBuilderPrintfEnv(continuation, builder) :> PrintfEnv<_, _, _> ) [] let kfprintf continuation textWriter format = - doPrintf format (fun _ -> + doPrintf format false (fun _ -> TextWriterPrintfEnv(continuation, textWriter) :> PrintfEnv<_, _, _> ) diff --git a/src/fsharp/FSharp.Core/printf.fsi b/src/fsharp/FSharp.Core/printf.fsi index 10e17ec68d8..1d406efcea9 100644 --- a/src/fsharp/FSharp.Core/printf.fsi +++ b/src/fsharp/FSharp.Core/printf.fsi @@ -214,6 +214,14 @@ module Printf = [] val sprintf : format:StringFormat<'T> -> 'T + /// Interpolated print to a string via an internal string buffer and return + /// the result as a string. Helper printers must return strings. + /// The input formatter. + /// The formatted string. + [] + [] + val isprintf: format:StringFormat<'T> -> 'T + /// bprintf, but call the given 'final' function to generate the result. /// See kprintf. /// The function called after formatting to generate the format result. diff --git a/src/fsharp/LanguageFeatures.fs b/src/fsharp/LanguageFeatures.fs index 05572cfb9e9..7b340029981 100644 --- a/src/fsharp/LanguageFeatures.fs +++ b/src/fsharp/LanguageFeatures.fs @@ -31,6 +31,7 @@ type LanguageFeature = | AndBang | NullableOptionalInterop | DefaultInterfaceMemberConsumption + | StringInterpolation /// LanguageVersion management type LanguageVersion (specifiedVersionAsString) = @@ -67,6 +68,7 @@ type LanguageVersion (specifiedVersionAsString) = LanguageFeature.AndBang, previewVersion LanguageFeature.NullableOptionalInterop, previewVersion LanguageFeature.DefaultInterfaceMemberConsumption, previewVersion + LanguageFeature.StringInterpolation, previewVersion ] let specified = @@ -135,6 +137,7 @@ type LanguageVersion (specifiedVersionAsString) = | LanguageFeature.AndBang -> FSComp.SR.featureAndBang() | LanguageFeature.NullableOptionalInterop -> FSComp.SR.featureNullableOptionalInterop() | LanguageFeature.DefaultInterfaceMemberConsumption -> FSComp.SR.featureDefaultInterfaceMemberConsumption() + | LanguageFeature.StringInterpolation -> FSComp.SR.featureStringInterpolation() /// Get a version string associated with the given feature. member _.GetFeatureVersionString feature = diff --git a/src/fsharp/LanguageFeatures.fsi b/src/fsharp/LanguageFeatures.fsi index fd367388bdd..0110454f67e 100644 --- a/src/fsharp/LanguageFeatures.fsi +++ b/src/fsharp/LanguageFeatures.fsi @@ -19,6 +19,7 @@ type LanguageFeature = | AndBang | NullableOptionalInterop | DefaultInterfaceMemberConsumption + | StringInterpolation /// LanguageVersion management type LanguageVersion = diff --git a/src/fsharp/LexFilter.fs b/src/fsharp/LexFilter.fs index b270910224b..1588a25cbe0 100644 --- a/src/fsharp/LexFilter.fs +++ b/src/fsharp/LexFilter.fs @@ -385,6 +385,10 @@ let parenTokensBalance t1 t2 = | (CLASS, END) | (SIG, END) | (STRUCT, END) + | (INTERP_STRING_BEGIN_PART _, INTERP_STRING_END _) + | (INTERP_STRING_BEGIN_PART _, INTERP_STRING_PART _) + | (INTERP_STRING_PART _, INTERP_STRING_PART _) + | (INTERP_STRING_PART _, INTERP_STRING_END _) | (LBRACK_BAR, BAR_RBRACK) | (LESS true, GREATER true) | (BEGIN, END) -> true @@ -1229,6 +1233,8 @@ type LexFilterImpl (lightSyntaxStatus: LightSyntaxStatus, compilingFsLib, lexer, | BAR_RBRACK | WITH | FINALLY + | INTERP_STRING_PART _ + | INTERP_STRING_END _ | RQUOTE _ -> not (tokenBalancesHeadContext token stack) && // Only close the context if some context is going to match at some point in the stack. @@ -1362,12 +1368,17 @@ type LexFilterImpl (lightSyntaxStatus: LightSyntaxStatus, compilingFsLib, lexer, hwTokenFetch useBlockRule // Balancing rule. Encountering a ')' or '}' balances with a '(' or '{', even if not offside - | ((END | RPAREN | RBRACE | BAR_RBRACE | RBRACK | BAR_RBRACK | RQUOTE _ | GREATER true) as t2), (CtxtParen (t1, _) :: _) + | ((END | RPAREN | RBRACE | BAR_RBRACE | RBRACK | BAR_RBRACK | RQUOTE _ | GREATER true | INTERP_STRING_END _ | INTERP_STRING_PART _) as t2), (CtxtParen (t1, _) :: _) when parenTokensBalance t1 t2 -> if debug then dprintf "RPAREN/RBRACE/BAR_RBRACE/RBRACK/BAR_RBRACK/RQUOTE/END at %a terminates CtxtParen()\n" outputPos tokenStartPos popCtxt() - // Queue a dummy token at this position to check if any closing rules apply - delayToken(pool.UseLocation(tokenTup, ODUMMY token)) + match t2 with + | INTERP_STRING_PART _ -> + pushCtxt tokenTup (CtxtParen (token, tokenStartPos)) + pushCtxtSeqBlock(false, NoAddBlockEnd) + | _ -> + // Queue a dummy token at this position to check if any closing rules apply + delayToken(pool.UseLocation(tokenTup, ODUMMY token)) returnToken tokenLexbufState token // Balancing rule. Encountering a 'end' can balance with a 'with' but only when not offside @@ -1852,6 +1863,7 @@ type LexFilterImpl (lightSyntaxStatus: LightSyntaxStatus, compilingFsLib, lexer, else returnToken tokenLexbufState token + // 'with id = ' ~~~> CtxtSeqBlock // 'with M.id = ' ~~~> CtxtSeqBlock // 'with id1 = 1 @@ -1889,7 +1901,9 @@ type LexFilterImpl (lightSyntaxStatus: LightSyntaxStatus, compilingFsLib, lexer, returnToken tokenLexbufState token // '(' tokens are balanced with ')' tokens and also introduce a CtxtSeqBlock - | (BEGIN | LPAREN | SIG | LBRACE | LBRACE_BAR | LBRACK | LBRACK_BAR | LQUOTE _ | LESS true), _ -> + // $".... { ... } ... { ....} " pushes a block context at first { + // $".... { ... } ... { ....} " pushes a block context at second { + | (BEGIN | LPAREN | SIG | LBRACE | LBRACE_BAR | LBRACK | LBRACK_BAR | LQUOTE _ | LESS true | INTERP_STRING_BEGIN_PART _), _ -> if debug then dprintf "LPAREN etc., pushes CtxtParen, pushing CtxtSeqBlock, tokenStartPos = %a\n" outputPos tokenStartPos pushCtxt tokenTup (CtxtParen (token, tokenStartPos)) pushCtxtSeqBlock(false, NoAddBlockEnd) diff --git a/src/fsharp/ParseHelpers.fs b/src/fsharp/ParseHelpers.fs index 0024246643b..5b3f3c59447 100644 --- a/src/fsharp/ParseHelpers.fs +++ b/src/fsharp/ParseHelpers.fs @@ -147,6 +147,16 @@ let rec LexerIfdefEval (lookup: string -> bool) = function // Parsing: continuations for whitespace tokens //------------------------------------------------------------------------ +[] +type LexerStringKind = + { IsByteString: bool + IsInterpolated: bool + IsInterpolatedFirst: bool } + static member String = { IsByteString = false; IsInterpolated = false; IsInterpolatedFirst=false } + static member ByteString = { IsByteString = true; IsInterpolated = false; IsInterpolatedFirst=false } + static member InterpolatedStringFirst = { IsByteString = false; IsInterpolated = true; IsInterpolatedFirst=true } + static member InterpolatedStringPart = { IsByteString = false; IsInterpolated = true; IsInterpolatedFirst=false } + /// The parser defines a number of tokens for whitespace and /// comments eliminated by the lexer. These carry a specification of /// a continuation for the lexer for continued processing after we've dealt with @@ -156,9 +166,9 @@ let rec LexerIfdefEval (lookup: string -> bool) = function type LexerWhitespaceContinuation = | Token of ifdef: LexerIfdefStackEntries | IfDefSkip of ifdef: LexerIfdefStackEntries * int * range: range - | String of ifdef: LexerIfdefStackEntries * range: range - | VerbatimString of ifdef: LexerIfdefStackEntries * range: range - | TripleQuoteString of ifdef: LexerIfdefStackEntries * range: range + | String of ifdef: LexerIfdefStackEntries * kind: LexerStringKind * range: range + | VerbatimString of ifdef: LexerIfdefStackEntries * kind: LexerStringKind * range: range + | TripleQuoteString of ifdef: LexerIfdefStackEntries * kind: LexerStringKind * range: range | Comment of ifdef: LexerIfdefStackEntries * int * range: range | SingleLineComment of ifdef: LexerIfdefStackEntries * int * range: range | StringInComment of ifdef: LexerIfdefStackEntries * int * range: range diff --git a/src/fsharp/PostInferenceChecks.fs b/src/fsharp/PostInferenceChecks.fs index 711bae6cbbc..60ca09c6781 100644 --- a/src/fsharp/PostInferenceChecks.fs +++ b/src/fsharp/PostInferenceChecks.fs @@ -799,7 +799,7 @@ and CheckForOverAppliedExceptionRaisingPrimitive (cenv: cenv) expr = | OptionalCoerce(Expr.Val (failwithfFunc, _, funcRange)) when valRefEq g failwithfFunc g.failwithf_vref -> match argsl with | Expr.App (Expr.Val (newFormat, _, _), _, [_; typB; typC; _; _], [Expr.Const (Const.String formatString, formatRange, _)], _) :: xs when valRefEq g newFormat g.new_format_vref -> - match CheckFormatStrings.TryCountFormatStringArguments formatRange g formatString typB typC with + match CheckFormatStrings.TryCountFormatStringArguments formatRange g false formatString typB typC with | Some n -> let expected = n + 1 let actual = List.length xs + 1 diff --git a/src/fsharp/SyntaxTree.fs b/src/fsharp/SyntaxTree.fs index a071f1c526b..d21611c24a4 100644 --- a/src/fsharp/SyntaxTree.fs +++ b/src/fsharp/SyntaxTree.fs @@ -1003,6 +1003,11 @@ type SynExpr = expr: SynExpr * range: range + /// F# syntax: interpolated string, e.g. "abc%{123}" + | InterpolatedString of + contents: Choice list * + range: range + /// Gets the syntax range of this constuct member e.Range = match e with @@ -1069,7 +1074,8 @@ type SynExpr = | SynExpr.LetOrUseBang (range=m) | SynExpr.MatchBang (range=m) | SynExpr.DoBang (range=m) - | SynExpr.Fixed (range=m) -> m + | SynExpr.Fixed (range=m) + | SynExpr.InterpolatedString (range=m) -> m | SynExpr.Ident id -> id.idRange /// Get the Range ignoring any (parse error) extra trailing dots diff --git a/src/fsharp/SyntaxTreeOps.fs b/src/fsharp/SyntaxTreeOps.fs index 43d473a137c..9e94afc0047 100644 --- a/src/fsharp/SyntaxTreeOps.fs +++ b/src/fsharp/SyntaxTreeOps.fs @@ -723,4 +723,7 @@ let rec synExprContainsError inpExpr = | SynExpr.LetOrUseBang (rhs=e1;body=e2;andBangs=es) -> walkExpr e1 || walkExprs [ for (_,_,_,_,e,_) in es do yield e ] || walkExpr e2 + | SynExpr.InterpolatedString (parts, _m) -> + walkExprs (parts |> List.choose (function Choice1Of2 _ -> None | Choice2Of2 x -> Some x)) + walkExpr inpExpr diff --git a/src/fsharp/TcGlobals.fs b/src/fsharp/TcGlobals.fs index 68ef24cb764..9eeef3abc01 100755 --- a/src/fsharp/TcGlobals.fs +++ b/src/fsharp/TcGlobals.fs @@ -428,6 +428,7 @@ type public TcGlobals(compilingFslib: bool, ilg:ILGlobals, fslibCcu: CcuThunk, d let fslib_MFQueryRunExtensionsLowPriority_nleref = mkNestedNonLocalEntityRef fslib_MFQueryRunExtensions_nleref "LowPriority" let fslib_MFQueryRunExtensionsHighPriority_nleref = mkNestedNonLocalEntityRef fslib_MFQueryRunExtensions_nleref "HighPriority" + let fslib_MFPrintfModule_nleref = mkNestedNonLocalEntityRef fslib_MFCore_nleref "PrintfModule" let fslib_MFSeqModule_nleref = mkNestedNonLocalEntityRef fslib_MFCollections_nleref "SeqModule" let fslib_MFListModule_nleref = mkNestedNonLocalEntityRef fslib_MFCollections_nleref "ListModule" let fslib_MFArrayModule_nleref = mkNestedNonLocalEntityRef fslib_MFCollections_nleref "ArrayModule" @@ -492,6 +493,7 @@ type public TcGlobals(compilingFslib: bool, ilg:ILGlobals, fslibCcu: CcuThunk, d fslib_MFQueryRunExtensionsLowPriority_nleref fslib_MFQueryRunExtensionsHighPriority_nleref + fslib_MFPrintfModule_nleref fslib_MFSeqModule_nleref fslib_MFListModule_nleref fslib_MFArrayModule_nleref @@ -703,6 +705,7 @@ type public TcGlobals(compilingFslib: bool, ilg:ILGlobals, fslibCcu: CcuThunk, d let v_seq_empty_info = makeIntrinsicValRef(fslib_MFSeqModule_nleref, "empty" , None , Some "Empty" , [vara], ([], mkSeqTy varaTy)) let v_new_format_info = makeIntrinsicValRef(fslib_MFCore_nleref, ".ctor" , Some "PrintfFormat`5", None , [vara;varb;varc;vard;vare], ([[v_string_ty]], mkPrintfFormatTy varaTy varbTy varcTy vardTy vareTy)) let v_sprintf_info = makeIntrinsicValRef(fslib_MFExtraTopLevelOperators_nleref, "sprintf" , None , Some "PrintFormatToStringThen", [vara], ([[mk_format4_ty varaTy v_unit_ty v_string_ty v_string_ty]], varaTy)) + let v_isprintf_info = makeIntrinsicValRef(fslib_MFPrintfModule_nleref, "isprintf" , None , Some "InterpolatedPrintFormatToStringThen", [vara], ([[mk_format4_ty varaTy v_unit_ty v_string_ty v_string_ty]], varaTy)) let v_lazy_force_info = makeIntrinsicValRef(fslib_MFLazyExtensions_nleref, "Force" , Some "Lazy`1" , None , [vara], ([[mkLazyTy varaTy]; []], varaTy)) let v_lazy_create_info = makeIntrinsicValRef(fslib_MFLazyExtensions_nleref, "Create" , Some "Lazy`1" , None , [vara], ([[v_unit_ty --> varaTy]], mkLazyTy varaTy)) @@ -1365,6 +1368,7 @@ type public TcGlobals(compilingFslib: bool, ilg:ILGlobals, fslibCcu: CcuThunk, d member val seq_empty_vref = ValRefForIntrinsic v_seq_empty_info member val new_format_vref = ValRefForIntrinsic v_new_format_info member val sprintf_vref = ValRefForIntrinsic v_sprintf_info + member val isprintf_vref = ValRefForIntrinsic v_isprintf_info member val unbox_vref = ValRefForIntrinsic v_unbox_info member val unbox_fast_vref = ValRefForIntrinsic v_unbox_fast_info member val istype_vref = ValRefForIntrinsic v_istype_info @@ -1390,6 +1394,8 @@ type public TcGlobals(compilingFslib: bool, ilg:ILGlobals, fslibCcu: CcuThunk, d member __.seq_map_info = v_seq_map_info member __.seq_singleton_info = v_seq_singleton_info member __.seq_empty_info = v_seq_empty_info + member __.sprintf_info = v_sprintf_info + member __.isprintf_info = v_isprintf_info member __.new_format_info = v_new_format_info member __.unbox_info = v_unbox_info member __.get_generic_comparer_info = v_get_generic_comparer_info diff --git a/src/fsharp/TypeChecker.fs b/src/fsharp/TypeChecker.fs index 2cb72a3047b..ae91c455012 100755 --- a/src/fsharp/TypeChecker.fs +++ b/src/fsharp/TypeChecker.fs @@ -5900,6 +5900,18 @@ and TcExprUndelayed cenv overallTy env tpenv (synExpr: SynExpr) = CallExprHasTypeSink cenv.tcSink (m, env.NameEnv, overallTy, env.AccessRights) TcConstStringExpr cenv overallTy env m tpenv s + | SynExpr.InterpolatedString (parts, m) -> + tryLanguageFeatureError cenv.g.langVersion LanguageFeature.StringInterpolation m + + // CHeck the library support is available in the referenced FSharp.Core + if cenv.g.isprintf_vref.TryDeref.IsNone then + languageFeatureNotSupportedInLibraryError cenv.g.langVersion LanguageFeature.StringInterpolation m + + CallExprHasTypeSink cenv.tcSink (m, env.NameEnv, overallTy, env.AccessRights) + let exprFills = parts |> List.choose(function Choice1Of2 _ -> None | Choice2Of2 e -> Some e) + let stringText = parts |> List.map(function Choice1Of2 s -> s | Choice2Of2 _ -> "%P") |> String.concat "" + TcConstStringFormatExpr cenv overallTy env m tpenv (Some exprFills) stringText + | SynExpr.Const (synConst, m) -> CallExprHasTypeSink cenv.tcSink (m, env.NameEnv, overallTy, env.AccessRights) TcConstExpr cenv overallTy env m tpenv synConst @@ -7081,20 +7093,38 @@ and TcObjectExpr cenv overallTy env tpenv (synObjTy, argopt, binds, extraImpls, and TcConstStringExpr cenv overallTy env m tpenv s = if (AddCxTypeEqualsTypeUndoIfFailed env.DisplayEnv cenv.css m overallTy cenv.g.string_ty) then - mkString cenv.g m s, tpenv + mkString cenv.g m s, tpenv else - let aty = NewInferenceType () - let bty = NewInferenceType () - let cty = NewInferenceType () - let dty = NewInferenceType () - let ety = NewInferenceType () - let ty' = mkPrintfFormatTy cenv.g aty bty cty dty ety - if (not (isObjTy cenv.g overallTy) && AddCxTypeMustSubsumeTypeUndoIfFailed env.DisplayEnv cenv.css m overallTy ty') then + TcConstStringFormatExpr cenv overallTy env m tpenv None s + +and TcConstStringFormatExpr cenv overallTy env m tpenv (optFills: SynExpr list option) s = + let g = cenv.g + let isInterp = optFills.IsSome + let aty = NewInferenceType () + let bty = if isInterp then g.unit_ty else NewInferenceType () + let cty = if isInterp then g.string_ty else NewInferenceType () + let dty = if isInterp then g.string_ty else NewInferenceType () + let ety = NewInferenceType () + let formatTy = mkPrintfFormatTy g aty bty cty dty ety + + let ok = + match optFills with + | Some _ -> + // If this is an interpolated string then the result must be a string + UnifyTypes cenv env m overallTy g.string_ty + true + | None -> + // This might qualify as a format string - check via a type directed rule + not (isObjTy g overallTy) && AddCxTypeMustSubsumeTypeUndoIfFailed env.DisplayEnv cenv.css m overallTy formatTy + + if ok then // Parse the format string to work out the phantom types let formatStringCheckContext = match cenv.tcSink.CurrentSink with None -> None | Some sink -> sink.FormatStringCheckContext let normalizedString = (s.Replace("\r\n", "\n").Replace("\r", "\n")) - let (aty', ety'), specifierLocations = (try CheckFormatStrings.ParseFormatString m cenv.g formatStringCheckContext normalizedString bty cty dty with Failure s -> error (Error(FSComp.SR.tcUnableToParseFormatString s, m))) + let (argtys, atyRequired, etyRequired), specifierLocations = + try CheckFormatStrings.ParseFormatString m g optFills.IsSome formatStringCheckContext normalizedString bty cty dty + with Failure s -> error (Error(FSComp.SR.tcUnableToParseFormatString s, m)) match cenv.tcSink.CurrentSink with | None -> () @@ -7102,12 +7132,23 @@ and TcConstStringExpr cenv overallTy env m tpenv s = for specifierLocation, numArgs in specifierLocations do sink.NotifyFormatSpecifierLocation(specifierLocation, numArgs) - UnifyTypes cenv env m aty aty' - UnifyTypes cenv env m ety ety' - mkCallNewFormat cenv.g m aty bty cty dty ety (mkString cenv.g m s), tpenv - else - UnifyTypes cenv env m overallTy cenv.g.string_ty - mkString cenv.g m s, tpenv + UnifyTypes cenv env m aty atyRequired + UnifyTypes cenv env m ety etyRequired + let fmtExpr = mkCallNewFormat g m aty bty cty dty ety (mkString g m s) + match optFills with + | None -> fmtExpr, tpenv + | Some synFillExprs -> + // Check the expressions filling the holes + if argtys.Length <> synFillExprs.Length then + error (Error(FSComp.SR.tcInterpolationMixedWithPercent(), m)) + let flexes = argtys |> List.map (fun _ -> false) + let fillExprs, tpenv = TcExprs cenv env m tpenv flexes argtys synFillExprs + // Make the call to isprintf + mkCall_isprintf g m aty fmtExpr fillExprs, tpenv + + else + UnifyTypes cenv env m overallTy g.string_ty + mkString g m s, tpenv //------------------------------------------------------------------------- // TcConstExpr @@ -9694,6 +9735,7 @@ and TcItemThen cenv overallTy env tpenv (item, mItem, rest, afterResolution) del | SynExpr.AddressOf (_, synExpr, _, _) | SynExpr.Quote (_, _, synExpr, _, _) -> isSimpleArgument synExpr + | SynExpr.InterpolatedString _ | SynExpr.Null _ | SynExpr.Ident _ | SynExpr.Const _ diff --git a/src/fsharp/TypedTreeOps.fs b/src/fsharp/TypedTreeOps.fs index 513727bf1c2..978f151ae6c 100644 --- a/src/fsharp/TypedTreeOps.fs +++ b/src/fsharp/TypedTreeOps.fs @@ -6972,6 +6972,9 @@ let mkCallSeqSingleton g m ty1 arg1 = let mkCallSeqEmpty g m ty1 = mkApps g (typedExprForIntrinsic g m g.seq_empty_info, [[ty1]], [ ], m) +let mkCall_isprintf (g: TcGlobals) m aty fmt es = + mkApps g (typedExprForIntrinsic g m g.isprintf_info, [[aty]], fmt::es , m) + let mkCallDeserializeQuotationFSharp20Plus g m e1 e2 e3 e4 = let args = [ e1; e2; e3; e4 ] mkApps g (typedExprForIntrinsic g m g.deserialize_quoted_FSharp_20_plus_info, [], [ mkRefTupledNoTypes g m args ], m) diff --git a/src/fsharp/TypedTreeOps.fsi b/src/fsharp/TypedTreeOps.fsi index dce873e1a8c..98c4e7eeaee 100755 --- a/src/fsharp/TypedTreeOps.fsi +++ b/src/fsharp/TypedTreeOps.fsi @@ -1969,6 +1969,8 @@ val mkCallSeqSingleton : TcGlobals -> range -> TType -> Expr -> Expr val mkCallSeqEmpty : TcGlobals -> range -> TType -> Expr +val mkCall_isprintf: g: TcGlobals -> m: range -> funcTy: TType -> fmtExpr: Expr -> fillExprs: Expr list -> Expr + val mkILAsmCeq : TcGlobals -> range -> Expr -> Expr -> Expr val mkILAsmClt : TcGlobals -> range -> Expr -> Expr -> Expr diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index 527f6ebd084..83f7209d9f1 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -115,19 +115,36 @@ let startString args (lexbuf: UnicodeLexing.Lexbuf) = let buf = ByteBuffer.Create 100 let m = lexbuf.LexemeRange let startp = lexbuf.StartPos - let fin = (fun _m2 b s -> - // Adjust the start-of-token mark back to the true start of the token - lexbuf.StartPos <- startp - if b then - if Lexhelp.stringBufferIsBytes buf then - BYTEARRAY (Lexhelp.stringBufferAsBytes buf) - else ( - fail args lexbuf (FSComp.SR.lexByteArrayCannotEncode()) () - BYTEARRAY (Lexhelp.stringBufferAsBytes buf) - ) - else - STRING (Lexhelp.stringBufferAsString s)) + let fin = + LexerStringFinisher (fun buf kind isPart -> + // Adjust the start-of-token mark back to the true start of the token + lexbuf.StartPos <- startp + if kind.IsByteString then + if kind.IsInterpolated then + fail args lexbuf (FSComp.SR.lexByteStringMayNotBeInterpolated()) () + BYTEARRAY (Lexhelp.stringBufferAsBytes buf) + elif Lexhelp.stringBufferIsBytes buf then + BYTEARRAY (Lexhelp.stringBufferAsBytes buf) + else + fail args lexbuf (FSComp.SR.lexByteArrayCannotEncode()) () + BYTEARRAY (Lexhelp.stringBufferAsBytes buf) + elif kind.IsInterpolated then + let s = Lexhelp.stringBufferAsString buf + if kind.IsInterpolatedFirst then + if isPart then + INTERP_STRING_BEGIN_PART s + else + INTERP_STRING_BEGIN_END s + else + if isPart then + INTERP_STRING_PART s + else + INTERP_STRING_END s + else + let s = Lexhelp.stringBufferAsString buf + STRING s) buf,fin,m + // Utility functions for processing XML documentation @@ -522,22 +539,31 @@ rule token args skip = parse | "(*IF-CAML*)" | "(*IF-OCAML*)" { let m = lexbuf.LexemeRange - if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack,m))) else mlOnly m args skip lexbuf } + if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack, m))) else mlOnly m args skip lexbuf } | '"' - { let buf,fin,m = startString args lexbuf - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string (buf,fin,m,args) skip lexbuf } + { let buf, fin, m = startString args lexbuf + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, LexerStringKind.String, m))) else string (buf, fin, m, LexerStringKind.String, args) skip lexbuf } - | '"' '"' '"' + | '$' '"' { let buf,fin,m = startString args lexbuf - if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack,m))) else tripleQuoteString (buf,fin,m,args) skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, LexerStringKind.InterpolatedStringFirst, m))) else string (buf, fin, m, LexerStringKind.InterpolatedStringFirst, args) skip lexbuf } + + | '"' '"' '"' + { let buf, fin, m = startString args lexbuf + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, LexerStringKind.String, m))) else tripleQuoteString (buf, fin, m, LexerStringKind.String, args) skip lexbuf } - | '$' '"' - { fail args lexbuf (FSComp.SR.lexTokenReserved()) (WHITESPACE (LexCont.Token args.ifdefStack)) } + | '$' '"' '"' '"' + { let buf, fin, m = startString args lexbuf + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, LexerStringKind.InterpolatedStringFirst, m))) else tripleQuoteString (buf, fin, m, LexerStringKind.InterpolatedStringFirst, args) skip lexbuf } | '@' '"' - { let buf,fin,m = startString args lexbuf - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString (buf,fin,m,args) skip lexbuf } + { let buf, fin, m = startString args lexbuf + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, LexerStringKind.String, m))) else verbatimString (buf, fin, m, LexerStringKind.String, args) skip lexbuf } + + | ("$@" | "@$") '"' + { let buf, fin, m = startString args lexbuf + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, LexerStringKind.InterpolatedStringFirst, m))) else verbatimString (buf, fin, m, LexerStringKind.InterpolatedStringFirst, args) skip lexbuf } | truewhite+ { if skip then token args skip lexbuf @@ -703,11 +729,26 @@ rule token args skip = parse | ">]" { GREATER_RBRACK } - | "{" { LBRACE } + | "{" { + if args.interpolatedStringNesting > 0 then + args.interpolatedStringNesting <- args.interpolatedStringNesting + 1 + LBRACE } | "|" { BAR } - | "}" { RBRACE } + | "}" + { + // We encounter a '}' in the expression token stream. First check if we're in an interpolated string expression + if args.interpolatedStringNesting = 1 then + let buf, fin, m = startString args lexbuf + args.interpolatedStringNesting <- args.interpolatedStringNesting - 1 + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, LexerStringKind.InterpolatedStringPart, m))) else string (buf, fin, m, LexerStringKind.InterpolatedStringPart, args) skip lexbuf + elif args.interpolatedStringNesting > 1 then + args.interpolatedStringNesting <- args.interpolatedStringNesting - 1 + RBRACE + else + RBRACE + } | "$" { DOLLAR } @@ -817,7 +858,7 @@ and ifdefSkip n m args skip = parse // If #if is the first thing on the line then increase depth, otherwise skip, because it is invalid (e.g. "(**) #if ...") if (m.StartColumn <> 0) then - if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf + if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, n, m))) else ifdefSkip n m args skip lexbuf else let tok = INACTIVECODE(LexCont.EndLine(LexerEndlineContinuation.Skip(args.ifdefStack,n+1,m))) if not skip then tok else endline (LexerEndlineContinuation.Skip(args.ifdefStack,n+1,m)) args skip lexbuf } @@ -828,7 +869,7 @@ and ifdefSkip n m args skip = parse // If #else is the first thing on the line then process it, otherwise ignore, because it is invalid (e.g. "(**) #else ...") if (m.StartColumn <> 0) then - if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf + if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, n, m))) else ifdefSkip n m args skip lexbuf elif n = 0 then match args.ifdefStack with | []-> LEX_FAILURE (FSComp.SR.lexHashElseNoMatchingIf()) @@ -838,7 +879,7 @@ and ifdefSkip n m args skip = parse args.ifdefStack <- (IfDefElse,m) :: rest if not skip then (HASH_ELSE(m,lexed,LexCont.EndLine(LexerEndlineContinuation.Token(args.ifdefStack)))) else endline (LexerEndlineContinuation.Token(args.ifdefStack)) args skip lexbuf else - if not skip then (INACTIVECODE(LexCont.EndLine(LexerEndlineContinuation.Skip(args.ifdefStack,n,m)))) else endline (LexerEndlineContinuation.Skip(args.ifdefStack,n,m)) args skip lexbuf } + if not skip then (INACTIVECODE(LexCont.EndLine(LexerEndlineContinuation.Skip(args.ifdefStack, n, m)))) else endline (LexerEndlineContinuation.Skip(args.ifdefStack, n, m)) args skip lexbuf } | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)? { let lexed = lexeme lexbuf @@ -846,7 +887,7 @@ and ifdefSkip n m args skip = parse // If #endif is the first thing on the line then process it, otherwise ignore, because it is invalid (e.g. "(**) #endif ...") if (m.StartColumn <> 0) then - if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf + if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, n, m))) else ifdefSkip n m args skip lexbuf elif n = 0 then match args.ifdefStack with | [] -> LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf()) @@ -869,10 +910,10 @@ and ifdefSkip n m args skip = parse | _ { // This tries to be nice and get tokens as 'words' because VS uses this when selecting stuff - if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf } + if not skip then (INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, n, m))) else ifdefSkip n m args skip lexbuf } | eof - { EOF (LexCont.IfDefSkip(args.ifdefStack,n,m)) } + { EOF (LexCont.IfDefSkip(args.ifdefStack, n, m)) } // Called after lexing #if IDENT/#else/#endif - this checks whether there is nothing except end of line // or end of file and then calls the lexing function specified by 'cont' - either token or ifdefSkip @@ -898,35 +939,35 @@ and endline cont args skip = parse and string sargs skip = parse | '\\' newline anywhite* - { let (_buf,_fin,m,args) = sargs + { let (_buf, _fin, m, kind, args) = sargs newline lexbuf - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | escape_char - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addByteChar buf (escape (lexeme lexbuf).[1]) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | trigraph - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs let s = lexeme lexbuf addByteChar buf (trigraph s.[1] s.[2] s.[3]) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | hexGraphShort - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeChar buf (int (hexGraphShort (lexemeTrimLeft lexbuf 2))) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | unicodeGraphShort - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeChar buf (int (unicodeGraphShort (lexemeTrimLeft lexbuf 2))) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | unicodeGraphLong - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs let hexChars = lexemeTrimLeft lexbuf 2 - let result () = if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf + let result () = if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf match unicodeGraphLong hexChars with | Invalid -> fail args lexbuf (FSComp.SR.lexInvalidUnicodeLiteral hexChars) (result ()) @@ -939,213 +980,237 @@ and string sargs skip = parse result () } | '"' - { let (buf,fin,_m,_args) = sargs - let m2 = lexbuf.LexemeRange - callStringFinisher fin buf m2 false } + { let (buf, fin, _m, kind, _args) = sargs + fin.Finish buf kind false } | '"''B' - { let (buf,fin,_m,_args) = sargs - let m2 = lexbuf.LexemeRange - callStringFinisher fin buf m2 true } + { let (buf, fin, _m, kind, _args) = sargs + fin.Finish buf { kind with IsByteString = true } false } + + | '{' + { let (buf, fin, m, kind, args) = sargs + if kind.IsInterpolated then + args.interpolatedStringNesting <- args.interpolatedStringNesting + 1 + fin.Finish buf kind true + else + addUnicodeString buf (lexeme lexbuf) + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf + } | newline - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs newline lexbuf addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | ident - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | integer | xinteger - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | anywhite + - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } | eof - { let (_buf,_fin,m,args) = sargs - EOF (LexCont.String(args.ifdefStack,m)) } + { let (_buf, _fin, m, kind, args) = sargs + EOF (LexCont.String(args.ifdefStack, kind, m)) } | surrogateChar surrogateChar // surrogate code points always come in pairs | _ - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack,m))) else string sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.String(args.ifdefStack, kind, m))) else string sargs skip lexbuf } and verbatimString sargs skip = parse | '"' '"' - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addByteChar buf '\"' - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf } | '"' - { let (buf,fin,_m,_args) = sargs - let m2 = lexbuf.LexemeRange - callStringFinisher fin buf m2 false } + { let (buf, fin, _m, kind, _args) = sargs + fin.Finish buf kind false } | '"''B' - { let (buf,fin,_m,_args) = sargs - let m2 = lexbuf.LexemeRange - callStringFinisher fin buf m2 true } + { let (buf, fin, _m, kind, _args) = sargs + fin.Finish buf { kind with IsByteString = true } false } | newline - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs newline lexbuf addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf } + + | '{' + { let (buf, fin, m, kind, args) = sargs + if kind.IsInterpolated then + fin.Finish buf kind true + else + addUnicodeString buf (lexeme lexbuf) + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf + } | ident - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf } | integer | xinteger - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf } | anywhite + - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf } | eof - { let (_buf,_fin,m,args) = sargs - EOF (LexCont.VerbatimString(args.ifdefStack,m)) } + { let (_buf, _fin, m, kind, args) = sargs + EOF (LexCont.VerbatimString(args.ifdefStack, kind, m)) } + | surrogateChar surrogateChar // surrogate code points always come in pairs | _ - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack,m))) else verbatimString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.VerbatimString(args.ifdefStack, kind, m))) else verbatimString sargs skip lexbuf } and tripleQuoteString sargs skip = parse | '"' '"' '"' - { let (buf,fin,_m,_args) = sargs - let m2 = lexbuf.LexemeRange - callStringFinisher fin buf m2 false } + { let (buf, fin, _m, kind, _args) = sargs + fin.Finish buf kind false } | newline - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs newline lexbuf addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack,m))) else tripleQuoteString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, kind, m))) else tripleQuoteString sargs skip lexbuf } // The rest is to break into pieces to allow double-click-on-word and other such things | ident - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack,m))) else tripleQuoteString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, kind, m))) else tripleQuoteString sargs skip lexbuf } | integer | xinteger - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack,m))) else tripleQuoteString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, kind, m))) else tripleQuoteString sargs skip lexbuf } | anywhite + - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack,m))) else tripleQuoteString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, kind, m))) else tripleQuoteString sargs skip lexbuf } + + | '{' + { let (buf, fin, m, kind, args) = sargs + if kind.IsInterpolated then + fin.Finish buf kind true + else + addUnicodeString buf (lexeme lexbuf) + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, kind, m))) else tripleQuoteString sargs skip lexbuf + } | eof - { let (_buf,_fin,m,args) = sargs - EOF (LexCont.TripleQuoteString(args.ifdefStack,m)) } + { let (_buf, _fin, m, kind, args) = sargs + EOF (LexCont.TripleQuoteString(args.ifdefStack, kind, m)) } | surrogateChar surrogateChar // surrogate code points always come in pairs | _ - { let (buf,_fin,m,args) = sargs + { let (buf, _fin, m, kind, args) = sargs addUnicodeString buf (lexeme lexbuf) - if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack,m))) else tripleQuoteString sargs skip lexbuf } + if not skip then (STRING_TEXT (LexCont.TripleQuoteString(args.ifdefStack, kind, m))) else tripleQuoteString sargs skip lexbuf } // Parsing single-line comment - we need to split it into words for Visual Studio IDE and singleLineComment cargs skip = parse | newline - { let buff,_n,_m,args = cargs + { let buff,_n, _m, args = cargs trySaveXmlDoc lexbuf buff newline lexbuf // Saves the documentation (if we're collecting any) into a buffer-local variable. if not skip then (LINE_COMMENT (LexCont.Token args.ifdefStack)) else token args skip lexbuf } | eof - { let _, _n,_m,args = cargs + { let _, _n, _m, args = cargs // NOTE: it is legal to end a file with this comment, so we'll return EOF as a token EOF (LexCont.Token args.ifdefStack) } | [^ ' ' '\n' '\r' ]+ | anywhite+ - { let buff,n,m,args = cargs + { let buff, n, m, args = cargs // Append the current token to the XML documentation if we're collecting it tryAppendXmlDoc buff (lexeme lexbuf) - if not skip then (LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack,n,m))) else singleLineComment (buff,n,m,args) skip lexbuf } + if not skip then (LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, n, m))) else singleLineComment (buff,n, m, args) skip lexbuf } | surrogateChar surrogateChar - | _ { let _, _n,_m,args = cargs - if not skip then (LINE_COMMENT (LexCont.Token args.ifdefStack)) else token args skip lexbuf } + | _ { let _, _n, _m, args = cargs + if not skip then (LINE_COMMENT (LexCont.Token args.ifdefStack)) else token args skip lexbuf } and comment cargs skip = parse | char - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment (n, m, args) skip lexbuf } | '"' - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack, n, m))) else stringInComment n m args skip lexbuf } | '"' '"' '"' - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack, n, m))) else tripleQuoteStringInComment n m args skip lexbuf } | '@' '"' - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack, n, m))) else verbatimStringInComment n m args skip lexbuf } | "(*)" - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment cargs skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment cargs skip lexbuf } | '(' '*' - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n+1,m))) else comment (n+1,m,args) skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n+1, m))) else comment (n+1,m,args) skip lexbuf } | newline - { let n,m,args = cargs + { let n, m, args = cargs newline lexbuf - if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment cargs skip lexbuf } + if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment cargs skip lexbuf } | "*)" { - let n,m,args = cargs + let n, m, args = cargs if n > 1 then if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n-1,m))) else comment (n-1,m,args) skip lexbuf else if not skip then (COMMENT (LexCont.Token args.ifdefStack)) else token args skip lexbuf } | anywhite+ | [^ '\'' '(' '*' '\n' '\r' '"' ')' '@' ' ' '\t' ]+ - { let n,m,args = cargs - if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment cargs skip lexbuf } + { let n, m, args = cargs + if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment cargs skip lexbuf } | eof - { let n,m,args = cargs - EOF (LexCont.Comment(args.ifdefStack,n,m)) } + { let n, m, args = cargs + EOF (LexCont.Comment(args.ifdefStack, n, m)) } | surrogateChar surrogateChar - | _ { let n,m,args = cargs - if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf } + | _ { let n, m, args = cargs + if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment (n, m, args) skip lexbuf } and stringInComment n m args skip = parse // Follow string lexing, skipping tokens until it finishes | '\\' newline anywhite* { newline lexbuf - if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf } + if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack, n, m))) else stringInComment n m args skip lexbuf } | escape_char | trigraph @@ -1156,80 +1221,80 @@ and stringInComment n m args skip = parse | integer | xinteger | anywhite + - { if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack, n, m))) else stringInComment n m args skip lexbuf } | '"' - { if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf } + { if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment (n, m, args) skip lexbuf } | newline { newline lexbuf - if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf } + if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack, n, m))) else stringInComment n m args skip lexbuf } | eof - { EOF (LexCont.StringInComment(args.ifdefStack,n,m)) } + { EOF (LexCont.StringInComment(args.ifdefStack, n, m)) } | surrogateChar surrogateChar | _ - { if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.StringInComment(args.ifdefStack, n, m))) else stringInComment n m args skip lexbuf } and verbatimStringInComment n m args skip = parse // Follow verbatimString lexing, in short, skip double-quotes and other chars until we hit a single quote | '"' '"' - { if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack, n, m))) else verbatimStringInComment n m args skip lexbuf } | '"' - { if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf } + { if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment (n, m, args) skip lexbuf } | ident | integer | xinteger | anywhite + - { if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack, n, m))) else verbatimStringInComment n m args skip lexbuf } | newline { newline lexbuf - if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf } + if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack, n, m))) else verbatimStringInComment n m args skip lexbuf } | eof - { EOF (LexCont.VerbatimStringInComment(args.ifdefStack,n,m)) } + { EOF (LexCont.VerbatimStringInComment(args.ifdefStack, n, m)) } | surrogateChar surrogateChar | _ - { if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.VerbatimStringInComment(args.ifdefStack, n, m))) else verbatimStringInComment n m args skip lexbuf } and tripleQuoteStringInComment n m args skip = parse // Follow tripleQuoteString lexing | '"' '"' '"' - { if not skip then (COMMENT (LexCont.Comment(args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf } + { if not skip then (COMMENT (LexCont.Comment(args.ifdefStack, n, m))) else comment (n, m, args) skip lexbuf } | ident | integer | xinteger | anywhite + - { if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack, n, m))) else tripleQuoteStringInComment n m args skip lexbuf } | newline { newline lexbuf - if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf } + if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack, n, m))) else tripleQuoteStringInComment n m args skip lexbuf } | eof - { EOF (LexCont.TripleQuoteStringInComment(args.ifdefStack,n,m)) } + { EOF (LexCont.TripleQuoteStringInComment(args.ifdefStack, n, m)) } | surrogateChar surrogateChar | _ - { if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf } + { if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(args.ifdefStack, n, m))) else tripleQuoteStringInComment n m args skip lexbuf } and mlOnly m args skip = parse | "\"" { let buf = ByteBuffer.Create 100 let m2 = lexbuf.LexemeRange - let _ = string (buf,defaultStringFinisher,m2,args) skip lexbuf - if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack,m))) else mlOnly m args skip lexbuf } + let _ = string (buf, LexerStringFinisher.Default, m2, LexerStringKind.String, args) skip lexbuf + if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack, m))) else mlOnly m args skip lexbuf } | newline - { newline lexbuf; if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack,m))) else mlOnly m args skip lexbuf } + { newline lexbuf; if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack, m))) else mlOnly m args skip lexbuf } | "(*ENDIF-CAML*)" { if not skip then (COMMENT (LexCont.Token args.ifdefStack)) else token args skip lexbuf } @@ -1238,11 +1303,11 @@ and mlOnly m args skip = parse { if not skip then (COMMENT (LexCont.Token args.ifdefStack)) else token args skip lexbuf } | [^ '(' '"' '\n' '\r' ]+ - { if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack,m))) else mlOnly m args skip lexbuf } + { if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack, m))) else mlOnly m args skip lexbuf } | eof - { EOF (LexCont.MLOnly(args.ifdefStack,m)) } + { EOF (LexCont.MLOnly(args.ifdefStack, m)) } | surrogateChar surrogateChar | _ - { if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack,m))) else mlOnly m args skip lexbuf } + { if not skip then (COMMENT (LexCont.MLOnly(args.ifdefStack, m))) else mlOnly m args skip lexbuf } diff --git a/src/fsharp/lexhelp.fs b/src/fsharp/lexhelp.fs index 707d7f0b499..c71d27c7240 100644 --- a/src/fsharp/lexhelp.fs +++ b/src/fsharp/lexhelp.fs @@ -56,6 +56,7 @@ type lexargs = lightSyntaxStatus : LightSyntaxStatus errorLogger: ErrorLogger applyLineDirectives: bool + mutable interpolatedStringNesting: int pathMap: PathMap } /// possible results of lexing a long Unicode escape sequence in a string literal, e.g. "\U0001F47D", @@ -72,6 +73,7 @@ let mkLexargs (_filename, defines, lightSyntaxStatus, resourceManager, ifdefStac resourceManager=resourceManager errorLogger=errorLogger applyLineDirectives=true + interpolatedStringNesting = 0 pathMap=pathMap } /// Register the lexbuf and call the given function @@ -95,20 +97,8 @@ let usingLexbufForParsing (lexbuf:UnicodeLexing.Lexbuf, filename) f = // Functions to manipulate lexer transient state //----------------------------------------------------------------------- -let defaultStringFinisher = (fun _endm _b s -> STRING (Encoding.Unicode.GetString(s, 0, s.Length))) - -let callStringFinisher fin (buf: ByteBuffer) endm b = fin endm b (buf.Close()) - -let addUnicodeString (buf: ByteBuffer) (x:string) = buf.EmitBytes (Encoding.Unicode.GetBytes x) - -let addIntChar (buf: ByteBuffer) c = - buf.EmitIntAsByte (c % 256) - buf.EmitIntAsByte (c / 256) - -let addUnicodeChar buf c = addIntChar buf (int c) -let addByteChar buf (c:char) = addIntChar buf (int32 c % 256) - -let stringBufferAsString (buf: byte[]) = +let stringBufferAsString (buf: ByteBuffer) = + let buf = buf.Close() if buf.Length % 2 <> 0 then failwith "Expected even number of bytes" let chars : char[] = Array.zeroCreate (buf.Length/2) for i = 0 to (buf.Length/2) - 1 do @@ -127,6 +117,44 @@ let stringBufferAsBytes (buf: ByteBuffer) = let bytes = buf.Close() Array.init (bytes.Length / 2) (fun i -> bytes.[i*2]) +type LexerStringFinisher = + | LexerStringFinisher of (ByteBuffer -> LexerStringKind -> bool -> token) + + member fin.Finish (buf: ByteBuffer) kind isPart = + let (LexerStringFinisher f) = fin + f buf kind isPart + + static member Default = + LexerStringFinisher (fun buf kind isPart -> + if kind.IsInterpolated then + let s = stringBufferAsString buf + if kind.IsInterpolatedFirst then + if isPart then + INTERP_STRING_BEGIN_PART s + else + INTERP_STRING_BEGIN_END s + else + if isPart then + INTERP_STRING_PART s + else + INTERP_STRING_END s + elif kind.IsByteString then + BYTEARRAY (stringBufferAsBytes buf) + else + STRING (stringBufferAsString buf) + ) + +let addUnicodeString (buf: ByteBuffer) (x:string) = + buf.EmitBytes (Encoding.Unicode.GetBytes x) + +let addIntChar (buf: ByteBuffer) c = + buf.EmitIntAsByte (c % 256) + buf.EmitIntAsByte (c / 256) + +let addUnicodeChar buf c = addIntChar buf (int c) + +let addByteChar buf (c:char) = addIntChar buf (int32 c % 256) + /// Sanity check that high bytes are zeros. Further check each low byte <= 127 let stringBufferIsBytes (buf: ByteBuffer) = let bytes = buf.Close() diff --git a/src/fsharp/lexhelp.fsi b/src/fsharp/lexhelp.fsi index 2bed4e89c60..22e5c2fffce 100644 --- a/src/fsharp/lexhelp.fsi +++ b/src/fsharp/lexhelp.fsi @@ -8,6 +8,7 @@ open Internal.Utilities.Text open FSharp.Compiler open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.ErrorLogger +open FSharp.Compiler.Parser open FSharp.Compiler.ParseHelpers open FSharp.Compiler.Range @@ -32,6 +33,7 @@ type lexargs = lightSyntaxStatus: LightSyntaxStatus errorLogger: ErrorLogger applyLineDirectives: bool + mutable interpolatedStringNesting: int pathMap: PathMap } type LongUnicodeLexResult = @@ -47,9 +49,12 @@ val reusingLexbufForParsing: UnicodeLexing.Lexbuf -> (unit -> 'a) -> 'a val usingLexbufForParsing: UnicodeLexing.Lexbuf * string -> (UnicodeLexing.Lexbuf -> 'a) -> 'a -val defaultStringFinisher: 'a -> 'b -> byte[] -> Parser.token +type LexerStringFinisher = + | LexerStringFinisher of (ByteBuffer -> LexerStringKind -> bool -> token) + + member Finish: buf: ByteBuffer -> kind: LexerStringKind -> isInterpolatedStringPart: bool -> token -val callStringFinisher: ('a -> 'b -> byte[] -> 'c) -> ByteBuffer -> 'a -> 'b -> 'c + static member Default: LexerStringFinisher val addUnicodeString: ByteBuffer -> string -> unit @@ -57,7 +62,7 @@ val addUnicodeChar: ByteBuffer -> int -> unit val addByteChar: ByteBuffer -> char -> unit -val stringBufferAsString: byte[] -> string +val stringBufferAsString: ByteBuffer -> string val stringBufferAsBytes: ByteBuffer -> byte[] @@ -85,9 +90,9 @@ exception IndentationProblem of string * Range.range module Keywords = - val KeywordOrIdentifierToken: lexargs -> UnicodeLexing.Lexbuf -> string -> Parser.token + val KeywordOrIdentifierToken: lexargs -> UnicodeLexing.Lexbuf -> string -> token - val IdentifierToken: lexargs -> UnicodeLexing.Lexbuf -> string -> Parser.token + val IdentifierToken: lexargs -> UnicodeLexing.Lexbuf -> string -> token val DoesIdentifierNeedQuotation: string -> bool diff --git a/src/fsharp/pars.fsy b/src/fsharp/pars.fsy index 343f737a9ac..e950010f548 100644 --- a/src/fsharp/pars.fsy +++ b/src/fsharp/pars.fsy @@ -94,9 +94,9 @@ let raiseParseErrorAt m s = let checkEndOfFileError t = match t with | LexCont.IfDefSkip(_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInHashIf()) - | LexCont.String (_, m) -> reportParseErrorAt m (FSComp.SR.parsEofInString()) - | LexCont.TripleQuoteString (_, m) -> reportParseErrorAt m (FSComp.SR.parsEofInTripleQuoteString()) - | LexCont.VerbatimString (_, m) -> reportParseErrorAt m (FSComp.SR.parsEofInVerbatimString()) + | LexCont.String (_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInString()) + | LexCont.TripleQuoteString (_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInTripleQuoteString()) + | LexCont.VerbatimString (_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInVerbatimString()) | LexCont.Comment (_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInComment()) | LexCont.SingleLineComment (_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInComment()) | LexCont.StringInComment (_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInStringInComment()) @@ -157,6 +157,10 @@ let rangeOfLongIdent(lid:LongIdent) = %token BYTEARRAY %token STRING +%token INTERP_STRING_BEGIN_END +%token INTERP_STRING_BEGIN_PART +%token INTERP_STRING_PART +%token INTERP_STRING_END %token KEYWORD_STRING // Like __SOURCE_DIRECTORY__ %token IDENT %token INFIX_STAR_STAR_OP @@ -382,6 +386,7 @@ let rangeOfLongIdent(lid:LongIdent) = */ %nonassoc prec_atompat_pathop %nonassoc INT8 UINT8 INT16 UINT16 INT32 UINT32 INT64 UINT64 NATIVEINT UNATIVEINT IEEE32 IEEE64 CHAR KEYWORD_STRING STRING BYTEARRAY BIGNUM DECIMAL +%nonassoc INTERP_STRING_BEGIN INTERP_STRING_PART INTERP_STRING_END %nonassoc LPAREN LBRACE LBRACK_BAR %nonassoc TRUE FALSE UNDERSCORE NULL @@ -4143,7 +4148,7 @@ rangeDeclExpr: if $1 <> "^" then reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsInvalidPrefixOperator()) $2, true } -/* the start et of atomicExprAfterType must not overlap with the valid postfix tokens of the type syntax, e.g. new List(...) */ +/* the start of atomicExprAfterType must not overlap with the valid postfix tokens of the type syntax, e.g. new List(...) */ atomicExprAfterType: | constant { SynExpr.Const ($1, $1.Range (lhs parseState)) } @@ -4157,6 +4162,9 @@ atomicExprAfterType: | braceBarExpr { $1 } + | interpolatedString + { SynExpr.InterpolatedString($1, rhs parseState 1) } + | NULL { SynExpr.Null (lhs parseState) } @@ -5419,11 +5427,28 @@ colonOrEquals: | COLON { mlCompatWarning (FSComp.SR.parsSyntaxModuleSigEndDeprecated()) (lhs parseState); } | EQUALS { } -/* A literal string or a string fromm a keyword like __SOURCE_FILE__ */ +/* A literal string or a string from a keyword like __SOURCE_FILE__ */ stringOrKeywordString: | STRING { $1 } | KEYWORD_STRING { $1 } +interpolatedStringParts: + | INTERP_STRING_END + { [ Choice1Of2 $1 ] } + + | INTERP_STRING_PART declExpr interpolatedStringParts + { Choice1Of2 $1 :: Choice2Of2 $2 :: $3 } + +/* INTERP_STRING_BEGIN_END */ +/* INTERP_STRING_BEGIN_PART int32 INTERP_STRING_END */ +/* INTERP_STRING_BEGIN_PART int32 INTERP_STRING_PART int32 INTERP_STRING_END */ +interpolatedString: + | INTERP_STRING_BEGIN_PART declExpr interpolatedStringParts + { Choice1Of2 $1 :: Choice2Of2 $2 :: $3 } + + | INTERP_STRING_BEGIN_END + { [ Choice1Of2 $1 ] } + opt_HIGH_PRECEDENCE_APP: | HIGH_PRECEDENCE_BRACK_APP { } | HIGH_PRECEDENCE_PAREN_APP { } diff --git a/src/fsharp/service/ServiceLexing.fs b/src/fsharp/service/ServiceLexing.fs index 3318358c1e3..7c4e1ac7550 100755 --- a/src/fsharp/service/ServiceLexing.fs +++ b/src/fsharp/service/ServiceLexing.fs @@ -310,6 +310,10 @@ module internal TokenClassifications = | KEYWORD_STRING _ -> (FSharpTokenColorKind.Keyword, FSharpTokenCharKind.Keyword, FSharpTokenTriggerClass.None) + | INTERP_STRING_BEGIN_END _ + | INTERP_STRING_BEGIN_PART _ + | INTERP_STRING_PART _ + | INTERP_STRING_END _ | BYTEARRAY _ | STRING _ | CHAR _ -> (FSharpTokenColorKind.String, FSharpTokenCharKind.String, FSharpTokenTriggerClass.None) @@ -363,7 +367,10 @@ module internal LexerStateEncoding = | LINE_COMMENT s | STRING_TEXT s | EOF s -> s - | BYTEARRAY _ | STRING _ -> LexCont.Token(prevLexcont.LexerIfdefStack) + + | BYTEARRAY _ + | STRING _ -> LexCont.Token(prevLexcont.LexerIfdefStack) + | _ -> prevLexcont // Note that this will discard all lexcont state, including the ifdefStack. @@ -374,23 +381,27 @@ module internal LexerStateEncoding = let hardwhiteNumBits = 1 let ifdefstackCountNumBits = 8 let ifdefstackNumBits = 24 // 0 means if, 1 means else + let stringKindBits = 3 let _ = assert (lexstateNumBits + ncommentsNumBits + hardwhiteNumBits + ifdefstackCountNumBits - + ifdefstackNumBits <= 64) + + ifdefstackNumBits + + stringKindBits <= 64) let lexstateStart = 0 let ncommentsStart = lexstateNumBits let hardwhitePosStart = lexstateNumBits+ncommentsNumBits let ifdefstackCountStart = lexstateNumBits+ncommentsNumBits+hardwhiteNumBits let ifdefstackStart = lexstateNumBits+ncommentsNumBits+hardwhiteNumBits+ifdefstackCountNumBits + let stringKindStart = lexstateNumBits+ncommentsNumBits+hardwhiteNumBits+ifdefstackCountNumBits+ifdefstackNumBits let lexstateMask = Bits.mask64 lexstateStart lexstateNumBits let ncommentsMask = Bits.mask64 ncommentsStart ncommentsNumBits let hardwhitePosMask = Bits.mask64 hardwhitePosStart hardwhiteNumBits let ifdefstackCountMask = Bits.mask64 ifdefstackCountStart ifdefstackCountNumBits let ifdefstackMask = Bits.mask64 ifdefstackStart ifdefstackNumBits + let stringKindMask = Bits.mask64 stringKindStart stringKindBits let bitOfBool b = if b then 1 else 0 let boolOfBit n = (n = 1L) @@ -401,7 +412,7 @@ module internal LexerStateEncoding = let inline lexStateOfColorState (state: FSharpTokenizerColorState) = (int64 state <<< lexstateStart) &&& lexstateMask - let encodeLexCont (colorState: FSharpTokenizerColorState) ncomments (b: pos) ifdefStack light = + let encodeLexCont (colorState: FSharpTokenizerColorState, numComments, b: pos, ifdefStack, light, stringKind: LexerStringKind) = let mutable ifdefStackCount = 0 let mutable ifdefStackBits = 0 for ifOrElse in ifdefStack do @@ -411,12 +422,19 @@ module internal LexerStateEncoding = ifdefStackBits <- (ifdefStackBits ||| (1 <<< ifdefStackCount)) ifdefStackCount <- ifdefStackCount + 1 + let stringKindValue = + (if stringKind.IsByteString then 0b100 else 0) ||| + (if stringKind.IsInterpolated then 0b010 else 0) ||| + (if stringKind.IsInterpolatedFirst then 0b001 else 0) + let bits = lexStateOfColorState colorState - ||| ((ncomments <<< ncommentsStart) &&& ncommentsMask) + ||| ((numComments <<< ncommentsStart) &&& ncommentsMask) ||| ((int64 (bitOfBool light) <<< hardwhitePosStart) &&& hardwhitePosMask) ||| ((int64 ifdefStackCount <<< ifdefstackCountStart) &&& ifdefstackCountMask) ||| ((int64 ifdefStackBits <<< ifdefstackStart) &&& ifdefstackMask) + ||| ((int64 stringKindValue <<< stringKindStart) &&& stringKindMask) + { PosBits = b.Encoding OtherBits = bits } @@ -424,6 +442,11 @@ module internal LexerStateEncoding = let decodeLexCont (state: FSharpTokenizerLexState) = let mutable ifDefs = [] let bits = state.OtherBits + + let colorState = colorStateOfLexState state + let ncomments = int32 ((bits &&& ncommentsMask) >>> ncommentsStart) + let pos = pos.Decode state.PosBits + let ifdefStackCount = int32 ((bits &&& ifdefstackCountMask) >>> ifdefstackCountStart) if ifdefStackCount>0 then let ifdefStack = int32 ((bits &&& ifdefstackMask) >>> ifdefstackStart) @@ -432,46 +455,50 @@ module internal LexerStateEncoding = let mask = 1 <<< bit let ifDef = (if ifdefStack &&& mask = 0 then IfDefIf else IfDefElse) ifDefs <- (ifDef, range0) :: ifDefs - colorStateOfLexState state, - int32 ((bits &&& ncommentsMask) >>> ncommentsStart), - pos.Decode state.PosBits, - ifDefs, - boolOfBit ((bits &&& hardwhitePosMask) >>> hardwhitePosStart) + + let stringKindValue = int32 ((bits &&& stringKindMask) >>> stringKindStart) + let stringKind : LexerStringKind = + { IsByteString = ((stringKindValue &&& 0b100) = 0b100) + IsInterpolated = ((stringKindValue &&& 0b010) = 0b010) + IsInterpolatedFirst = ((stringKindValue &&& 0b001) = 0b001) } + + let hardwhite = boolOfBit ((bits &&& hardwhitePosMask) >>> hardwhitePosStart) + + (colorState, ncomments, pos, ifDefs, hardwhite, stringKind) let encodeLexInt lightSyntaxStatus (lexcont: LexerWhitespaceContinuation) = - let tag, n1, p1, ifd = + let tag, n1, p1, ifd, stringKind = match lexcont with - | LexCont.Token ifd -> FSharpTokenizerColorState.Token, 0L, pos0, ifd - | LexCont.IfDefSkip (ifd, n, m) -> FSharpTokenizerColorState.IfDefSkip, int64 n, m.Start, ifd - | LexCont.EndLine(LexerEndlineContinuation.Skip(ifd, n, m)) -> FSharpTokenizerColorState.EndLineThenSkip, int64 n, m.Start, ifd - | LexCont.EndLine(LexerEndlineContinuation.Token ifd) -> FSharpTokenizerColorState.EndLineThenToken, 0L, pos0, ifd - | LexCont.String (ifd, m) -> FSharpTokenizerColorState.String, 0L, m.Start, ifd - | LexCont.Comment (ifd, n, m) -> FSharpTokenizerColorState.Comment, int64 n, m.Start, ifd - | LexCont.SingleLineComment (ifd, n, m) -> FSharpTokenizerColorState.SingleLineComment, int64 n, m.Start, ifd - | LexCont.StringInComment (ifd, n, m) -> FSharpTokenizerColorState.StringInComment, int64 n, m.Start, ifd - | LexCont.VerbatimStringInComment (ifd, n, m) -> FSharpTokenizerColorState.VerbatimStringInComment, int64 n, m.Start, ifd - | LexCont.TripleQuoteStringInComment (ifd, n, m) -> FSharpTokenizerColorState.TripleQuoteStringInComment, int64 n, m.Start, ifd - | LexCont.MLOnly (ifd, m) -> FSharpTokenizerColorState.CamlOnly, 0L, m.Start, ifd - | LexCont.VerbatimString (ifd, m) -> FSharpTokenizerColorState.VerbatimString, 0L, m.Start, ifd - | LexCont.TripleQuoteString (ifd, m) -> FSharpTokenizerColorState.TripleQuoteString, 0L, m.Start, ifd - encodeLexCont tag n1 p1 ifd lightSyntaxStatus - + | LexCont.Token ifd -> FSharpTokenizerColorState.Token, 0L, pos0, ifd, LexerStringKind.String + | LexCont.IfDefSkip (ifd, n, m) -> FSharpTokenizerColorState.IfDefSkip, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.EndLine(LexerEndlineContinuation.Skip(ifd, n, m)) -> FSharpTokenizerColorState.EndLineThenSkip, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.EndLine(LexerEndlineContinuation.Token ifd) -> FSharpTokenizerColorState.EndLineThenToken, 0L, pos0, ifd, LexerStringKind.String + | LexCont.String (ifd, kind, m) -> FSharpTokenizerColorState.String, 0L, m.Start, ifd, kind + | LexCont.Comment (ifd, n, m) -> FSharpTokenizerColorState.Comment, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.SingleLineComment (ifd, n, m) -> FSharpTokenizerColorState.SingleLineComment, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.StringInComment (ifd, n, m) -> FSharpTokenizerColorState.StringInComment, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.VerbatimStringInComment (ifd, n, m) -> FSharpTokenizerColorState.VerbatimStringInComment, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.TripleQuoteStringInComment (ifd, n, m) -> FSharpTokenizerColorState.TripleQuoteStringInComment, int64 n, m.Start, ifd, LexerStringKind.String + | LexCont.MLOnly (ifd, m) -> FSharpTokenizerColorState.CamlOnly, 0L, m.Start, ifd, LexerStringKind.String + | LexCont.VerbatimString (ifd, kind, m) -> FSharpTokenizerColorState.VerbatimString, 0L, m.Start, ifd, kind + | LexCont.TripleQuoteString (ifd, kind, m) -> FSharpTokenizerColorState.TripleQuoteString, 0L, m.Start, ifd, kind + encodeLexCont (tag, n1, p1, ifd, lightSyntaxStatus, stringKind) let decodeLexInt (state: FSharpTokenizerLexState) = - let tag, n1, p1, ifd, lightSyntaxStatusInitial = decodeLexCont state + let tag, n1, p1, ifd, lightSyntaxStatusInitial, stringKind = decodeLexCont state let lexcont = match tag with | FSharpTokenizerColorState.Token -> LexCont.Token ifd | FSharpTokenizerColorState.IfDefSkip -> LexCont.IfDefSkip (ifd, n1, mkRange "file" p1 p1) - | FSharpTokenizerColorState.String -> LexCont.String (ifd, mkRange "file" p1 p1) + | FSharpTokenizerColorState.String -> LexCont.String (ifd, stringKind, mkRange "file" p1 p1) | FSharpTokenizerColorState.Comment -> LexCont.Comment (ifd, n1, mkRange "file" p1 p1) | FSharpTokenizerColorState.SingleLineComment -> LexCont.SingleLineComment (ifd, n1, mkRange "file" p1 p1) | FSharpTokenizerColorState.StringInComment -> LexCont.StringInComment (ifd, n1, mkRange "file" p1 p1) | FSharpTokenizerColorState.VerbatimStringInComment -> LexCont.VerbatimStringInComment (ifd, n1, mkRange "file" p1 p1) | FSharpTokenizerColorState.TripleQuoteStringInComment -> LexCont.TripleQuoteStringInComment (ifd, n1, mkRange "file" p1 p1) | FSharpTokenizerColorState.CamlOnly -> LexCont.MLOnly (ifd, mkRange "file" p1 p1) - | FSharpTokenizerColorState.VerbatimString -> LexCont.VerbatimString (ifd, mkRange "file" p1 p1) - | FSharpTokenizerColorState.TripleQuoteString -> LexCont.TripleQuoteString (ifd, mkRange "file" p1 p1) + | FSharpTokenizerColorState.VerbatimString -> LexCont.VerbatimString (ifd, stringKind, mkRange "file" p1 p1) + | FSharpTokenizerColorState.TripleQuoteString -> LexCont.TripleQuoteString (ifd, stringKind, mkRange "file" p1 p1) | FSharpTokenizerColorState.EndLineThenSkip -> LexCont.EndLine(LexerEndlineContinuation.Skip(ifd, n1, mkRange "file" p1 p1)) | FSharpTokenizerColorState.EndLineThenToken -> LexCont.EndLine(LexerEndlineContinuation.Token ifd) | _ -> LexCont.Token [] @@ -488,7 +515,7 @@ module internal LexerStateEncoding = | LexCont.Token ifd -> Lexer.token (argsWithIfDefs ifd) skip lexbuf | LexCont.IfDefSkip (ifd, n, m) -> Lexer.ifdefSkip n m (argsWithIfDefs ifd) skip lexbuf // Q: What's this magic 100 number for? Q: it's just an initial buffer size. - | LexCont.String (ifd, m) -> Lexer.string (ByteBuffer.Create 100, defaultStringFinisher, m, (argsWithIfDefs ifd)) skip lexbuf + | LexCont.String (ifd, kind, m) -> Lexer.string (ByteBuffer.Create 100, LexerStringFinisher.Default, m, kind, argsWithIfDefs ifd) skip lexbuf | LexCont.Comment (ifd, n, m) -> Lexer.comment (n, m, (argsWithIfDefs ifd)) skip lexbuf // The first argument is 'None' because we don't need XML comments when called from VS | LexCont.SingleLineComment (ifd, n, m) -> Lexer.singleLineComment (None, n, m, (argsWithIfDefs ifd)) skip lexbuf @@ -496,8 +523,8 @@ module internal LexerStateEncoding = | LexCont.VerbatimStringInComment (ifd, n, m) -> Lexer.verbatimStringInComment n m (argsWithIfDefs ifd) skip lexbuf | LexCont.TripleQuoteStringInComment (ifd, n, m) -> Lexer.tripleQuoteStringInComment n m (argsWithIfDefs ifd) skip lexbuf | LexCont.MLOnly (ifd, m) -> Lexer.mlOnly m (argsWithIfDefs ifd) skip lexbuf - | LexCont.VerbatimString (ifd, m) -> Lexer.verbatimString (ByteBuffer.Create 100, defaultStringFinisher, m, (argsWithIfDefs ifd)) skip lexbuf - | LexCont.TripleQuoteString (ifd, m) -> Lexer.tripleQuoteString (ByteBuffer.Create 100, defaultStringFinisher, m, (argsWithIfDefs ifd)) skip lexbuf + | LexCont.VerbatimString (ifd, kind, m) -> Lexer.verbatimString (ByteBuffer.Create 100, LexerStringFinisher.Default, m, kind, argsWithIfDefs ifd) skip lexbuf + | LexCont.TripleQuoteString (ifd, kind, m) -> Lexer.tripleQuoteString (ByteBuffer.Create 100, LexerStringFinisher.Default, m, kind, argsWithIfDefs ifd) skip lexbuf //---------------------------------------------------------------------------- // Colorization @@ -588,8 +615,6 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, // ---------------------------------------------------------------------------------- - - do match filename with | None -> lexbuf.EndPos <- Internal.Utilities.Text.Lexing.Position.Empty | Some value -> resetLexbufPos value lexbuf @@ -696,13 +721,33 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, | _ -> // Get the information about the token let (colorClass, charClass, triggerClass) = TokenClassifications.tokenInfo token + let lexcontFinal = // If we're using token from cache, we don't move forward with lexing if isCached then lexcontInitial else LexerStateEncoding.computeNextLexState token lexcontInitial - let tokenTag = tagOfToken token + + let tokenTag = + // Tokenization just reports repeated STRING_TEXT then STRING for each part of an interpolated string + match token with + | INTERP_STRING_BEGIN_END _ + | INTERP_STRING_BEGIN_PART _ + | INTERP_STRING_PART _ + | INTERP_STRING_END _ -> FSharpTokenTag.STRING + | _ -> tagOfToken token + + let tokenName = + // Tokenization just reports repeated STRING_TEXT then STRING for each part of an interpolated string + match token with + | INTERP_STRING_BEGIN_END _ + | INTERP_STRING_BEGIN_PART _ + | INTERP_STRING_PART _ + | INTERP_STRING_END _ -> token_to_string (STRING "") + | _ -> token_to_string token + let fullMatchedLength = lexbuf.EndPos.AbsoluteOffset - lexbuf.StartPos.AbsoluteOffset + let tokenData = - { TokenName = token_to_string token + { TokenName = tokenName LeftColumn=leftc RightColumn=rightc ColorClass=colorClass @@ -1211,6 +1256,10 @@ module Lexer = | INFIX_STAR_STAR_OP _ -> FSharpSyntaxTokenKind.InfixStarStarOperator | IDENT _ -> FSharpSyntaxTokenKind.Identifier | KEYWORD_STRING _ -> FSharpSyntaxTokenKind.KeywordString + | INTERP_STRING_BEGIN_END _ + | INTERP_STRING_BEGIN_PART _ + | INTERP_STRING_PART _ + | INTERP_STRING_END _ | STRING _ -> FSharpSyntaxTokenKind.String | BYTEARRAY _ -> FSharpSyntaxTokenKind.ByteArray | _ -> FSharpSyntaxTokenKind.None diff --git a/src/fsharp/service/ServiceParseTreeWalk.fs b/src/fsharp/service/ServiceParseTreeWalk.fs index 7bf60285087..fd068ecd984 100755 --- a/src/fsharp/service/ServiceParseTreeWalk.fs +++ b/src/fsharp/service/ServiceParseTreeWalk.fs @@ -200,25 +200,36 @@ module public AstTraversal = let path = TraverseStep.Expr e :: path let traverseSynExpr = traverseSynExpr path match e with + | SynExpr.Paren (synExpr, _, _, _parenRange) -> traverseSynExpr synExpr + | SynExpr.Quote (_synExpr, _, synExpr2, _, _range) -> [//dive synExpr synExpr.Range traverseSynExpr // TODO, what is this? dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.Const (_synConst, _range) -> None - | SynExpr.Typed (synExpr, synType, _range) -> [ traverseSynExpr synExpr; traverseSynType synType ] |> List.tryPick id + + | SynExpr.InterpolatedString (es, _) -> + [ for e in es do match e with Choice1Of2 _ -> () | Choice2Of2 e -> yield dive e e.Range traverseSynExpr ] + |> pick expr + + | SynExpr.Typed (synExpr, synType, _range) -> + [ traverseSynExpr synExpr; traverseSynType synType ] |> List.tryPick id + | SynExpr.Tuple (_, synExprList, _, _range) - | SynExpr.ArrayOrList (_, synExprList, _range) -> synExprList |> List.map (fun x -> dive x x.Range traverseSynExpr) |> pick expr + | SynExpr.ArrayOrList (_, synExprList, _range) -> + synExprList |> List.map (fun x -> dive x x.Range traverseSynExpr) |> pick expr | SynExpr.AnonRecd (_isStruct, copyOpt, synExprList, _range) -> [ match copyOpt with - | Some(expr, (withRange, _)) -> - yield dive expr expr.Range traverseSynExpr + | Some(origExpr, (withRange, _)) -> + yield dive origExpr origExpr.Range traverseSynExpr yield dive () withRange (fun () -> if posGeq pos withRange.End then // special case: caret is after WITH // { x with $ } - visitor.VisitRecordField (path, Some expr, None) + visitor.VisitRecordField (path, Some origExpr, None) else None ) @@ -226,6 +237,7 @@ module public AstTraversal = for (_,x) in synExprList do yield dive x x.Range traverseSynExpr ] |> pick expr + | SynExpr.Record (inheritOpt,copyOpt,fields, _range) -> [ let diveIntoSeparator offsideColumn scPosOpt copyOpt = @@ -312,6 +324,7 @@ module public AstTraversal = | _ -> () ] |> pick expr + | SynExpr.New (_, _synType, synExpr, _range) -> traverseSynExpr synExpr | SynExpr.ObjExpr (ty,baseCallOpt,binds,ifaces,_range1,_range2) -> let result = @@ -335,21 +348,26 @@ module public AstTraversal = for b in binds do yield dive b b.RangeOfBindingAndRhs (traverseSynBinding path) ] |> pick expr + | SynExpr.While (_sequencePointInfoForWhileLoop, synExpr, synExpr2, _range) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.For (_sequencePointInfoForForLoop, _ident, synExpr, _, synExpr2, synExpr3, _range) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr dive synExpr3 synExpr3.Range traverseSynExpr] |> pick expr + | SynExpr.ForEach (_sequencePointInfoForForLoop, _seqExprOnly, _isFromSource, synPat, synExpr, synExpr2, _range) -> [dive synPat synPat.Range traversePat dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.ArrayOrListOfSeqExpr (_, synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.CompExpr (_, _, synExpr, _range) -> // now parser treats this syntactic expression as computation expression // { identifier } @@ -367,6 +385,7 @@ module public AstTraversal = if ok.IsSome then ok else traverseSynExpr synExpr + | SynExpr.Lambda (_, _, synSimplePats, synExpr, _range) -> match synSimplePats with | SynSimplePats.SimplePats(pats,_) -> @@ -374,17 +393,23 @@ module public AstTraversal = | Some x -> Some x | None -> traverseSynExpr synExpr | _ -> traverseSynExpr synExpr + | SynExpr.MatchLambda (_isExnMatch,_argm,synMatchClauseList,_spBind,_wholem) -> synMatchClauseList |> List.map (fun x -> dive x x.Range (traverseSynMatchClause path)) |> pick expr + | SynExpr.Match (_sequencePointInfoForBinding, synExpr, synMatchClauseList, _range) -> [yield dive synExpr synExpr.Range traverseSynExpr yield! synMatchClauseList |> List.map (fun x -> dive x x.RangeOfGuardAndRhs (traverseSynMatchClause path))] |> pick expr + | SynExpr.Do (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.Assert (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.Fixed (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.App (_exprAtomicFlag, isInfix, synExpr, synExpr2, _range) -> if isInfix then [dive synExpr2 synExpr2.Range traverseSynExpr @@ -394,7 +419,9 @@ module public AstTraversal = [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.TypeApp (synExpr, _, _synTypeList, _commas, _, _, _range) -> traverseSynExpr synExpr + | SynExpr.LetOrUse (_, _, synBindingList, synExpr, range) -> match visitor.VisitLetOrUse(path, traverseSynBinding path, synBindingList, range) with | Some x -> Some x @@ -402,20 +429,26 @@ module public AstTraversal = [yield! synBindingList |> List.map (fun x -> dive x x.RangeOfBindingAndRhs (traverseSynBinding path)) yield dive synExpr synExpr.Range traverseSynExpr] |> pick expr + | SynExpr.TryWith (synExpr, _range, synMatchClauseList, _range2, _range3, _sequencePointInfoForTry, _sequencePointInfoForWith) -> [yield dive synExpr synExpr.Range traverseSynExpr yield! synMatchClauseList |> List.map (fun x -> dive x x.Range (traverseSynMatchClause path))] |> pick expr + | SynExpr.TryFinally (synExpr, synExpr2, _range, _sequencePointInfoForTry, _sequencePointInfoForFinally) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.Lazy (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.SequentialOrImplicitYield (_sequencePointInfoForSequential, synExpr, synExpr2, _, _range) + | SynExpr.Sequential (_sequencePointInfoForSequential, _, synExpr, synExpr2, _range) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.IfThenElse (synExpr, synExpr2, synExprOpt, _sequencePointInfoForBinding, _isRecovery, _range, _range2) -> [yield dive synExpr synExpr.Range traverseSynExpr yield dive synExpr2 synExpr2.Range traverseSynExpr @@ -423,21 +456,29 @@ module public AstTraversal = | None -> () | Some(x) -> yield dive x x.Range traverseSynExpr] |> pick expr + | SynExpr.Ident (_ident) -> None + | SynExpr.LongIdent (_, _longIdent, _altNameRefCell, _range) -> None + | SynExpr.LongIdentSet (_longIdent, synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.DotGet (synExpr, _dotm, _longIdent, _range) -> traverseSynExpr synExpr + | SynExpr.Set (synExpr, synExpr2, _) + | SynExpr.DotSet (synExpr, _, synExpr2, _) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.DotIndexedGet (synExpr, synExprList, _range, _range2) -> [yield dive synExpr synExpr.Range traverseSynExpr for synExpr in synExprList do for x in synExpr.Exprs do yield dive x x.Range traverseSynExpr] |> pick expr + | SynExpr.DotIndexedSet (synExpr, synExprList, synExpr2, _, _range, _range2) -> [yield dive synExpr synExpr.Range traverseSynExpr for synExpr in synExprList do @@ -445,33 +486,48 @@ module public AstTraversal = yield dive x x.Range traverseSynExpr yield dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.JoinIn (synExpr1, _range, synExpr2, _range2) -> [dive synExpr1 synExpr1.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.NamedIndexedPropertySet (_longIdent, synExpr, synExpr2, _range) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr] |> pick expr + | SynExpr.DotNamedIndexedPropertySet (synExpr, _longIdent, synExpr2, synExpr3, _range) -> [dive synExpr synExpr.Range traverseSynExpr dive synExpr2 synExpr2.Range traverseSynExpr dive synExpr3 synExpr3.Range traverseSynExpr] |> pick expr + | SynExpr.TypeTest (synExpr, synType, _range) + | SynExpr.Upcast (synExpr, synType, _range) + | SynExpr.Downcast (synExpr, synType, _range) -> [dive synExpr synExpr.Range traverseSynExpr dive synType synType.Range traverseSynType] |> pick expr + | SynExpr.InferredUpcast (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.InferredDowncast (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.Null (_range) -> None + | SynExpr.AddressOf (_, synExpr, _range, _range2) -> traverseSynExpr synExpr + | SynExpr.TraitCall (_synTyparList, _synMemberSig, synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.ImplicitZero (_range) -> None + | SynExpr.YieldOrReturn (_, synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.YieldOrReturnFrom (_, synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.LetOrUseBang(_sequencePointInfoForBinding, _, _, synPat, synExpr, andBangSynExprs, synExpr2, _range) -> [ yield dive synPat synPat.Range traversePat @@ -483,17 +539,26 @@ module public AstTraversal = yield dive synExpr2 synExpr2.Range traverseSynExpr ] |> pick expr + | SynExpr.MatchBang (_sequencePointInfoForBinding, synExpr, synMatchClauseList, _range) -> [yield dive synExpr synExpr.Range traverseSynExpr yield! synMatchClauseList |> List.map (fun x -> dive x x.RangeOfGuardAndRhs (traverseSynMatchClause path))] |> pick expr + | SynExpr.DoBang (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.LibraryOnlyILAssembly _ -> None + | SynExpr.LibraryOnlyStaticOptimization _ -> None + | SynExpr.LibraryOnlyUnionCaseFieldGet _ -> None + | SynExpr.LibraryOnlyUnionCaseFieldSet _ -> None + | SynExpr.ArbitraryAfterError (_debugStr, _range) -> None + | SynExpr.FromParseError (synExpr, _range) -> traverseSynExpr synExpr + | SynExpr.DiscardAfterMissingQualificationAfterDot (synExpr, _range) -> traverseSynExpr synExpr visitor.VisitExpr(path, traverseSynExpr path, defaultTraverse, expr) diff --git a/src/fsharp/service/ServiceUntypedParse.fs b/src/fsharp/service/ServiceUntypedParse.fs index 684f3a04e06..2667a10e8ac 100755 --- a/src/fsharp/service/ServiceUntypedParse.fs +++ b/src/fsharp/service/ServiceUntypedParse.fs @@ -209,6 +209,9 @@ type FSharpParseFileResults(errors: FSharpErrorInfo[], input: ParsedInput option | SynExpr.Paren (e, _, _, _) -> yield! walkExpr false e + | SynExpr.InterpolatedString (es, _) -> + yield! walkExprs [ for e in es do match e with Choice1Of2 _ -> () | Choice2Of2 e -> yield e ] + | SynExpr.YieldOrReturn (_, e, _) | SynExpr.YieldOrReturnFrom (_, e, _) | SynExpr.DoBang (e, _) -> diff --git a/tests/FSharp.Core.UnitTests/SurfaceArea.coreclr.fs b/tests/FSharp.Core.UnitTests/SurfaceArea.coreclr.fs index e496f677f6c..2815d8594cc 100644 --- a/tests/FSharp.Core.UnitTests/SurfaceArea.coreclr.fs +++ b/tests/FSharp.Core.UnitTests/SurfaceArea.coreclr.fs @@ -2246,6 +2246,7 @@ Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringBuilder[T](System.Text. Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringThenFail[T,TResult](Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,TResult]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringThen[TResult,T](Microsoft.FSharp.Core.FSharpFunc`2[System.String,TResult], Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,TResult]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringThen[T](Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,System.String]) +Microsoft.FSharp.Core.PrintfModule: T InterpolatedPrintFormatToStringThen[T](Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,System.String]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToTextWriterThen[TResult,T](Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,TResult], System.IO.TextWriter, Microsoft.FSharp.Core.PrintfFormat`4[T,System.IO.TextWriter,Microsoft.FSharp.Core.Unit,TResult]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToTextWriter[T](System.IO.TextWriter, Microsoft.FSharp.Core.PrintfFormat`4[T,System.IO.TextWriter,Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Core.Unit]) Microsoft.FSharp.Core.PrintfModule: T PrintFormat[T](Microsoft.FSharp.Core.PrintfFormat`4[T,System.IO.TextWriter,Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Core.Unit]) diff --git a/tests/FSharp.Core.UnitTests/SurfaceArea.net40.fs b/tests/FSharp.Core.UnitTests/SurfaceArea.net40.fs index e2bc4a25a85..cb77401ddfc 100644 --- a/tests/FSharp.Core.UnitTests/SurfaceArea.net40.fs +++ b/tests/FSharp.Core.UnitTests/SurfaceArea.net40.fs @@ -2246,6 +2246,7 @@ Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringBuilder[T](System.Text. Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringThenFail[T,TResult](Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,TResult]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringThen[TResult,T](Microsoft.FSharp.Core.FSharpFunc`2[System.String,TResult], Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,TResult]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToStringThen[T](Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,System.String]) +Microsoft.FSharp.Core.PrintfModule: T InterpolatedPrintFormatToStringThen[T](Microsoft.FSharp.Core.PrintfFormat`4[T,Microsoft.FSharp.Core.Unit,System.String,System.String]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToTextWriterThen[TResult,T](Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,TResult], System.IO.TextWriter, Microsoft.FSharp.Core.PrintfFormat`4[T,System.IO.TextWriter,Microsoft.FSharp.Core.Unit,TResult]) Microsoft.FSharp.Core.PrintfModule: T PrintFormatToTextWriter[T](System.IO.TextWriter, Microsoft.FSharp.Core.PrintfFormat`4[T,System.IO.TextWriter,Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Core.Unit]) Microsoft.FSharp.Core.PrintfModule: T PrintFormat[T](Microsoft.FSharp.Core.PrintfFormat`4[T,System.IO.TextWriter,Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Core.Unit]) diff --git a/tests/fsharp/FSharpSuite.Tests.fsproj b/tests/fsharp/FSharpSuite.Tests.fsproj index 347dacb464c..570357f8002 100644 --- a/tests/fsharp/FSharpSuite.Tests.fsproj +++ b/tests/fsharp/FSharpSuite.Tests.fsproj @@ -70,6 +70,7 @@ + diff --git a/tests/service/TokenizerTests.fs b/tests/service/TokenizerTests.fs index 0827ff35027..c8b024a67ca 100644 --- a/tests/service/TokenizerTests.fs +++ b/tests/service/TokenizerTests.fs @@ -54,5 +54,39 @@ let ``Tokenizer test 1``() = ("STRING_TEXT", "\""); ("STRING_TEXT", "Hello"); ("STRING_TEXT", " "); ("STRING_TEXT", "world"); ("STRING", "\""); ("WHITESPACE", " ")])] - Assert.AreEqual(actual, expected) + if actual <> expected then + printfn "actual = %A" actual + printfn "expected = %A" expected + Assert.Fail(sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected) + +[] +let ``Tokenizer test 2``() = + let tokenizedLines = + tokenizeLines + [| "// Tests tokenizing string interpolation" + "let hello = $\"Hello world {1+1} = {2}\" " |] + + let actual = + [ for lineNo, lineToks in tokenizedLines do + yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] + let expected = + [(0, + [("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Tests"); + ("LINE_COMMENT", " "); ("LINE_COMMENT", "tokenizing"); ("LINE_COMMENT", " "); + ("LINE_COMMENT", "string"); ("LINE_COMMENT", " "); + ("LINE_COMMENT", "interpolation")]); + (1, + [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello"); ("WHITESPACE", " "); + ("EQUALS", "="); ("WHITESPACE", " "); ("STRING_TEXT", "$\""); + ("STRING_TEXT", "Hello"); ("STRING_TEXT", " "); ("STRING_TEXT", "world"); + ("STRING_TEXT", " "); ("STRING", "{"); ("STRING_TEXT", "1"); + ("STRING_TEXT", "+"); ("STRING_TEXT", "1"); ("STRING_TEXT", "}"); + ("STRING_TEXT", " "); ("STRING_TEXT", "="); ("STRING_TEXT", " "); + ("STRING", "{"); ("STRING_TEXT", "2"); ("STRING_TEXT", "}"); ("STRING", "\""); + ("STRING_TEXT", " ")])] + + if actual <> expected then + printfn "actual = %A" actual + printfn "expected = %A" expected + Assert.Fail(sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)