Remove LBRACK, LBRACK_BAR, RBRACK and BAR_RBRACK from TokenParser. (#…

…2033)

src/Fantomas/AstTransformer.fs

@@ -129,16 +129,21 @@ module private Ast =
                     |> finalContinuation
 
                 Continuation.sequence continuations finalContinuation
-            | SynExpr.ArrayOrList (_, exprs, range) ->
+            | SourceParser.ArrayOrList (startRange, _isArray, exprs, endRange, range) ->
                 let continuations: ((TriviaNodeAssigner list -> TriviaNodeAssigner list) -> TriviaNodeAssigner list) list =
                     exprs |> List.map visit
 
                 let finalContinuation (nodes: TriviaNodeAssigner list list) : TriviaNodeAssigner list =
-                    mkNode SynExpr_ArrayOrList range
-                    :: (List.collect id nodes)
+                    [ yield mkNode SynExpr_ArrayOrList range
+                      yield mkNode SynExpr_ArrayOrList_OpeningDelimiter startRange
+                      yield! List.collect id nodes
+                      yield mkNode SynExpr_ArrayOrList_ClosingDelimiter endRange ]
                     |> finalContinuation
 
                 Continuation.sequence continuations finalContinuation
+            // Captured above
+            | SynExpr.ArrayOrList _
+            | SynExpr.ArrayOrListComputed _ -> []
             | SynExpr.Record (_, _, recordFields, range) ->
                 mkNode SynExpr_Record range
                 :: (List.collect visitRecordField recordFields)
@@ -194,10 +199,6 @@ module private Ast =
                     |> finalContinuation
 
                 Continuation.sequence continuations finalContinuation
-            | SynExpr.ArrayOrListComputed (_, expr, range) ->
-                visit expr (fun nodes ->
-                    mkNode SynExpr_ArrayOrListComputed range :: nodes
-                    |> finalContinuation)
             | SynExpr.ComputationExpr (_, expr, _) -> visit expr finalContinuation
             | SynExpr.Lambda (_, _, args, arrowRange, body, _parsedData, range) ->
                 visit body (fun nodes ->
@@ -888,6 +889,7 @@ module private Ast =
                 visit pat (fun nodes ->
                     mkNode SynPat_Paren range :: nodes
                     |> finalContinuation)
+
             | SynPat.ArrayOrList (_, pats, range) ->
                 let continuations: ((TriviaNodeAssigner list -> TriviaNodeAssigner list) -> TriviaNodeAssigner list) list =
                     pats |> List.map visit
@@ -898,6 +900,7 @@ module private Ast =
                     |> finalContinuation
 
                 Continuation.sequence continuations finalContinuation
+
             | SynPat.Record (pats, range) ->
                 let continuations: ((TriviaNodeAssigner list -> TriviaNodeAssigner list) -> TriviaNodeAssigner list) list =
                     pats |> List.map (snd >> visit)

src/Fantomas/RangeHelpers.fs

@@ -30,3 +30,12 @@ module RangeHelpers =
         r1.FileName = r2.FileName
         && r1.End.Line = r2.Start.Line
         && r1.EndColumn = r2.StartColumn
+
+    let rec mkStartEndRange (size: int) (r: range) : range * range =
+        let startRange =
+            Range.mkRange r.FileName r.Start (Position.mkPos r.StartLine (r.StartColumn + size))
+
+        let endRange =
+            Range.mkRange r.FileName (Position.mkPos r.EndLine (r.EndColumn - size)) r.End
+
+        startRange, endRange

src/Fantomas/SourceParser.fs

@@ -691,10 +691,18 @@ let (|NamedComputationExpr|_|) =
 
 /// Combines all ArrayOrList patterns
 let (|ArrayOrList|_|) =
+    let (|Size|) isArray = if isArray then 2 else 1
+
     function
-    | SynExpr.ArrayOrListComputed (isArray, Sequentials xs, _) -> Some(isArray, xs)
-    | SynExpr.ArrayOrListComputed (isArray, singleExpr, _) -> Some(isArray, [ singleExpr ])
-    | SynExpr.ArrayOrList (isArray, xs, _) -> Some(isArray, xs)
+    | SynExpr.ArrayOrListComputed (Size size as isArray, Sequentials xs, range) ->
+        let sr, er = RangeHelpers.mkStartEndRange size range
+        Some(sr, isArray, xs, er, range)
+    | SynExpr.ArrayOrListComputed (Size size as isArray, singleExpr, range) ->
+        let sr, er = RangeHelpers.mkStartEndRange size range
+        Some(sr, isArray, [ singleExpr ], er, range)
+    | SynExpr.ArrayOrList (Size size as isArray, xs, range) ->
+        let sr, er = RangeHelpers.mkStartEndRange size range
+        Some(sr, isArray, xs, er, range)
     | _ -> None
 
 let (|Tuple|_|) =
@@ -1653,17 +1661,17 @@ let rec (|UppercaseSynType|LowercaseSynType|) (synType: SynType) =
 let (|ElmishReactWithoutChildren|_|) e =
     match e with
     | IndexWithoutDotExpr _ -> None
-    | SynExpr.App (_, false, OptVar (ident, _, _), (ArrayOrList (isArray, children) as aolEx), _) ->
-        Some(ident, isArray, children, aolEx.Range)
+    | SynExpr.App (_, false, OptVar (ident, _, _), ArrayOrList (sr, isArray, children, er, _), _) ->
+        Some(ident, sr, isArray, children, er)
     | _ -> None
 
 let (|ElmishReactWithChildren|_|) (e: SynExpr) =
     match e with
     | SynExpr.App (_,
                    false,
                    SynExpr.App (_, false, OptVar ident, (ArrayOrList _ as attributes), _),
-                   (ArrayOrList (isArray, children) as childrenNode),
-                   _) -> Some(ident, attributes, (isArray, children, childrenNode.Range))
+                   ArrayOrList (sr, isArray, children, er, _),
+                   _) -> Some(ident, attributes, (isArray, sr, children, er))
     | _ -> None
 
 let isIfThenElseWithYieldReturn e =

src/Fantomas/SourceTransformer.fs

@@ -206,7 +206,8 @@ let rec synExprToFsAstType (expr: SynExpr) : FsAstType * Range =
     | SynExpr.DoBang _ -> SynExpr_DoBang, expr.Range
     | SynExpr.Paren _ -> SynExpr_Paren, expr.Range
     | SynExpr.AnonRecd _ -> SynExpr_AnonRecd, expr.Range
-    | SynExpr.ArrayOrListComputed _ -> SynExpr_ArrayOrListComputed, expr.Range
+    | SynExpr.ArrayOrList _ -> SynExpr_ArrayOrList, expr.Range
+    | SynExpr.ArrayOrListComputed _ -> SynExpr_ArrayOrList, expr.Range
     | SynExpr.LongIdentSet _ -> SynExpr_LongIdentSet, expr.Range
     | SynExpr.New _ -> SynExpr_New, expr.Range
     | SynExpr.Quote _ -> SynExpr_Quote, expr.Range
@@ -226,7 +227,6 @@ let rec synExprToFsAstType (expr: SynExpr) : FsAstType * Range =
     | SynExpr.Do _ -> SynExpr_Do, expr.Range
     | SynExpr.AddressOf _ -> SynExpr_AddressOf, expr.Range
     | SynExpr.Typed (e, _, _) -> synExprToFsAstType e
-    | SynExpr.ArrayOrList _ -> SynExpr_ArrayOrList, expr.Range
     | SynExpr.ObjExpr _ -> SynExpr_ObjExpr, expr.Range
     | SynExpr.For _ -> SynExpr_For, expr.Range
     | SynExpr.ForEach _ -> SynExpr_ForEach, expr.Range

src/Fantomas/TokenParser.fs

@@ -1124,10 +1124,6 @@ let private tokenNames =
       "RBRACE"
       "LPAREN"
       "RPAREN"
-      "LBRACK"
-      "RBRACK"
-      "LBRACK_BAR"
-      "BAR_RBRACK"
       "EQUALS"
       "BAR"
       "TRY"
@@ -1146,7 +1142,6 @@ let internal getFsToken tokenName =
     | "AND_BANG" -> AND_BANG
     | "BAR" -> BAR
     | "BAR_BAR" -> BAR_BAR
-    | "BAR_RBRACK" -> BAR_RBRACK
     | "COLON_COLON" -> COLON_COLON
     | "COLON_EQUALS" -> COLON_EQUALS
     | "COLON_GREATER" -> COLON_GREATER
@@ -1171,8 +1166,6 @@ let internal getFsToken tokenName =
     | "INFIX_STAR_STAR_OP" -> INFIX_STAR_STAR_OP
     | "INT32_DOT_DOT" -> INT32_DOT_DOT
     | "LBRACE" -> LBRACE
-    | "LBRACK" -> LBRACK
-    | "LBRACK_BAR" -> LBRACK_BAR
     | "LESS" -> LESS
     | "LPAREN" -> LPAREN
     | "LPAREN_STAR_RPAREN" -> LPAREN_STAR_RPAREN
@@ -1184,7 +1177,6 @@ let internal getFsToken tokenName =
     | "QMARK" -> QMARK
     | "QMARK_QMARK" -> QMARK_QMARK
     | "RBRACE" -> RBRACE
-    | "RBRACK" -> RBRACK
     | "RPAREN" -> RPAREN
     | "THEN" -> THEN
     | "TRY" -> TRY

src/Fantomas/TriviaTypes.fs

@@ -9,7 +9,6 @@ type FsTokenType =
     | AND_BANG
     | BAR
     | BAR_BAR
-    | BAR_RBRACK
     | COLON_COLON
     | COLON_EQUALS
     | COLON_GREATER
@@ -34,8 +33,6 @@ type FsTokenType =
     | INFIX_STAR_STAR_OP
     | INT32_DOT_DOT
     | LBRACE
-    | LBRACK
-    | LBRACK_BAR
     | LESS
     | LPAREN
     | LPAREN_STAR_RPAREN
@@ -47,7 +44,6 @@ type FsTokenType =
     | QMARK
     | QMARK_QMARK
     | RBRACE
-    | RBRACK
     | RPAREN
     | THEN
     | TRY
@@ -128,8 +124,10 @@ type FsAstType =
     | SynExpr_While
     | SynExpr_For
     | SynExpr_ForEach
-    | SynExpr_ArrayOrListComputed
+    // | SynExpr_ArrayOrListComputed generalized in SynExpr_ArrayOrList
     | SynExpr_ArrayOrList
+    | SynExpr_ArrayOrList_OpeningDelimiter
+    | SynExpr_ArrayOrList_ClosingDelimiter
     // | SynExpr_ComputationExpr use first nested SynExpr
     | SynExpr_Lambda
     | SynExpr_Lambda_Arrow