From 3a1d25e8b1bd43f6ead88bd7e6b378b5983c752c Mon Sep 17 00:00:00 2001
From: Stephen Weatherford <stephwe@microsoft.com>
Date: Tue, 16 Jul 2019 15:01:30 -0700
Subject: [PATCH] Fix #250, strings vs expressions boundary cases

---
 .vscode/launch.json                           |  2 +-
 .vscode/settings.json                         |  2 +-
 .../arm-expression-string.tmLanguage.json     | 25 ++++---
 gulpfile.ts                                   | 19 +++--
 .../inputs/TODO string.INVALID.jsonc          |  8 ---
 .../inputs/double-bracket.NOT-EXPR.jsonc      |  5 --
 .../inputs/double-brackets.NOT-EXPR.json      |  5 --
 .../end-bracket-wrong-place.INVALID.jsonc     |  4 --
 .../inputs/expr-vs-string.is-expression.jsonc | 16 +++++
 .../expr-vs-string.is-string.NOT-EXPR.jsonc   | 23 ++++++
 .../expr-vs-string.multiline.NOT-EXPR.jsonc   | 15 ++++
 .../inputs/no-ending-bracket.INVALID.jsonc    |  4 --
 .../results/double-bracket.NOT-EXPR.jsonc.txt |  4 --
 .../results/double-brackets.NOT-EXPR.json.txt |  4 --
 .../end-bracket-wrong-place.INVALID.jsonc.txt | 20 ------
 .../expr-vs-string.is-expression.jsonc.txt    | 72 +++++++++++++++++++
 ...xpr-vs-string.is-string.NOT-EXPR.jsonc.txt | 46 ++++++++++++
 ...xpr-vs-string.multiline.NOT-EXPR.jsonc.txt | 17 +++++
 .../no-ending-bracket.INVALID.jsonc.txt       | 18 -----
 19 files changed, 222 insertions(+), 87 deletions(-)
 delete mode 100644 test/colorization/inputs/TODO string.INVALID.jsonc
 delete mode 100644 test/colorization/inputs/double-bracket.NOT-EXPR.jsonc
 delete mode 100644 test/colorization/inputs/double-brackets.NOT-EXPR.json
 delete mode 100644 test/colorization/inputs/end-bracket-wrong-place.INVALID.jsonc
 create mode 100644 test/colorization/inputs/expr-vs-string.is-expression.jsonc
 create mode 100644 test/colorization/inputs/expr-vs-string.is-string.NOT-EXPR.jsonc
 create mode 100644 test/colorization/inputs/expr-vs-string.multiline.NOT-EXPR.jsonc
 delete mode 100644 test/colorization/inputs/no-ending-bracket.INVALID.jsonc
 delete mode 100644 test/colorization/results/double-bracket.NOT-EXPR.jsonc.txt
 delete mode 100644 test/colorization/results/double-brackets.NOT-EXPR.json.txt
 delete mode 100644 test/colorization/results/end-bracket-wrong-place.INVALID.jsonc.txt
 create mode 100644 test/colorization/results/expr-vs-string.is-expression.jsonc.txt
 create mode 100644 test/colorization/results/expr-vs-string.is-string.NOT-EXPR.jsonc.txt
 create mode 100644 test/colorization/results/expr-vs-string.multiline.NOT-EXPR.jsonc.txt
 delete mode 100644 test/colorization/results/no-ending-bracket.INVALID.jsonc.txt

diff --git a/.vscode/launch.json b/.vscode/launch.json
index 5d5dbf13c..50e390a2b 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -60,7 +60,7 @@
             "preLaunchTask": "npm: compile",
             "env": {
                 "AZCODE_ARM_IGNORE_BUNDLE": "1",
-                "MOCHA_grep": "", // RegExp of tests to run (empty for all)
+                "MOCHA_grep": "coloriz", // RegExp of tests to run (empty for all)
                 "MOCHA_enableTimeouts": "0", // Disable time-outs
                 "DEBUGTELEMETRY": "1", // Set to verbose to see telemetry in console
                 "NODE_DEBUG": ""
diff --git a/.vscode/settings.json b/.vscode/settings.json
index a05cbcdc8..11614dcfd 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,5 +1,5 @@
 {
-    "editor.formatOnSave": true,
+    //"editor.formatOnSave": true,
     "editor.insertSpaces": true,
     "editor.tabSize": 4,
     "files.insertFinalNewline": true,
diff --git a/grammars/arm-expression-string.tmLanguage.json b/grammars/arm-expression-string.tmLanguage.json
index fa510c519..191ddb154 100644
--- a/grammars/arm-expression-string.tmLanguage.json
+++ b/grammars/arm-expression-string.tmLanguage.json
@@ -2,14 +2,21 @@
 	"name": "ARM Template",
 	"scopeName": "source.tle.arm",
 	"uuid": "3ADA43CD-2258-4907-8477-169A7FDDF216",
-	"$preprocessComment1": "Items in the preprocess section get replaced during gulp build-grammars",
-	"$preprocessComment2": "The 'builtin' key/value are special - they're created during build from our function metadata",
-	"$preprocessComment3": "Currently you need to run 'build grammars' after changing this file",
 	"preprocess": {
+		"$preprocessComment1": "Items in the preprocess section get replaced during gulp build-grammars",
+		"$preprocessComment2": "Items beginning with $ are ignored",
+		"$preprocessComment3": "---",
+		"$preprocessComment4": "A preprocess item with the key 'builtin-functions' is automatically created during build from our function metadata",
+		"$preprocessComment6": "---",
+		"$preprocessComment7": "IMPORTANT!  You need to run 'build grammars' after changing this file, F5 currently doesn't handle that",
+		"$1": "--------------- Regular expressions ---------------",
+		"escaped-json-character": "(?:\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4}))",
+		"escaped-apostrophe": "''",
 		"idchar": "[_$[:alnum:]]",
 		"id": "(?:[_$[:alpha:]]{{idchar}}*)",
 		"ns-userfunc": "(?:({{id}})\\s*(\\.)\\s*({{id}}))",
 		"logical": "(?:if|and|or|not)",
+		"$2": "--------------- Scope Names ---------------",
 		"scope-expression-start": "support.function.expression.begin.tle.arm",
 		"scope-expression-end": "support.function.expression.end.tle.arm",
 		"scope-builtin": "support.function.builtin.tle.arm",
@@ -42,8 +49,10 @@
 	"repository": {
 		"expressionstring": {
 			"name": "meta.expression.tle.arm",
-			"begin": "\"\\[(?!\\[)",
-			"$beginComment": "A string that starts with [[ is not an expression",
+			"begin": "(?x)  \"\\[(?!\\[)  (?= (?:{{escaped-apostrophe}}|{{escaped-json-character}}|[^\"])* ( \\]\"|$ ) )",
+			"$beginComment": "An expression must start with '[' (no whitespace before), not start with '[[', and end with ']' (no whitespace after)",
+			"$beginComment2": "Since we can't check if an expression ends with ']' for multi-line strings, we also assume a string which starts with '[' and",
+			"$beginComment3": "   isn't terminated on the first line is an expression",
 			"end": "\\]\"",
 			"beginCaptures": {
 				"0": {
@@ -123,13 +132,13 @@
 			"patterns": [
 				{
 					"$comment": "Escaped JSON string characters",
-					"match": "(?x)                # turn on extended mode\n  \\\\                # a literal backslash\n  (?:               # ...followed by...\n    [\"\\\\/bfnrt]     # one of these characters\n    |               # ...or...\n    u               # a u\n    [0-9a-fA-F]{4}) # and four hex digits",
+					"match": "{{escaped-json-character}}",
 					"name": "{{scope-json-escape-chars}}"
 				},
 				{
 					"$comment": "Escaped apostrophe (interpreted by ARM backend)",
 					"name": "{{scope-escapedapostrophe}}",
-					"match": "['][']"
+					"match": "{{escaped-apostrophe}}"
 				},
 				{
 					"match": "\\\\.",
@@ -186,7 +195,7 @@
 			}
 		},
 		"functionname": {
-			"match": "(?x) \\s* (?: {{ns-userfunc}} | (parameters) | (variables) | ({{logical}}) | ({{builtin}}) | ({{id}}) ) (?!{{idchar}} (?# Make sure we don't match a well-known name like 'add' inside something like 'add2'))",
+			"match": "(?x) \\s* (?: {{ns-userfunc}} | (parameters) | (variables) | ({{logical}}) | ({{builtin-functions}}) | ({{id}}) ) (?!{{idchar}} (?# Make sure we don't match a well-known name like 'add' inside something like 'add2'))",
 			"captures": {
 				"1": {
 					"$comment": "user namespace (capturing group inside ns-userfunc)",
diff --git a/gulpfile.ts b/gulpfile.ts
index a87a602b9..f80459988 100644
--- a/gulpfile.ts
+++ b/gulpfile.ts
@@ -22,7 +22,7 @@ export const tleGrammarBuiltPath: string = path.resolve('dist/grammars/arm-expre
 
 export interface IGrammar {
     preprocess?: {
-        builtin: string;
+        "builtin-functions": string;
         [key: string]: string;
     };
     [key: string]: unknown;
@@ -51,15 +51,19 @@ function buildTLEGrammar(): void {
     let builtinFunctions: string[] = expressionMetadata.functionSignatures.map(sig => sig.name);
     let grammarAsObject = <IGrammar>JSON.parse(grammar);
     grammarAsObject.preprocess = {
-        builtin: `(?:(?i)${builtinFunctions.join('|')})`,
+        "builtin-functions": `(?:(?i)${builtinFunctions.join('|')})`,
         ... (grammarAsObject.preprocess || {})
     };
+
     grammarAsObject = {
-        $comment: `DO NOT EDIT - This file was built from ${path.relative(__dirname, tleGrammarBuiltPath)}`,
+        $comment: `DO NOT EDIT - This file was built from ${path.relative(__dirname, tleGrammarSourcePath)}`,
         ...grammarAsObject
     };
+
     grammar = JSON.stringify(grammarAsObject, null, 4);
-    const replacementKeys = Object.getOwnPropertyNames((<IGrammar>JSON.parse(grammar)).preprocess);
+
+    // Get the replacement keys from the preprocess section (ignore those that start with "$")
+    const replacementKeys = Object.getOwnPropertyNames((<IGrammar>JSON.parse(grammar)).preprocess).filter(key => !key.startsWith("$"));
 
     // Build grammar: Make replacements specified
     for (let key of replacementKeys) {
@@ -70,11 +74,16 @@ function buildTLEGrammar(): void {
         // remove quotes
         valueString = valueString.slice(1, valueString.length - 1);
         if (!sourceGrammar.includes(replacementKey)) {
-            console.log(`WARNING: Preprocess key ${replacementKey} not found in ${tleGrammarSourcePath}`);
+            console.log(`WARNING: Preprocess key ${replacementKey} is never referenced in ${tleGrammarSourcePath}`);
         }
         grammar = grammar.replace(new RegExp(replacementKey, "g"), valueString);
     }
 
+    // Remove preprocess section from the output grammar file
+    let outputGrammarAsObject = (<IGrammar>JSON.parse(grammar));
+    delete outputGrammarAsObject.preprocess;
+    grammar = JSON.stringify(outputGrammarAsObject, null, 4);
+
     fs.writeFileSync(tleGrammarBuiltPath, grammar);
     console.log(`Built ${tleGrammarBuiltPath}`);
 
diff --git a/test/colorization/inputs/TODO string.INVALID.jsonc b/test/colorization/inputs/TODO string.INVALID.jsonc
deleted file mode 100644
index 81ef1bd0e..000000000
--- a/test/colorization/inputs/TODO string.INVALID.jsonc	
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
-    "parameters": {
-        "guid": {
-            "$TEST": "['abc'def]"
-        }
-    }
-}
diff --git a/test/colorization/inputs/double-bracket.NOT-EXPR.jsonc b/test/colorization/inputs/double-bracket.NOT-EXPR.jsonc
deleted file mode 100644
index 47761d78b..000000000
--- a/test/colorization/inputs/double-bracket.NOT-EXPR.jsonc
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
-    // Not an expression if it starts with [[
-    "$TEST": "[[2]"
-}
diff --git a/test/colorization/inputs/double-brackets.NOT-EXPR.json b/test/colorization/inputs/double-brackets.NOT-EXPR.json
deleted file mode 100644
index 81c7d431d..000000000
--- a/test/colorization/inputs/double-brackets.NOT-EXPR.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
-    // Not an expression if it starts with [[
-    "$TEST": "[[2]]"
-}
diff --git a/test/colorization/inputs/end-bracket-wrong-place.INVALID.jsonc b/test/colorization/inputs/end-bracket-wrong-place.INVALID.jsonc
deleted file mode 100644
index 2c2923b0a..000000000
--- a/test/colorization/inputs/end-bracket-wrong-place.INVALID.jsonc
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
-    "test": "[2]a"
-}
diff --git a/test/colorization/inputs/expr-vs-string.is-expression.jsonc b/test/colorization/inputs/expr-vs-string.is-expression.jsonc
new file mode 100644
index 000000000..69b8f7351
--- /dev/null
+++ b/test/colorization/inputs/expr-vs-string.is-expression.jsonc
@@ -0,0 +1,16 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
+    //
+    // Handle escaped characters - this should be an expression with a literal string inside it
+    //
+    "$TEST1": "['I said Hi! He''s here']",
+    "$TEST2": "['I said \"Hi! He''s here!\"']",
+    //
+    // Multi-line expressions - the colorization can't peek onto another line to determine whether the
+    // string ends with "]" (and therefore know it's an expression and not a string), so assume it is an expression
+    // if a multi-line string starts with "["
+    "$TEST10": "[concat('This is a ', 1, '-line ', 'expression ', 4, 'you!')]", //asdf
+    "$TEST11": "[concat('This is a ',
+        3, '-line ',
+        'expression ', 4, ' you!')]"
+}
diff --git a/test/colorization/inputs/expr-vs-string.is-string.NOT-EXPR.jsonc b/test/colorization/inputs/expr-vs-string.is-string.NOT-EXPR.jsonc
new file mode 100644
index 000000000..51d9860ba
--- /dev/null
+++ b/test/colorization/inputs/expr-vs-string.is-string.NOT-EXPR.jsonc
@@ -0,0 +1,23 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
+    //
+    // Not an expression if it starts with [[
+    //
+    "$TEST1": "[[2]]",
+    "$TEST2": "[[[four]five]",
+    "$TEST3": "[['I said \"Hi! He''s here!\"]",
+    //
+    // Not an expression if it doesn't end with ]
+    //
+    "$TEST10": "[pre]post",
+    "$TEST11": "[[three]four",
+    //
+    // Not an expression if it doesn't start immediately with [ or the very last character is not ] (even if it's whitespace)
+    //
+    "$TEST20": " [starts with whitespace]",
+    "$TEST21": "[ends with whitespace] ",
+    //
+    // Real examples
+    //
+    "$TEST100": "[ChefInSpec]InstalledApplicationLinuxResource1;AttributesYmlContent"
+}
diff --git a/test/colorization/inputs/expr-vs-string.multiline.NOT-EXPR.jsonc b/test/colorization/inputs/expr-vs-string.multiline.NOT-EXPR.jsonc
new file mode 100644
index 000000000..ca8ca0860
--- /dev/null
+++ b/test/colorization/inputs/expr-vs-string.multiline.NOT-EXPR.jsonc
@@ -0,0 +1,15 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
+    //
+    // Not an expression if it starts with [[
+    //
+    "$TEST1": "[[2
+        ]]",
+    "$TEST2": "[[[four]
+        five]",
+    //
+    // Not an expression if it doesn't start immediately with [ or the very last character is not ] (even if it's whitespace)
+    //
+    "$TEST20": "
+[starts with whitespace]"
+}
diff --git a/test/colorization/inputs/no-ending-bracket.INVALID.jsonc b/test/colorization/inputs/no-ending-bracket.INVALID.jsonc
deleted file mode 100644
index 0e33a1500..000000000
--- a/test/colorization/inputs/no-ending-bracket.INVALID.jsonc
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
-    "test": "[2"
-}
diff --git a/test/colorization/results/double-bracket.NOT-EXPR.jsonc.txt b/test/colorization/results/double-bracket.NOT-EXPR.jsonc.txt
deleted file mode 100644
index 92061c99e..000000000
--- a/test/colorization/results/double-bracket.NOT-EXPR.jsonc.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-TEST STRING: "[[2]"
-"                   punctuation.definition.string.begin.json.arm
-[[2]                string.quoted.double.json.arm
-"                   punctuation.definition.string.end.json.arm
\ No newline at end of file
diff --git a/test/colorization/results/double-brackets.NOT-EXPR.json.txt b/test/colorization/results/double-brackets.NOT-EXPR.json.txt
deleted file mode 100644
index cb681b152..000000000
--- a/test/colorization/results/double-brackets.NOT-EXPR.json.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-TEST STRING: "[[2]]"
-"                   punctuation.definition.string.begin.json.arm
-[[2]]               string.quoted.double.json.arm
-"                   punctuation.definition.string.end.json.arm
\ No newline at end of file
diff --git a/test/colorization/results/end-bracket-wrong-place.INVALID.jsonc.txt b/test/colorization/results/end-bracket-wrong-place.INVALID.jsonc.txt
deleted file mode 100644
index 7cfd83c48..000000000
--- a/test/colorization/results/end-bracket-wrong-place.INVALID.jsonc.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-{                   punctuation.definition.dictionary.begin.json.arm
-"                   punctuation.support.type.property-name.begin.json.arm
-$schema             support.type.property-name.json.arm
-"                   punctuation.support.type.property-name.end.json.arm
-:                   punctuation.separator.dictionary.key-value.json.arm
-"                   punctuation.definition.string.begin.json.arm
-http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#
-                    string.quoted.double.json.arm
-"                   punctuation.definition.string.end.json.arm
-,                   punctuation.separator.dictionary.pair.json.arm
-"                   punctuation.support.type.property-name.begin.json.arm
-test                support.type.property-name.json.arm
-"                   punctuation.support.type.property-name.end.json.arm
-:                   punctuation.separator.dictionary.key-value.json.arm
-"[                  {{scope-expression-start}}
-2                   {{scope-number}}
-]                   invalid.illegal.expected-expression.tle.arm
-a                   {{scope-unknownfunction}}
-"                   invalid.illegal.expected-expression.tle.arm
-}                   invalid.illegal.expected-expression.tle.arm
\ No newline at end of file
diff --git a/test/colorization/results/expr-vs-string.is-expression.jsonc.txt b/test/colorization/results/expr-vs-string.is-expression.jsonc.txt
new file mode 100644
index 000000000..a0a67860f
--- /dev/null
+++ b/test/colorization/results/expr-vs-string.is-expression.jsonc.txt
@@ -0,0 +1,72 @@
+TEST STRING: "['I said Hi! He''s here']"
+"[                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-start}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-string-start}}
+I said Hi! He       source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}}
+''                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-escapedapostrophe}}
+s here              source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-string-end}}
+]"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-end}}
+
+TEST STRING: "['I said\"Hi! He''s here!\"']"
+"[                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-start}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-string-start}}
+I said              source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}}
+\"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-json-escape-chars}}
+Hi! He              source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}}
+''                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-escapedapostrophe}}
+s here!             source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}}
+\"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-json-escape-chars}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-string-contents}} {{scope-string-end}}
+]"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-end}}
+
+TEST STRING: "[concat('This is a',1,'-line','expression',4,'you!')]"
+"[                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-start}}
+concat              source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-builtin}}
+(                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm {{scope-parentheses-funccall}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+This is a           source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+1                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-number}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+-line               source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+expression          source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+4                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-number}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+you!                source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+)                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm {{scope-parentheses-funccall}}
+]"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-end}}
+
+TEST STRING: "[concat('This is a',3,'-line','expression',4,'you!')]"
+"[                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-start}}
+concat              source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-builtin}}
+(                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm {{scope-parentheses-funccall}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+This is a           source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+3                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-number}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+-line               source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+expression          source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+4                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-number}}
+,                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-funcargs-separator}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-start}}
+you!                source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}}
+'                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm meta.function-arguments.tle.arm {{scope-string-contents}} {{scope-string-end}}
+)                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm meta.function-call.tle.arm {{scope-parentheses-funccall}}
+]"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments meta.expression.tle.arm {{scope-expression-end}}
\ No newline at end of file
diff --git a/test/colorization/results/expr-vs-string.is-string.NOT-EXPR.jsonc.txt b/test/colorization/results/expr-vs-string.is-string.NOT-EXPR.jsonc.txt
new file mode 100644
index 000000000..be2a13d3b
--- /dev/null
+++ b/test/colorization/results/expr-vs-string.is-string.NOT-EXPR.jsonc.txt
@@ -0,0 +1,46 @@
+TEST STRING: "[[2]]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[[2]]               source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[[[four]five]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[[[four]five]       source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[['I said\"Hi! He''s here!\"]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[['I said           source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+\"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm constant.character.escape.json.arm
+Hi! He''s here!     source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+\"                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm constant.character.escape.json.arm
+]                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[pre]post"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[pre]post           source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[[three]four"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[[three]four        source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[starts with whitespace]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[starts with whitespace]
+                    source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[ends with whitespace]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[ends with whitespace]
+                    source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[ChefInSpec]InstalledApplicationLinuxResource1;AttributesYmlContent"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[ChefInSpec]InstalledApplicationLinuxResource1;AttributesYmlContent
+                    source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
\ No newline at end of file
diff --git a/test/colorization/results/expr-vs-string.multiline.NOT-EXPR.jsonc.txt b/test/colorization/results/expr-vs-string.multiline.NOT-EXPR.jsonc.txt
new file mode 100644
index 000000000..cc0973015
--- /dev/null
+++ b/test/colorization/results/expr-vs-string.multiline.NOT-EXPR.jsonc.txt
@@ -0,0 +1,17 @@
+TEST STRING: "[[2]]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[[2                 source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+]]                  source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[[[four]five]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[[[four]            source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+five]               source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
+
+TEST STRING: "[starts with whitespace]"
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.begin.json.arm
+[starts with whitespace]
+                    source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm
+"                   source.json.comments meta.structure.dictionary.json.comments meta.arm-deployment-template.json.comments meta.structure.dictionary.value.json.comments string.quoted.double.json.arm punctuation.definition.string.end.json.arm
\ No newline at end of file
diff --git a/test/colorization/results/no-ending-bracket.INVALID.jsonc.txt b/test/colorization/results/no-ending-bracket.INVALID.jsonc.txt
deleted file mode 100644
index 890ca02b2..000000000
--- a/test/colorization/results/no-ending-bracket.INVALID.jsonc.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-{                   punctuation.definition.dictionary.begin.json.arm
-"                   punctuation.support.type.property-name.begin.json.arm
-$schema             support.type.property-name.json.arm
-"                   punctuation.support.type.property-name.end.json.arm
-:                   punctuation.separator.dictionary.key-value.json.arm
-"                   punctuation.definition.string.begin.json.arm
-https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#
-                    string.quoted.double.json.arm
-"                   punctuation.definition.string.end.json.arm
-,                   punctuation.separator.dictionary.pair.json.arm
-"                   punctuation.support.type.property-name.begin.json.arm
-test                support.type.property-name.json.arm
-"                   punctuation.support.type.property-name.end.json.arm
-:                   punctuation.separator.dictionary.key-value.json.arm
-"[                  {{scope-expression-start}}
-2                   {{scope-number}}
-"                   invalid.illegal.expected-expression.tle.arm
-}                   invalid.illegal.expected-expression.tle.arm
\ No newline at end of file