[Console] Console with better SQL support (#51446) (#54091)

* First iteration of PoC for SQL highlighting and better completion

* Alternate implementation (no """sql markers)

* xLang markers

* - Remove 'start-sql'
- no flow :c

* Revert "- Remove 'start-sql'"

This reverts commit 2d585ee.

* Revert "xLang markers"

This reverts commit f019616.

* Revert "xLang markers" and add comment

This reverts commit f019616.

* Add elasticsearch sql highlight rules

* Add links to sources of data

* Update colors

* Redo built in functions

# Conflicts:
#	src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/input_highlight_rules.js
#	src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/x_json_highlight_rules.js

src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/elasticsearch_sql_highlight_rules.ts

@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import ace from 'brace';
+
+const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
+const oop = ace.acequire('ace/lib/oop');
+
+export const ElasticsearchSqlHighlightRules = function(this: any) {
+  // See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-commands.html
+  const keywords =
+    'describe|between|in|like|not|and|or|desc|select|from|where|having|group|by|order' +
+    'asc|desc|pivot|for|in|as|show|columns|include|frozen|tables|escape|limit|rlike|all|distinct|is';
+
+  const builtinConstants = 'true|false';
+
+  // See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-syntax-show-functions.html
+  const builtinFunctions =
+    'avg|count|first|first_value|last|last_value|max|min|sum|kurtosis|mad|percentile|percentile_rank|skewness' +
+    '|stddev_pop|sum_of_squares|var_pop|histogram|case|coalesce|greatest|ifnull|iif|isnull|least|nullif|nvl' +
+    '|curdate|current_date|current_time|current_timestamp|curtime|dateadd|datediff|datepart|datetrunc|date_add' +
+    '|date_diff|date_part|date_trunc|day|dayname|dayofmonth|dayofweek|dayofyear|day_name|day_of_month|day_of_week' +
+    '|day_of_year|dom|dow|doy|hour|hour_of_day|idow|isodayofweek|isodow|isoweek|isoweekofyear|iso_day_of_week|iso_week_of_year' +
+    '|iw|iwoy|minute|minute_of_day|minute_of_hour|month|monthname|month_name|month_of_year|now|quarter|second|second_of_minute' +
+    '|timestampadd|timestampdiff|timestamp_add|timestamp_diff|today|week|week_of_year|year|abs|acos|asin|atan|atan2|cbrt' +
+    '|ceil|ceiling|cos|cosh|cot|degrees|e|exp|expm1|floor|log|log10|mod|pi|power|radians|rand|random|round|sign|signum|sin' +
+    '|sinh|sqrt|tan|truncate|ascii|bit_length|char|character_length|char_length|concat|insert|lcase|left|length|locate' +
+    '|ltrim|octet_length|position|repeat|replace|right|rtrim|space|substring|ucase|cast|convert|database|user|st_astext|st_aswkt' +
+    '|st_distance|st_geometrytype|st_geomfromtext|st_wkttosql|st_x|st_y|st_z|score';
+
+  // See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-data-types.html
+  const dataTypes =
+    'null|boolean|byte|short|integer|long|double|float|half_float|scaled_float|keyword|text|binary|date|ip|object|nested|time' +
+    '|interval_year|interval_month|interval_day|interval_hour|interval_minute|interval_second|interval_year_to_month' +
+    'inteval_day_to_hour|interval_day_to_minute|interval_day_to_second|interval_hour_to_minute|interval_hour_to_second' +
+    'interval_minute_to_second|geo_point|geo_shape|shape';
+
+  const keywordMapper = this.createKeywordMapper(
+    {
+      keyword: [keywords, builtinFunctions, builtinConstants, dataTypes].join('|'),
+    },
+    'identifier',
+    true
+  );
+
+  this.$rules = {
+    start: [
+      {
+        token: 'comment',
+        regex: '--.*$',
+      },
+      {
+        token: 'comment',
+        start: '/\\*',
+        end: '\\*/',
+      },
+      {
+        token: 'string', // " string
+        regex: '".*?"',
+      },
+      {
+        token: 'constant', // ' string
+        regex: "'.*?'",
+      },
+      {
+        token: 'string', // ` string (apache drill)
+        regex: '`.*?`',
+      },
+      {
+        token: 'entity.name.function', // float
+        regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',
+      },
+      {
+        token: keywordMapper,
+        regex: '[a-zA-Z_$][a-zA-Z0-9_$]*\\b',
+      },
+      {
+        token: 'keyword.operator',
+        regex: '⇐|<⇒|\\*|\\.|\\:\\:|\\+|\\-|\\/|\\/\\/|%|&|\\^|~|<|>|<=|=>|==|!=|<>|=',
+      },
+      {
+        token: 'paren.lparen',
+        regex: '[\\(]',
+      },
+      {
+        token: 'paren.rparen',
+        regex: '[\\)]',
+      },
+      {
+        token: 'text',
+        regex: '\\s+',
+      },
+    ],
+  };
+  this.normalizeRules();
+};
+
+oop.inherits(ElasticsearchSqlHighlightRules, TextHighlightRules);

src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/input_highlight_rules.js

@@ -20,27 +20,33 @@
 const ace = require('brace');
 import { addToRules } from './x_json_highlight_rules';
 
-const oop = ace.acequire('ace/lib/oop');
-const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
-export function InputHighlightRules() {
-  function mergeTokens(/* ... */) {
-    return [].concat.apply([], arguments);
+export function addEOL(tokens, reg, nextIfEOL, normalNext) {
+  if (typeof reg === 'object') {
+    reg = reg.source;
   }
+  return [
+    { token: tokens.concat(['whitespace']), regex: reg + '(\\s*)$', next: nextIfEOL },
+    { token: tokens, regex: reg, next: normalNext },
+  ];
+}
 
-  function addEOL(tokens, reg, nextIfEOL, normalNext) {
-    if (typeof reg === 'object') {
-      reg = reg.source;
-    }
-    return [
-      { token: tokens.concat(['whitespace']), regex: reg + '(\\s*)$', next: nextIfEOL },
-      { token: tokens, regex: reg, next: normalNext },
-    ];
-  }
+export function mergeTokens(/* ... */) {
+  return [].concat.apply([], arguments);
+}
 
+const oop = ace.acequire('ace/lib/oop');
+const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
+
+export function InputHighlightRules() {
   // regexp must not have capturing parentheses. Use (?:) instead.
   // regexps are ordered -> the first match is used
   /*jshint -W015 */
   this.$rules = {
+    'start-sql': [
+      { token: 'whitespace', regex: '\\s+' },
+      { token: 'paren.lparen', regex: '{', next: 'json-sql', push: true },
+      { regex: '', next: 'start' },
+    ],
     start: mergeTokens(
       [
         { token: 'warning', regex: '#!.*$' },
@@ -71,6 +77,7 @@ export function InputHighlightRules() {
       addEOL(['whitespace'], /(\s+)/, 'start', 'url')
     ),
     url: mergeTokens(
+      addEOL(['url.part'], /(_sql)/, 'start-sql', 'url-sql'),
       addEOL(['url.part'], /([^?\/,\s]+)/, 'start'),
       addEOL(['url.comma'], /(,)/, 'start'),
       addEOL(['url.slash'], /(\/)/, 'start'),
@@ -81,6 +88,16 @@ export function InputHighlightRules() {
       addEOL(['url.param'], /([^&=]+)/, 'start'),
       addEOL(['url.amp'], /(&)/, 'start')
     ),
+    'url-sql': mergeTokens(
+      addEOL(['url.comma'], /(,)/, 'start-sql'),
+      addEOL(['url.slash'], /(\/)/, 'start-sql'),
+      addEOL(['url.questionmark'], /(\?)/, 'start-sql', 'urlParams-sql')
+    ),
+    'urlParams-sql': mergeTokens(
+      addEOL(['url.param', 'url.equal', 'url.value'], /([^&=]+)(=)([^&]*)/, 'start-sql'),
+      addEOL(['url.param'], /([^&=]+)/, 'start-sql'),
+      addEOL(['url.amp'], /(&)/, 'start-sql')
+    ),
   };
 
   addToRules(this);

src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/x_json_highlight_rules.js

@@ -18,12 +18,14 @@
  */
 
 const _ = require('lodash');
-const ScriptHighlightRules = require('./script_highlight_rules').ScriptHighlightRules;
+
+import { ElasticsearchSqlHighlightRules } from './elasticsearch_sql_highlight_rules';
+const { ScriptHighlightRules } = require('./script_highlight_rules');
 
 const jsonRules = function(root) {
   root = root ? root : 'json';
   const rules = {};
-  rules[root] = [
+  const xJsonRules = [
     {
       token: [
         'variable',
@@ -112,6 +114,24 @@ const jsonRules = function(root) {
       regex: '.+?',
     },
   ];
+
+  rules[root] = xJsonRules;
+  rules[root + '-sql'] = [
+    {
+      token: [
+        'variable',
+        'whitespace',
+        'ace.punctuation.colon',
+        'whitespace',
+        'punctuation.start_triple_quote',
+      ],
+      regex: '("query")(\\s*?)(:)(\\s*?)(""")',
+      next: 'sql-start',
+      merge: false,
+      push: true,
+    },
+  ].concat(xJsonRules);
+
   rules.string_literal = [
     {
       token: 'punctuation.end_triple_quote',
@@ -135,4 +155,11 @@ export function addToRules(otherRules, embedUnder) {
       next: 'pop',
     },
   ]);
+  otherRules.embedRules(ElasticsearchSqlHighlightRules, 'sql-', [
+    {
+      token: 'punctuation.end_triple_quote',
+      regex: '"""',
+      next: 'pop',
+    },
+  ]);
 }

src/legacy/core_plugins/console/public/np_ready/lib/autocomplete/autocomplete.ts

@@ -990,7 +990,7 @@ export default function({ coreEditor: editor, parser }: { coreEditor: CoreEditor
               score: 0,
               context,
             };
-            // we only need out custom insertMatch behavior for the body
+            // we only need our custom insertMatch behavior for the body
             if (context.autoCompleteType === 'body') {
               defaults.completer = {
                 insertMatch() {

src/legacy/core_plugins/console/public/np_ready/lib/autocomplete/components/full_request_component.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// @ts-ignore
+import { ConstantComponent } from './constant_component';
+
+export class FullRequestComponent extends ConstantComponent {
+  private readonly name: string;
+  constructor(name: string, parent: any, private readonly template: string) {
+    super(name, parent);
+    this.name = name;
+  }
+
+  getTerms() {
+    return [{ name: this.name, snippet: this.template }];
+  }
+}

src/legacy/core_plugins/console/public/np_ready/lib/autocomplete/components/url_pattern_matcher.js

@@ -25,6 +25,8 @@ import {
   SimpleParamComponent,
 } from './index';
 
+import { FullRequestComponent } from './full_request_component';
+
 /**
  * @param parametrizedComponentFactories a dict of the following structure
  * that will be used as a fall back for pattern parameters (i.e.: {indices})
@@ -54,6 +56,9 @@ export class UrlPatternMatcher {
     endpoint.methods.forEach(method => {
       let c;
       let activeComponent = this[method].rootComponent;
+      if (endpoint.template) {
+        new FullRequestComponent(pattern + '[body]', activeComponent, endpoint.template);
+      }
       const endpointComponents = endpoint.url_components || {};
       const partList = pattern.split('/');
       _.each(

src/legacy/core_plugins/console/public/np_ready/lib/row_parser.ts

@@ -44,7 +44,10 @@ export default class RowParser {
       return MODE.BETWEEN_REQUESTS;
     } // shouldn't really happen
 
-    if (mode !== 'start') {
+    // If another "start" mode is added here because we want to allow for new language highlighting
+    // please see https://github.com/elastic/kibana/pull/51446 for a discussion on why
+    // should consider a different approach.
+    if (mode !== 'start' && mode !== 'start-sql') {
       return MODE.IN_REQUEST;
     }
     let line = (this.editor.getLineValue(lineNumber) || '').trim();

x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json

@@ -18,6 +18,7 @@
         "cbor",
         "smile"
       ]
-    }
+    },
+    "template": "_sql?format=json\n{\n  \"query\": \"\"\"\n  SELECT * FROM \"${1:TABLE}\"\n  \"\"\"\n}\n"
   }
 }