Introduce simple kibana query language (#15646)

* Introduce simple kuery language

* Rename to kql and add modules

src/core_plugins/kibana/public/dashboard/dashboard_app.html

@@ -39,7 +39,7 @@
     ng-show="showFilterBar()"
     state="state"
     index-patterns="indexPatterns"
-    ng-if="model.query.language === 'lucene'"
+    ng-if="['lucene', 'kql'].includes(model.query.language)"
   ></filter-bar>
 
   <div

src/core_plugins/kibana/public/discover/index.html

@@ -39,7 +39,7 @@ <h1 tabindex="0" id="kui_local_breadcrumb" class="kuiLocalBreadcrumb">
       <filter-bar
         state="state"
         index-patterns="[indexPattern]"
-        ng-if="state.query.language === 'lucene'"
+        ng-if="['lucene', 'kql'].includes(state.query.language)"
       ></filter-bar>
     </div>
     <div class="row">

src/core_plugins/kibana/public/visualize/editor/editor.html

@@ -45,7 +45,7 @@
 
   <!-- Filters. -->
   <filter-bar
-    ng-if="vis.type.options.showFilterBar && state.query.language === 'lucene'"
+    ng-if="vis.type.options.showFilterBar && ['lucene', 'kql'].includes(state.query.language)"
     state="state"
     index-patterns="[indexPattern]"
   ></filter-bar>

src/core_plugins/tile_map/public/coordinate_maps_visualization.js

@@ -120,7 +120,7 @@ export function CoordinateMapsVisualizationProvider(Notifier, Private) {
       const query = this.vis.API.queryManager.getQuery();
       const language = query.language;
 
-      if (language === 'lucene') {
+      if (['lucene', 'kql'].includes(language)) {
         const filter = { meta: { negate: false, index: indexPatternName } };
         filter[filterName] = { ignore_unmapped: true };
         filter[filterName][field] = filterData;
@@ -198,4 +198,3 @@ export function CoordinateMapsVisualizationProvider(Notifier, Private) {
 
   return CoordinateMapsVisualization;
 }
-

src/ui/public/courier/data_source/build_query/build_es_query.js

@@ -1,6 +1,6 @@
 import { groupBy, has } from 'lodash';
 import { DecorateQueryProvider } from '../_decorate_query';
-import { buildQueryFromKuery } from './from_kuery';
+import { buildQueryFromKuery, buildQueryFromKql } from './from_kuery';
 import { buildQueryFromFilters } from './from_filters';
 import { buildQueryFromLucene } from './from_lucene';
 
@@ -17,15 +17,16 @@ export function BuildESQueryProvider(Private) {
     const queriesByLanguage = groupBy(validQueries, 'language');
 
     const kueryQuery = buildQueryFromKuery(indexPattern, queriesByLanguage.kuery);
+    const kqlQuery = buildQueryFromKql(indexPattern, queriesByLanguage.kql);
     const luceneQuery = buildQueryFromLucene(queriesByLanguage.lucene, decorateQuery);
     const filterQuery = buildQueryFromFilters(filters, decorateQuery);
 
     return {
       bool: {
-        must: [].concat(kueryQuery.must, luceneQuery.must, filterQuery.must),
-        filter: [].concat(kueryQuery.filter, luceneQuery.filter, filterQuery.filter),
-        should: [].concat(kueryQuery.should, luceneQuery.should, filterQuery.should),
-        must_not: [].concat(kueryQuery.must_not, luceneQuery.must_not, filterQuery.must_not),
+        must: [].concat(kueryQuery.must, kqlQuery.must, luceneQuery.must, filterQuery.must),
+        filter: [].concat(kueryQuery.filter, kqlQuery.filter, luceneQuery.filter, filterQuery.filter),
+        should: [].concat(kueryQuery.should, kqlQuery.should, luceneQuery.should, filterQuery.should),
+        must_not: [].concat(kueryQuery.must_not, kqlQuery.must_not, luceneQuery.must_not, filterQuery.must_not),
       }
     };
   }

src/ui/public/courier/data_source/build_query/from_kuery.js

@@ -1,8 +1,17 @@
 import _ from 'lodash';
-import { fromKueryExpression, toElasticsearchQuery, nodeTypes } from '../../../kuery';
+import { fromKueryExpression, fromKqlExpression, toElasticsearchQuery, nodeTypes } from '../../../kuery';
 
 export function buildQueryFromKuery(indexPattern, queries) {
   const queryASTs = _.map(queries, query => fromKueryExpression(query.query));
+  return buildQuery(indexPattern, queryASTs);
+}
+
+export function buildQueryFromKql(indexPattern, queries) {
+  const queryASTs = _.map(queries, query => fromKqlExpression(query.query));
+  return buildQuery(indexPattern, queryASTs);
+}
+
+function buildQuery(indexPattern, queryASTs) {
   const compoundQueryAST = nodeTypes.function.buildNode('and', queryASTs);
   const kueryQuery = toElasticsearchQuery(compoundQueryAST, indexPattern);
   return {
@@ -13,5 +22,3 @@ export function buildQueryFromKuery(indexPattern, queries) {
     ...kueryQuery.bool
   };
 }
-
-

src/ui/public/doc_table/actions/filter.js

@@ -8,7 +8,7 @@ export function addFilter(field, values = [], operation, index, state, filterMan
     values = [values];
   }
 
-  if (state.query.language === 'lucene') {
+  if (['lucene', 'kql'].includes(state.query.language)) {
     filterManager.add(field, values, operation, index);
   }
 

src/ui/public/filter_bar/filter_bar_click_handler.js

@@ -64,7 +64,7 @@ export function FilterBarClickHandlerProvider(Notifier, Private) {
         filters = dedupFilters($state.filters, uniqFilters(filters), { negate: true });
 
         if (!simulate) {
-          if ($state.query.language === 'lucene') {
+          if (['lucene', 'kql'].includes($state.query.language)) {
             $state.$newFilters = filters;
           }
           else if ($state.query.language === 'kuery') {
@@ -81,4 +81,3 @@ export function FilterBarClickHandlerProvider(Notifier, Private) {
     };
   };
 }
-

src/ui/public/kuery/ast/ast.js

@@ -1,11 +1,21 @@
 import grammar from 'raw-loader!./kuery.peg';
+import kqlGrammar from 'raw-loader!./kql.peg';
 import PEG from 'pegjs';
 import _ from 'lodash';
 import { nodeTypes } from '../node_types/index';
 
 const kueryParser = PEG.buildParser(grammar);
+const kqlParser = PEG.buildParser(kqlGrammar);
 
-export function fromKueryExpression(expression, parseOptions = {}) {
+export function fromKueryExpression(expression, parseOptions) {
+  return fromExpression(expression, parseOptions, kueryParser);
+}
+
+export function fromKqlExpression(expression, parseOptions) {
+  return fromExpression(expression, parseOptions, kqlParser);
+}
+
+function fromExpression(expression, parseOptions = {}, parser = kqlParser) {
   if (_.isUndefined(expression)) {
     throw new Error('expression must be a string, got undefined instead');
   }
@@ -15,7 +25,7 @@ export function fromKueryExpression(expression, parseOptions = {}) {
     helpers: { nodeTypes }
   };
 
-  return kueryParser.parse(expression, parseOptions);
+  return parser.parse(expression, parseOptions);
 }
 
 export function toKueryExpression(node) {

src/ui/public/kuery/ast/index.js

@@ -1 +1 @@
-export { fromKueryExpression, toKueryExpression, toElasticsearchQuery } from './ast';
+export { fromKueryExpression, fromKqlExpression, toKueryExpression, toElasticsearchQuery } from './ast';

src/ui/public/kuery/ast/kql.peg

@@ -0,0 +1,123 @@
+/*
+ * Kuery parser
+ */
+
+/*
+ * Initialization block
+ */
+{
+  var nodeTypes = options.helpers.nodeTypes;
+
+  if (options.includeMetadata === undefined) {
+    options.includeMetadata = true;
+  }
+
+  function addMeta(source, text, location) {
+    if (options.includeMetadata) {
+      return Object.assign(
+        {},
+        source,
+        {
+          text: text,
+          location: simpleLocation(location),
+        }
+      );
+    }
+
+    return source;
+  }
+
+  function simpleLocation(location) {
+    // Returns an object representing the position of the function within the expression,
+    // demarcated by the position of its first character and last character. We calculate these values
+    // using the offset because the expression could span multiple lines, and we don't want to deal
+    // with column and line values.
+    return {
+      min: location.start.offset,
+      max: location.end.offset
+    }
+  }
+}
+
+start
+  = Query
+  / space* {
+    return addMeta(nodeTypes.function.buildNode('and', []), text(), location());
+  }
+
+Query
+  = space? query:OrQuery space? {
+    if (query.type === 'literal') {
+      return addMeta(nodeTypes.function.buildNode('and', [query]), text(), location());
+    }
+    return query;
+  }
+
+OrQuery
+  = left:AndQuery space 'or'i space right:OrQuery {
+    return addMeta(nodeTypes.function.buildNode('or', [left, right]), text(), location());
+  }
+  / AndQuery
+
+AndQuery
+  = left:NotQuery space 'and'i space right:AndQuery {
+    return addMeta(nodeTypes.function.buildNode('and', [left, right]), text(), location());
+  }
+  / NotQuery
+
+NotQuery
+  = 'not'i space clause:Clause {
+    return addMeta(nodeTypes.function.buildNode('not', clause), text(), location());
+  }
+  / Clause
+
+Clause
+  = '(' subQuery:Query ')' {
+    return subQuery;
+  }
+  / Term
+
+Term
+  = field:literal_arg_type space? ':' space? value:literal_arg_type {
+    return addMeta(nodeTypes.function.buildNodeWithArgumentNodes('is', [field, value]), text(), location());
+  }
+  / field:literal_arg_type space? ':' space? '[' space? gt:literal_arg_type space 'to'i space lt:literal_arg_type space? ']' {
+    return addMeta(nodeTypes.function.buildNodeWithArgumentNodes('range', [field, gt, lt]), text(), location());
+  }
+  / !Keywords literal:literal_arg_type { return literal; }
+
+literal_arg_type
+ = literal:literal {
+   var result = addMeta(nodeTypes.literal.buildNode(literal), text(), location());
+   return result;
+ }
+
+Keywords
+  = 'or'i / 'and'i / 'not'i
+
+ /* ----- Core types ----- */
+
+literal "literal"
+  =  '"' chars:dq_char* '"' { return chars.join(''); } // double quoted string
+  / "'" chars:sq_char* "'" { return chars.join(''); } // single quoted string
+  / 'true'  { return true; } // unquoted literals from here down
+  / 'false' { return false; }
+  / 'null'  { return null; }
+  / string:[^\[\]()"',:=\ \t]+ { // this also matches numbers via Number()
+   var result = string.join('');
+   // Sort of hacky, but PEG doesn't have backtracking so
+   // a number rule is hard to read, and performs worse
+   if (isNaN(Number(result))) return result;
+   return Number(result)
+  }
+
+space
+ = [\ \t\r\n]+
+
+dq_char
+  = "\\" sequence:('"' / "\\") { return sequence; }
+  / [^"] // everything except "
+
+sq_char
+  = "\\" sequence:("'" / "\\") { return sequence; }
+  / [^'] // everything except '

src/ui/public/query_bar/directive/query_bar.html

@@ -67,6 +67,23 @@
         </div>
       </div>
 
+      <!-- kql input -->
+      <div class="kuiLocalSearchAssistedInput" ng-if="queryBar.localQuery.language === 'kql'">
+        <input
+          ng-model="queryBar.localQuery.query"
+          input-focus
+          disable-input-focus="queryBar.disableAutoFocus"
+          kbn-typeahead-input
+          placeholder="Search... (e.g. status:200 AND extension:PHP)"
+          aria-label="Search input"
+          aria-describedby="discoverKuerySyntaxHint"
+          type="text"
+          class="kuiLocalSearchInput"
+          ng-class="{'kuiLocalSearchInput-isInvalid': queryBarForm.$invalid}"
+          data-test-subj="queryInput"
+        />
+      </div>
+
       <select
         class="kuiLocalSearchSelect"
         ng-options="option for option in queryBar.availableQueryLanguages"

src/ui/public/query_bar/lib/queryLanguages.js

@@ -1,4 +1,5 @@
 export const queryLanguages = [
   'lucene',
   'kuery',
+  // 'kql'
 ];