[6.x] Enable KQL support for TSVB (#26006) (#29828)

* Enable KQL support for TSVB (#26006)

* Use buildEsQuery for table, series and annotations

* Fix query test.

Using the buildEsQuery changed a bit the order of the must.bool array on the query

* Remove console.log

* Remove console.error and comment

* Fix wrong merge of PR #26510

* Fix default/empty index_pattern

When the user save the visualization without configuring
manually an index_pattern, a default empty string is used
instead of the default pattern. This leads to an empty
visualization after the refactoring done in #24832.
Now it will update the index_pattern field with the default
index pattern in visualize editor and on dashboard.

* Remove unnecessary wrapping query in an array

* Enable query bar on tsvb

* Remove unnecessary setDefaultIndexPattern

After fixing the null index pattern issue in kql
there is no need to use set the default index pattern
before rendering.

* fix(tsvb-server): Ignore query bar search on ignore_global_filters param

This commit mimic the behaviour of the ignore_global_filters currently used with lucene syntax:
if you enable the ignore_global_filter option on data or on annotations, data or annotations
are filtered out when using the filter bar and the search bar

* Disable showQueryBar

* Fix non array query value on request

src/legacy/core_plugins/metrics/public/components/_error.scss

@@ -5,7 +5,7 @@
   display: flex;
   align-items: center;
   justify-content: center;
-
+  flex-direction: column;
   // Calculate colors similar to EuiCallout
   $tempBackgroundColor: tintOrShade($euiColorDanger, 90%, 70%);
   background-color: $tempBackgroundColor;

src/legacy/core_plugins/metrics/public/kbn_vis_types/request_handler.js

@@ -20,7 +20,6 @@
 import { validateInterval } from '../lib/validate_interval';
 import { timezoneProvider } from 'ui/vis/lib/timezone';
 import { timefilter } from 'ui/timefilter';
-import { buildEsQuery } from '@kbn/es-query';
 
 const MetricsRequestHandlerProvider = function (Private, Notifier, config, $http, i18n) {
   const notify = new Notifier({ location: i18n('tsvb.requestHandler.notifier.locationNameTitle', { defaultMessage: 'Metrics' }) });
@@ -35,14 +34,11 @@ const MetricsRequestHandlerProvider = function (Private, Notifier, config, $http
         const parsedTimeRange = timefilter.calculateBounds(timeRange);
         const scaledDataFormat = config.get('dateFormat:scaled');
         const dateFormat = config.get('dateFormat');
-        const esQueryConfigs = {
-          allowLeadingWildcards: config.get('query:allowLeadingWildcards'),
-          queryStringOptions: config.get('query:queryString:options'),
-        };
         if (panel && panel.id) {
           const params = {
             timerange: { timezone, ...parsedTimeRange },
-            filters: [buildEsQuery(undefined, [query], filters, esQueryConfigs)],
+            query: Array.isArray(query) ? query : [query],
+            filters,
             panels: [panel],
             state: uiStateObj
           };

src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js

@@ -20,8 +20,8 @@
 import buildProcessorFunction from './build_processor_function';
 import processors from './request_processors/annotations';
 
-export default function buildAnnotationRequest(req, panel, annotation) {
-  const processor = buildProcessorFunction(processors, req, panel, annotation);
+export default function buildAnnotationRequest(req, panel, annotation,  esQueryConfig, indexPattern) {
+  const processor = buildProcessorFunction(processors, req, panel, annotation,  esQueryConfig, indexPattern);
   const doc = processor({});
   return doc;
 }

src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js

@@ -19,6 +19,7 @@
 
 import buildAnnotationRequest from './build_annotation_request';
 import handleAnnotationResponse from './handle_annotation_response';
+import { getIndexPatternObject } from './helpers/get_index_pattern';
 
 function validAnnotation(annotation) {
   return annotation.index_pattern &&
@@ -28,27 +29,19 @@ function validAnnotation(annotation) {
     annotation.template;
 }
 
-export default async (req, panel) => {
+export default async (req, panel, esQueryConfig) => {
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data');
-  const bodies = panel.annotations
+  const bodiesPromises = panel.annotations
     .filter(validAnnotation)
     .map(annotation => {
-
-      const indexPattern = annotation.index_pattern;
-      const bodies = [];
-
-      bodies.push({
-        index: indexPattern,
-        ignoreUnavailable: true,
-      });
-
-      const body = buildAnnotationRequest(req, panel, annotation);
-      body.timeout = '90s';
-      bodies.push(body);
-      return bodies;
+      return getAnnotationBody(req, panel, annotation, esQueryConfig);
     });
-
-  if (!bodies.length) return { responses: [] };
+  const bodies = await Promise.all(bodiesPromises);
+  if (!bodies.length) {
+    return {
+      responses: [],
+    };
+  }
   try {
     const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen');
     const resp = await callWithRequest(req, 'msearch', {
@@ -68,6 +61,18 @@ export default async (req, panel) => {
     if (error.message === 'missing-indices') return { responses: [] };
     throw error;
   }
-
 };
 
+async function getAnnotationBody(req, panel, annotation, esQueryConfig) {
+  const indexPatternString = annotation.index_pattern;
+  const indexPatternObject = await getIndexPatternObject(req, indexPatternString);
+  const request = buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPatternObject);
+  request.timeout = '90s';
+  return [
+    {
+      index: indexPatternString,
+      ignoreUnavailable: true,
+    },
+    request,
+  ];
+}

src/legacy/core_plugins/metrics/server/lib/vis_data/get_panel_data.js

@@ -21,7 +21,9 @@ import { getTableData } from './get_table_data';
 import { getSeriesData } from './get_series_data';
 export default function getPanelData(req) {
   return panel => {
-    if (panel.type === 'table') return getTableData(req, panel);
+    if (panel.type === 'table') {
+      return getTableData(req, panel);
+    }
     return getSeriesData(req, panel);
   };
 }

src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js

@@ -21,36 +21,45 @@ import getRequestParams from './series/get_request_params';
 import handleResponseBody from './series/handle_response_body';
 import handleErrorResponse from './handle_error_response';
 import getAnnotations from './get_annotations';
+import { getEsQueryConfig } from './helpers/get_es_query_uisettings';
+
 export async function getSeriesData(req, panel) {
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data');
   const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen');
-  const bodies = panel.series.map(series => getRequestParams(req, panel, series));
-  const params = {
-    rest_total_hits_as_int: true,
-    ignore_throttled: !includeFrozen,
-    body: bodies.reduce((acc, items) => acc.concat(items), [])
-  };
-  return callWithRequest(req, 'msearch', params)
-    .then(resp => {
-      const series = resp.responses.map(handleResponseBody(panel));
-      return {
-        [panel.id]: {
-          id: panel.id,
-          series: series.reduce((acc, series) => acc.concat(series), [])
-        }
-      };
-    })
-    .then(resp => {
-      if (!panel.annotations || panel.annotations.length === 0) return resp;
-      return getAnnotations(req, panel).then(annotations => {
-        resp[panel.id].annotations = annotations;
+  const esQueryConfig = await getEsQueryConfig(req);
+
+  try {
+    const bodiesPromises = panel.series.map(series => getRequestParams(req, panel, series, esQueryConfig));
+    const bodies = await Promise.all(bodiesPromises);
+    const params = {
+      rest_total_hits_as_int: true,
+      ignore_throttled: !includeFrozen,
+      body: bodies.reduce((acc, items) => acc.concat(items), [])
+    };
+    return callWithRequest(req, 'msearch', params)
+      .then(resp => {
+        const series = resp.responses.map(handleResponseBody(panel));
+        return {
+          [panel.id]: {
+            id: panel.id,
+            series: series.reduce((acc, series) => acc.concat(series), [])
+          }
+        };
+      })
+      .then(resp => {
+        if (!panel.annotations || panel.annotations.length === 0) return resp;
+        return getAnnotations(req, panel, esQueryConfig).then(annotations => {
+          resp[panel.id].annotations = annotations;
+          return resp;
+        });
+      })
+      .then(resp => {
+        resp.type = panel.type;
         return resp;
-      });
-    })
-    .then(resp => {
-      resp.type = panel.type;
-      return resp;
-    })
-    .catch(handleErrorResponse(panel));
+      })
+      .catch(handleErrorResponse(panel));
+  } catch(e) {
+    return handleErrorResponse(e);
+  }
 }
 

src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js

@@ -16,18 +16,25 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+import { get } from 'lodash';
 import buildRequestBody from './table/build_request_body';
 import handleErrorResponse from './handle_error_response';
-import { get } from 'lodash';
 import processBucket from './table/process_bucket';
+import { getIndexPatternObject } from './helpers/get_index_pattern';
+import { getEsQueryConfig } from './helpers/get_es_query_uisettings';
+
+
 export async function getTableData(req, panel) {
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data');
   const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen');
+  const indexPatternString = panel.index_pattern;
+
+  const esQueryConfig = await getEsQueryConfig(req);
+  const indexPatternObject = await getIndexPatternObject(req, indexPatternString);
   const params = {
-    index: panel.index_pattern,
+    index: indexPatternString,
     ignore_throttled: !includeFrozen,
-    body: buildRequestBody(req, panel)
+    body: buildRequestBody(req, panel, esQueryConfig, indexPatternObject)
   };
   try {
     const resp = await callWithRequest(req, 'search', params);

src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_query_uisettings.js

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export async function getEsQueryConfig(req) {
+  const uiSettings = req.getUiSettingsService();
+  const allowLeadingWildcards = await uiSettings.get('query:allowLeadingWildcards');
+  const queryStringOptions = await uiSettings.get('query:queryString:options');
+  return {
+    allowLeadingWildcards,
+    queryStringOptions: JSON.parse(queryStringOptions),
+  };
+}

src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_index_pattern.js

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export async function getIndexPatternObject(req, indexPatternString) {
+  // getting the matching index pattern
+  const savedObjectClient = req.getSavedObjectsClient();
+  const indexPatternObjects = await savedObjectClient.find({
+    type: 'index-pattern',
+    fields: ['title', 'fields'],
+    search: `"${indexPatternString}"`,
+    search_fields: ['title'],
+  });
+
+  // getting the index pattern fields
+  const indexPatterns = indexPatternObjects.saved_objects
+    .filter(obj => obj.attributes.title === indexPatternString)
+    .map(indexPattern => {
+      const { title, fields } = indexPattern.attributes;
+      return {
+        title,
+        fields: JSON.parse(fields),
+      };
+    });
+  return indexPatterns.length === 1 ? indexPatterns[0] : null;
+}

src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js

@@ -19,52 +19,44 @@
 
 import getBucketSize from '../../helpers/get_bucket_size';
 import getTimerange from '../../helpers/get_timerange';
-export default function query(req, panel, annotation) {
+import { buildEsQuery } from '@kbn/es-query';
+
+export default function query(req, panel, annotation, esQueryConfig, indexPattern) {
   return next => doc => {
     const timeField = annotation.time_field;
-    const {
-      bucketSize
-    } = getBucketSize(req, 'auto');
+    const { bucketSize } = getBucketSize(req, 'auto');
     const { from, to } = getTimerange(req);
 
     doc.size = 0;
-    doc.query = {
-      bool: {
-        must: []
-      }
-    };
-
+    const queries = !annotation.ignore_global_filters ? req.payload.query : [];
+    const filters = !annotation.ignore_global_filters ? req.payload.filters : [];
+    doc.query = buildEsQuery(indexPattern, queries, filters, esQueryConfig);
     const timerange = {
       range: {
         [timeField]: {
           gte: from.valueOf(),
-          lte: to.valueOf() - (bucketSize * 1000),
+          lte: to.valueOf() - bucketSize * 1000,
           format: 'epoch_millis',
-        }
-      }
+        },
+      },
     };
     doc.query.bool.must.push(timerange);
 
     if (annotation.query_string) {
       doc.query.bool.must.push({
         query_string: {
           query: annotation.query_string,
-          analyze_wildcard: true
-        }
+          analyze_wildcard: true,
+        },
       });
     }
 
-    const globalFilters = req.payload.filters;
-    if (!annotation.ignore_global_filters) {
-      doc.query.bool.must = doc.query.bool.must.concat(globalFilters);
-    }
-
     if (!annotation.ignore_panel_filters && panel.filter) {
       doc.query.bool.must.push({
         query_string: {
           query: panel.filter,
-          analyze_wildcard: true
-        }
+          analyze_wildcard: true,
+        },
       });
     }
 
@@ -76,7 +68,5 @@ export default function query(req, panel, annotation) {
     }
 
     return next(doc);
-
   };
 }
-