[Discover] Supports SQL query language (#134429)

docs/management/advanced-options.asciidoc

@@ -319,6 +319,9 @@ The default sort direction for time-based data views.
 [[doctable-hidetimecolumn]]`doc_table:hideTimeColumn`::
 Hides the "Time" column in *Discover* and in all saved searches on dashboards.
 
+[[discover:enableSql]]`discover:enableSql`::
+When enabled, allows SQL queries for search.
+
 [[doctable-highlight]]`doc_table:highlight`::
 Highlights results in *Discover* and saved searches on dashboards. Highlighting
 slows requests when working on big documents.

packages/kbn-es-query/src/es_query/build_es_query.ts

@@ -11,11 +11,13 @@ import { SerializableRecord } from '@kbn/utility-types';
 import { buildQueryFromKuery } from './from_kuery';
 import { buildQueryFromFilters } from './from_filters';
 import { buildQueryFromLucene } from './from_lucene';
-import { Filter, Query } from '../filters';
+import { Filter, Query, AggregateQuery } from '../filters';
+import { isOfQueryType } from './es_query_sql';
 import { BoolQuery, DataViewBase } from './types';
 import type { KueryQueryOptions } from '../kuery';
 import type { EsQueryFiltersConfig } from './from_filters';
 
+type AnyQuery = Query | AggregateQuery;
 /**
  * Configurations to be used while constructing an ES query.
  * @public
@@ -44,7 +46,7 @@ function removeMatchAll<T>(filters: T[]) {
  */
 export function buildEsQuery(
   indexPattern: DataViewBase | undefined,
-  queries: Query | Query[],
+  queries: AnyQuery | AnyQuery[],
   filters: Filter | Filter[],
   config: EsQueryConfig = {
     allowLeadingWildcards: false,
@@ -55,7 +57,7 @@ export function buildEsQuery(
   queries = Array.isArray(queries) ? queries : [queries];
   filters = Array.isArray(filters) ? filters : [filters];
 
-  const validQueries = queries.filter((query) => has(query, 'query'));
+  const validQueries = queries.filter(isOfQueryType).filter((query) => has(query, 'query'));
   const queriesByLanguage = groupBy(validQueries, 'language');
   const kueryQuery = buildQueryFromKuery(
     indexPattern,

packages/kbn-es-query/src/es_query/es_query_sql.test.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+  isOfQueryType,
+  isOfAggregateQueryType,
+  getAggregateQueryMode,
+  getIndexPatternFromSQLQuery,
+} from './es_query_sql';
+
+describe('sql query helpers', () => {
+  describe('isOfQueryType', () => {
+    it('should return true for a Query type query', () => {
+      const flag = isOfQueryType({ query: 'foo', language: 'test' });
+      expect(flag).toBe(true);
+    });
+
+    it('should return false for an Aggregate type query', () => {
+      const flag = isOfQueryType({ sql: 'SELECT * FROM foo' });
+      expect(flag).toBe(false);
+    });
+  });
+
+  describe('isOfAggregateQueryType', () => {
+    it('should return false for a Query type query', () => {
+      const flag = isOfAggregateQueryType({ query: 'foo', language: 'test' });
+      expect(flag).toBe(false);
+    });
+
+    it('should return true for an Aggregate type query', () => {
+      const flag = isOfAggregateQueryType({ sql: 'SELECT * FROM foo' });
+      expect(flag).toBe(true);
+    });
+  });
+
+  describe('getAggregateQueryMode', () => {
+    it('should return sql for an SQL AggregateQuery type', () => {
+      const mode = getAggregateQueryMode({ sql: 'SELECT * FROM foo' });
+      expect(mode).toBe('sql');
+    });
+
+    it('should return esql for an ESQL AggregateQuery type', () => {
+      const mode = getAggregateQueryMode({ esql: 'foo | where field > 100' });
+      expect(mode).toBe('esql');
+    });
+  });
+
+  describe('getIndexPatternFromSQLQuery', () => {
+    it('should return the index pattern string from sql queries', () => {
+      const idxPattern1 = getIndexPatternFromSQLQuery('SELECT * FROM foo');
+      expect(idxPattern1).toBe('foo');
+
+      const idxPattern2 = getIndexPatternFromSQLQuery('SELECT woof, meow FROM "foo"');
+      expect(idxPattern2).toBe('foo');
+
+      const idxPattern3 = getIndexPatternFromSQLQuery('SELECT woof, meow FROM "the_index_pattern"');
+      expect(idxPattern3).toBe('the_index_pattern');
+
+      const idxPattern4 = getIndexPatternFromSQLQuery('SELECT woof, meow FROM "the-index-pattern"');
+      expect(idxPattern4).toBe('the-index-pattern');
+
+      const idxPattern5 = getIndexPatternFromSQLQuery('SELECT woof, meow from "the-index-pattern"');
+      expect(idxPattern5).toBe('the-index-pattern');
+
+      const idxPattern6 = getIndexPatternFromSQLQuery('SELECT woof, meow from "logstash-*"');
+      expect(idxPattern6).toBe('logstash-*');
+
+      const idxPattern7 = getIndexPatternFromSQLQuery(
+        'SELECT woof, meow from logstash-1234! WHERE field > 100'
+      );
+      expect(idxPattern7).toBe('logstash-1234!');
+    });
+  });
+});

packages/kbn-es-query/src/es_query/es_query_sql.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import type { Query, AggregateQuery } from '../filters';
+
+type Language = keyof AggregateQuery;
+
+// Checks if the query is of type Query
+export function isOfQueryType(arg?: Query | AggregateQuery): arg is Query {
+  return Boolean(arg && 'query' in arg);
+}
+
+// Checks if the query is of type AggregateQuery
+// currently only supports the sql query type
+// should be enhanced to support other query types
+export function isOfAggregateQueryType(
+  query: AggregateQuery | Query | { [key: string]: any }
+): query is AggregateQuery {
+  return Boolean(query && ('sql' in query || 'esql' in query));
+}
+
+// returns the language of the aggregate Query, sql, esql etc
+export function getAggregateQueryMode(query: AggregateQuery): Language {
+  return Object.keys(query)[0] as Language;
+}
+
+// retrieves the index pattern from the aggregate query
+export function getIndexPatternFromSQLQuery(sqlQuery?: string): string {
+  const sql = sqlQuery?.replaceAll('"', '').replaceAll("'", '');
+  // case insensitive match for the index pattern
+  const regex = new RegExp(/FROM\s+([\w*-.!@$^()~;]+)/, 'i');
+  const matches = sql?.match(regex);
+  if (matches) {
+    return matches[1];
+  }
+  return '';
+}

packages/kbn-es-query/src/es_query/index.ts

@@ -13,6 +13,12 @@ export { buildEsQuery } from './build_es_query';
 export { buildQueryFromFilters } from './from_filters';
 export { luceneStringToDsl } from './lucene_string_to_dsl';
 export { decorateQuery } from './decorate_query';
+export {
+  isOfQueryType,
+  isOfAggregateQueryType,
+  getAggregateQueryMode,
+  getIndexPatternFromSQLQuery,
+} from './es_query_sql';
 export type {
   IFieldSubType,
   BoolQuery,

packages/kbn-es-query/src/filters/build_filters/types.ts

@@ -82,6 +82,8 @@ export type Query = {
   language: string;
 };
 
+export type AggregateQuery = { sql: string } | { esql: string };
+
 /**
  * An interface for a latitude-longitude pair
  * @public

packages/kbn-es-query/src/filters/index.ts

@@ -59,6 +59,7 @@ export {
 
 export type {
   Query,
+  AggregateQuery,
   Filter,
   LatLon,
   FieldFilter,

packages/kbn-es-query/src/index.ts

@@ -29,6 +29,7 @@ export type {
   PhraseFilter,
   PhrasesFilter,
   Query,
+  AggregateQuery,
   QueryStringFilter,
   RangeFilter,
   RangeFilterMeta,
@@ -52,6 +53,10 @@ export {
   decorateQuery,
   luceneStringToDsl,
   migrateFilter,
+  isOfQueryType,
+  isOfAggregateQueryType,
+  getAggregateQueryMode,
+  getIndexPatternFromSQLQuery,
 } from './es_query';
 
 export {

src/plugins/dashboard/public/application/lib/dashboard_session_restoration.ts

@@ -8,6 +8,7 @@
 
 import { History } from 'history';
 import { createQueryParamObservable } from '@kbn/kibana-utils-plugin/public';
+import type { Query } from '@kbn/es-query';
 import { DashboardAppLocatorParams, DashboardConstants } from '../..';
 import { DashboardState } from '../../types';
 import { getDashboardTitle } from '../../dashboard_strings';
@@ -113,7 +114,7 @@ function getLocatorParams({
     timeRange: shouldRestoreSearchSession ? timefilter.getAbsoluteTime() : timefilter.getTime(),
     searchSessionId: shouldRestoreSearchSession ? data.search.session.getSessionId() : undefined,
     panels: getDashboardId() ? undefined : appState.panels,
-    query: queryString.formatQuery(appState.query),
+    query: queryString.formatQuery(appState.query) as Query,
     filters: filterManager.getFilters(),
     savedQuery: appState.savedQuery,
     dashboardId: getDashboardId(),

src/plugins/dashboard/public/application/lib/sync_dashboard_filter_state.ts

@@ -9,7 +9,6 @@
 import _ from 'lodash';
 import { merge } from 'rxjs';
 import { debounceTime, finalize, map, switchMap, tap } from 'rxjs/operators';
-
 import { setQuery } from '../state';
 import { DashboardBuildContext, DashboardState } from '../../types';
 import { DashboardSavedObject } from '../../saved_dashboards';
@@ -100,7 +99,7 @@ export const syncDashboardFilterState = ({
   // apply filters when the filter manager changes
   const filterManagerSubscription = merge(filterManager.getUpdates$(), queryString.getUpdates$())
     .pipe(debounceTime(100))
-    .subscribe(() => applyFilters(queryString.getQuery(), filterManager.getFilters()));
+    .subscribe(() => applyFilters(queryString.getQuery() as Query, filterManager.getFilters()));
 
   const timeRefreshSubscription = merge(
     timefilterService.getRefreshIntervalUpdate$(),

src/plugins/data/common/query/query_state.ts

@@ -8,7 +8,7 @@
 
 import type { Filter } from '@kbn/es-query';
 import type { TimeRange, RefreshInterval } from './timefilter/types';
-import type { Query } from './types';
+import type { Query, AggregateQuery } from './types';
 
 /**
  * All query state service state
@@ -22,5 +22,5 @@ export type QueryState = {
   time?: TimeRange;
   refreshInterval?: RefreshInterval;
   filters?: Filter[];
-  query?: Query;
+  query?: Query | AggregateQuery;
 };