From e5c1a64e5c04e035811b3cad84af94a6d744709e Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Thu, 15 Feb 2024 18:46:44 +0000 Subject: [PATCH] [SQL] initial commit for SQL language selector Only adds the quick startup for OpenSearch cluster with a SQL plugin and observability with: ``` yarn opensearch snapshot --sql ``` Also, adds SQL to the language selector stolen shamelessly from https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5623 Next steps to intercept and send to SQL API or just transform basic syntax to DSL Signed-off-by: Kawika Avilla --- .../src/cli_commands/snapshot.js | 7 + packages/osd-opensearch/src/cluster.js | 25 ++- src/cli/serve/serve.js | 1 + .../build_opensearch_query.ts | 13 +- .../opensearch_query/from_sql.ts | 21 +++ .../opensearch_query/index.ts | 1 + .../opensearch_query/sql_string_to_dsl.ts | 23 +++ .../search/expressions/opensearchdsl.ts | 2 + .../data/public/search/search_interceptor.ts | 7 + .../public/ui/query_string_input/_index.scss | 1 + .../_language_switcher.scss | 4 + .../query_string_input/language_switcher.tsx | 146 ++++++------------ 12 files changed, 150 insertions(+), 101 deletions(-) create mode 100644 src/plugins/data/common/opensearch_query/opensearch_query/from_sql.ts create mode 100644 src/plugins/data/common/opensearch_query/opensearch_query/sql_string_to_dsl.ts create mode 100644 src/plugins/data/public/ui/query_string_input/_language_switcher.scss diff --git a/packages/osd-opensearch/src/cli_commands/snapshot.js b/packages/osd-opensearch/src/cli_commands/snapshot.js index ff21dbe851c8..84d6acee104e 100644 --- a/packages/osd-opensearch/src/cli_commands/snapshot.js +++ b/packages/osd-opensearch/src/cli_commands/snapshot.js @@ -50,6 +50,7 @@ exports.help = (defaults = {}) => { --download-only Download the snapshot but don't actually start it --ssl Sets up SSL on OpenSearch --security Installs and sets up the OpenSearch Security plugin on the cluster + --sql Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster --P OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin or for local zip files, use file:. @@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => { boolean: ['security'], + boolean: ['sql'], + default: defaults, }); @@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => { await cluster.setupSecurity(installPath, options.version ?? defaults.version); } + if (options.sql) { + await cluster.setupSql(installPath, options.version ?? defaults.version); + } + options.bundledJDK = true; await cluster.run(installPath, options); diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js index 455a1e5f919f..4b1c8b38259d 100644 --- a/packages/osd-opensearch/src/cluster.js +++ b/packages/osd-opensearch/src/cluster.js @@ -70,10 +70,11 @@ const first = (stream, map) => }); exports.Cluster = class Cluster { - constructor({ log = defaultLog, ssl = false, security = false } = {}) { + constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) { this._log = log; this._ssl = ssl; this._security = security; + this._sql = sql; this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined; } @@ -224,6 +225,28 @@ exports.Cluster = class Cluster { } } + /** + * Setups cluster with SQL/PPL plugins + * + * @param {string} installPath + * @property {String} version - version of OpenSearch + */ + async setupSql(installPath, version) { + await this.installSqlPlugin(installPath, version, 'opensearch-sql'); + await this.installSqlPlugin(installPath, version, 'opensearch-observability'); + } + + async installSqlPlugin(installPath, version, id) { + this._log.info(`Setting up: ${id}`); + try { + const pluginUrl = generateEnginePluginUrl(version, id); + await this.installOpenSearchPlugins(installPath, pluginUrl); + this._log.info(`Completed setup: ${id}`); + } catch (ex) { + this._log.warning(`Failed to setup: ${id}`); + } + } + /** * Starts OpenSearch and returns resolved promise once started * diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index aed5d74a2c01..c1e489afb4ef 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -249,6 +249,7 @@ export default function (program) { .option('--dev', 'Run the server with development mode defaults') .option('--ssl', 'Run the dev server using HTTPS') .option('--security', 'Run the dev server using security defaults') + .option('--sql', 'Run the dev server using SQL/PPL defaults') .option('--dist', 'Use production assets from osd/optimizer') .option( '--no-base-path', diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts index 481eae12d121..4d9b381b1d97 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts @@ -35,6 +35,7 @@ import { buildQueryFromLucene } from './from_lucene'; import { IIndexPattern } from '../../index_patterns'; import { Filter } from '../filters'; import { Query } from '../../query/types'; +import { buildQueryFromSql } from './from_sql'; export interface OpenSearchQueryConfig { allowLeadingWildcards: boolean; @@ -64,7 +65,17 @@ export function buildOpenSearchQuery( queries = Array.isArray(queries) ? queries : [queries]; filters = Array.isArray(filters) ? filters : [filters]; - const validQueries = queries.filter((query) => has(query, 'query')); + // TODO: SQL make this combinable. SQL needs to support DSL + // console.log('queries', queries); + const sqlQueries = queries.filter((query) => query.language === 'SQL'); + if (sqlQueries.length > 0) { + // console.log('sqlQueries', sqlQueries); + return buildQueryFromSql(sqlQueries, config.dateFormatTZ); + } + + const validQueries = queries + .filter((query) => query.language !== 'SQL') + .filter((query) => has(query, 'query')); const queriesByLanguage = groupBy(validQueries, 'language'); const kueryQuery = buildQueryFromKuery( indexPattern, diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/from_sql.ts b/src/plugins/data/common/opensearch_query/opensearch_query/from_sql.ts new file mode 100644 index 000000000000..9d802bc95e3c --- /dev/null +++ b/src/plugins/data/common/opensearch_query/opensearch_query/from_sql.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { decorateQuery } from './decorate_query'; +import { getIndexPatternFromSql, sqlStringToDsl } from './sql_string_to_dsl'; +import { Query } from '../../query/types'; + +export function buildQueryFromSql(queries: Query[], dateFormatTZ?: string) { + const combinedQueries = (queries || []).map((query) => { + const indexPattern = getIndexPatternFromSql(query.query); + const queryDsl = sqlStringToDsl(query.query); + + return decorateQuery(queryDsl, indexPattern, dateFormatTZ); + }); + + return { + combinedQueries, + }; +} diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/index.ts b/src/plugins/data/common/opensearch_query/opensearch_query/index.ts index ba3fe8817006..263bb1970a65 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/index.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/index.ts @@ -31,5 +31,6 @@ export { buildOpenSearchQuery, OpenSearchQueryConfig } from './build_opensearch_query'; export { buildQueryFromFilters } from './from_filters'; export { luceneStringToDsl } from './lucene_string_to_dsl'; +export { getIndexPatternFromSql, sqlStringToDsl } from './sql_string_to_dsl'; export { decorateQuery } from './decorate_query'; export { getOpenSearchQueryConfig } from './get_opensearch_query_config'; diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/sql_string_to_dsl.ts b/src/plugins/data/common/opensearch_query/opensearch_query/sql_string_to_dsl.ts new file mode 100644 index 000000000000..19cdca1945dc --- /dev/null +++ b/src/plugins/data/common/opensearch_query/opensearch_query/sql_string_to_dsl.ts @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { isString } from 'lodash'; +import { DslQuery } from './opensearch_query_dsl'; + +export function getIndexPatternFromSql(query: string | any) { + const from = query.match(new RegExp(/FROM\s+([\w*-.!@$^()~;]+)/, 'i')); + if (from) { + return from[1]; + } + return ''; +} + +export function sqlStringToDsl(query: string | any): DslQuery { + if (isString(query)) { + return { query_string: { query } }; + } + + return query; +} diff --git a/src/plugins/data/public/search/expressions/opensearchdsl.ts b/src/plugins/data/public/search/expressions/opensearchdsl.ts index 5c8d8350f260..57e014890956 100644 --- a/src/plugins/data/public/search/expressions/opensearchdsl.ts +++ b/src/plugins/data/public/search/expressions/opensearchdsl.ts @@ -28,6 +28,8 @@ * under the License. */ +// TODO: SQL this file seems important + import { i18n } from '@osd/i18n'; import { OpenSearchDashboardsContext, diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index 153ac80a249c..f77d7e05c8f9 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -28,6 +28,8 @@ * under the License. */ +// TODO: SQL this file seems important + import { get, trimEnd, debounce } from 'lodash'; import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs'; import { catchError, finalize } from 'rxjs/operators'; @@ -236,6 +238,11 @@ export class SearchInterceptor { }); this.pendingCount$.next(this.pendingCount$.getValue() + 1); + // TODO: SQL this isn't the right place but if core includes SQL then we dont need to do this + // TODO: hack setting to undefined + // console.log(request); + // console.log(options); + return this.runSearch( request, combinedSignal, diff --git a/src/plugins/data/public/ui/query_string_input/_index.scss b/src/plugins/data/public/ui/query_string_input/_index.scss index 8686490016c5..f21b9cbb4327 100644 --- a/src/plugins/data/public/ui/query_string_input/_index.scss +++ b/src/plugins/data/public/ui/query_string_input/_index.scss @@ -1 +1,2 @@ @import "./query_bar"; +@import "./language_switcher" diff --git a/src/plugins/data/public/ui/query_string_input/_language_switcher.scss b/src/plugins/data/public/ui/query_string_input/_language_switcher.scss new file mode 100644 index 000000000000..9ee71b9fc7e2 --- /dev/null +++ b/src/plugins/data/public/ui/query_string_input/_language_switcher.scss @@ -0,0 +1,4 @@ +// From: https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5623 +.languageSwitcher { + max-width: 150px; +} diff --git a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx index 816c21bc0848..31faf1e57662 100644 --- a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx +++ b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx @@ -28,21 +28,9 @@ * under the License. */ -import { - EuiButtonEmpty, - EuiForm, - EuiFormRow, - EuiLink, - EuiPopover, - EuiPopoverTitle, - EuiSpacer, - EuiSwitch, - EuiText, - PopoverAnchorPosition, -} from '@elastic/eui'; -import { FormattedMessage } from '@osd/i18n/react'; -import React, { useState } from 'react'; -import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; +import { EuiComboBox, EuiComboBoxOptionOption, PopoverAnchorPosition } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import React from 'react'; interface Props { language: string; @@ -51,93 +39,53 @@ interface Props { } export function QueryLanguageSwitcher(props: Props) { - const osdDQLDocs = useOpenSearchDashboards().services.docLinks?.links.opensearchDashboards.dql - .base; - const [isPopoverOpen, setIsPopoverOpen] = useState(false); - const luceneLabel = ( - - ); - const dqlLabel = ( - - ); - const dqlFullName = ( - - ); + const luceneLabel = i18n.translate('data.query.queryBar.luceneLanguageName', { + defaultMessage: 'Lucene', + }); + const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', { + defaultMessage: 'DQL', + }); + const sqlLabel = i18n.translate('data.query.queryBar.sqlLanguageName', { + defaultMessage: 'SQL', + }); - const button = ( - setIsPopoverOpen(!isPopoverOpen)} - className="euiFormControlLayout__append dqlQueryBar__languageSwitcherButton" - data-test-subj={'switchQueryLanguageButton'} - > - {props.language === 'lucene' ? luceneLabel : dqlLabel} - - ); + const languageOptions: EuiComboBoxOptionOption[] = [ + { + label: luceneLabel, + value: 'lucene', + }, + { + label: dqlLabel, + value: 'dql', + }, + { + label: sqlLabel, + value: 'sql', + // TODO: add option to disable if SQL is not supported + // disabled: true, + }, + ]; - return ( - setIsPopoverOpen(false)} - repositionOnScroll - > - - - -
- -

- - {dqlFullName} - - ), - }} - /> -

- + const selectedLanguage = { + label: props.language === 'kuery' ? 'DQL' : props.language, + }; - + const handleLanguageChange = (newLanguage: EuiComboBoxOptionOption[]) => { + const queryLanguage = newLanguage[0].label === 'DQL' ? 'kuery' : newLanguage[0].label; + props.onSelectLanguage(queryLanguage); + }; - - - - ) : ( - - ) - } - checked={props.language === 'kuery'} - onChange={() => { - const newLanguage = props.language === 'lucene' ? 'kuery' : 'lucene'; - props.onSelectLanguage(newLanguage); - }} - data-test-subj="languageToggle" - /> - - -
- + return ( + ); }