[SQL] initial commit for SQL language selector

Only adds the quick startup for OpenSearch cluster with a SQL plugin and observability with: ``` yarn opensearch snapshot --sql ``` Also, adds SQL to the language selector stolen shamelessly from opensearch-project#5623 Next steps to intercept and send to SQL API or just transform basic syntax to DSL Signed-off-by: Kawika Avilla <[email protected]> Working query enhance Signed-off-by: Kawika Avilla <[email protected]> Reget search enhance Signed-off-by: Kawika Avilla <[email protected]> data frames Signed-off-by: Kawika Avilla <[email protected]> temp state need to pass df Signed-off-by: Kawika Avilla <[email protected]> sending data frame Signed-off-by: Kawika Avilla <[email protected]> gets df type Signed-off-by: Kawika Avilla <[email protected]> add some small ui things Signed-off-by: Kawika Avilla <[email protected]> move back query language switcher Signed-off-by: Kawika Avilla <[email protected]> updating side panel Signed-off-by: Kawika Avilla <[email protected]> add ui config on query enhancement Signed-off-by: Kawika Avilla <[email protected]> support query string input Signed-off-by: Kawika Avilla <[email protected]> update to say search bar Signed-off-by: Kawika Avilla <[email protected]> update ppl to call once Signed-off-by: Kawika Avilla <[email protected]> update when fields update Signed-off-by: Kawika Avilla <[email protected]> add unknown Signed-off-by: Kawika Avilla <[email protected]>
kavilla · Apr 22, 2024 · 307056c · 307056c
1 parent b117fd3
commit 307056c
Show file tree

Hide file tree

Showing 62 changed files with 1,080 additions and 167 deletions.
diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml
@@ -313,3 +313,5 @@
 
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
+opensearch_alerting.enabled: false
+opensearch_security.enabled: false
diff --git a/package.json b/package.json
@@ -136,7 +136,7 @@
   },
   "dependencies": {
     "@aws-crypto/client-node": "^3.1.1",
-    "@elastic/datemath": "5.0.3",
+    "@elastic/datemath": "link:packages/opensearch-datemath",
     "@elastic/eui": "npm:@opensearch-project/[email protected]",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/numeral": "npm:@amoo-miki/[email protected]",

diff --git a/packages/opensearch-datemath/index.d.ts b/packages/opensearch-datemath/index.d.ts
@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

diff --git a/packages/opensearch-datemath/index.js b/packages/opensearch-datemath/index.js
@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d) => moment.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

diff --git a/packages/osd-opensearch/src/cli_commands/snapshot.js b/packages/osd-opensearch/src/cli_commands/snapshot.js
@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js
@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js
@@ -249,6 +249,7 @@ export default function (program) {
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
       .option('--security', 'Run the dev server using security defaults')
+      .option('--sql', 'Run the dev server using SQL/PPL defaults')
       .option('--dist', 'Use production assets from osd/optimizer')
       .option(
         '--no-base-path',

diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts
@@ -60,4 +60,5 @@ export const UI_SETTINGS = {
   INDEXPATTERN_PLACEHOLDER: 'indexPattern:placeholder',
   FILTERS_PINNED_BY_DEFAULT: 'filters:pinnedByDefault',
   FILTERS_EDITOR_SUGGEST_VALUES: 'filterEditor:suggestValues',
+  DATAFRAME_HYDRATION_STRATEGY: 'dataframe:hydrationStrategy',
 } as const;
diff --git a/src/plugins/data/common/data_frames/_df_cache.ts b/src/plugins/data/common/data_frames/_df_cache.ts
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import { IDataFrame } from '..';
+
+export interface DfCache {
+  get: () => IDataFrame | undefined;
+  set: (value: IDataFrame) => IDataFrame;
+  clear: () => void;
+}
+
+export function createDataFrameCache(): DfCache {
+  let df: IDataFrame | undefined;
+  const cache: DfCache = {
+    get: () => {
+      return df;
+    },
+    set: (prom: IDataFrame) => {
+      df = prom;
+      return prom;
+    },
+    clear: () => {
+      df = undefined;
+    },
+  };
+  return cache;
+}
diff --git a/src/plugins/data/common/data_frames/fields/index.ts b/src/plugins/data/common/data_frames/fields/index.ts
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+export * from './types';
diff --git a/src/plugins/data/common/data_frames/fields/types.ts b/src/plugins/data/common/data_frames/fields/types.ts
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+export interface IFieldType {
+  name: string;
+  type: string;
+  values: any[];
+  count?: number;
+  aggregatable?: boolean;
+  filterable?: boolean;
+  searchable?: boolean;
+  sortable?: boolean;
+  visualizable?: boolean;
+  displayName?: string;
+  format?: any;
+}
diff --git a/src/plugins/data/common/data_frames/index.ts b/src/plugins/data/common/data_frames/index.ts
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+export * from './types';
+export * from './utils';
diff --git a/src/plugins/data/common/data_frames/types.ts b/src/plugins/data/common/data_frames/types.ts
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import { IFieldType } from './fields';
+
+export * from './_df_cache';
+
+export interface IDataFrame {
+  name?: string;
+  schema?: Array<Partial<IFieldType>>;
+  fields: IFieldType[];
+  size: number;
+}
+
+export interface DataFrameAgg {
+  key: string;
+  value: number;
+}
+
+export interface PartialDataFrame extends Omit<IDataFrame, 'fields' | 'size'> {
+  fields: Array<Partial<IFieldType>>;
+}
+
+/**
+ * To be utilize with aggregations and will map to buckets
+ * Plugins can get the aggreted value by their own logic
+ * Setting to null will disable the aggregation if plugin wishes
+ * In future, if the plugin doesn't intentionally set the value to null,
+ * we can calculate the value based on the fields.
+ */
+export interface IDataFrameWithAggs extends IDataFrame {
+  aggs: DataFrameAgg[] | null;
+}