[MQL] support different query languages

package.json

@@ -136,7 +136,7 @@
   },
   "dependencies": {
     "@aws-crypto/client-node": "^3.1.1",
-    "@elastic/datemath": "5.0.3",
+    "@elastic/datemath": "link:packages/opensearch-datemath",
     "@elastic/eui": "npm:@opensearch-project/[email protected]",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/numeral": "npm:@amoo-miki/[email protected]",

packages/opensearch-datemath/index.d.ts

@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

packages/opensearch-datemath/index.js

@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d) => moment.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

packages/osd-opensearch/src/cli_commands/snapshot.js

@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

packages/osd-opensearch/src/cluster.js

@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

src/cli/serve/serve.js

@@ -249,6 +249,7 @@ export default function (program) {
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
       .option('--security', 'Run the dev server using security defaults')
+      .option('--sql', 'Run the dev server using SQL/PPL defaults')
       .option('--dist', 'Use production assets from osd/optimizer')
       .option(
         '--no-base-path',

src/plugins/data/common/constants.ts

@@ -60,4 +60,5 @@ export const UI_SETTINGS = {
   INDEXPATTERN_PLACEHOLDER: 'indexPattern:placeholder',
   FILTERS_PINNED_BY_DEFAULT: 'filters:pinnedByDefault',
   FILTERS_EDITOR_SUGGEST_VALUES: 'filterEditor:suggestValues',
+  DATAFRAME_HYDRATION_STRATEGY: 'dataframe:hydrationStrategy',
 } as const;

src/plugins/data/common/data_frames/_df_cache.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IDataFrame } from '..';
+
+export interface DfCache {
+  get: () => IDataFrame | undefined;
+  set: (value: IDataFrame) => IDataFrame;
+  clear: () => void;
+}
+
+export function createDataFrameCache(): DfCache {
+  let df: IDataFrame | undefined;
+  const cache: DfCache = {
+    get: () => {
+      return df;
+    },
+    set: (prom: IDataFrame) => {
+      df = prom;
+      return prom;
+    },
+    clear: () => {
+      df = undefined;
+    },
+  };
+  return cache;
+}

src/plugins/data/common/data_frames/fields/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './types';

src/plugins/data/common/data_frames/fields/types.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface IFieldType {
+  name: string;
+  type: string;
+  values: any[];
+  count?: number;
+  aggregatable?: boolean;
+  filterable?: boolean;
+  searchable?: boolean;
+  sortable?: boolean;
+  visualizable?: boolean;
+  displayName?: string;
+  format?: any;
+}

src/plugins/data/common/data_frames/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './types';
+export * from './utils';

src/plugins/data/common/data_frames/types.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IFieldType } from './fields';
+
+export * from './_df_cache';
+
+export interface IDataFrame {
+  name?: string;
+  schema?: Array<Partial<IFieldType>>;
+  fields: IFieldType[];
+  size: number;
+}
+
+export interface DataFrameAgg {
+  key: string;
+  value: number;
+}
+
+export interface PartialDataFrame extends Omit<IDataFrame, 'fields' | 'size'> {
+  fields: Array<Partial<IFieldType>>;
+}
+
+/**
+ * To be utilize with aggregations and will map to buckets
+ * Plugins can get the aggreted value by their own logic
+ * Setting to null will disable the aggregation if plugin wishes
+ * In future, if the plugin doesn't intentionally set the value to null,
+ * we can calculate the value based on the fields.
+ */
+export interface IDataFrameWithAggs extends IDataFrame {
+  aggs: DataFrameAgg[] | null;
+}

src/plugins/data/common/data_frames/utils.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SearchResponse } from 'elasticsearch';
+import datemath from '@elastic/datemath';
+import { IDataFrame, IDataFrameWithAggs, PartialDataFrame } from './types';
+import { IFieldType } from './fields';
+import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns';
+import { IOpenSearchDashboardsSearchRequest } from '../search';
+
+const name = 'data_frame';
+
+export interface IDataFrameResponse extends SearchResponse<any> {
+  type: typeof name;
+  body: IDataFrame | IDataFrameWithAggs;
+  took: number;
+}
+
+export const getRawQueryString = (
+  searchRequest: IOpenSearchDashboardsSearchRequest
+): string | undefined => {
+  return searchRequest.params?.body?.query?.queries[0]?.query;
+};
+
+export const convertResult = (response: IDataFrameResponse): SearchResponse<any> => {
+  const data = response.body;
+  const hits: any[] = [];
+  for (let index = 0; index < data.size; index++) {
+    const hit: { [key: string]: any } = {};
+    data.fields.forEach((field) => {
+      hit[field.name] = field.values[index];
+    });
+    hits.push({
+      _index: data.name ?? '',
+      _type: '',
+      _id: '',
+      _score: 0,
+      _source: hit,
+    });
+  }
+  const searchResponse: SearchResponse<any> = {
+    took: response.took,
+    timed_out: false,
+    _shards: {
+      total: 1,
+      successful: 1,
+      skipped: 0,
+      failed: 0,
+    },
+    hits: {
+      total: 0,
+      max_score: 0,
+      hits,
+    },
+  };
+
+  if (data.hasOwnProperty('aggs')) {
+    const dataWithAggs = data as IDataFrameWithAggs;
+    if (!dataWithAggs.aggs) {
+      // TODO: SQL best guess, get timestamp field and caculate it here
+      return searchResponse;
+    }
+    searchResponse.aggregations = {
+      2: {
+        buckets: dataWithAggs.aggs.map((agg) => {
+          searchResponse.hits.total += agg.value;
+          return {
+            key: new Date(agg.key).getTime(),
+            key_as_string: agg.key,
+            doc_count: agg.value,
+          };
+        }),
+      },
+    };
+  }
+
+  return searchResponse;
+};
+
+export const formatFieldValue = (field: IFieldType | Partial<IFieldType>, value: any): any => {
+  return field.format && field.format.convert ? field.format.convert(value) : value;
+};
+
+export const getFieldType = (field: IFieldType | Partial<IFieldType>): string | undefined => {
+  if (field.name) {
+    const fieldName = field.name.toLowerCase();
+    // TODO: feels little biased to check if timestamp.
+    // Has to be a better way to know so to be fair to all data sources
+    if (fieldName.includes('date') || fieldName.includes('timestamp')) {
+      return 'date';
+    }
+  }
+  if (!field.values) return field.type;
+  const firstValue = field.values.filter((value) => value !== null && value !== undefined)[0];
+  if (firstValue instanceof Date || datemath.isDateTime(firstValue)) {
+    return 'date';
+  }
+  return field.type;
+};
+
+export const getTimeField = (data: IDataFrame): IFieldType | undefined => {
+  return data.fields.find((field) => field.type === 'date');
+};
+
+export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFrameWithAggs => {
+  let size = 0;
+  const fields = partial.fields.map((field) => {
+    if (!field.values) {
+      field.values = new Array(size);
+    } else if (field.values.length > size) {
+      size = field.values.length;
+    }
+    field.type = getFieldType(field);
+    // if (!field.type) {
+    // need to think if this needs to be mapped to OSD field type for example
+    // PPL type for date is TIMESTAMP
+    // OSD is expecting date
+    //   field.type = get type
+    // }
+    // get timeseries field
+    return field as IFieldType;
+  });
+
+  return {
+    ...partial,
+    fields,
+    size,
+  };
+};
+
+export const dataFrameToSpec = (dataFrame: IDataFrame): IndexPatternSpec => {
+  return {
+    id: 'data_frame',
+    title: dataFrame.name,
+    timeFieldName: getTimeField(dataFrame)?.name,
+    fields: dataFrame.fields.reduce((acc, field) => {
+      acc[field.name] = {
+        name: field.name,
+        type: field.type,
+        aggregatable: true,
+        searchable: true,
+      };
+      return acc;
+    }, {} as IndexPatternFieldMap),
+    // TODO: SQL dataSourceRef
+  };
+};

src/plugins/data/common/index.ts

@@ -30,6 +30,7 @@
 
 export * from './constants';
 export * from './opensearch_query';
+export * from './data_frames';
 export * from './field_formats';
 export * from './field_mapping';
 export * from './index_patterns';

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -208,6 +208,10 @@ export class IndexPatternsService {
     return this.savedObjectsCache;
   };
 
+  getIndexPatternCache = () => {
+    return indexPatternCache;
+  };
+
   /**
    * Get default index pattern
    */