Moving query module into superset-ui from incubator-superset (apache-…

…superset#48)

* migrate query code from incubator-superset

* Getting tests to pass

- Up build-config version to pick up a fix to eslint-typescript-parser
- Remove usage of default exports in favor of named exports unless the export is the only thing being exported out of a module
- Fixing up a few linting errors

* - Remove DatasourceKey interface in favor of readonly id and type properties on the DatasourceKey class directly.
- Adding tests for DatasourceKey.

package.json

@@ -8,7 +8,7 @@
     "build": "yarn run build:cjs && yarn run build:esm && yarn run build:ts",
     "build:cjs": "NODE_ENV=production beemo babel ./src --out-dir lib/ --minify --workspaces=\"@superset-ui/!(demo|generator-superset)\"",
     "build:esm": "NODE_ENV=production beemo babel ./src --out-dir esm/ --esm --minify --workspaces=\"@superset-ui/!(demo|generator-superset)\"",
-    "build:ts": "NODE_ENV=production beemo typescript --workspaces=\"@superset-ui/connection\"",
+    "build:ts": "NODE_ENV=production beemo typescript --workspaces=\"@superset-ui/(connection|chart)\"",
     "lint": "beemo create-config prettier && beemo eslint \"./packages/*/{src,test,storybook}/**/*.{js,jsx,ts,tsx}\"",
     "jest": "beemo jest --color --coverage",
     "postrelease": "lerna run gh-pages",
@@ -35,7 +35,7 @@
   ],
   "license": "Apache-2.0",
   "devDependencies": {
-    "@data-ui/build-config": "^0.0.31",
+    "@data-ui/build-config": "^0.0.33",
     "husky": "^1.1.2",
     "lerna": "^3.2.1",
     "lint-staged": "^8.0.4",

packages/superset-ui-chart/src/query/Column.ts

@@ -0,0 +1,24 @@
+export enum ColumnType {
+  DOUBLE = 'DOUBLE',
+  FLOAT = 'FLOAT',
+  INT = 'INT',
+  BIGINT = 'BIGINT',
+  LONG = 'LONG',
+  REAL = 'REAL',
+  NUMERIC = 'NUMERIC',
+  DECIMAL = 'DECIMAL',
+  MONEY = 'MONEY',
+  DATE = 'DATE',
+  TIME = 'TIME',
+  DATETIME = 'DATETIME',
+  VARCHAR = 'VARCHAR',
+  STRING = 'STRING',
+  CHAR = 'CHAR',
+}
+
+// TODO: fill out additional fields of the Column interface
+export interface Column {
+  id: number;
+  type: ColumnType;
+  columnName: string;
+}

packages/superset-ui-chart/src/query/DatasourceKey.ts

@@ -0,0 +1,26 @@
+export enum DatasourceType {
+  Table = 'table',
+  Druid = 'druid',
+}
+
+export class DatasourceKey {
+  readonly id: number;
+  readonly type: DatasourceType;
+
+  constructor(key: string) {
+    const [idStr, typeStr] = key.split('__');
+    this.id = parseInt(idStr, 10);
+    this.type = typeStr === 'table' ? DatasourceType.Table : DatasourceType.Druid;
+  }
+
+  public toString() {
+    return `${this.id}__${this.type}`;
+  }
+
+  public toObject() {
+    return {
+      id: this.id,
+      type: this.type,
+    };
+  }
+}

packages/superset-ui-chart/src/query/FormData.ts

@@ -0,0 +1,32 @@
+import { AdhocMetric, MetricKey } from './Metric';
+
+// Type signature and utility functions for formData shared by all viz types
+// It will be gradually filled out as we build out the query object
+
+// Define mapped type separately to work around a limitation of TypeScript
+// https://github.com/Microsoft/TypeScript/issues/13573
+// The Metrics in formData is either a string or a proper metric. It will be
+// unified into a proper Metric type during buildQuery (see `/query/Metrics.ts`).
+type Metrics = Partial<Record<MetricKey, AdhocMetric | string>>;
+
+type BaseFormData = {
+  datasource: string;
+} & Metrics;
+
+// FormData is either sqla-based or druid-based
+type SqlaFormData = {
+  // FormData uses snake_cased keys.
+  // eslint-disable-next-line camelcase
+  granularity_sqla: string;
+} & BaseFormData;
+
+type DruidFormData = {
+  granularity: string;
+} & BaseFormData;
+
+type FormData = SqlaFormData | DruidFormData;
+export default FormData;
+
+export function getGranularity(formData: FormData): string {
+  return 'granularity_sqla' in formData ? formData.granularity_sqla : formData.granularity;
+}

packages/superset-ui-chart/src/query/Metric.ts

@@ -0,0 +1,96 @@
+import { Column } from './Column';
+import FormData from './FormData';
+
+export const LABEL_MAX_LENGTH = 43;
+
+// Note that the values of MetricKeys are lower_snake_case because they're
+// used as keys of form data jsons.
+export enum MetricKey {
+  METRIC = 'metric',
+  METRICS = 'metrics',
+  PERCENT_METRICS = 'percent_metrics',
+  RIGHT_AXIS_METRIC = 'metric_2',
+  SECONDARY_METRIC = 'secondary_metric',
+  X = 'x',
+  Y = 'y',
+  SIZE = 'size',
+}
+
+export enum Aggregate {
+  AVG = 'AVG',
+  COUNT = 'COUNT ',
+  COUNT_DISTINCT = 'COUNT_DISTINCT',
+  MAX = 'MAX',
+  MIN = 'MIN',
+  SUM = 'SUM',
+}
+
+export enum ExpressionType {
+  SIMPLE = 'SIMPLE',
+  SQL = 'SQL',
+}
+
+interface AdhocMetricSimple {
+  expressionType: ExpressionType.SIMPLE;
+  column: Column;
+  aggregate: Aggregate;
+}
+
+interface AdhocMetricSQL {
+  expressionType: ExpressionType.SQL;
+  sqlExpression: string;
+}
+
+export type AdhocMetric = {
+  label?: string;
+  optionName?: string;
+} & (AdhocMetricSimple | AdhocMetricSQL);
+
+export type Metric = {
+  label: string;
+} & Partial<AdhocMetric>;
+
+export class Metrics {
+  // Use Array to maintain insertion order for metrics that are order sensitive
+  private metrics: Metric[];
+
+  constructor(formData: FormData) {
+    this.metrics = Object.keys(MetricKey)
+      .map(key => formData[MetricKey[key as any] as MetricKey])
+      .filter(metric => metric)
+      .map(metric => {
+        if (typeof metric === 'string') {
+          return { label: metric };
+        }
+
+        // Note we further sanitize the metric label for BigQuery datasources
+        // TODO: move this logic to the client once client has more info on the
+        // the datasource
+        return {
+          ...metric,
+          label: (metric as Metric).label || this.getDefaultLabel(metric as AdhocMetric),
+        };
+      });
+  }
+
+  public getMetrics() {
+    return this.metrics;
+  }
+
+  public getLabels() {
+    return this.metrics.map(m => m.label);
+  }
+
+  private getDefaultLabel(metric: AdhocMetric) {
+    let label: string;
+    if (metric.expressionType === ExpressionType.SIMPLE) {
+      label = `${metric.aggregate}(${metric.column.columnName})`;
+    } else {
+      label = metric.sqlExpression;
+    }
+
+    return label.length <= LABEL_MAX_LENGTH
+      ? label
+      : `${label.substring(0, LABEL_MAX_LENGTH - 3)}...`;
+  }
+}

packages/superset-ui-chart/src/query/buildQueryContext.ts

@@ -0,0 +1,16 @@
+import buildQueryObject, { QueryObject } from './buildQueryObject';
+import { DatasourceKey } from './DatasourceKey';
+import FormData from './FormData';
+
+const WRAP_IN_ARRAY = (baseQueryObject: QueryObject) => [baseQueryObject];
+
+// Note: let TypeScript infer the return type
+export default function buildQueryContext(
+  formData: FormData,
+  buildQuery: (baseQueryObject: QueryObject) => QueryObject[] = WRAP_IN_ARRAY,
+) {
+  return {
+    datasource: new DatasourceKey(formData.datasource).toObject(),
+    queries: buildQuery(buildQueryObject(formData)),
+  };
+}

packages/superset-ui-chart/src/query/buildQueryObject.ts

@@ -0,0 +1,21 @@
+import FormData, { getGranularity } from './FormData';
+import { Metric, Metrics } from './Metric';
+
+// TODO: fill out the rest of the query object
+export interface QueryObject {
+  granularity: string;
+  groupby?: string[];
+  metrics?: Metric[];
+}
+
+// Build the common segments of all query objects (e.g. the granularity field derived from
+// either sql alchemy or druid). The segments specific to each viz type is constructed in the
+// buildQuery method for each viz type (see `wordcloud/buildQuery.ts` for an example).
+// Note the type of the formData argument passed in here is the type of the formData for a
+// specific viz, which is a subtype of the generic formData shared among all viz types.
+export default function buildQueryObject<T extends FormData>(formData: T): QueryObject {
+  return {
+    granularity: getGranularity(formData),
+    metrics: new Metrics(formData).getMetrics(),
+  };
+}

packages/superset-ui-chart/src/query/index.ts

@@ -0,0 +1,3 @@
+// Public API of the query module
+export { default } from './buildQueryContext';
+export { default as FormData } from './FormData';

packages/superset-ui-chart/test/query/DatasourceKey.test.ts

@@ -0,0 +1,18 @@
+import { DatasourceKey } from '../../src/query/DatasourceKey';
+
+describe('DatasourceKey', () => {
+  const tableKey = '5__table';
+  const druidKey = '5__druid';
+
+  it('should handle table data sources', () => {
+    const datasourceKey = new DatasourceKey(tableKey);
+    expect(datasourceKey.toString()).toBe(tableKey);
+    expect(datasourceKey.toObject()).toEqual({ id: 5, type: 'table' });
+  });
+
+  it('should handle druid data sources', () => {
+    const datasourceKey = new DatasourceKey(druidKey);
+    expect(datasourceKey.toString()).toBe(druidKey);
+    expect(datasourceKey.toObject()).toEqual({ id: 5, type: 'druid' });
+  });
+});

packages/superset-ui-chart/test/query/Metric.test.ts

@@ -0,0 +1,105 @@
+import { ColumnType } from '../../src/query/Column';
+import {
+  AdhocMetric,
+  Aggregate,
+  ExpressionType,
+  LABEL_MAX_LENGTH,
+  Metrics,
+} from '../../src/query/Metric';
+
+describe('Metrics', () => {
+  let metrics: Metrics;
+  const formData = {
+    datasource: '5__table',
+    granularity_sqla: 'ds',
+  };
+
+  it('should build metrics for built-in metric keys', () => {
+    metrics = new Metrics({
+      ...formData,
+      metric: 'sum__num',
+    });
+    expect(metrics.getMetrics()).toEqual([{ label: 'sum__num' }]);
+    expect(metrics.getLabels()).toEqual(['sum__num']);
+  });
+
+  it('should build metrics for simple adhoc metrics', () => {
+    const adhocMetric: AdhocMetric = {
+      aggregate: Aggregate.AVG,
+      column: {
+        columnName: 'sum_girls',
+        id: 5,
+        type: ColumnType.BIGINT,
+      },
+      expressionType: ExpressionType.SIMPLE,
+    };
+    metrics = new Metrics({
+      ...formData,
+      metric: adhocMetric,
+    });
+    expect(metrics.getMetrics()).toEqual([
+      {
+        aggregate: 'AVG',
+        column: {
+          columnName: 'sum_girls',
+          id: 5,
+          type: ColumnType.BIGINT,
+        },
+        expressionType: 'SIMPLE',
+        label: 'AVG(sum_girls)',
+      },
+    ]);
+    expect(metrics.getLabels()).toEqual(['AVG(sum_girls)']);
+  });
+
+  it('should build metrics for SQL adhoc metrics', () => {
+    const adhocMetric: AdhocMetric = {
+      expressionType: ExpressionType.SQL,
+      sqlExpression: 'COUNT(sum_girls)',
+    };
+    metrics = new Metrics({
+      ...formData,
+      metric: adhocMetric,
+    });
+    expect(metrics.getMetrics()).toEqual([
+      {
+        expressionType: 'SQL',
+        label: 'COUNT(sum_girls)',
+        sqlExpression: 'COUNT(sum_girls)',
+      },
+    ]);
+    expect(metrics.getLabels()).toEqual(['COUNT(sum_girls)']);
+  });
+
+  it('should build metrics for adhoc metrics with custom labels', () => {
+    const adhocMetric: AdhocMetric = {
+      expressionType: ExpressionType.SQL,
+      label: 'foo',
+      sqlExpression: 'COUNT(sum_girls)',
+    };
+    metrics = new Metrics({
+      ...formData,
+      metric: adhocMetric,
+    });
+    expect(metrics.getMetrics()).toEqual([
+      {
+        expressionType: 'SQL',
+        label: 'foo',
+        sqlExpression: 'COUNT(sum_girls)',
+      },
+    ]);
+    expect(metrics.getLabels()).toEqual(['foo']);
+  });
+
+  it('should truncate labels if they are too long', () => {
+    const adhocMetric: AdhocMetric = {
+      expressionType: ExpressionType.SQL,
+      sqlExpression: 'COUNT(verrrrrrrrry_loooooooooooooooooooooong_string)',
+    };
+    metrics = new Metrics({
+      ...formData,
+      metric: adhocMetric,
+    });
+    expect(metrics.getLabels()[0].length).toBeLessThanOrEqual(LABEL_MAX_LENGTH);
+  });
+});

packages/superset-ui-chart/test/query/buildQueryContext.test.ts

@@ -0,0 +1,22 @@
+import build from '../../src/query/buildQueryContext';
+import * as queryObjectBuilder from '../../src/query/buildQueryObject';
+
+describe('queryContextBuilder', () => {
+  it('should build datasource for table sources', () => {
+    const queryContext = build({ datasource: '5__table', granularity_sqla: 'ds' });
+    expect(queryContext.datasource.id).toBe(5);
+    expect(queryContext.datasource.type).toBe('table');
+  });
+
+  it('should build datasource for druid sources', () => {
+    const queryContext = build({ datasource: '5__druid', granularity: 'ds' });
+    expect(queryContext.datasource.id).toBe(5);
+    expect(queryContext.datasource.type).toBe('druid');
+  });
+
+  it('should call queryObjectBuilder to build queries', () => {
+    const buildQueryObjectSpy = jest.spyOn(queryObjectBuilder, 'default');
+    build({ datasource: '5__table', granularity_sqla: 'ds' });
+    expect(buildQueryObjectSpy).toHaveBeenCalledTimes(1);
+  });
+});