[Lens][Agg based table]Navigate to lens for agg based table vis (#140791

)

* Removed console.log.

* Other aggs to formula convertion added.

* Filter ratio added.

* Add XY Configuration

* Added last_value convertion.

* Updated ColumnWithMeta types.

* Fixed for loop.

* Added StaticValue convertion.

* Added default agg convertion.

* Add filters column

* Refactored filter part.

* Added meta to all the cols.

* Fixed problem with columns.

* Added date_histogram column, fixed layer configuration

* Added terms support.

* Refactored.

* Refactored for consistancy.

* Removed `getVisualizationSuggestionFromContext` fn.

* Added references.

* Fixed percentile and percentile ranks.

* removed unused return statement.

* Refactoring 'getEmptyLayersSuggestionsForVisualizeCharts' method

* TimeScaleUnit fix.

* Fixed references.

* Fixed types.

* Fixed terms column

* Fixed count custom label.

* Fix multi series

* Fixed custom labels.

* Fixed 'insertReferences' method

* Revert "Fixed custom labels."

This reverts commit 9bd61fb.

* Fixed custom labels.

* Fixed filter ratio and derivative metrics

* fix last_value and series agg

* Added CounterRate formula support.

* Fix types

* Replaced usage of timeseries.

* Added window everywhere.

* Add std_deviation and value_count, fix some issues

* Fix interval for date_histogram and fix getting filter for metrics and difference

* Fix tests for 'getDatasourceSuggestionsForVisualizeCharts' except test for 'format'

* Refactored code.

* Refactoring 'createNewLayerWithMetricAggregationFromVizEditor'

* Removed not used code.

* Removed unused import.

* Fixed types.

* Removed tests with type troubles.

* Some code refactoring + formatter.

* Removed not used comment.

* Added formatter validation.

* Added tests for getFormat.

* Fix `getFormat` method

* createColumn test added.

* Added tests for `excludeMetaFromColumn` and `isColumnWithMeta`.

* Added tests for counter_rate.

* Added tests for cumulative_sum.

* Updated cumulative_sum tests.

* Added date_histogram tests.

* Added filter ratio tests.

* Filters test added.

* createFormulaColumn test added.

* convertMathToFormulaColumn test added.

* Added one more case to check.

* Added test for convertOtherAggsToFormulaColumn

* added more tests for formula.

* Added last_value tests.

* Added tests for moving average.

* Style fix.

* Added test for convertMetricAggregationColumnWithoutSpecialParams

* Add tests for metrics_helper. Update tests for filter ratio and counter rate formulas

* Added tests for convertMetricAggregationToColumn

* Added tests for computeParentPipelineColumns

* Added tests for convertParentPipelineAggToColumns

* Add tests for getParentPipelineSeriesFormula

* Added tests for createParentPipelineAggregationColumn

* Added tests for isPercentileRanksColumnWithMeta

* Added tests for convertToPercentileRankParams

* Update and add tests for getSiblingPipelineSeriesFormula

* Fixed tests.

* Added tests for convertToPercentileRankColumn.

* Added tests for convertToPercentileRankColumns.

* Added tests for percentile.

* Added tests for static_value.

* Add tests for isValidMetrics

* Added std_deviation tests.

* Added tests for terms.

* Added tests.

* Add tests for getLayers in xy configuration

* Rename window to reducedTimeRange

* Added test for getBucketsColumns

* Fixed tests for parent_pipeline.

* Added tests for getValidColumns

* Added tests for getMetricsColumns

* Removed empty new.index.ts of top_n.

* Configuration refactored.

* Added tests for getConfigurationForTimeseries and getConfigurationForTopN

* Added tests for timeseries/index.ts

* Added test for top_n.

* Refactored getSuggestionFromConvertToLensContext at suggestion_helpers.

* Fix format

* Fix types

* Remove unused code

* Fix tests

* Fix bucket columns test

* Init comment. Add column and supported metric, update types for vis schemas

* Added percentile.

* Added percentile_rank.

* Add metric column without special params

* Added new type for AggParamsMapping.

* Added cumulative_sum converter.

* Replaced cumulative_sum with common parent_pipeline.

* Add sibling pipeline agg and date histogram column converter

* Added getColumnsFromVis method.

* Fixed formulas nad added support of value_count.

* Added navigateToLens functionality to visEditor table.

* Fixed eslint errors.

* Add convertBucketToColumns and some small updates

* Fixed problems with converting.

* Fixed Avg.

* Added median.

* Fixed percentile and percentile_ranks.

* Add terms and filters column, fix problem with changing index pattern, refactor sibling pipelline aggs

* Fixed std_dev.

* update convert to lens for datatable

* Fixed dateHistogram labels.

* Fix terms converter

* Fixed percentile/percentile_ranks/std_deviation.

* Fixed sum.

* Fixed cumulative_sum with value count.

* Fix counter rate metric and remove unused comments

* Added validation for *_bucket metrics.

* Added timeshift, where it is possible.

* Fixed type.

* Add support of percentage column for datatable

* Fixed formats.

* Add trigger for agg based visualizations

* Fix formats

* Some refactoring

* Added configuration.

* Fixed suggestions.

* Add dropEmptyRowsInDateHistogram param

* excluded meta.

* Refactored imports.

* Fixed label.

* Added last_value.

* Provide collapseFn for bucket columns

* Added percentile/percentile_rank labels fix.

* Fix sibling aggs converter

* Refactored convertToSchemaConfig

* Fixed terms

* Fixed problem with showing sub metrics on UI

* Moved out convertToSchemaConfig.

* Fixed types.

* Converted IAggConfig to SchemaConfig.

* Fixed pipeline and terms aggs. Added timeShift arg.

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* Fixed tests

* Added support of nested parent pipeline metrics.

* Fixed formats.

* Fixed cumulative sum by value count.

* Fixed percentile and percentile ranks.

* Fixed tests.

* One more fix.

* Fixed timeShift

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* Fixed std_deviation labels.

* Fixed percentage column and labels for sibling pipeline aggs

* Fixed getDataSourceInfo method

* added track if the rendered table is able to be converted to Lens

* Added support of nested parent pipeline aggs.

* Fixed unit tests

* Fix column order

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* Added shard_delay to SUPPORTED_AGGREGATIONS array and fixed limits

* Fixed mock for xy/vislib unti tests

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* Small refactoring of getColumnsFromVis.

* Refactored for tests purpose.

* Added tests for utils.

* Added tests for isValidVis.

* Added tests for getMetricsWithoutDuplicates

* Added tests for sortColumns.

* Added getColumnIds.

* Added unit tests for table configuration

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* Added tests for schemas.

* Reduced bundle size.

* Fixed annotations.

* Removed export.

* Fixed terms column

* Update src/plugins/vis_types/table/public/convert_to_lens/configurations/index.ts

Co-authored-by: Marta Bondyra <[email protected]>

* Added support of ranges and histograms

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* Added supporting of split series + sibling pipeline aggs

* Fix getBucketCollapseFn method

* Fixed problem with invalid fields.

* Update src/plugins/visualizations/common/convert_to_lens/lib/convert/last_value.ts

Co-authored-by: Marta Bondyra <[email protected]>

* Fixed problem with removed index pattern.

* Fixed types in timeseries supported metrics.

* Fixed tests

* Provided dataView to all the formula functions.

* Added checking field type for formula

* Fixed bundle size.

Co-authored-by: Yaroslav Kuznietsov <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>
Co-authored-by: Joe Reuter <[email protected]>
Co-authored-by: Marta Bondyra <[email protected]>

packages/kbn-optimizer/limits.yml

@@ -71,7 +71,7 @@ pageLoadAssetSize:
   kibanaUsageCollection: 16463
   kibanaUtils: 79713
   kubernetesSecurity: 77234
-  lens: 36000
+  lens: 36500
   licenseManagement: 41817
   licensing: 29004
   lists: 22900

src/plugins/data/common/search/aggs/buckets/index.ts

@@ -48,3 +48,4 @@ export * from './sampler_fn';
 export * from './sampler';
 export * from './diversified_sampler_fn';
 export * from './diversified_sampler';
+export { SHARD_DELAY_AGG_NAME } from './shard_delay';

src/plugins/data/common/search/aggs/buckets/multi_terms.ts

@@ -13,7 +13,7 @@ import { BucketAggType } from './bucket_agg_type';
 import { BUCKET_TYPES } from './bucket_agg_types';
 import { createFilterMultiTerms } from './create_filter/multi_terms';
 import { aggMultiTermsFnName } from './multi_terms_fn';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
 import { MultiFieldKey } from './multi_field_key';
 import {
@@ -26,10 +26,9 @@ const termsTitle = i18n.translate('data.search.aggs.buckets.multiTermsTitle', {
   defaultMessage: 'Multi-Terms',
 });
 
-export interface AggParamsMultiTerms extends BaseAggParams {
+interface CommonAggParamsMultiTerms extends BaseAggParams {
   fields: string[];
   orderBy: string;
-  orderAgg?: AggConfigSerialized;
   order?: 'asc' | 'desc';
   size?: number;
   shardSize?: number;
@@ -38,6 +37,14 @@ export interface AggParamsMultiTerms extends BaseAggParams {
   separatorLabel?: string;
 }
 
+export interface AggParamsMultiTermsSerialized extends CommonAggParamsMultiTerms {
+  orderAgg?: AggConfigSerialized;
+}
+
+export interface AggParamsMultiTerms extends CommonAggParamsMultiTerms {
+  orderAgg?: IAggConfig;
+}
+
 export const getMultiTermsBucketAgg = () => {
   const keyCaches = new WeakMap();
   return new BucketAggType({

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -17,7 +17,7 @@ import {
   migrateIncludeExcludeFormat,
 } from './migrate_include_exclude_format';
 import { aggTermsFnName } from './terms_fn';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
 import { KBN_FIELD_TYPES } from '../../..';
 
@@ -33,24 +33,35 @@ const termsTitle = i18n.translate('data.search.aggs.buckets.termsTitle', {
   defaultMessage: 'Terms',
 });
 
-export interface AggParamsTerms extends BaseAggParams {
+export interface CommonAggParamsTerms extends BaseAggParams {
   field: string;
   orderBy: string;
-  orderAgg?: AggConfigSerialized;
-  order?: 'asc' | 'desc';
   size?: number;
   shardSize?: number;
   missingBucket?: boolean;
   missingBucketLabel?: string;
   otherBucket?: boolean;
   otherBucketLabel?: string;
   // advanced
-  exclude?: string[] | number[];
-  include?: string[] | number[];
+  exclude?: string[] | string | number[];
+  include?: string[] | string | number[];
   includeIsRegex?: boolean;
   excludeIsRegex?: boolean;
 }
 
+export interface AggParamsTermsSerialized extends CommonAggParamsTerms {
+  orderAgg?: AggConfigSerialized;
+  order?: 'asc' | 'desc';
+}
+
+export interface AggParamsTerms extends CommonAggParamsTerms {
+  orderAgg?: IAggConfig;
+  order?: {
+    value: 'asc' | 'desc';
+    text: string;
+  };
+}
+
 export const getTermsBucketAgg = () =>
   new BucketAggType({
     name: BUCKET_TYPES.TERMS,

src/plugins/data/common/search/aggs/metrics/bucket_avg.ts

@@ -13,13 +13,18 @@ import { MetricAggType } from './metric_agg_type';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsBucketAvg extends BaseAggParams {
+export interface AggParamsBucketAvgSerialized extends BaseAggParams {
   customMetric?: AggConfigSerialized;
   customBucket?: AggConfigSerialized;
 }
 
+export interface AggParamsBucketAvg extends BaseAggParams {
+  customMetric?: IAggConfig;
+  customBucket?: IAggConfig;
+}
+
 const overallAverageLabel = i18n.translate('data.search.aggs.metrics.overallAverageLabel', {
   defaultMessage: 'overall average',
 });

src/plugins/data/common/search/aggs/metrics/bucket_max.ts

@@ -12,13 +12,18 @@ import { MetricAggType } from './metric_agg_type';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsBucketMax extends BaseAggParams {
+export interface AggParamsBucketMaxSerialized extends BaseAggParams {
   customMetric?: AggConfigSerialized;
   customBucket?: AggConfigSerialized;
 }
 
+export interface AggParamsBucketMax extends BaseAggParams {
+  customMetric?: IAggConfig;
+  customBucket?: IAggConfig;
+}
+
 const overallMaxLabel = i18n.translate('data.search.aggs.metrics.overallMaxLabel', {
   defaultMessage: 'overall max',
 });

src/plugins/data/common/search/aggs/metrics/bucket_min.ts

@@ -12,13 +12,18 @@ import { MetricAggType } from './metric_agg_type';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsBucketMin extends BaseAggParams {
+export interface AggParamsBucketMinSerialized extends BaseAggParams {
   customMetric?: AggConfigSerialized;
   customBucket?: AggConfigSerialized;
 }
 
+export interface AggParamsBucketMin extends BaseAggParams {
+  customMetric?: IAggConfig;
+  customBucket?: IAggConfig;
+}
+
 const overallMinLabel = i18n.translate('data.search.aggs.metrics.overallMinLabel', {
   defaultMessage: 'overall min',
 });

src/plugins/data/common/search/aggs/metrics/bucket_sum.ts

@@ -12,13 +12,18 @@ import { MetricAggType } from './metric_agg_type';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsBucketSum extends BaseAggParams {
+export interface AggParamsBucketSumSerialized extends BaseAggParams {
   customMetric?: AggConfigSerialized;
   customBucket?: AggConfigSerialized;
 }
 
+export interface AggParamsBucketSum extends BaseAggParams {
+  customMetric?: IAggConfig;
+  customBucket?: IAggConfig;
+}
+
 const overallSumLabel = i18n.translate('data.search.aggs.metrics.overallSumLabel', {
   defaultMessage: 'overall sum',
 });

src/plugins/data/common/search/aggs/metrics/cumulative_sum.ts

@@ -12,14 +12,21 @@ import { MetricAggType } from './metric_agg_type';
 import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsCumulativeSum extends BaseAggParams {
+export interface CommonAggParamsCumulativeSum extends BaseAggParams {
   buckets_path?: string;
-  customMetric?: AggConfigSerialized;
   metricAgg?: string;
 }
 
+export interface AggParamsCumulativeSumSerialized extends CommonAggParamsCumulativeSum {
+  customMetric?: AggConfigSerialized;
+}
+
+export interface AggParamsCumulativeSum extends CommonAggParamsCumulativeSum {
+  customMetric?: IAggConfig;
+}
+
 const cumulativeSumLabel = i18n.translate('data.search.aggs.metrics.cumulativeSumLabel', {
   defaultMessage: 'cumulative sum',
 });

src/plugins/data/common/search/aggs/metrics/derivative.ts

@@ -12,14 +12,21 @@ import { MetricAggType } from './metric_agg_type';
 import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsDerivative extends BaseAggParams {
+export interface CommonAggParamsDerivative extends BaseAggParams {
   buckets_path?: string;
-  customMetric?: AggConfigSerialized;
   metricAgg?: string;
 }
 
+export interface AggParamsDerivativeSerialized extends CommonAggParamsDerivative {
+  customMetric?: AggConfigSerialized;
+}
+
+export interface AggParamsDerivative extends CommonAggParamsDerivative {
+  customMetric?: IAggConfig;
+}
+
 const derivativeLabel = i18n.translate('data.search.aggs.metrics.derivativeLabel', {
   defaultMessage: 'derivative',
 });

src/plugins/data/common/search/aggs/metrics/filtered_metric.ts

@@ -13,14 +13,19 @@ import { MetricAggType } from './metric_agg_type';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 import { aggFilteredMetricFnName } from './filtered_metric_fn';
 
-export interface AggParamsFilteredMetric extends BaseAggParams {
+export interface AggParamsFilteredMetricSerialized extends BaseAggParams {
   customMetric?: AggConfigSerialized;
   customBucket?: AggConfigSerialized;
 }
 
+export interface AggParamsFilteredMetric extends BaseAggParams {
+  customMetric?: IAggConfig;
+  customBucket?: IAggConfig;
+}
+
 const filteredMetricLabel = i18n.translate('data.search.aggs.metrics.filteredMetricLabel', {
   defaultMessage: 'filtered',
 });

src/plugins/data/common/search/aggs/metrics/moving_avg.ts

@@ -12,16 +12,23 @@ import { aggMovingAvgFnName } from './moving_avg_fn';
 import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsMovingAvg extends BaseAggParams {
+export interface CommonAggParamsMovingAvg extends BaseAggParams {
   buckets_path?: string;
   window?: number;
   script?: string;
-  customMetric?: AggConfigSerialized;
   metricAgg?: string;
 }
 
+export interface AggParamsMovingAvgSerialized extends CommonAggParamsMovingAvg {
+  customMetric?: AggConfigSerialized;
+}
+
+export interface AggParamsMovingAvg extends CommonAggParamsMovingAvg {
+  customMetric?: IAggConfig;
+}
+
 const movingAvgTitle = i18n.translate('data.search.aggs.metrics.movingAvgTitle', {
   defaultMessage: 'Moving Avg',
 });

src/plugins/data/common/search/aggs/metrics/serial_diff.ts

@@ -12,14 +12,21 @@ import { aggSerialDiffFnName } from './serial_diff_fn';
 import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper';
 import { makeNestedLabel } from './lib/make_nested_label';
 import { METRIC_TYPES } from './metric_agg_types';
-import { AggConfigSerialized, BaseAggParams } from '../types';
+import { AggConfigSerialized, BaseAggParams, IAggConfig } from '../types';
 
-export interface AggParamsSerialDiff extends BaseAggParams {
+export interface CommonAggParamsSerialDiff extends BaseAggParams {
   buckets_path?: string;
-  customMetric?: AggConfigSerialized;
   metricAgg?: string;
 }
 
+export interface AggParamsSerialDiffSerialized extends CommonAggParamsSerialDiff {
+  customMetric?: AggConfigSerialized;
+}
+
+export interface AggParamsSerialDiff extends CommonAggParamsSerialDiff {
+  customMetric?: IAggConfig;
+}
+
 const serialDiffTitle = i18n.translate('data.search.aggs.metrics.serialDiffTitle', {
   defaultMessage: 'Serial Diff',
 });

src/plugins/data/common/search/aggs/metrics/top_hit.ts

@@ -8,18 +8,30 @@
 
 import _ from 'lodash';
 import { i18n } from '@kbn/i18n';
+import { DataViewField } from '@kbn/data-views-plugin/common';
 import { aggTopHitFnName } from './top_hit_fn';
 import { IMetricAggConfig, MetricAggType } from './metric_agg_type';
 import { METRIC_TYPES } from './metric_agg_types';
 import { flattenHit, KBN_FIELD_TYPES } from '../../..';
 import { BaseAggParams } from '../types';
 
-export interface AggParamsTopHit extends BaseAggParams {
+export interface BaseAggParamsTopHit extends BaseAggParams {
   field: string;
   aggregate: 'min' | 'max' | 'sum' | 'average' | 'concat';
-  sortField?: string;
   size?: number;
+}
+
+export interface AggParamsTopHitSerialized extends BaseAggParamsTopHit {
   sortOrder?: 'desc' | 'asc';
+  sortField?: string;
+}
+
+export interface AggParamsTopHit extends BaseAggParamsTopHit {
+  sortOrder?: {
+    value: 'desc' | 'asc';
+    text: string;
+  };
+  sortField?: DataViewField;
 }
 
 const isNumericFieldSelected = (agg: IMetricAggConfig) => {

src/plugins/data/common/search/aggs/metrics/top_metrics.ts

@@ -12,16 +12,27 @@ import { i18n } from '@kbn/i18n';
 import { aggTopMetricsFnName } from './top_metrics_fn';
 import { IMetricAggConfig, MetricAggType } from './metric_agg_type';
 import { METRIC_TYPES } from './metric_agg_types';
-import { KBN_FIELD_TYPES } from '../../..';
+import { DataViewField, KBN_FIELD_TYPES } from '../../..';
 import { BaseAggParams } from '../types';
 
-export interface AggParamsTopMetrics extends BaseAggParams {
+export interface BaseAggParamsTopMetrics extends BaseAggParams {
   field: string;
-  sortField?: string;
-  sortOrder?: 'desc' | 'asc';
   size?: number;
 }
 
+export interface AggParamsTopMetricsSerialized extends BaseAggParamsTopMetrics {
+  sortOrder?: 'desc' | 'asc';
+  sortField?: string;
+}
+
+export interface AggParamsTopMetrics extends BaseAggParamsTopMetrics {
+  sortOrder?: {
+    value: 'desc' | 'asc';
+    text: string;
+  };
+  sortField?: DataViewField;
+}
+
 export const getTopMetricsMetricAgg = () => {
   return new MetricAggType({
     name: METRIC_TYPES.TOP_METRICS,