diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index cd0495a3f78c6..c3b7ffe937b35 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -16,6 +16,15 @@ import type { DataView } from '@kbn/data-views-plugin/common'; import { stubIndexPattern } from '../../stubs'; import { IEsSearchResponse } from '..'; +// Mute moment.tz warnings about not finding a mock timezone +jest.mock('../utils', () => { + const original = jest.requireActual('../utils'); + return { + ...original, + getUserTimeZone: jest.fn(() => 'US/Pacific'), + }; +}); + describe('AggConfigs', () => { const indexPattern: DataView = stubIndexPattern; let typesRegistry: AggTypesRegistryStart; @@ -563,6 +572,82 @@ describe('AggConfigs', () => { '1-bucket>_count' ); }); + + it('prepends a sampling agg whenever sampling is enabled', () => { + const configStates = [ + { + enabled: true, + id: '1', + type: 'avg_bucket', + schema: 'metric', + params: { + customBucket: { + id: '1-bucket', + type: 'date_histogram', + schema: 'bucketAgg', + params: { + field: '@timestamp', + interval: '10s', + }, + }, + customMetric: { + id: '1-metric', + type: 'count', + schema: 'metricAgg', + params: {}, + }, + }, + }, + { + enabled: true, + id: '2', + type: 'terms', + schema: 'bucket', + params: { + field: 'clientip', + }, + }, + { + enabled: true, + id: '3', + type: 'terms', + schema: 'bucket', + params: { + field: 'machine.os.raw', + }, + }, + ]; + + const ac = new AggConfigs( + indexPattern, + configStates, + { typesRegistry, hierarchical: true, probability: 0.5 }, + jest.fn() + ); + const topLevelDsl = ac.toDsl(); + + expect(Object.keys(topLevelDsl)).toContain('sampling'); + expect(Object.keys(topLevelDsl.sampling)).toEqual(['random_sampler', 'aggs']); + expect(Object.keys(topLevelDsl.sampling.aggs)).toContain('2'); + expect(Object.keys(topLevelDsl.sampling.aggs['2'].aggs)).toEqual(['1', '3', '1-bucket']); + }); + + it('should not prepend a sampling agg when no nested agg is avaialble', () => { + const ac = new AggConfigs( + indexPattern, + [ + { + enabled: true, + type: 'count', + schema: 'metric', + }, + ], + { typesRegistry, probability: 0.5 }, + jest.fn() + ); + const topLevelDsl = ac.toDsl(); + expect(Object.keys(topLevelDsl)).not.toContain('sampling'); + }); }); describe('#postFlightTransform', () => { @@ -854,4 +939,74 @@ describe('AggConfigs', () => { `); }); }); + + describe('isSamplingEnabled', () => { + it('should return false if probability is 1', () => { + const ac = new AggConfigs( + indexPattern, + [{ enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }], + { typesRegistry, probability: 1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeFalsy(); + }); + + it('should return true if probability is less than 1', () => { + const ac = new AggConfigs( + indexPattern, + [{ enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }], + { typesRegistry, probability: 0.1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeTruthy(); + }); + + it('should return false when all aggs have hasNoDsl flag enabled', () => { + const ac = new AggConfigs( + indexPattern, + [ + { + enabled: true, + type: 'count', + schema: 'metric', + }, + ], + { typesRegistry, probability: 1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeFalsy(); + }); + + it('should return false when no nested aggs are avaialble', () => { + const ac = new AggConfigs( + indexPattern, + [{ enabled: false, type: 'avg', schema: 'metric', params: { field: 'bytes' } }], + { typesRegistry, probability: 1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeFalsy(); + }); + + it('should return true if at least one nested agg is available and probability < 1', () => { + const ac = new AggConfigs( + indexPattern, + [ + { + enabled: true, + type: 'count', + schema: 'metric', + }, + { enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }, + ], + { typesRegistry, probability: 0.1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeTruthy(); + }); + }); }); diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index c61ca69e0c6df..7cab863fba11d 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -26,6 +26,7 @@ import { AggTypesDependencies, GetConfigFn, getUserTimeZone } from '../..'; import { getTime, calculateBounds } from '../..'; import type { IBucketAggConfig } from './buckets'; import { insertTimeShiftSplit, mergeTimeShifts } from './utils/time_splits'; +import { createSamplerAgg, isSamplingEnabled } from './utils/sampler'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -55,6 +56,8 @@ export interface AggConfigsOptions { hierarchical?: boolean; aggExecutionContext?: AggTypesDependencies['aggExecutionContext']; partialRows?: boolean; + probability?: number; + samplerSeed?: number; } export type CreateAggConfigParams = Assign; @@ -107,6 +110,17 @@ export class AggConfigs { return this.opts.partialRows ?? false; } + public get samplerConfig() { + return { probability: this.opts.probability ?? 1, seed: this.opts.samplerSeed }; + } + + isSamplingEnabled() { + return ( + isSamplingEnabled(this.opts.probability) && + this.getRequestAggs().filter((agg) => !agg.type.hasNoDsl).length > 0 + ); + } + setTimeFields(timeFields: string[] | undefined) { this.timeFields = timeFields; } @@ -225,7 +239,7 @@ export class AggConfigs { } toDsl(): Record { - const dslTopLvl = {}; + const dslTopLvl: Record = {}; let dslLvlCursor: Record; let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; @@ -254,10 +268,21 @@ export class AggConfigs { (config) => 'splitForTimeShift' in config.type && config.type.splitForTimeShift(config, this) ); + if (this.isSamplingEnabled()) { + dslTopLvl.sampling = createSamplerAgg({ + probability: this.opts.probability ?? 1, + seed: this.opts.samplerSeed, + }); + } + requestAggs.forEach((config: AggConfig, i: number, list) => { if (!dslLvlCursor) { // start at the top level dslLvlCursor = dslTopLvl; + // when sampling jump directly to the aggs + if (this.isSamplingEnabled()) { + dslLvlCursor = dslLvlCursor.sampling.aggs; + } } else { const prevConfig: AggConfig = list[i - 1]; const prevDsl = dslLvlCursor[prevConfig.id]; @@ -452,7 +477,12 @@ export class AggConfigs { doc_count: response.rawResponse.hits?.total as estypes.AggregationsAggregate, }; } - const aggCursor = transformedRawResponse.aggregations!; + const aggCursor = this.isSamplingEnabled() + ? (transformedRawResponse.aggregations!.sampling! as Record< + string, + estypes.AggregationsAggregate + >) + : transformedRawResponse.aggregations!; mergeTimeShifts(this, aggCursor); return { @@ -531,6 +561,8 @@ export class AggConfigs { metricsAtAllLevels: this.hierarchical, partialRows: this.partialRows, aggs: this.aggs.map((agg) => buildExpression(agg.toExpressionAst())), + probability: this.opts.probability, + samplerSeed: this.opts.samplerSeed, }), ]).toAst(); } diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 6ce588b98fa9c..be2e969f279b1 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -18,6 +18,8 @@ import { AggConfigs, CreateAggConfigParams } from '../agg_configs'; import { BUCKET_TYPES } from './bucket_agg_types'; import { IBucketAggConfig } from './bucket_agg_type'; import { mockAggTypesRegistry } from '../test_helpers'; +import { estypes } from '@elastic/elasticsearch'; +import { isSamplingEnabled } from '../utils/sampler'; const indexPattern = { id: '1234', @@ -281,71 +283,63 @@ const nestedOtherResponse = { describe('Terms Agg Other bucket helper', () => { const typesRegistry = mockAggTypesRegistry(); - const getAggConfigs = (aggs: CreateAggConfigParams[] = []) => { - return new AggConfigs(indexPattern, [...aggs], { typesRegistry }, jest.fn()); - }; - - describe('buildOtherBucketAgg', () => { - test('returns a function', () => { - const aggConfigs = getAggConfigs(singleTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig, - singleTermResponse - ); - expect(typeof agg).toBe('function'); - }); - - test('correctly builds query with single terms agg', () => { - const aggConfigs = getAggConfigs(singleTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig, - singleTermResponse - ); - const expectedResponse = { - aggs: undefined, - filters: { - filters: { - '': { - bool: { - must: [], - filter: [{ exists: { field: 'machine.os.raw' } }], - should: [], - must_not: [ - { match_phrase: { 'machine.os.raw': 'ios' } }, - { match_phrase: { 'machine.os.raw': 'win xp' } }, - ], - }, - }, + for (const probability of [1, 0.5, undefined]) { + function getTitlePostfix() { + if (!isSamplingEnabled(probability)) { + return ''; + } + return ` - with sampling (probability = ${probability})`; + } + function enrichResponseWithSampling(response: any) { + if (!isSamplingEnabled(probability)) { + return response; + } + return { + ...response, + aggregations: { + sampling: { + ...response.aggregations, }, }, }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()['other-filter']).toEqual(expectedResponse); - } - }); + } - test('correctly builds query for nested terms agg', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponse - ); - const expectedResponse = { - 'other-filter': { + function getAggConfigs(aggs: CreateAggConfigParams[] = []) { + return new AggConfigs(indexPattern, [...aggs], { typesRegistry, probability }, jest.fn()); + } + + function getTopAggregations(updatedResponse: estypes.SearchResponse) { + return !isSamplingEnabled(probability) + ? updatedResponse.aggregations! + : (updatedResponse.aggregations!.sampling as Record); + } + + describe(`buildOtherBucketAgg${getTitlePostfix()}`, () => { + test('returns a function', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + enrichResponseWithSampling(singleTermResponse) + ); + expect(typeof agg).toBe('function'); + }); + + test('correctly builds query with single terms agg', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + enrichResponseWithSampling(singleTermResponse) + ); + const expectedResponse = { aggs: undefined, filters: { filters: { - [`${SEP}IN-with-dash`]: { + '': { bool: { must: [], - filter: [ - { match_phrase: { 'geo.src': 'IN-with-dash' } }, - { exists: { field: 'machine.os.raw' } }, - ], + filter: [{ exists: { field: 'machine.os.raw' } }], should: [], must_not: [ { match_phrase: { 'machine.os.raw': 'ios' } }, @@ -353,272 +347,322 @@ describe('Terms Agg Other bucket helper', () => { ], }, }, - [`${SEP}US-with-dash`]: { - bool: { - must: [], - filter: [ - { match_phrase: { 'geo.src': 'US-with-dash' } }, - { exists: { field: 'machine.os.raw' } }, - ], - should: [], - must_not: [ - { match_phrase: { 'machine.os.raw': 'ios' } }, - { match_phrase: { 'machine.os.raw': 'win xp' } }, - ], + }, + }, + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + expect(topAgg['other-filter']).toEqual(expectedResponse); + } + }); + + test('correctly builds query for nested terms agg', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(nestedTermResponse) + ); + const expectedResponse = { + 'other-filter': { + aggs: undefined, + filters: { + filters: { + [`${SEP}IN-with-dash`]: { + bool: { + must: [], + filter: [ + { match_phrase: { 'geo.src': 'IN-with-dash' } }, + { exists: { field: 'machine.os.raw' } }, + ], + should: [], + must_not: [ + { match_phrase: { 'machine.os.raw': 'ios' } }, + { match_phrase: { 'machine.os.raw': 'win xp' } }, + ], + }, + }, + [`${SEP}US-with-dash`]: { + bool: { + must: [], + filter: [ + { match_phrase: { 'geo.src': 'US-with-dash' } }, + { exists: { field: 'machine.os.raw' } }, + ], + should: [], + must_not: [ + { match_phrase: { 'machine.os.raw': 'ios' } }, + { match_phrase: { 'machine.os.raw': 'win xp' } }, + ], + }, }, }, }, }, - }, - }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()).toEqual(expectedResponse); - } - }); + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + // console.log({ probability }, JSON.stringify(topAgg, null, 2)); + expect(topAgg).toEqual(expectedResponse); + } + }); - test('correctly builds query for nested terms agg with one disabled', () => { - const oneDisabledNestedTerms = { - aggs: [ - { - id: '2', - type: BUCKET_TYPES.TERMS, - enabled: false, - params: { - field: { - name: 'machine.os.raw', - indexPattern, - filterable: true, + test('correctly builds query for nested terms agg with one disabled', () => { + const oneDisabledNestedTerms = { + aggs: [ + { + id: '2', + type: BUCKET_TYPES.TERMS, + enabled: false, + params: { + field: { + name: 'machine.os.raw', + indexPattern, + filterable: true, + }, + size: 2, + otherBucket: false, + missingBucket: true, }, - size: 2, - otherBucket: false, - missingBucket: true, }, - }, - { - id: '1', - type: BUCKET_TYPES.TERMS, - params: { - field: { - name: 'geo.src', - indexPattern, - filterable: true, + { + id: '1', + type: BUCKET_TYPES.TERMS, + params: { + field: { + name: 'geo.src', + indexPattern, + filterable: true, + }, + size: 2, + otherBucket: true, + missingBucket: false, }, - size: 2, - otherBucket: true, - missingBucket: false, }, - }, - ], - }; - const aggConfigs = getAggConfigs(oneDisabledNestedTerms.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - singleTermResponse - ); - const expectedResponse = { - 'other-filter': { - aggs: undefined, - filters: { + ], + }; + const aggConfigs = getAggConfigs(oneDisabledNestedTerms.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(singleTermResponse) + ); + const expectedResponse = { + 'other-filter': { + aggs: undefined, filters: { - '': { - bool: { - filter: [ - { - exists: { - field: 'geo.src', + filters: { + '': { + bool: { + filter: [ + { + exists: { + field: 'geo.src', + }, }, - }, - ], - must: [], - must_not: [ - { - match_phrase: { - 'geo.src': 'ios', + ], + must: [], + must_not: [ + { + match_phrase: { + 'geo.src': 'ios', + }, }, - }, - { - match_phrase: { - 'geo.src': 'win xp', + { + match_phrase: { + 'geo.src': 'win xp', + }, }, - }, - ], - should: [], + ], + should: [], + }, }, }, }, }, - }, - }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()).toEqual(expectedResponse); - } - }); + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + expect(topAgg).toEqual(expectedResponse); + } + }); - test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - expect( - buildOtherBucketAgg( + test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + expect( + buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(exhaustiveNestedTermResponse) + ) + ).toBeFalsy(); + }); + + test('excludes exists filter for scripted fields', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + aggConfigs.aggs[1].params.field = { + ...aggConfigs.aggs[1].params.field, + scripted: true, + }; + const agg = buildOtherBucketAgg( aggConfigs, aggConfigs.aggs[1] as IBucketAggConfig, - exhaustiveNestedTermResponse - ) - ).toBeFalsy(); - }); - - test('excludes exists filter for scripted fields', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - aggConfigs.aggs[1].params.field.scripted = true; - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponse - ); - const expectedResponse = { - 'other-filter': { - aggs: undefined, - filters: { + enrichResponseWithSampling(nestedTermResponse) + ); + const expectedResponse = { + 'other-filter': { + aggs: undefined, filters: { - [`${SEP}IN-with-dash`]: { - bool: { - must: [], - filter: [{ match_phrase: { 'geo.src': 'IN-with-dash' } }], - should: [], - must_not: [ - { - script: { + filters: { + [`${SEP}IN-with-dash`]: { + bool: { + must: [], + filter: [{ match_phrase: { 'geo.src': 'IN-with-dash' } }], + should: [], + must_not: [ + { script: { - lang: undefined, - params: { value: 'ios' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'ios' }, + source: '(undefined) == value', + }, }, }, - }, - { - script: { + { script: { - lang: undefined, - params: { value: 'win xp' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'win xp' }, + source: '(undefined) == value', + }, }, }, - }, - ], + ], + }, }, - }, - [`${SEP}US-with-dash`]: { - bool: { - must: [], - filter: [{ match_phrase: { 'geo.src': 'US-with-dash' } }], - should: [], - must_not: [ - { - script: { + [`${SEP}US-with-dash`]: { + bool: { + must: [], + filter: [{ match_phrase: { 'geo.src': 'US-with-dash' } }], + should: [], + must_not: [ + { script: { - lang: undefined, - params: { value: 'ios' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'ios' }, + source: '(undefined) == value', + }, }, }, - }, - { - script: { + { script: { - lang: undefined, - params: { value: 'win xp' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'win xp' }, + source: '(undefined) == value', + }, }, }, - }, - ], + ], + }, }, }, }, }, - }, - }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()).toEqual(expectedResponse); - } - }); + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + expect(topAgg).toEqual(expectedResponse); + } + }); - test('returns false when nested terms agg has no buckets', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponseNoResults - ); + test('returns false when nested terms agg has no buckets', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(nestedTermResponseNoResults) + ); - expect(agg).toEqual(false); + expect(agg).toEqual(false); + }); }); - }); - describe('mergeOtherBucketAggResponse', () => { - test('correctly merges other bucket with single terms agg', () => { - const aggConfigs = getAggConfigs(singleTerm.aggs); - const otherAggConfig = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig, - singleTermResponse - ); - - expect(otherAggConfig).toBeDefined(); - if (otherAggConfig) { - const mergedResponse = mergeOtherBucketAggResponse( + describe(`mergeOtherBucketAggResponse${getTitlePostfix()}`, () => { + test('correctly merges other bucket with single terms agg', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const otherAggConfig = buildOtherBucketAgg( aggConfigs, - singleTermResponse, - singleOtherResponse, aggConfigs.aggs[0] as IBucketAggConfig, - otherAggConfig(), - constructSingleTermOtherFilter + enrichResponseWithSampling(singleTermResponse) ); - expect((mergedResponse!.aggregations!['1'] as any).buckets[3].key).toEqual('__other__'); - } - }); - test('correctly merges other bucket with nested terms agg', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const otherAggConfig = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponse - ); + expect(otherAggConfig).toBeDefined(); + if (otherAggConfig) { + const mergedResponse = mergeOtherBucketAggResponse( + aggConfigs, + enrichResponseWithSampling(singleTermResponse), + enrichResponseWithSampling(singleOtherResponse), + aggConfigs.aggs[0] as IBucketAggConfig, + otherAggConfig(), + constructSingleTermOtherFilter + ); - expect(otherAggConfig).toBeDefined(); - if (otherAggConfig) { - const mergedResponse = mergeOtherBucketAggResponse( + const topAgg = getTopAggregations(mergedResponse); + expect((topAgg['1'] as any).buckets[3].key).toEqual('__other__'); + } + }); + + test('correctly merges other bucket with nested terms agg', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const otherAggConfig = buildOtherBucketAgg( aggConfigs, - nestedTermResponse, - nestedOtherResponse, aggConfigs.aggs[1] as IBucketAggConfig, - otherAggConfig(), - constructSingleTermOtherFilter + enrichResponseWithSampling(nestedTermResponse) ); - expect((mergedResponse!.aggregations!['1'] as any).buckets[1]['2'].buckets[3].key).toEqual( - '__other__' - ); - } + expect(otherAggConfig).toBeDefined(); + if (otherAggConfig) { + const mergedResponse = mergeOtherBucketAggResponse( + aggConfigs, + enrichResponseWithSampling(nestedTermResponse), + enrichResponseWithSampling(nestedOtherResponse), + aggConfigs.aggs[1] as IBucketAggConfig, + otherAggConfig(), + constructSingleTermOtherFilter + ); + + const topAgg = getTopAggregations(mergedResponse); + expect((topAgg['1'] as any).buckets[1]['2'].buckets[3].key).toEqual('__other__'); + } + }); }); - }); - describe('updateMissingBucket', () => { - test('correctly updates missing bucket key', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const updatedResponse = updateMissingBucket( - singleTermResponse, - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig - ); - expect( - (updatedResponse!.aggregations!['1'] as any).buckets.find( - (bucket: Record) => bucket.key === '__missing__' - ) - ).toBeDefined(); + describe(`updateMissingBucket${getTitlePostfix()}`, () => { + test('correctly updates missing bucket key', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const updatedResponse = updateMissingBucket( + enrichResponseWithSampling(singleTermResponse), + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig + ); + const topAgg = getTopAggregations(updatedResponse); + expect( + (topAgg['1'] as any).buckets.find( + (bucket: Record) => bucket.key === '__missing__' + ) + ).toBeDefined(); + }); }); - }); + } }); diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index f695dc1b1d399..68c64f67ef27f 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -20,6 +20,7 @@ import { AggGroupNames } from '../agg_groups'; import { IAggConfigs } from '../agg_configs'; import { IAggType } from '../agg_type'; import { IAggConfig } from '../agg_config'; +import { createSamplerAgg } from '../utils/sampler'; export const OTHER_BUCKET_SEPARATOR = '╰┄►'; @@ -128,6 +129,28 @@ const getOtherAggTerms = (requestAgg: Record, key: string, otherAgg .map((filter: Record) => filter.match_phrase[otherAgg.params.field.name]); }; +/** + * Helper function to handle sampling case and get the correct cursor agg from a request object + */ +const getCorrectAggCursorFromRequest = ( + requestAgg: Record, + aggConfigs: IAggConfigs +) => { + return aggConfigs.isSamplingEnabled() ? requestAgg.sampling.aggs : requestAgg; +}; + +/** + * Helper function to handle sampling case and get the correct cursor agg from a response object + */ +const getCorrectAggregationsCursorFromResponse = ( + response: estypes.SearchResponse, + aggConfigs: IAggConfigs +) => { + return aggConfigs.isSamplingEnabled() + ? (response.aggregations?.sampling as Record) + : response.aggregations; +}; + export const buildOtherBucketAgg = ( aggConfigs: IAggConfigs, aggWithOtherBucket: IAggConfig, @@ -234,7 +257,13 @@ export const buildOtherBucketAgg = ( bool: buildQueryFromFilters(filters, indexPattern), }; }; - walkBucketTree(0, response.aggregations, bucketAggs[0].id, [], ''); + walkBucketTree( + 0, + getCorrectAggregationsCursorFromResponse(response, aggConfigs), + bucketAggs[0].id, + [], + '' + ); // bail if there were no bucket results if (noAggBucketResults || exhaustiveBuckets) { @@ -242,6 +271,14 @@ export const buildOtherBucketAgg = ( } return () => { + if (aggConfigs.isSamplingEnabled()) { + return { + sampling: { + ...createSamplerAgg(aggConfigs.samplerConfig), + aggs: { 'other-filter': resultAgg }, + }, + }; + } return { 'other-filter': resultAgg, }; @@ -257,16 +294,27 @@ export const mergeOtherBucketAggResponse = ( otherFilterBuilder: (requestAgg: Record, key: string, otherAgg: IAggConfig) => Filter ): estypes.SearchResponse => { const updatedResponse = cloneDeep(response); - each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => { + const aggregationsRoot = getCorrectAggregationsCursorFromResponse(otherResponse, aggsConfig); + const updatedAggregationsRoot = getCorrectAggregationsCursorFromResponse( + updatedResponse, + aggsConfig + ); + const buckets = + 'buckets' in aggregationsRoot!['other-filter'] ? aggregationsRoot!['other-filter'].buckets : {}; + each(buckets, (bucket, key) => { if (!bucket.doc_count || key === undefined) return; const bucketKey = key.replace(new RegExp(`^${OTHER_BUCKET_SEPARATOR}`), ''); const aggResultBuckets = getAggResultBuckets( aggsConfig, - updatedResponse.aggregations, + updatedAggregationsRoot, otherAgg, bucketKey ); - const otherFilter = otherFilterBuilder(requestAgg, key, otherAgg); + const otherFilter = otherFilterBuilder( + getCorrectAggCursorFromRequest(requestAgg, aggsConfig), + key, + otherAgg + ); bucket.filters = [otherFilter]; bucket.key = '__other__'; @@ -290,7 +338,10 @@ export const updateMissingBucket = ( agg: IAggConfig ) => { const updatedResponse = cloneDeep(response); - const aggResultBuckets = getAggConfigResultMissingBuckets(updatedResponse.aggregations, agg.id); + const aggResultBuckets = getAggConfigResultMissingBuckets( + getCorrectAggregationsCursorFromResponse(updatedResponse, aggConfigs), + agg.id + ); aggResultBuckets.forEach((bucket) => { bucket.key = '__missing__'; }); diff --git a/src/plugins/data/common/search/aggs/utils/sampler.ts b/src/plugins/data/common/search/aggs/utils/sampler.ts new file mode 100644 index 0000000000000..5a6fde63b0a29 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/sampler.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export function createSamplerAgg({ + type = 'random_sampler', + probability, + seed, +}: { + type?: string; + probability: number; + seed?: number; +}) { + return { + [type]: { + probability, + seed, + }, + aggs: {}, + }; +} + +export function isSamplingEnabled(probability: number | undefined) { + return probability != null && probability !== 1; +} diff --git a/src/plugins/data/common/search/aggs/utils/time_splits.ts b/src/plugins/data/common/search/aggs/utils/time_splits.ts index c2fe8aaca0fb2..b1262683446f3 100644 --- a/src/plugins/data/common/search/aggs/utils/time_splits.ts +++ b/src/plugins/data/common/search/aggs/utils/time_splits.ts @@ -427,11 +427,11 @@ export function insertTimeShiftSplit( const timeRange = aggConfigs.timeRange; const filters: Record = {}; const timeField = aggConfigs.timeFields[0]; + const timeFilter = getTime(aggConfigs.indexPattern, timeRange, { + fieldName: timeField, + forceNow: aggConfigs.forceNow, + }) as RangeFilter; Object.entries(timeShifts).forEach(([key, shift]) => { - const timeFilter = getTime(aggConfigs.indexPattern, timeRange, { - fieldName: timeField, - forceNow: aggConfigs.forceNow, - }) as RangeFilter; if (timeFilter) { filters[key] = { range: { diff --git a/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts b/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts index c8296b3c06557..f0b55de8ffeff 100644 --- a/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts +++ b/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts @@ -32,6 +32,8 @@ interface Arguments { metricsAtAllLevels?: boolean; partialRows?: boolean; timeFields?: string[]; + probability?: number; + samplerSeed?: number; } export type EsaggsExpressionFunctionDefinition = ExpressionFunctionDefinition< @@ -94,6 +96,21 @@ export const getEsaggsMeta: () => Omit defaultMessage: 'Provide time fields to get the resolved time ranges for the query', }), }, + probability: { + types: ['number'], + default: 1, + help: i18n.translate('data.search.functions.esaggs.probability.help', { + defaultMessage: + 'The probability that a document will be included in the aggregated data. Uses random sampler.', + }), + }, + samplerSeed: { + types: ['number'], + help: i18n.translate('data.search.functions.esaggs.samplerSeed.help', { + defaultMessage: + 'The seed to generate the random sampling of documents. Uses random sampler.', + }), + }, }, }); diff --git a/src/plugins/data/common/search/tabify/tabify.test.ts b/src/plugins/data/common/search/tabify/tabify.test.ts index d7d983fabfdaf..90ef53623c298 100644 --- a/src/plugins/data/common/search/tabify/tabify.test.ts +++ b/src/plugins/data/common/search/tabify/tabify.test.ts @@ -11,184 +11,217 @@ import type { DataView } from '@kbn/data-views-plugin/common'; import { AggConfigs, BucketAggParam, IAggConfig, IAggConfigs } from '../aggs'; import { mockAggTypesRegistry } from '../aggs/test_helpers'; import { metricOnly, threeTermBuckets } from './fixtures/fake_hierarchical_data'; +import { isSamplingEnabled } from '../aggs/utils/sampler'; describe('tabifyAggResponse Integration', () => { const typesRegistry = mockAggTypesRegistry(); - const createAggConfigs = (aggs: IAggConfig[] = []) => { - const field = { - name: '@timestamp', - }; - - const indexPattern = { - id: '1234', - title: 'logstash-*', - fields: { - getByName: () => field, - filter: () => [field], - }, - getFormatterForField: () => ({ - toJSON: () => '{}', - }), - } as unknown as DataView; - - return new AggConfigs(indexPattern, aggs, { typesRegistry }, jest.fn()); - }; - - const mockAggConfig = (agg: any): IAggConfig => agg as unknown as IAggConfig; - - test('transforms a simple response properly', () => { - const aggConfigs = createAggConfigs([{ type: 'count' } as any]); - - const resp = tabifyAggResponse(aggConfigs, metricOnly, { - metricsAtAllLevels: true, - }); - - expect(resp).toHaveProperty('rows'); - expect(resp).toHaveProperty('columns'); + for (const probability of [1, 0.5, undefined]) { + function getTitlePostfix() { + if (!isSamplingEnabled(probability)) { + return ''; + } + return ` - with sampling (probability = ${probability})`; + } - expect(resp.rows).toHaveLength(1); - expect(resp.columns).toHaveLength(1); + function enrichResponseWithSampling(response: any) { + if (!isSamplingEnabled(probability)) { + return response; + } + return { + ...response, + aggregations: { + sampling: { + ...response.aggregations, + }, + }, + }; + } - expect(resp.rows[0]).toEqual({ 'col-0-1': 1000 }); - expect(resp.columns[0]).toHaveProperty('name', aggConfigs.aggs[0].makeLabel()); + const createAggConfigs = (aggs: IAggConfig[] = []) => { + const field = { + name: '@timestamp', + }; + + const indexPattern = { + id: '1234', + title: 'logstash-*', + fields: { + getByName: () => field, + filter: () => [field], + }, + getFormatterForField: () => ({ + toJSON: () => '{}', + }), + } as unknown as DataView; + + return new AggConfigs(indexPattern, aggs, { typesRegistry, probability }, jest.fn()); + }; - expect(resp).toHaveProperty('meta.type', 'esaggs'); - expect(resp).toHaveProperty('meta.source', '1234'); - expect(resp).toHaveProperty('meta.statistics.totalCount', 1000); - }); + const mockAggConfig = (agg: any): IAggConfig => agg as unknown as IAggConfig; - describe('scaleMetricValues performance check', () => { - beforeAll(() => { - typesRegistry.get('count').params.push({ - name: 'scaleMetricValues', - default: false, - write: () => {}, - advanced: true, - } as any as BucketAggParam); - }); - test('does not call write if scaleMetricValues is not set', () => { + test(`transforms a simple response properly${getTitlePostfix()}`, () => { const aggConfigs = createAggConfigs([{ type: 'count' } as any]); - const writeMock = jest.fn(); - aggConfigs.getRequestAggs()[0].write = writeMock; - - tabifyAggResponse(aggConfigs, metricOnly, { + const resp = tabifyAggResponse(aggConfigs, enrichResponseWithSampling(metricOnly), { metricsAtAllLevels: true, }); - expect(writeMock).not.toHaveBeenCalled(); - }); - test('does call write if scaleMetricValues is set', () => { - const aggConfigs = createAggConfigs([ - { type: 'count', params: { scaleMetricValues: true } } as any, - ]); + expect(resp).toHaveProperty('rows'); + expect(resp).toHaveProperty('columns'); - const writeMock = jest.fn(() => ({})); - aggConfigs.getRequestAggs()[0].write = writeMock; - - tabifyAggResponse(aggConfigs, metricOnly, { - metricsAtAllLevels: true, - }); - expect(writeMock).toHaveBeenCalled(); - }); - }); - - describe('transforms a complex response', () => { - let esResp: typeof threeTermBuckets; - let aggConfigs: IAggConfigs; - let avg: IAggConfig; - let ext: IAggConfig; - let src: IAggConfig; - let os: IAggConfig; - - beforeEach(() => { - aggConfigs = createAggConfigs([ - mockAggConfig({ type: 'avg', schema: 'metric', params: { field: '@timestamp' } }), - mockAggConfig({ type: 'terms', schema: 'split', params: { field: '@timestamp' } }), - mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), - mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), - ]); - - [avg, ext, src, os] = aggConfigs.aggs; - - esResp = threeTermBuckets; - esResp.aggregations.agg_2.buckets[1].agg_3.buckets[0].agg_4.buckets = []; - }); + expect(resp.rows).toHaveLength(1); + expect(resp.columns).toHaveLength(1); - // check that the columns of a table are formed properly - function expectColumns(table: ReturnType, aggs: IAggConfig[]) { - expect(table.columns).toHaveLength(aggs.length); + expect(resp.rows[0]).toEqual({ 'col-0-1': 1000 }); + expect(resp.columns[0]).toHaveProperty('name', aggConfigs.aggs[0].makeLabel()); - aggs.forEach((agg, i) => { - expect(table.columns[i]).toHaveProperty('name', agg.makeLabel()); - }); - } + expect(resp).toHaveProperty('meta.type', 'esaggs'); + expect(resp).toHaveProperty('meta.source', '1234'); + expect(resp).toHaveProperty('meta.statistics.totalCount', 1000); + }); - // check that a row has expected values - function expectRow( - row: Record, - asserts: Array<(val: string | number) => void> - ) { - expect(typeof row).toBe('object'); - - asserts.forEach((assert, i: number) => { - if (row[`col-${i}`]) { - assert(row[`col-${i}`]); - } + describe(`scaleMetricValues performance check${getTitlePostfix()}`, () => { + beforeAll(() => { + typesRegistry.get('count').params.push({ + name: 'scaleMetricValues', + default: false, + write: () => {}, + advanced: true, + } as any as BucketAggParam); }); - } + test('does not call write if scaleMetricValues is not set', () => { + const aggConfigs = createAggConfigs([{ type: 'count' } as any]); - // check for two character country code - function expectCountry(val: string | number) { - expect(typeof val).toBe('string'); - expect(val).toHaveLength(2); - } + const writeMock = jest.fn(); + aggConfigs.getRequestAggs()[0].write = writeMock; - // check for an OS term - function expectExtension(val: string | number) { - expect(val).toMatch(/^(js|png|html|css|jpg)$/); - } + tabifyAggResponse(aggConfigs, enrichResponseWithSampling(metricOnly), { + metricsAtAllLevels: true, + }); + expect(writeMock).not.toHaveBeenCalled(); + }); - // check for an OS term - function expectOS(val: string | number) { - expect(val).toMatch(/^(win|mac|linux)$/); - } + test('does call write if scaleMetricValues is set', () => { + const aggConfigs = createAggConfigs([ + { type: 'count', params: { scaleMetricValues: true } } as any, + ]); - // check for something like an average bytes result - function expectAvgBytes(val: string | number) { - expect(typeof val).toBe('number'); - expect(val === 0 || val > 1000).toBeDefined(); - } + const writeMock = jest.fn(() => ({})); + aggConfigs.getRequestAggs()[0].write = writeMock; - test('for non-hierarchical vis', () => { - // the default for a non-hierarchical vis is to display - // only complete rows, and only put the metrics at the end. + tabifyAggResponse(aggConfigs, enrichResponseWithSampling(metricOnly), { + metricsAtAllLevels: true, + }); + expect(writeMock).toHaveBeenCalled(); + }); + }); - const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: false }); + describe(`transforms a complex response${getTitlePostfix()}`, () => { + let esResp: typeof threeTermBuckets; + let aggConfigs: IAggConfigs; + let avg: IAggConfig; + let ext: IAggConfig; + let src: IAggConfig; + let os: IAggConfig; + + function getTopAggregations( + rawResp: typeof threeTermBuckets + ): typeof threeTermBuckets['aggregations'] { + return !isSamplingEnabled(probability) + ? rawResp.aggregations! + : // @ts-ignore + rawResp.aggregations!.sampling!; + } + + beforeEach(() => { + aggConfigs = createAggConfigs([ + mockAggConfig({ type: 'avg', schema: 'metric', params: { field: '@timestamp' } }), + mockAggConfig({ type: 'terms', schema: 'split', params: { field: '@timestamp' } }), + mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), + mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), + ]); - expectColumns(tabbed, [ext, src, os, avg]); + [avg, ext, src, os] = aggConfigs.aggs; - tabbed.rows.forEach((row) => { - expectRow(row, [expectExtension, expectCountry, expectOS, expectAvgBytes]); + esResp = enrichResponseWithSampling(threeTermBuckets); + getTopAggregations(esResp).agg_2.buckets[1].agg_3.buckets[0].agg_4.buckets = []; }); - }); - - test('for hierarchical vis', () => { - const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: true }); - expectColumns(tabbed, [ext, avg, src, avg, os, avg]); + // check that the columns of a table are formed properly + function expectColumns(table: ReturnType, aggs: IAggConfig[]) { + expect(table.columns).toHaveLength(aggs.length); + + aggs.forEach((agg, i) => { + expect(table.columns[i]).toHaveProperty('name', agg.makeLabel()); + }); + } + + // check that a row has expected values + function expectRow( + row: Record, + asserts: Array<(val: string | number) => void> + ) { + expect(typeof row).toBe('object'); + + asserts.forEach((assert, i: number) => { + if (row[`col-${i}`]) { + assert(row[`col-${i}`]); + } + }); + } + + // check for two character country code + function expectCountry(val: string | number) { + expect(typeof val).toBe('string'); + expect(val).toHaveLength(2); + } + + // check for an OS term + function expectExtension(val: string | number) { + expect(val).toMatch(/^(js|png|html|css|jpg)$/); + } + + // check for an OS term + function expectOS(val: string | number) { + expect(val).toMatch(/^(win|mac|linux)$/); + } + + // check for something like an average bytes result + function expectAvgBytes(val: string | number) { + expect(typeof val).toBe('number'); + expect(val === 0 || val > 1000).toBeDefined(); + } + + test('for non-hierarchical vis', () => { + // the default for a non-hierarchical vis is to display + // only complete rows, and only put the metrics at the end. + + const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: false }); + + expectColumns(tabbed, [ext, src, os, avg]); + + tabbed.rows.forEach((row) => { + expectRow(row, [expectExtension, expectCountry, expectOS, expectAvgBytes]); + }); + }); - tabbed.rows.forEach((row) => { - expectRow(row, [ - expectExtension, - expectAvgBytes, - expectCountry, - expectAvgBytes, - expectOS, - expectAvgBytes, - ]); + test('for hierarchical vis', () => { + const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: true }); + + expectColumns(tabbed, [ext, avg, src, avg, os, avg]); + + tabbed.rows.forEach((row) => { + expectRow(row, [ + expectExtension, + expectAvgBytes, + expectCountry, + expectAvgBytes, + expectOS, + expectAvgBytes, + ]); + }); }); }); - }); + } }); diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index ea26afd30129a..2c332c1ad6a75 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -147,7 +147,9 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { - ...esResponse.aggregations, + ...(aggConfigs.isSamplingEnabled() + ? esResponse.aggregations.sampling + : esResponse.aggregations), doc_count: esResponse.aggregations?.doc_count || esResponse.hits?.total, }; diff --git a/src/plugins/data/public/search/expressions/esaggs.test.ts b/src/plugins/data/public/search/expressions/esaggs.test.ts index aa26ceab9ae6b..1a04e4ffeb839 100644 --- a/src/plugins/data/public/search/expressions/esaggs.test.ts +++ b/src/plugins/data/public/search/expressions/esaggs.test.ts @@ -50,6 +50,7 @@ describe('esaggs expression function - public', () => { metricsAtAllLevels: true, partialRows: false, timeFields: ['@timestamp', 'utc_time'], + probability: 1, }; beforeEach(() => { @@ -88,7 +89,7 @@ describe('esaggs expression function - public', () => { expect(startDependencies.aggs.createAggConfigs).toHaveBeenCalledWith( {}, args.aggs.map((agg) => agg.value), - { hierarchical: true, partialRows: false } + { hierarchical: true, partialRows: false, probability: 1, samplerSeed: undefined } ); }); @@ -98,6 +99,8 @@ describe('esaggs expression function - public', () => { expect(startDependencies.aggs.createAggConfigs).toHaveBeenCalledWith({}, [], { hierarchical: true, partialRows: false, + probability: 1, + samplerSeed: undefined, }); }); diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index d944dcd5c1a5d..b82401d4d8caf 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -48,7 +48,12 @@ export function getFunctionDefinition({ const aggConfigs = aggs.createAggConfigs( indexPattern, args.aggs?.map((agg) => agg.value) ?? [], - { hierarchical: args.metricsAtAllLevels, partialRows: args.partialRows } + { + hierarchical: args.metricsAtAllLevels, + partialRows: args.partialRows, + probability: args.probability, + samplerSeed: args.samplerSeed, + } ); const { handleEsaggsRequest } = await import('../../../common/search/expressions'); diff --git a/src/plugins/data/server/search/expressions/esaggs.test.ts b/src/plugins/data/server/search/expressions/esaggs.test.ts index 166f2719e356d..9954fb2457968 100644 --- a/src/plugins/data/server/search/expressions/esaggs.test.ts +++ b/src/plugins/data/server/search/expressions/esaggs.test.ts @@ -51,6 +51,7 @@ describe('esaggs expression function - server', () => { metricsAtAllLevels: true, partialRows: false, timeFields: ['@timestamp', 'utc_time'], + probability: 1, }; beforeEach(() => { diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts index 3f30dff6fd1a7..251d83201f468 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts @@ -315,7 +315,9 @@ describe('IndexPattern Data Source', () => { describe('#toExpression', () => { it('should generate an empty expression when no columns are selected', async () => { const state = FormBasedDatasource.initialize(); - expect(FormBasedDatasource.toExpression(state, 'first', indexPatterns)).toEqual(null); + expect( + FormBasedDatasource.toExpression(state, 'first', indexPatterns, 'testing-seed') + ).toEqual(null); }); it('should create a table when there is a formula without aggs', async () => { @@ -338,7 +340,9 @@ describe('IndexPattern Data Source', () => { }, }, }; - expect(FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns)).toEqual({ + expect( + FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns, 'testing-seed') + ).toEqual({ chain: [ { function: 'createTable', @@ -385,8 +389,9 @@ describe('IndexPattern Data Source', () => { }, }; - expect(FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns)) - .toMatchInlineSnapshot(` + expect( + FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns, 'testing-seed') + ).toMatchInlineSnapshot(` Object { "chain": Array [ Object { @@ -487,6 +492,12 @@ describe('IndexPattern Data Source', () => { "partialRows": Array [ false, ], + "probability": Array [ + 1, + ], + "samplerSeed": Array [ + 1889181588, + ], "timeFields": Array [ "timestamp", ], @@ -560,7 +571,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.timeFields).toEqual(['timestamp', 'another_datefield']); }); @@ -595,7 +611,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect((ast.chain[1].arguments.aggs[1] as Ast).chain[0].arguments.timeShift).toEqual(['1d']); }); @@ -802,7 +823,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const count = (ast.chain[1].arguments.aggs[1] as Ast).chain[0]; const sum = (ast.chain[1].arguments.aggs[2] as Ast).chain[0]; const average = (ast.chain[1].arguments.aggs[3] as Ast).chain[0]; @@ -866,7 +892,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.aggs[0]).toMatchInlineSnapshot(` Object { "chain": Array [ @@ -990,7 +1021,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const timeScaleCalls = ast.chain.filter((fn) => fn.function === 'lens_time_scale'); const formatCalls = ast.chain.filter((fn) => fn.function === 'lens_format_column'); expect(timeScaleCalls).toHaveLength(1); @@ -1055,7 +1091,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const filteredMetricAgg = (ast.chain[1].arguments.aggs[0] as Ast).chain[0].arguments; const metricAgg = (filteredMetricAgg.customMetric[0] as Ast).chain[0].arguments; const bucketAgg = (filteredMetricAgg.customBucket[0] as Ast).chain[0].arguments; @@ -1106,7 +1147,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const formatIndex = ast.chain.findIndex((fn) => fn.function === 'lens_format_column'); const calculationIndex = ast.chain.findIndex((fn) => fn.function === 'moving_average'); expect(calculationIndex).toBeLessThan(formatIndex); @@ -1154,7 +1200,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.metricsAtAllLevels).toEqual([false]); expect(JSON.parse(ast.chain[2].arguments.idMap[0] as string)).toEqual({ 'col-0-0': [expect.objectContaining({ id: 'bucket1' })], @@ -1193,7 +1244,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.timeFields).toEqual(['timestamp']); expect(ast.chain[1].arguments.timeFields).not.toContain('timefield'); }); @@ -1250,7 +1306,7 @@ describe('IndexPattern Data Source', () => { const optimizeMock = jest.spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs'); - FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns); + FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns, 'testing-seed'); expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); @@ -1318,7 +1374,12 @@ describe('IndexPattern Data Source', () => { return { aggs: aggs.reverse(), esAggsIdMap }; }); - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); @@ -1382,7 +1443,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const idMap = JSON.parse(ast.chain[2].arguments.idMap as unknown as string); @@ -1487,7 +1553,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; // @ts-expect-error we can't isolate just the reference type expect(operationDefinitionMap.testReference.toExpression).toHaveBeenCalled(); expect(ast.chain[3]).toEqual('mock'); @@ -1520,7 +1591,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(JSON.parse(ast.chain[2].arguments.idMap[0] as string)).toEqual({ 'col-0-0': [ @@ -1607,7 +1683,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const chainLength = ast.chain.length; expect(ast.chain[chainLength - 2].arguments.name).toEqual(['math']); expect(ast.chain[chainLength - 1].arguments.id).toEqual(['formula']); @@ -1631,6 +1712,7 @@ describe('IndexPattern Data Source', () => { }, }, currentIndexPatternId: '1', + sampling: 1, }; expect(FormBasedDatasource.insertLayer(state, 'newLayer', ['link-to-id'])).toEqual({ ...state, @@ -1640,6 +1722,7 @@ describe('IndexPattern Data Source', () => { indexPatternId: '1', columnOrder: [], columns: {}, + sampling: 1, linkToLayers: ['link-to-id'], }, }, diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx index 62c77ec5225fe..f772e432268c2 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx @@ -36,6 +36,7 @@ import type { IndexPatternField, IndexPattern, IndexPatternRef, + DatasourceLayerSettingsProps, } from '../../types'; import { changeIndexPattern, @@ -96,6 +97,7 @@ import { getStateTimeShiftWarningMessages } from './time_shift_utils'; import { getPrecisionErrorWarningMessages } from './utils'; import { DOCUMENT_FIELD_NAME } from '../../../common/constants'; import { isColumnOfType } from './operations/definitions/helpers'; +import { LayerSettingsPanel } from './layer_settings'; import { FormBasedLayer } from '../..'; export type { OperationType, GenericIndexPatternColumn } from './operations'; export { deleteColumn } from './operations'; @@ -392,8 +394,34 @@ export function getFormBasedDatasource({ return fields; }, - toExpression: (state, layerId, indexPatterns) => - toExpression(state, layerId, indexPatterns, uiSettings), + toExpression: (state, layerId, indexPatterns, searchSessionId) => + toExpression(state, layerId, indexPatterns, uiSettings, searchSessionId), + + renderLayerSettings( + domElement: Element, + props: DatasourceLayerSettingsProps + ) { + render( + + + + + + + , + domElement + ); + }, renderDataPanel(domElement: Element, props: DatasourceDataPanelProps) { const { onChangeIndexPattern, ...otherProps } = props; @@ -569,6 +597,22 @@ export function getFormBasedDatasource({ getDropProps, onDrop, + getSupportedActionsForLayer(layerId, state, _, openLayerSettings) { + if (!openLayerSettings) { + return []; + } + return [ + { + displayName: i18n.translate('xpack.lens.indexPattern.layerSettingsAction', { + defaultMessage: 'Layer settings', + }), + execute: openLayerSettings, + icon: 'gear', + isCompatible: Boolean(state.layers[layerId]), + 'data-test-subj': 'lnsLayerSettings', + }, + ]; + }, getCustomWorkspaceRenderer: ( state: FormBasedPrivateState, @@ -931,5 +975,6 @@ function blankLayer(indexPatternId: string, linkToLayers?: string[]): FormBasedL linkToLayers, columns: {}, columnOrder: [], + sampling: 1, }; } diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx index 2489659f0da55..eca0c032ee224 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx @@ -176,6 +176,7 @@ function testInitialState(): FormBasedPrivateState { currentIndexPatternId: '1', layers: { first: { + sampling: 1, indexPatternId: '1', columnOrder: ['col1'], columns: { @@ -458,7 +459,7 @@ describe('IndexPattern Data Source suggestions', () => { }); describe('with a previous empty layer', () => { - function stateWithEmptyLayer() { + function stateWithEmptyLayer(): FormBasedPrivateState { const state = testInitialState(); return { ...state, @@ -761,6 +762,35 @@ describe('IndexPattern Data Source suggestions', () => { }) ); }); + + it('should inherit the sampling rate when generating new layer, if avaialble', () => { + const state = stateWithEmptyLayer(); + state.layers.previousLayer.sampling = 0.001; + const suggestions = getDatasourceSuggestionsForField( + state, + '1', + { + name: 'bytes', + displayName: 'bytes', + type: 'number', + aggregatable: true, + searchable: true, + }, + expectedIndexPatterns + ); + + expect(suggestions).toContainEqual( + expect.objectContaining({ + state: expect.objectContaining({ + layers: { + previousLayer: expect.objectContaining({ + sampling: 0.001, + }), + }, + }), + }) + ); + }); }); describe('suggesting extensions to non-empty tables', () => { diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts index 52008f10bcdeb..81ce81bb49053 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts @@ -529,6 +529,12 @@ function getEmptyLayerSuggestionsForField( newLayer = createNewLayerWithMetricAggregation(indexPattern, field); } + // copy the sampling rate to the new layer + // or just default to 1 + if (newLayer) { + newLayer.sampling = state.layers[layerId]?.sampling ?? 1; + } + const newLayerSuggestions = newLayer ? [ buildSuggestion({ diff --git a/x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx b/x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx new file mode 100644 index 0000000000000..7d02ac98f23a4 --- /dev/null +++ b/x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFormRow, EuiRange, EuiBetaBadge } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import type { DatasourceLayerSettingsProps } from '../../types'; +import type { FormBasedPrivateState } from './types'; + +const samplingValue = [0.0001, 0.001, 0.01, 0.1, 1]; + +export function LayerSettingsPanel({ + state, + setState, + layerId, +}: DatasourceLayerSettingsProps) { + const samplingIndex = samplingValue.findIndex((v) => v === state.layers[layerId].sampling); + const currentSamplingIndex = samplingIndex > -1 ? samplingIndex : samplingValue.length - 1; + return ( + + {i18n.translate('xpack.lens.xyChart.randomSampling.label', { + defaultMessage: 'Sampling', + })}{' '} + + + } + > + { + setState({ + ...state, + layers: { + ...state.layers, + [layerId]: { + ...state.layers[layerId], + sampling: samplingValue[Number(e.currentTarget.value)], + }, + }, + }); + }} + showInput={false} + showRange={false} + showTicks + step={1} + min={0} + max={samplingValue.length - 1} + ticks={samplingValue.map((v, i) => ({ label: `${v}`, value: i }))} + /> + + ); +} diff --git a/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts b/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts index fc77aa6520bd0..365d80a3d8285 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts @@ -7,6 +7,7 @@ import type { IUiSettingsClient } from '@kbn/core/public'; import { partition, uniq } from 'lodash'; +import seedrandom from 'seedrandom'; import { AggFunctionsMapping, EsaggsExpressionFunctionDefinition, @@ -52,7 +53,8 @@ const updatePositionIndex = (currentId: string, newIndex: number) => { function getExpressionForLayer( layer: FormBasedLayer, indexPattern: IndexPattern, - uiSettings: IUiSettingsClient + uiSettings: IUiSettingsClient, + searchSessionId?: string ): ExpressionAstExpression | null { const { columnOrder } = layer; if (columnOrder.length === 0 || !indexPattern) { @@ -392,6 +394,8 @@ function getExpressionForLayer( metricsAtAllLevels: false, partialRows: false, timeFields: allDateHistogramFields, + probability: layer.sampling || 1, + samplerSeed: seedrandom(searchSessionId).int32(), }).toAst(), { type: 'function', @@ -441,13 +445,15 @@ export function toExpression( state: FormBasedPrivateState, layerId: string, indexPatterns: IndexPatternMap, - uiSettings: IUiSettingsClient + uiSettings: IUiSettingsClient, + searchSessionId?: string ) { if (state.layers[layerId]) { return getExpressionForLayer( state.layers[layerId], indexPatterns[state.layers[layerId].indexPatternId], - uiSettings + uiSettings, + searchSessionId ); } diff --git a/x-pack/plugins/lens/public/datasources/form_based/types.ts b/x-pack/plugins/lens/public/datasources/form_based/types.ts index 3c695d5064e3f..0846c96d76dc8 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/types.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/types.ts @@ -54,6 +54,7 @@ export interface FormBasedLayer { linkToLayers?: string[]; // Partial columns represent the temporary invalid states incompleteColumns?: Record; + sampling?: number; } export interface FormBasedPersistedState { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss index b8ae7fa515190..7b74d0e966410 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss @@ -1,52 +1,3 @@ -@import '@elastic/eui/src/components/flyout/variables'; -@import '@elastic/eui/src/components/flyout/mixins'; - -.lnsDimensionContainer { - // Use the EuiFlyout style - @include euiFlyout; - // But with custom positioning to keep it within the sidebar contents - animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; - left: 0; - max-width: none !important; - z-index: $euiZContentMenu; - - @include euiBreakpoint('l', 'xl') { - height: 100% !important; - position: absolute; - top: 0 !important; - } - - .lnsFrameLayout__sidebar-isFullscreen & { - border-left: $euiBorderThin; // Force border regardless of theme in fullscreen - box-shadow: none; - } -} - -.lnsDimensionContainer__header { - padding: $euiSize; - - .lnsFrameLayout__sidebar-isFullscreen & { - display: none; - } -} - -.lnsDimensionContainer__content { - @include euiYScroll; - flex: 1; -} - -.lnsDimensionContainer__footer { - padding: $euiSize; - - .lnsFrameLayout__sidebar-isFullscreen & { - display: none; - } -} - -.lnsBody--overflowHidden { - overflow: hidden; -} - .lnsLayerAddButton:hover { text-decoration: none; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index 9d71e74eff473..13ba032f9b902 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -7,44 +7,12 @@ import './dimension_container.scss'; -import React, { useState, useEffect, useCallback } from 'react'; -import { - EuiFlyoutHeader, - EuiFlyoutFooter, - EuiTitle, - EuiButtonIcon, - EuiButtonEmpty, - EuiFlexGroup, - EuiFlexItem, - EuiFocusTrap, -} from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import { DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS } from '../../../utils'; - -function fromExcludedClickTarget(event: Event) { - for ( - let node: HTMLElement | null = event.target as HTMLElement; - node !== null; - node = node!.parentElement - ) { - if ( - node.classList!.contains(DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS) || - node.classList!.contains('euiBody-hasPortalContent') || - node.getAttribute('data-euiportal') === 'true' - ) { - return true; - } - } - return false; -} +import React from 'react'; +import { FlyoutContainer } from './flyout_container'; export function DimensionContainer({ - isOpen, - groupLabel, - handleClose, panel, - isFullscreen, - panelRef, + ...props }: { isOpen: boolean; handleClose: () => boolean; @@ -53,107 +21,5 @@ export function DimensionContainer({ isFullscreen: boolean; panelRef: (el: HTMLDivElement) => void; }) { - const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); - - const closeFlyout = useCallback(() => { - const canClose = handleClose(); - if (canClose) { - setFocusTrapIsEnabled(false); - } - return canClose; - }, [handleClose]); - - useEffect(() => { - document.body.classList.toggle('lnsBody--overflowHidden', isOpen); - return () => { - if (isOpen) { - setFocusTrapIsEnabled(false); - } - document.body.classList.remove('lnsBody--overflowHidden'); - }; - }, [isOpen]); - - if (!isOpen) { - return null; - } - - return ( -
- { - if (isFullscreen || fromExcludedClickTarget(event)) { - return; - } - closeFlyout(); - }} - onEscapeKey={closeFlyout} - > -
{ - if (isOpen) { - // EuiFocusTrap interferes with animating elements with absolute position: - // running this onAnimationEnd, otherwise the flyout pushes content when animating - setFocusTrapIsEnabled(true); - } - }} - > - - - - -

- - {i18n.translate('xpack.lens.configure.configurePanelTitle', { - defaultMessage: '{groupLabel}', - values: { - groupLabel, - }, - })} - -

- - - - - - - - - -
{panel}
- - - - {i18n.translate('xpack.lens.dimensionContainer.close', { - defaultMessage: 'Close', - })} - - -
- -
- ); + return {panel}; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss new file mode 100644 index 0000000000000..b08eb6281fa0e --- /dev/null +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss @@ -0,0 +1,48 @@ +@import '@elastic/eui/src/components/flyout/variables'; +@import '@elastic/eui/src/components/flyout/mixins'; + +.lnsDimensionContainer { + // Use the EuiFlyout style + @include euiFlyout; + // But with custom positioning to keep it within the sidebar contents + animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; + left: 0; + max-width: none !important; + z-index: $euiZContentMenu; + + @include euiBreakpoint('l', 'xl') { + height: 100% !important; + position: absolute; + top: 0 !important; + } + + .lnsFrameLayout__sidebar-isFullscreen & { + border-left: $euiBorderThin; // Force border regardless of theme in fullscreen + box-shadow: none; + } +} + +.lnsDimensionContainer__header { + padding: $euiSize; + + .lnsFrameLayout__sidebar-isFullscreen & { + display: none; + } +} + +.lnsDimensionContainer__content { + @include euiYScroll; + flex: 1; +} + +.lnsDimensionContainer__footer { + padding: $euiSize; + + .lnsFrameLayout__sidebar-isFullscreen & { + display: none; + } +} + +.lnsBody--overflowHidden { + overflow: hidden; +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx new file mode 100644 index 0000000000000..041f59df332f4 --- /dev/null +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import './flyout_container.scss'; + +import React, { useState, useEffect, useCallback } from 'react'; +import { + EuiFlyoutHeader, + EuiFlyoutFooter, + EuiTitle, + EuiButtonIcon, + EuiButtonEmpty, + EuiFlexGroup, + EuiFlexItem, + EuiFocusTrap, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS } from '../../../utils'; + +function fromExcludedClickTarget(event: Event) { + for ( + let node: HTMLElement | null = event.target as HTMLElement; + node !== null; + node = node!.parentElement + ) { + if ( + node.classList!.contains(DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS) || + node.classList!.contains('euiBody-hasPortalContent') || + node.getAttribute('data-euiportal') === 'true' + ) { + return true; + } + } + return false; +} + +export function FlyoutContainer({ + isOpen, + groupLabel, + handleClose, + isFullscreen, + panelRef, + children, +}: { + isOpen: boolean; + handleClose: () => boolean; + children: React.ReactElement | null; + groupLabel: string; + isFullscreen: boolean; + panelRef: (el: HTMLDivElement) => void; +}) { + const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); + + const closeFlyout = useCallback(() => { + const canClose = handleClose(); + if (canClose) { + setFocusTrapIsEnabled(false); + } + return canClose; + }, [handleClose]); + + useEffect(() => { + document.body.classList.toggle('lnsBody--overflowHidden', isOpen); + return () => { + if (isOpen) { + setFocusTrapIsEnabled(false); + } + document.body.classList.remove('lnsBody--overflowHidden'); + }; + }, [isOpen]); + + if (!isOpen) { + return null; + } + + return ( +
+ { + if (isFullscreen || fromExcludedClickTarget(event)) { + return; + } + closeFlyout(); + }} + onEscapeKey={closeFlyout} + > +
{ + if (isOpen) { + // EuiFocusTrap interferes with animating elements with absolute position: + // running this onAnimationEnd, otherwise the flyout pushes content when animating + setFocusTrapIsEnabled(true); + } + }} + > + + + + +

+ {i18n.translate('xpack.lens.configure.configurePanelTitle', { + defaultMessage: '{groupLabel}', + values: { + groupLabel, + }, + })} +

+ + + + + + + + + +
{children}
+ + + + {i18n.translate('xpack.lens.dimensionContainer.close', { + defaultMessage: 'Close', + })} + + +
+ +
+ ); +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx index bc1c41caa650b..9c32a24eac4dd 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx @@ -104,6 +104,9 @@ const InContextMenuActions = (props: LayerActionsProps) => { closePopover={closePopover} panelPaddingSize="none" anchorPosition="downLeft" + panelProps={{ + 'data-test-subj': 'lnsLayerActionsMenu', + }} > ( initialActiveDimensionState ); + const [isPanelSettingsOpen, setPanelSettingsOpen] = useState(false); + const [hideTooltip, setHideTooltip] = useState(false); const { @@ -120,6 +123,7 @@ export function LayerPanel( }, [activeVisualization.id]); const panelRef = useRef(null); + const settingsPanelRef = useRef(null); const registerLayerRef = useCallback( (el) => registerNewLayerRef(layerId, el), [layerId, registerNewLayerRef] @@ -316,7 +320,14 @@ export function LayerPanel( ...(activeVisualization.getSupportedActionsForLayer?.( layerId, visualizationState, - updateVisualization + updateVisualization, + () => setPanelSettingsOpen(true) + ) || []), + ...(layerDatasource?.getSupportedActionsForLayer?.( + layerId, + layerDatasourceState, + (newState) => updateDatasource(datasourceId, newState), + () => setPanelSettingsOpen(true) ) || []), ...getSharedActions({ activeVisualization, @@ -332,12 +343,16 @@ export function LayerPanel( [ activeVisualization, core, + datasourceId, isOnlyLayer, isTextBasedLanguage, + layerDatasource, + layerDatasourceState, layerId, layerIndex, onCloneLayer, onRemoveLayer, + updateDatasource, updateVisualization, visualizationState, ] @@ -624,7 +639,42 @@ export function LayerPanel( })} - + {(layerDatasource?.renderLayerSettings || activeVisualization?.renderLayerSettings) && ( + (settingsPanelRef.current = el)} + isOpen={isPanelSettingsOpen} + isFullscreen={false} + groupLabel={i18n.translate('xpack.lens.editorFrame.layerSettingsTitle', { + defaultMessage: 'Layer settings', + })} + handleClose={() => { + // update the current layer settings + setPanelSettingsOpen(false); + return true; + }} + > +
+
+ {layerDatasource?.renderLayerSettings && ( + + )} + {activeVisualization?.renderLayerSettings && ( + + )} +
+
+ + )} (panelRef.current = el)} isOpen={isDimensionPanelOpen} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts index c9b0358b81a0b..aee10196d5156 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts @@ -11,7 +11,8 @@ import { Visualization, DatasourceMap, DatasourceLayers, IndexPatternMap } from export function getDatasourceExpressionsByLayers( datasourceMap: DatasourceMap, datasourceStates: DatasourceStates, - indexPatterns: IndexPatternMap + indexPatterns: IndexPatternMap, + searchSessionId?: string ): null | Record { const datasourceExpressions: Array<[string, Ast | string]> = []; @@ -24,7 +25,7 @@ export function getDatasourceExpressionsByLayers( const layers = datasource.getLayers(state); layers.forEach((layerId) => { - const result = datasource.toExpression(state, layerId, indexPatterns); + const result = datasource.toExpression(state, layerId, indexPatterns, searchSessionId); if (result) { datasourceExpressions.push([layerId, result]); } @@ -53,6 +54,7 @@ export function buildExpression({ title, description, indexPatterns, + searchSessionId, }: { title?: string; description?: string; @@ -62,6 +64,7 @@ export function buildExpression({ datasourceStates: DatasourceStates; datasourceLayers: DatasourceLayers; indexPatterns: IndexPatternMap; + searchSessionId?: string; }): Ast | null { if (visualization === null) { return null; @@ -70,7 +73,8 @@ export function buildExpression({ const datasourceExpressionsByLayers = getDatasourceExpressionsByLayers( datasourceMap, datasourceStates, - indexPatterns + indexPatterns, + searchSessionId ); const visualizationExpression = visualization.toExpression( diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 38f62b6e928b6..2f2003970171a 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -162,6 +162,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ const changesApplied = useLensSelector(selectChangesApplied); const triggerApply = useLensSelector(selectTriggerApplyChanges); const datasourceLayers = useLensSelector((state) => selectDatasourceLayers(state, datasourceMap)); + const searchSessionId = useLensSelector(selectSearchSessionId); const [localState, setLocalState] = useState({ expressionBuildError: undefined, @@ -317,6 +318,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ datasourceStates, datasourceLayers, indexPatterns: dataViews.indexPatterns, + searchSessionId, }); if (ast) { @@ -349,16 +351,17 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ })); } }, [ + configurationValidationError?.length, + missingRefsErrors.length, + unknownVisError, activeVisualization, visualization.state, + visualization.activeId, datasourceMap, datasourceStates, datasourceLayers, - configurationValidationError?.length, - missingRefsErrors.length, - unknownVisError, - visualization.activeId, dataViews.indexPatterns, + searchSessionId, ]); useEffect(() => { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index e01c35f3457d9..dd5d12809c94a 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -301,6 +301,10 @@ export interface Datasource { }) => T; getSelectedFields?: (state: T) => string[]; + renderLayerSettings?: ( + domElement: Element, + props: DatasourceLayerSettingsProps + ) => ((cleanupElement: Element) => void) | void; renderDataPanel: ( domElement: Element, props: DatasourceDataPanelProps @@ -360,7 +364,8 @@ export interface Datasource { toExpression: ( state: T, layerId: string, - indexPatterns: IndexPatternMap + indexPatterns: IndexPatternMap, + searchSessionId?: string ) => ExpressionAstExpression | string | null; getDatasourceSuggestionsForField: ( @@ -458,6 +463,13 @@ export interface Datasource { * Get all the used DataViews from state */ getUsedDataViews: (state: T) => string[]; + + getSupportedActionsForLayer?: ( + layerId: string, + state: T, + setState: StateSetter, + openLayerSettings?: () => void + ) => LayerAction[]; } export interface DatasourceFixAction { @@ -509,6 +521,12 @@ export interface DatasourcePublicAPI { hasDefaultTimeField: () => boolean; } +export interface DatasourceLayerSettingsProps { + layerId: string; + state: T; + setState: StateSetter; +} + export interface DatasourceDataPanelProps { state: T; dragDropContext: DragContextState; @@ -715,6 +733,11 @@ export interface VisualizationToolbarProps { state: T; } +export type VisualizationLayerSettingsProps = VisualizationConfigProps & { + setState(newState: T | ((currState: T) => T)): void; + panelRef: MutableRefObject; +}; + export type VisualizationDimensionEditorProps = VisualizationConfigProps & { groupId: string; accessor: string; @@ -1010,7 +1033,8 @@ export interface Visualization { getSupportedActionsForLayer?: ( layerId: string, state: T, - setState: StateSetter + setState: StateSetter, + openLayerSettings?: () => void ) => LayerAction[]; /** returns the type string of the given layer */ getLayerType: (layerId: string, state?: T) => LayerType | undefined; @@ -1090,6 +1114,11 @@ export interface Visualization { dropProps: GetDropPropsArgs ) => { dropTypes: DropType[]; nextLabel?: string } | undefined; + renderLayerSettings?: ( + domElement: Element, + props: VisualizationLayerSettingsProps + ) => ((cleanupElement: Element) => void) | void; + /** * Additional editor that gets rendered inside the dimension popover. * This can be used to configure dimension-specific options diff --git a/x-pack/test/functional/apps/lens/group1/index.ts b/x-pack/test/functional/apps/lens/group1/index.ts index aa2b078a50a6b..47f08a59e7341 100644 --- a/x-pack/test/functional/apps/lens/group1/index.ts +++ b/x-pack/test/functional/apps/lens/group1/index.ts @@ -79,6 +79,7 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext loadTestFile(require.resolve('./table_dashboard')); loadTestFile(require.resolve('./table')); loadTestFile(require.resolve('./text_based_languages')); + loadTestFile(require.resolve('./layer_actions')); } }); }; diff --git a/x-pack/test/functional/apps/lens/group1/layer_actions.ts b/x-pack/test/functional/apps/lens/group1/layer_actions.ts new file mode 100644 index 0000000000000..be22e4ad62511 --- /dev/null +++ b/x-pack/test/functional/apps/lens/group1/layer_actions.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const find = getService('find'); + const testSubjects = getService('testSubjects'); + + describe('lens layer actions tests', () => { + it('should allow creation of lens xy chart', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.openLayerContextMenu(); + + // should be 3 actions available + expect( + (await find.allByCssSelector('[data-test-subj=lnsLayerActionsMenu] button')).length + ).to.eql(3); + }); + + it('should open layer settings for a data layer', async () => { + // click on open layer settings + await testSubjects.click('lnsLayerSettings'); + // random sampling available + await testSubjects.existOrFail('lns-indexPattern-random-sampling-row'); + // tweak the value + await PageObjects.lens.dragRangeInput('lns-indexPattern-random-sampling', 2, 'left'); + + expect(await PageObjects.lens.getRangeInputValue('lns-indexPattern-random-sampling')).to.eql( + 2 // 0.01 + ); + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + }); + + it('should add an annotation layer and settings shoud not be available', async () => { + // configure a date histogram + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'average', + field: 'bytes', + }); + // add annotation layer + await testSubjects.click('lnsLayerAddButton'); + await testSubjects.click(`lnsLayerAddButton-annotations`); + await PageObjects.lens.openLayerContextMenu(1); + // layer settings not available + await testSubjects.missingOrFail('lnsLayerSettings'); + }); + + it('should switch to pie chart and have layer settings available', async () => { + await PageObjects.lens.switchToVisualization('pie'); + await PageObjects.lens.openLayerContextMenu(); + // layer settings still available + // open the panel + await testSubjects.click('lnsLayerSettings'); + // check the sampling value + expect(await PageObjects.lens.getRangeInputValue('lns-indexPattern-random-sampling')).to.eql( + 2 // 0.01 + ); + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + }); + + it('should switch to table and still have layer settings', async () => { + await PageObjects.lens.switchToVisualization('lnsDatatable'); + await PageObjects.lens.openLayerContextMenu(); + // layer settings still available + // open the panel + await testSubjects.click('lnsLayerSettings'); + // check the sampling value + expect(await PageObjects.lens.getRangeInputValue('lns-indexPattern-random-sampling')).to.eql( + 2 // 0.01 + ); + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index a336a0da3e4ba..8dd95aa107929 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -600,6 +600,19 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await this.waitForVisualization(); }, + async dragRangeInput(testId: string, steps: number = 1, direction: 'left' | 'right' = 'right') { + const inputEl = await testSubjects.find(testId); + await inputEl.focus(); + const browserKey = direction === 'left' ? browser.keys.LEFT : browser.keys.RIGHT; + while (steps--) { + await browser.pressKeys(browserKey); + } + }, + + async getRangeInputValue(testId: string) { + return (await testSubjects.find(testId)).getAttribute('value'); + }, + async isTopLevelAggregation() { return await testSubjects.isEuiSwitchChecked('indexPattern-nesting-switch'); }, @@ -1117,6 +1130,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ); }, + async openLayerContextMenu(index: number = 0) { + await testSubjects.click(`lnsLayerSplitButton--${index}`); + }, + async toggleColumnVisibility(dimension: string, no = 1) { await this.openDimensionEditor(dimension); const id = 'lns-table-column-hidden';