From f42ad16408d2bfa420b5acbef12377ad5085bb65 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Thu, 29 Sep 2022 16:43:15 -0500 Subject: [PATCH 1/4] Add functional tests for random sampler controls --- .../document_count_content.tsx | 12 ++- .../constants/random_sampler.ts | 9 ++- .../apps/ml/data_visualizer/index.ts | 1 + .../index_data_visualizer_random_sampler.ts | 80 +++++++++++++++++++ .../ml/data_visualizer_index_based.ts | 72 +++++++++++++++++ 5 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx index 911eb851924e3..a2eb458bbdf46 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx @@ -110,7 +110,7 @@ export const DocumentCountContent: FC = ({ const ProbabilityUsed = randomSamplerPreference !== RANDOM_SAMPLER_OPTION.OFF && isDefined(samplingProbability) ? ( - <> +
= ({ defaultMessage="Probability used: {samplingProbability}%" values={{ samplingProbability: samplingProbability * 100 }} /> - +
) : null; return ( @@ -127,7 +127,8 @@ export const DocumentCountContent: FC = ({ = ({ size="xs" iconType="gear" onClick={onShowSamplingOptions} - data-test-subj="discoverSamplingOptionsToggle" + data-test-subj="dvRandomSamplerOptionsButton" aria-label={i18n.translate('xpack.dataVisualizer.samplingOptionsButton', { defaultMessage: 'Sampling options', })} @@ -157,6 +158,7 @@ export const DocumentCountContent: FC = ({ = ({ )} > @@ -212,6 +215,7 @@ export const DocumentCountContent: FC = ({ } }} step={RANDOM_SAMPLER_STEP} + data-test-subj="dvRandomSamplerProbabilityRange" /> diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/constants/random_sampler.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/constants/random_sampler.ts index c2188bab87fe5..310ccde5f9e29 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/constants/random_sampler.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/constants/random_sampler.ts @@ -24,20 +24,27 @@ export const RANDOM_SAMPLER_OPTION = { export type RandomSamplerOption = typeof RANDOM_SAMPLER_OPTION[keyof typeof RANDOM_SAMPLER_OPTION]; -export const RANDOM_SAMPLER_SELECT_OPTIONS: Array<{ value: RandomSamplerOption; text: string }> = [ +export const RANDOM_SAMPLER_SELECT_OPTIONS: Array<{ + value: RandomSamplerOption; + text: string; + 'data-test-subj': string; +}> = [ { + 'data-test-subj': 'dvRandomSamplerOptionOnAutomatic', value: RANDOM_SAMPLER_OPTION.ON_AUTOMATIC, text: i18n.translate('xpack.dataVisualizer.randomSamplerPreference.onAutomaticLabel', { defaultMessage: 'On - automatic', }), }, { + 'data-test-subj': 'dvRandomSamplerOptionOnManual', value: RANDOM_SAMPLER_OPTION.ON_MANUAL, text: i18n.translate('xpack.dataVisualizer.randomSamplerPreference.onManualLabel', { defaultMessage: 'On - manual', }), }, { + 'data-test-subj': 'dvRandomSamplerOptionOff', value: RANDOM_SAMPLER_OPTION.OFF, text: i18n.translate('xpack.dataVisualizer.randomSamplerPreference.offLabel', { defaultMessage: 'Off', diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index.ts b/x-pack/test/functional/apps/ml/data_visualizer/index.ts index ab14b7f3c86c0..13ed76a002ca6 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index.ts @@ -33,6 +33,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { }); loadTestFile(require.resolve('./index_data_visualizer')); + loadTestFile(require.resolve('./index_data_visualizer_random_sampler')); loadTestFile(require.resolve('./index_data_visualizer_filters')); loadTestFile(require.resolve('./index_data_visualizer_grid_in_discover')); loadTestFile(require.resolve('./index_data_visualizer_grid_in_dashboard')); diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts new file mode 100644 index 0000000000000..00efec9cca818 --- /dev/null +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { farequoteDataViewTestData, farequoteLuceneSearchTestData } from './index_test_data'; + +export default function ({ getPageObject, getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + const browser = getService('browser'); + async function goToSourceForIndexBasedDataVisualizer(sourceIndexOrSavedSearch: string) { + await ml.testExecution.logTestStep(`navigates to Data Visualizer page`); + await ml.navigation.navigateToDataVisualizer(); + + await ml.testExecution.logTestStep(`loads the saved search selection page`); + await ml.dataVisualizer.navigateToIndexPatternSelection(); + + await ml.testExecution.logTestStep(`loads the index data visualizer page`); + await ml.jobSourceSelection.selectSourceForIndexBasedDataVisualizer(sourceIndexOrSavedSearch); + } + describe('index based random sampler controls', function () { + this.tags(['ml']); + before(async () => { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/module_sample_logs'); + + await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.createIndexPatternIfNeeded('ft_module_sample_logs', '@timestamp'); + await ml.testResources.createSavedSearchFarequoteLuceneIfNeeded(); + await ml.testResources.setKibanaTimeZoneToUTC(); + + await ml.securityUI.loginAsMlPowerUser(); + // Start navigation from the base of the ML app. + await ml.navigation.navigateToMl(); + }); + + after(async () => { + await ml.testResources.deleteSavedSearches(); + await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_logs'); + await browser.removeLocalStorageItem('dataVisualizer.randomSamplerPreference'); + }); + + describe('with small data sets', function () { + it(`has random sampler 'on - automatic' by default`, async () => { + await goToSourceForIndexBasedDataVisualizer( + farequoteDataViewTestData.sourceIndexOrSavedSearch + ); + + await ml.dataVisualizerIndexBased.assertRandomSamplingOption( + 'dvRandomSamplerOptionOnAutomatic', + 100 + ); + }); + + it(`retains random sampler 'off' setting`, async () => { + await ml.dataVisualizerIndexBased.setRandomSamplingOption('dvRandomSamplerOptionOff'); + + await goToSourceForIndexBasedDataVisualizer( + farequoteLuceneSearchTestData.sourceIndexOrSavedSearch + ); + await ml.dataVisualizerIndexBased.assertRandomSamplingOption('dvRandomSamplerOptionOff'); + }); + + it(`retains random sampler 'on - manual' setting`, async () => { + await ml.dataVisualizerIndexBased.setRandomSamplingOption('dvRandomSamplerOptionOnManual'); + + await goToSourceForIndexBasedDataVisualizer('ft_module_sample_logs'); + await ml.dataVisualizerIndexBased.assertRandomSamplingOption( + 'dvRandomSamplerOptionOnManual', + 100 + ); + }); + }); + }); +} diff --git a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts index 0e1860de4dab9..1ba83280b8bd6 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts @@ -17,6 +17,7 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ const PageObjects = getPageObjects(['discover']); const queryBar = getService('queryBar'); const filterBar = getService('filterBar'); + const browser = getService('browser'); return { async assertTimeRangeSelectorSectionExists() { @@ -231,5 +232,76 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ } ); }, + + async assertRandomSamplingOptionsButtonExists() { + await testSubjects.existOrFail('dvRandomSamplerOptionsButton'); + }, + + async assertRandomSamplingOption( + expectedOption: + | 'dvRandomSamplerOptionOnAutomatic' + | 'dvRandomSamplerOptionOnManual' + | 'dvRandomSamplerOptionOff', + expectedProbability?: number + ) { + await browser.pressKeys(browser.keys.ESCAPE); + await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); + await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); + + if (expectedOption === 'dvRandomSamplerOptionOff') { + await testSubjects.existOrFail('dvRandomSamplerOptionOff'); + await testSubjects.missingOrFail('dvRandomSamplerProbabilityRange'); + await testSubjects.missingOrFail('dvRandomSamplerAutomaticProbabilityMsg'); + } + + if (expectedOption === 'dvRandomSamplerOptionOnManual') { + await testSubjects.existOrFail('dvRandomSamplerOptionOnManual'); + await testSubjects.existOrFail('dvRandomSamplerProbabilityRange'); + if (expectedProbability !== undefined) { + const probability = await testSubjects.getAttribute( + 'dvRandomSamplerProbabilityRange', + 'value' + ); + expect(probability).to.eql( + `${probability}`, + `Expected probability to be ${expectedProbability}, got ${probability}` + ); + } + } + + if (expectedOption === 'dvRandomSamplerOptionOnAutomatic') { + await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic'); + await testSubjects.existOrFail('dvRandomSamplerAutomaticProbabilityMsg'); + + if (expectedProbability !== undefined) { + const probabilityText = await testSubjects.getVisibleText( + 'dvRandomSamplerAutomaticProbabilityMsg' + ); + expect(probabilityText).to.contain( + `${expectedProbability}`, + `Expected probability to be ${expectedProbability}, got ${probabilityText}` + ); + } + } + }, + + async setRandomSamplingOption( + option: + | 'dvRandomSamplerOptionOnAutomatic' + | 'dvRandomSamplerOptionOnManual' + | 'dvRandomSamplerOptionOff' + ) { + // escape popover + await browser.pressKeys(browser.keys.ESCAPE); + await this.assertRandomSamplingOptionsButtonExists(); + await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); + await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); + + await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsSelect'); + await testSubjects.click(option); + await retry.tryForTime(2000, async () => { + await this.assertRandomSamplingOption(option); + }); + }, }; } From d30e8ced311828876c68650bff5f5c18c88b6f29 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 30 Sep 2022 09:50:21 -0500 Subject: [PATCH 2/4] Stabilize tests, ifx expected probability --- .../index_data_visualizer_random_sampler.ts | 2 +- .../ml/data_visualizer_index_based.ts | 94 ++++++++++--------- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts index 00efec9cca818..7df4e9c18eee7 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_random_sampler.ts @@ -72,7 +72,7 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await goToSourceForIndexBasedDataVisualizer('ft_module_sample_logs'); await ml.dataVisualizerIndexBased.assertRandomSamplingOption( 'dvRandomSamplerOptionOnManual', - 100 + 50 ); }); }); diff --git a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts index 1ba83280b8bd6..47242f165eaef 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts @@ -244,45 +244,47 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ | 'dvRandomSamplerOptionOff', expectedProbability?: number ) { - await browser.pressKeys(browser.keys.ESCAPE); - await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); - await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); - - if (expectedOption === 'dvRandomSamplerOptionOff') { - await testSubjects.existOrFail('dvRandomSamplerOptionOff'); - await testSubjects.missingOrFail('dvRandomSamplerProbabilityRange'); - await testSubjects.missingOrFail('dvRandomSamplerAutomaticProbabilityMsg'); - } - - if (expectedOption === 'dvRandomSamplerOptionOnManual') { - await testSubjects.existOrFail('dvRandomSamplerOptionOnManual'); - await testSubjects.existOrFail('dvRandomSamplerProbabilityRange'); - if (expectedProbability !== undefined) { - const probability = await testSubjects.getAttribute( - 'dvRandomSamplerProbabilityRange', - 'value' - ); - expect(probability).to.eql( - `${probability}`, - `Expected probability to be ${expectedProbability}, got ${probability}` - ); + await retry.tryForTime(2000, async () => { + await browser.pressKeys(browser.keys.ESCAPE); + await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); + await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); + + if (expectedOption === 'dvRandomSamplerOptionOff') { + await testSubjects.existOrFail('dvRandomSamplerOptionOff'); + await testSubjects.missingOrFail('dvRandomSamplerProbabilityRange'); + await testSubjects.missingOrFail('dvRandomSamplerAutomaticProbabilityMsg'); } - } - if (expectedOption === 'dvRandomSamplerOptionOnAutomatic') { - await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic'); - await testSubjects.existOrFail('dvRandomSamplerAutomaticProbabilityMsg'); + if (expectedOption === 'dvRandomSamplerOptionOnManual') { + await testSubjects.existOrFail('dvRandomSamplerOptionOnManual'); + await testSubjects.existOrFail('dvRandomSamplerProbabilityRange'); + if (expectedProbability !== undefined) { + const probability = await testSubjects.getAttribute( + 'dvRandomSamplerProbabilityRange', + 'value' + ); + expect(expectedProbability).to.eql( + `${probability}`, + `Expected probability to be ${expectedProbability}, got ${probability}` + ); + } + } - if (expectedProbability !== undefined) { - const probabilityText = await testSubjects.getVisibleText( - 'dvRandomSamplerAutomaticProbabilityMsg' - ); - expect(probabilityText).to.contain( - `${expectedProbability}`, - `Expected probability to be ${expectedProbability}, got ${probabilityText}` - ); + if (expectedOption === 'dvRandomSamplerOptionOnAutomatic') { + await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic'); + await testSubjects.existOrFail('dvRandomSamplerAutomaticProbabilityMsg'); + + if (expectedProbability !== undefined) { + const probabilityText = await testSubjects.getVisibleText( + 'dvRandomSamplerAutomaticProbabilityMsg' + ); + expect(probabilityText).to.contain( + `${expectedProbability}`, + `Expected probability text to contain ${expectedProbability}, got ${probabilityText}` + ); + } } - } + }); }, async setRandomSamplingOption( @@ -291,15 +293,21 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ | 'dvRandomSamplerOptionOnManual' | 'dvRandomSamplerOptionOff' ) { - // escape popover - await browser.pressKeys(browser.keys.ESCAPE); - await this.assertRandomSamplingOptionsButtonExists(); - await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); - await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); - - await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsSelect'); - await testSubjects.click(option); await retry.tryForTime(2000, async () => { + // escape popover + await browser.pressKeys(browser.keys.ESCAPE); + await this.assertRandomSamplingOptionsButtonExists(); + await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); + await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); + + await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsSelect'); + + await testSubjects.existOrFail('dvRandomSamplerOptionOff'); + await testSubjects.existOrFail('dvRandomSamplerOptionOnManual'); + await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic'); + + await testSubjects.click(option); + await this.assertRandomSamplingOption(option); }); }, From 91964466475af8eddaa364bacc027e4b80ae417f Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Fri, 30 Sep 2022 10:09:18 -0500 Subject: [PATCH 3/4] Fix prob --- .../functional/services/ml/data_visualizer_index_based.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts index 47242f165eaef..784144d6275d5 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts @@ -263,8 +263,8 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ 'dvRandomSamplerProbabilityRange', 'value' ); - expect(expectedProbability).to.eql( - `${probability}`, + expect(probability).to.eql( + `${expectedProbability}`, `Expected probability to be ${expectedProbability}, got ${probability}` ); } From 0b7d3d9cee24b2c16c91034533dcaf847a2176b6 Mon Sep 17 00:00:00 2001 From: Quynh Nguyen Date: Sun, 16 Oct 2022 20:06:52 -0500 Subject: [PATCH 4/4] Change timeouts & retry durations --- .../ml/data_visualizer_index_based.ts | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts index 784144d6275d5..600790294a539 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts @@ -244,20 +244,22 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ | 'dvRandomSamplerOptionOff', expectedProbability?: number ) { - await retry.tryForTime(2000, async () => { + await retry.tryForTime(20000, async () => { await browser.pressKeys(browser.keys.ESCAPE); await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); if (expectedOption === 'dvRandomSamplerOptionOff') { - await testSubjects.existOrFail('dvRandomSamplerOptionOff'); - await testSubjects.missingOrFail('dvRandomSamplerProbabilityRange'); - await testSubjects.missingOrFail('dvRandomSamplerAutomaticProbabilityMsg'); + await testSubjects.existOrFail('dvRandomSamplerOptionOff', { timeout: 1000 }); + await testSubjects.missingOrFail('dvRandomSamplerProbabilityRange', { timeout: 1000 }); + await testSubjects.missingOrFail('dvRandomSamplerAutomaticProbabilityMsg', { + timeout: 1000, + }); } if (expectedOption === 'dvRandomSamplerOptionOnManual') { - await testSubjects.existOrFail('dvRandomSamplerOptionOnManual'); - await testSubjects.existOrFail('dvRandomSamplerProbabilityRange'); + await testSubjects.existOrFail('dvRandomSamplerOptionOnManual', { timeout: 1000 }); + await testSubjects.existOrFail('dvRandomSamplerProbabilityRange', { timeout: 1000 }); if (expectedProbability !== undefined) { const probability = await testSubjects.getAttribute( 'dvRandomSamplerProbabilityRange', @@ -271,8 +273,10 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ } if (expectedOption === 'dvRandomSamplerOptionOnAutomatic') { - await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic'); - await testSubjects.existOrFail('dvRandomSamplerAutomaticProbabilityMsg'); + await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic', { timeout: 1000 }); + await testSubjects.existOrFail('dvRandomSamplerAutomaticProbabilityMsg', { + timeout: 1000, + }); if (expectedProbability !== undefined) { const probabilityText = await testSubjects.getVisibleText( @@ -293,18 +297,18 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ | 'dvRandomSamplerOptionOnManual' | 'dvRandomSamplerOptionOff' ) { - await retry.tryForTime(2000, async () => { + await retry.tryForTime(20000, async () => { // escape popover await browser.pressKeys(browser.keys.ESCAPE); await this.assertRandomSamplingOptionsButtonExists(); await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsButton'); - await testSubjects.existOrFail('dvRandomSamplerOptionsPopover'); + await testSubjects.existOrFail('dvRandomSamplerOptionsPopover', { timeout: 1000 }); await testSubjects.clickWhenNotDisabled('dvRandomSamplerOptionsSelect'); - await testSubjects.existOrFail('dvRandomSamplerOptionOff'); - await testSubjects.existOrFail('dvRandomSamplerOptionOnManual'); - await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic'); + await testSubjects.existOrFail('dvRandomSamplerOptionOff', { timeout: 1000 }); + await testSubjects.existOrFail('dvRandomSamplerOptionOnManual', { timeout: 1000 }); + await testSubjects.existOrFail('dvRandomSamplerOptionOnAutomatic', { timeout: 1000 }); await testSubjects.click(option);