Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.5] [ML] Adds extra a11y tests for anomaly detection and DFA jobs (#142589) #142710

Merged
merged 1 commit into from
Oct 5, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 193 additions & 10 deletions x-pack/test/accessibility/apps/ml.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,22 +60,36 @@ export default function ({ getService }: FtrProviderContext) {

describe('with data loaded', function () {
const adJobId = 'fq_single_a11y';
const dfaOutlierJobId = 'iph_outlier_a11y';
const dfaOutlierResultsJobId = 'iph_outlier_a11y';
const calendarId = 'calendar_a11y';
const eventDescription = 'calendar_event_a11y';
const filterId = 'filter_a11y';
const filterItems = ['filter_item_a11y'];
const fqIndexPattern = 'ft_farequote';
const ecIndexPattern = 'ft_module_sample_ecommerce';
const ihpIndexPattern = 'ft_ihp_outlier';
const egsIndexPattern = 'ft_egs_regression';
const bmIndexPattern = 'ft_bank_marketing';
const ecExpectedTotalCount = '287';

const adJobAggAndFieldIdentifier = 'Mean(responsetime)';
const adJobBucketSpan = '30m';
const adSingleMetricJobId = `fq_single_a11y_${Date.now()}`;

const dfaJobType = 'outlier_detection';
const dfaJobId = `ihp_ally_${Date.now()}`;
const adMultiSplitField = 'airline';
const adMultiMetricJobId = `fq_multi_a11y_${Date.now()}`;
const adMultiMetricJobDescription =
'Multi metric job based on the farequote dataset with 30m bucketspan and mean(responsetime) split by airline';

const dfaOutlierJobType = 'outlier_detection';
const dfaOutlierJobId = `ihp_outlier_ally_${Date.now()}`;
const dfaRegressionJobType = 'regression';
const dfaRegressionJobId = `egs_regression_ally_${Date.now()}`;
const dfaRegressionJobDepVar = 'stab';
const dfaRegressionJobTrainingPercent = 30;
const dfaClassificationJobType = 'classification';
const dfaClassificationJobId = `bm_classification_ally_${Date.now()}`;
const dfaClassificationJobDepVar = 'y';
const dfaClassificationJobTrainingPercent = 30;

const uploadFilePath = require.resolve(
'../../functional/apps/ml/data_visualizer/files_to_import/artificial_server_log'
Expand All @@ -84,11 +98,15 @@ export default function ({ getService }: FtrProviderContext) {
before(async () => {
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ihp_outlier');
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/egs_regression');
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification');
await esArchiver.loadIfNeeded(
'x-pack/test/functional/es_archives/ml/module_sample_ecommerce'
);
await ml.testResources.createIndexPatternIfNeeded(fqIndexPattern, '@timestamp');
await ml.testResources.createIndexPatternIfNeeded(ihpIndexPattern, '@timestamp');
await ml.testResources.createIndexPatternIfNeeded(ihpIndexPattern);
await ml.testResources.createIndexPatternIfNeeded(egsIndexPattern);
await ml.testResources.createIndexPatternIfNeeded(bmIndexPattern);
await ml.testResources.createIndexPatternIfNeeded(ecIndexPattern, 'order_date');
await ml.testResources.setKibanaTimeZoneToUTC();

Expand All @@ -98,7 +116,7 @@ export default function ({ getService }: FtrProviderContext) {
);

await ml.api.createAndRunDFAJob(
ml.commonConfig.getDFAIhpOutlierDetectionJobConfig(dfaOutlierJobId)
ml.commonConfig.getDFAIhpOutlierDetectionJobConfig(dfaOutlierResultsJobId)
);

await ml.api.createCalendar(calendarId, {
Expand All @@ -122,15 +140,19 @@ export default function ({ getService }: FtrProviderContext) {

after(async () => {
await ml.api.cleanMlIndices();
await ml.api.deleteIndices(`user-${dfaOutlierJobId}`);
await ml.api.deleteIndices(`user-${dfaOutlierResultsJobId}`);
await ml.api.deleteCalendar(calendarId);
await ml.api.deleteFilter(filterId);

await ml.testResources.deleteIndexPatternByTitle(fqIndexPattern);
await ml.testResources.deleteIndexPatternByTitle(ihpIndexPattern);
await ml.testResources.deleteIndexPatternByTitle(egsIndexPattern);
await ml.testResources.deleteIndexPatternByTitle(bmIndexPattern);
await ml.testResources.deleteIndexPatternByTitle(ecIndexPattern);
await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/ihp_outlier');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/egs_regression');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/bm_classification');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_sample_ecommerce');
await ml.testResources.resetKibanaTimeZone();
});
Expand Down Expand Up @@ -196,6 +218,55 @@ export default function ({ getService }: FtrProviderContext) {
await a11y.testAppSnapshot();
});

it('anomaly detection create multi metric job and move to time range step', async () => {
// Proceed all the way to the step for selecting the time range
// as the other steps have already been tested for the single metric job
await ml.navigation.navigateToAnomalyDetection();
await ml.jobManagement.navigateToNewJobSourceSelection();
await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob(fqIndexPattern);
await ml.jobTypeSelection.selectMultiMetricJob();
await ml.testExecution.logTestStep('job creation set the time range');
await ml.jobWizardCommon.clickUseFullDataButton(
'Feb 7, 2016 @ 00:00:00.000',
'Feb 11, 2016 @ 23:59:54.000'
);
await a11y.testAppSnapshot();
});

it('anomaly detection create multi metric job pick fields step', async () => {
await ml.jobWizardCommon.advanceToPickFieldsSection();
await ml.testExecution.logTestStep('job creation selects field and aggregation');
await ml.jobWizardCommon.selectAggAndField(adJobAggAndFieldIdentifier, false);
await ml.testExecution.logTestStep('job creation selects split field');
await ml.jobWizardMultiMetric.selectSplitField(adMultiSplitField);
await ml.testExecution.logTestStep('job creation inputs the bucket span');
await ml.jobWizardCommon.setBucketSpan(adJobBucketSpan);
await a11y.testAppSnapshot();
});

it('anomaly detection create multi metric job details step', async () => {
await ml.jobWizardCommon.advanceToJobDetailsSection();
await ml.testExecution.logTestStep('job creation inputs the job id');
await ml.jobWizardCommon.setJobId(adMultiMetricJobId);
await ml.testExecution.logTestStep('job creation inputs the job description');
await ml.jobWizardCommon.setJobDescription(adMultiMetricJobDescription);
await ml.testExecution.logTestStep('job creation opens the additional settings section');
await ml.jobWizardCommon.ensureAdditionalSettingsSectionOpen();
await ml.testExecution.logTestStep('job creation opens the advanced section');
await ml.jobWizardCommon.ensureAdvancedSectionOpen();
await a11y.testAppSnapshot();
});

it('anomaly detection create multi metric job validation step', async () => {
await ml.jobWizardCommon.advanceToValidationSection();
await a11y.testAppSnapshot();
});

it('anomaly detection create multi metric job summary step', async () => {
await ml.jobWizardCommon.advanceToSummarySection();
await a11y.testAppSnapshot();
});

it('anomaly detection Single Metric Viewer page', async () => {
await ml.navigation.navigateToMl();
await ml.navigation.navigateToAnomalyDetection();
Expand All @@ -210,6 +281,22 @@ export default function ({ getService }: FtrProviderContext) {
await a11y.testAppSnapshot();
});

it('anomaly detection forecasting from Single Metric Viewer page', async () => {
await ml.testExecution.logTestStep('opens the forecasting modal showing no forecasts');
await ml.forecast.openForecastModal();
await a11y.testAppSnapshot();

await ml.testExecution.logTestStep('run the forecast and close the modal');
await ml.forecast.clickForecastModalRunButton();

await ml.testExecution.logTestStep('opens the forecasting modal showing a forecast');
await ml.forecast.openForecastModal();
await a11y.testAppSnapshot();

await ml.testExecution.logTestStep('closes the forecasting modal');
await ml.forecast.closeForecastModal();
});

it('anomaly detection Anomaly Explorer page', async () => {
await ml.singleMetricViewer.openAnomalyExplorer();
await ml.commonUI.waitForMlLoadingIndicatorToDisappear();
Expand All @@ -222,7 +309,7 @@ export default function ({ getService }: FtrProviderContext) {
});

it('data frame analytics outlier job exploration page', async () => {
await ml.dataFrameAnalyticsTable.openResultsView(dfaOutlierJobId);
await ml.dataFrameAnalyticsTable.openResultsView(dfaOutlierResultsJobId);
await ml.dataFrameAnalyticsResults.assertOutlierTablePanelExists();
await ml.dataFrameAnalyticsResults.assertResultsTableExists();
await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
Expand All @@ -248,7 +335,7 @@ export default function ({ getService }: FtrProviderContext) {
it('data frame analytics create job configuration step for outlier job', async () => {
await ml.testExecution.logTestStep('selects the outlier job type');
await ml.dataFrameAnalyticsCreation.assertJobTypeSelectExists();
await ml.dataFrameAnalyticsCreation.selectJobType(dfaJobType);
await ml.dataFrameAnalyticsCreation.selectJobType(dfaOutlierJobType);
await ml.testExecution.logTestStep('displays the source data preview');
await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists();
await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewHistogramChartEnabled(true);
Expand All @@ -264,7 +351,7 @@ export default function ({ getService }: FtrProviderContext) {

it('data frame analytics create job additional options step for outlier job', async () => {
await ml.dataFrameAnalyticsCreation.continueToDetailsStep();
await ml.dataFrameAnalyticsCreation.setJobId(dfaJobId);
await ml.dataFrameAnalyticsCreation.setJobId(dfaOutlierJobId);
await a11y.testAppSnapshot();
});

Expand All @@ -279,6 +366,102 @@ export default function ({ getService }: FtrProviderContext) {
await a11y.testAppSnapshot();
});

it('data frame analytics create job configuration step for regression job', async () => {
await ml.testExecution.logTestStep(
'job creation selects the source data and loads the DFA job wizard page'
);
await ml.navigation.navigateToMl();
await ml.navigation.navigateToDataFrameAnalytics();
await ml.dataFrameAnalytics.startAnalyticsCreation();
await ml.jobSourceSelection.selectSourceForAnalyticsJob(egsIndexPattern);
await ml.dataFrameAnalyticsCreation.assertConfigurationStepActive();
await ml.testExecution.logTestStep('selects the regression job type');
await ml.dataFrameAnalyticsCreation.assertJobTypeSelectExists();
await ml.dataFrameAnalyticsCreation.selectJobType(dfaRegressionJobType);
await ml.testExecution.logTestStep('inputs the dependent variable');
await ml.dataFrameAnalyticsCreation.assertDependentVariableInputExists();
await ml.dataFrameAnalyticsCreation.selectDependentVariable(dfaRegressionJobDepVar);
await ml.testExecution.logTestStep('inputs the training percent');
await ml.dataFrameAnalyticsCreation.assertTrainingPercentInputExists();
await ml.dataFrameAnalyticsCreation.setTrainingPercent(dfaRegressionJobTrainingPercent);
await ml.testExecution.logTestStep('displays the source data preview');
await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists();
await ml.testExecution.logTestStep('displays the include fields selection');
await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
await a11y.testAppSnapshot();
});

it('data frame analytics create job additional options step for regression job', async () => {
await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep();
await a11y.testAppSnapshot();
});

it('data frame analytics create job additional options step for regression job', async () => {
await ml.dataFrameAnalyticsCreation.continueToDetailsStep();
await ml.dataFrameAnalyticsCreation.setJobId(dfaRegressionJobId);
await a11y.testAppSnapshot();
});

it('data frame analytics create job validation step for regression job', async () => {
await ml.dataFrameAnalyticsCreation.continueToValidationStep();
await ml.dataFrameAnalyticsCreation.assertValidationCalloutsExists();
await a11y.testAppSnapshot();
});

it('data frame analytics create job create step for regression job', async () => {
await ml.dataFrameAnalyticsCreation.continueToCreateStep();
await a11y.testAppSnapshot();
});

it('data frame analytics create job configuration step for classification job', async () => {
await ml.testExecution.logTestStep(
'job creation selects the source data and loads the DFA job wizard page'
);
await ml.navigation.navigateToMl();
await ml.navigation.navigateToDataFrameAnalytics();
await ml.dataFrameAnalytics.startAnalyticsCreation();
await ml.jobSourceSelection.selectSourceForAnalyticsJob(bmIndexPattern);
await ml.dataFrameAnalyticsCreation.assertConfigurationStepActive();
await ml.testExecution.logTestStep('selects the classification job type');
await ml.dataFrameAnalyticsCreation.assertJobTypeSelectExists();
await ml.dataFrameAnalyticsCreation.selectJobType(dfaClassificationJobType);
await ml.testExecution.logTestStep('inputs the dependent variable');
await ml.dataFrameAnalyticsCreation.assertDependentVariableInputExists();
await ml.dataFrameAnalyticsCreation.selectDependentVariable(dfaClassificationJobDepVar);
await ml.testExecution.logTestStep('inputs the training percent');
await ml.dataFrameAnalyticsCreation.assertTrainingPercentInputExists();
await ml.dataFrameAnalyticsCreation.setTrainingPercent(
dfaClassificationJobTrainingPercent
);
await ml.testExecution.logTestStep('displays the source data preview');
await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists();
await ml.testExecution.logTestStep('displays the include fields selection');
await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
await a11y.testAppSnapshot();
});

it('data frame analytics create job additional options step for classification job', async () => {
await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep();
await a11y.testAppSnapshot();
});

it('data frame analytics create job additional options step for classification job', async () => {
await ml.dataFrameAnalyticsCreation.continueToDetailsStep();
await ml.dataFrameAnalyticsCreation.setJobId(dfaClassificationJobId);
await a11y.testAppSnapshot();
});

it('data frame analytics create job validation step for classification job', async () => {
await ml.dataFrameAnalyticsCreation.continueToValidationStep();
await ml.dataFrameAnalyticsCreation.assertValidationCalloutsExists();
await a11y.testAppSnapshot();
});

it('data frame analytics create job create step for classification job', async () => {
await ml.dataFrameAnalyticsCreation.continueToCreateStep();
await a11y.testAppSnapshot();
});

it('settings page', async () => {
await ml.navigation.navigateToMl();
await ml.navigation.navigateToSettings();
Expand Down