Skip to content

Commit

Permalink
[ML] Add tests for anomaly embeddables migrations (elastic#116520)
Browse files Browse the repository at this point in the history
* [ML] Add tests for anomaly charts embeddable migrations

* [ML] Broaden tests for anomaly swimlane as well

* [ML] Fix function rename

* [ML] Update tests to use bulk api

* [ML] Remove override

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
qn895 and kibanamachine authored Nov 3, 2021
1 parent c2d7f33 commit 38a511b
Show file tree
Hide file tree
Showing 7 changed files with 198 additions and 39 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
it('can open job selection flyout', async () => {
await PageObjects.dashboard.clickCreateDashboardPrompt();
await ml.dashboardEmbeddables.assertDashboardIsEmpty();
await ml.dashboardEmbeddables.openJobSelectionFlyout();
await ml.dashboardEmbeddables.openAnomalyJobSelectionFlyout('ml_anomaly_charts');
await a11y.testAppSnapshot();
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,42 +6,16 @@
*/

import { FtrProviderContext } from '../../../ftr_provider_context';
import { Job, Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs';

// @ts-expect-error not full interface
const JOB_CONFIG: Job = {
job_id: `fq_multi_1_ae`,
description:
'mean/min/max(responsetime) partition=airline on farequote dataset with 1h bucket span',
groups: ['farequote', 'automated', 'multi-metric'],
analysis_config: {
bucket_span: '1h',
influencers: ['airline'],
detectors: [
{ function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'min', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'max', field_name: 'responsetime', partition_field_name: 'airline' },
],
},
data_description: { time_field: '@timestamp' },
analysis_limits: { model_memory_limit: '20mb' },
model_plot_config: { enabled: true },
};

// @ts-expect-error not full interface
const DATAFEED_CONFIG: Datafeed = {
datafeed_id: 'datafeed-fq_multi_1_ae',
indices: ['ft_farequote'],
job_id: 'fq_multi_1_ae',
query: { bool: { must: [{ match_all: {} }] } },
};
import { JOB_CONFIG, DATAFEED_CONFIG, ML_EMBEDDABLE_TYPES } from './constants';

const testDataList = [
{
type: 'testData',
suiteSuffix: 'with multi metric job',
panelTitle: `ML anomaly charts for ${JOB_CONFIG.job_id}`,
jobConfig: JOB_CONFIG,
datafeedConfig: DATAFEED_CONFIG,
dashboardTitle: `ML anomaly charts for fq_multi_1_ae ${Date.now()}`,
expected: {
influencers: [
{
Expand All @@ -59,7 +33,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const ml = getService('ml');
const PageObjects = getPageObjects(['common', 'timePicker', 'dashboard']);

describe('anomaly charts', function () {
describe('anomaly charts in dashboard', function () {
this.tags(['mlqa']);

before(async () => {
Expand All @@ -69,6 +43,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await ml.securityUI.loginAsMlPowerUser();
});

after(async () => {
await ml.api.cleanMlIndices();
});

for (const testData of testDataList) {
describe(testData.suiteSuffix, function () {
before(async () => {
Expand All @@ -79,14 +57,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.common.navigateToApp('dashboard');
});

after(async () => {
await ml.api.cleanMlIndices();
});

it('can open job selection flyout', async () => {
await PageObjects.dashboard.clickCreateDashboardPrompt();
await PageObjects.dashboard.clickNewDashboard();
await ml.dashboardEmbeddables.assertDashboardIsEmpty();
await ml.dashboardEmbeddables.openJobSelectionFlyout();
await ml.dashboardEmbeddables.openAnomalyJobSelectionFlyout(
ML_EMBEDDABLE_TYPES.ANOMALY_CHARTS
);
});

it('can select jobs', async () => {
Expand All @@ -109,6 +85,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.timePicker.pauseAutoRefresh();
await ml.dashboardEmbeddables.assertAnomalyChartsSeverityThresholdControlExists();
await ml.dashboardEmbeddables.assertAnomalyChartsExists();
await PageObjects.dashboard.saveDashboard(testData.dashboardTitle);
});
});
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { FtrProviderContext } from '../../../ftr_provider_context';
import { JOB_CONFIG, DATAFEED_CONFIG, ML_EMBEDDABLE_TYPES } from './constants';

const testDataList = [
{
type: ML_EMBEDDABLE_TYPES.ANOMALY_SWIMLANE,
panelTitle: 'ML anomaly swim lane',
dashboardSavedObject: {
type: 'dashboard',
attributes: {
title: `7.15.2 ML anomaly swimlane dashboard ${Date.now()}`,
description: '',
panelsJSON: `[{"version":"7.15.2","type":"ml_anomaly_swimlane","gridData":{"x":0,"y":0,"w":24,"h":15,"i":"c177ed0a-dea0-40f8-8980-cfb0c6bc13a8"},"panelIndex":"c177ed0a-dea0-40f8-8980-cfb0c6bc13a8","embeddableConfig":{"jobIds":["fq_multi_1_ae"],"swimlaneType":"viewBy","viewBy":"airline","enhancements":{}},"title":"ML anomaly swim lane"}]`,
optionsJSON: '{"useMargins":true,"syncColors":false,"hidePanelTitles":false}',
timeRestore: true,
timeTo: '2016-02-11T00:00:00.000Z',
timeFrom: '2016-02-07T00:00:00.000Z',
refreshInterval: {
pause: true,
value: 0,
},
kibanaSavedObjectMeta: {
searchSourceJSON: '{"query":{"query":"","language":"kuery"},"filter":[]}',
},
},
coreMigrationVersion: '7.15.2',
},
},
{
type: ML_EMBEDDABLE_TYPES.ANOMALY_CHARTS,
panelTitle: 'ML anomaly charts',
dashboardSavedObject: {
type: 'dashboard',
attributes: {
title: `7.15.2 ML anomaly charts dashboard ${Date.now()}`,
description: '',
panelsJSON:
'[{"version":"7.15.2","type":"ml_anomaly_charts","gridData":{"x":0,"y":0,"w":38,"h":21,"i":"1155890b-c19c-4d98-8153-50e6434612f1"},"panelIndex":"1155890b-c19c-4d98-8153-50e6434612f1","embeddableConfig":{"jobIds":["fq_multi_1_ae"],"maxSeriesToPlot":6,"severityThreshold":0,"enhancements":{}},"title":"ML anomaly charts"}]',
optionsJSON: '{"useMargins":true,"syncColors":false,"hidePanelTitles":false}',
timeRestore: true,
timeTo: '2016-02-11T00:00:00.000Z',
timeFrom: '2016-02-07T00:00:00.000Z',
refreshInterval: {
pause: true,
value: 0,
},
kibanaSavedObjectMeta: {
searchSourceJSON: '{"query":{"query":"","language":"kuery"},"filter":[]}',
},
},
coreMigrationVersion: '7.15.2',
},
},
];

export default function ({ getService, getPageObjects }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const ml = getService('ml');
const PageObjects = getPageObjects(['common', 'timePicker', 'dashboard']);

describe('anomaly embeddables migration in Dashboard', function () {
this.tags(['mlqa']);

before(async () => {
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');
await ml.testResources.setKibanaTimeZoneToUTC();
await ml.securityUI.loginAsMlPowerUser();

await ml.api.createAndRunAnomalyDetectionLookbackJob(JOB_CONFIG, DATAFEED_CONFIG);
// Using bulk API because create API might return 400 for conflict errors
await ml.testResources.createBulkSavedObjects(
testDataList.map((d) => d.dashboardSavedObject)
);

await PageObjects.common.navigateToApp('dashboard');
});

after(async () => {
await ml.api.cleanMlIndices();
});

for (const testData of testDataList) {
const { dashboardSavedObject, panelTitle, type } = testData;
describe(`for ${panelTitle}`, function () {
before(async () => {
await PageObjects.common.navigateToApp('dashboard');
});

after(async () => {
await ml.testResources.deleteDashboardByTitle(dashboardSavedObject.attributes.title);
});

it(`loads saved dashboard from version ${dashboardSavedObject.coreMigrationVersion}`, async () => {
await PageObjects.dashboard.loadSavedDashboard(dashboardSavedObject.attributes.title);

await ml.dashboardEmbeddables.assertDashboardPanelExists(panelTitle);

if (type === ML_EMBEDDABLE_TYPES.ANOMALY_CHARTS) {
await ml.dashboardEmbeddables.assertAnomalyChartsSeverityThresholdControlExists();
await ml.dashboardEmbeddables.assertAnomalyChartsExists();
}

if (type === ML_EMBEDDABLE_TYPES.ANOMALY_SWIMLANE) {
await ml.dashboardEmbeddables.assertAnomalySwimlaneExists();
}
});
});
}
});
}
41 changes: 41 additions & 0 deletions x-pack/test/functional/apps/ml/embeddables/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { Datafeed, Job } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs';

// @ts-expect-error not full interface
export const JOB_CONFIG: Job = {
job_id: `fq_multi_1_ae`,
description:
'mean/min/max(responsetime) partition=airline on farequote dataset with 1h bucket span',
groups: ['farequote', 'automated', 'multi-metric'],
analysis_config: {
bucket_span: '1h',
influencers: ['airline'],
detectors: [
{ function: 'mean', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'min', field_name: 'responsetime', partition_field_name: 'airline' },
{ function: 'max', field_name: 'responsetime', partition_field_name: 'airline' },
],
},
data_description: { time_field: '@timestamp' },
analysis_limits: { model_memory_limit: '20mb' },
model_plot_config: { enabled: true },
};

// @ts-expect-error not full interface
export const DATAFEED_CONFIG: Datafeed = {
datafeed_id: 'datafeed-fq_multi_1_ae',
indices: ['ft_farequote'],
job_id: 'fq_multi_1_ae',
query: { bool: { must: [{ match_all: {} }] } },
};

export const ML_EMBEDDABLE_TYPES = {
ANOMALY_SWIMLANE: 'ml_anomaly_swimlane',
ANOMALY_CHARTS: 'ml_anomaly_charts',
} as const;
1 change: 1 addition & 0 deletions x-pack/test/functional/apps/ml/embeddables/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('embeddables', function () {
this.tags(['skipFirefox']);
loadTestFile(require.resolve('./anomaly_charts_dashboard_embeddables'));
loadTestFile(require.resolve('./anomaly_embeddables_migration'));
});
}
12 changes: 10 additions & 2 deletions x-pack/test/functional/services/ml/dashboard_embeddables.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,21 @@ export function MachineLearningDashboardEmbeddablesProvider(
});
},

async openJobSelectionFlyout() {
async assertAnomalySwimlaneExists() {
await retry.tryForTime(60 * 1000, async () => {
await testSubjects.existOrFail(`mlAnomalySwimlaneEmbeddableWrapper`);
});
},

async openAnomalyJobSelectionFlyout(
mlEmbeddableType: 'ml_anomaly_swimlane' | 'ml_anomaly_charts'
) {
await retry.tryForTime(60 * 1000, async () => {
await dashboardAddPanel.clickEditorMenuButton();
await testSubjects.existOrFail('dashboardEditorContextMenu', { timeout: 2000 });

await dashboardAddPanel.clickEmbeddableFactoryGroupButton('ml');
await dashboardAddPanel.clickAddNewEmbeddableLink('ml_anomaly_charts');
await dashboardAddPanel.clickAddNewEmbeddableLink(mlEmbeddableType);

await mlDashboardJobSelectionTable.assertJobSelectionTableExists();
});
Expand Down
14 changes: 14 additions & 0 deletions x-pack/test/functional/services/ml/test_resources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,20 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider
return createResponse.id;
},

async createBulkSavedObjects(body: object[]): Promise<string> {
log.debug(`Creating bulk saved objects'`);

const createResponse = await supertest
.post(`/api/saved_objects/_bulk_create`)
.set(COMMON_REQUEST_HEADERS)
.send(body)
.expect(200)
.then((res: any) => res.body);

log.debug(` > Created bulk saved objects'`);
return createResponse;
},

async createIndexPatternIfNeeded(title: string, timeFieldName?: string): Promise<string> {
const indexPatternId = await this.getIndexPatternId(title);
if (indexPatternId !== undefined) {
Expand Down

0 comments on commit 38a511b

Please sign in to comment.