From 93dac5435ff51a3b28c5b3dd30bc4c24d1cf302c Mon Sep 17 00:00:00 2001 From: Robert Oskamp Date: Fri, 22 Nov 2024 17:54:01 +0100 Subject: [PATCH] [ML] Functional tests - cleanMlIndices without system index access (#199653) ## Summary This PR updates the `cleanMlIndices` service method to no longer run with `esDeleteAllIndices` and thus no longer requires system index superuser privileges. ### Details / other changes - Not all ML items can be cleaned up through APIs (e.g. notifications), so tests have been adjusted to deal with pre-existing data - Some cleanup steps had to be re-ordered - Basic license tests didn't need the `cleanMlIndices` in their `before` so it was removed there - Observability serverless tests can't use `cleanMlIndices` as the APIs for DFA are not available for that project type, so the cleanup is changed to `cleanAnomalyDetection` for the AD tests and the `cleanMlIndices` is removed from the AI assistant helpers as the existing cleanup there should be enough --- .../ml/notifications/count_notifications.ts | 27 ++-- .../ml/notifications/get_notifications.ts | 10 +- .../apis/ml/trained_models/get_models.ts | 2 +- x-pack/test/functional/services/ml/api.ts | 152 +++++++++++++++++- .../apps/ml/permissions/full_ml_access.ts | 2 - .../apps/ml/permissions/read_ml_access.ts | 2 - .../tests/knowledge_base/helpers.ts | 1 - .../ml/anomaly_detection_jobs_list.ts | 2 +- .../ml/anomaly_detection_jobs_list.ts | 2 +- .../ml/data_frame_analytics_jobs_list.ts | 2 +- 10 files changed, 175 insertions(+), 27 deletions(-) diff --git a/x-pack/test/api_integration/apis/ml/notifications/count_notifications.ts b/x-pack/test/api_integration/apis/ml/notifications/count_notifications.ts index a066999a08b5..e1ddf399d572 100644 --- a/x-pack/test/api_integration/apis/ml/notifications/count_notifications.ts +++ b/x-pack/test/api_integration/apis/ml/notifications/count_notifications.ts @@ -6,7 +6,6 @@ */ import expect from '@kbn/expect'; -import moment from 'moment'; import type { FtrProviderContext } from '../../../ftr_provider_context'; import { USER } from '../../../../functional/services/ml/security_common'; import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api'; @@ -14,20 +13,25 @@ import { getCommonRequestHeader } from '../../../../functional/services/ml/commo export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertestWithoutAuth'); const ml = getService('ml'); + let testStart: number; describe('GET notifications count', () => { + before(async () => { + testStart = Date.now(); + }); + describe('when no ML entities present', () => { it('return a default response', async () => { const { body, status } = await supertest .get(`/internal/ml/notifications/count`) - .query({ lastCheckedAt: moment().subtract(7, 'd').valueOf() }) + .query({ lastCheckedAt: testStart }) .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) .set(getCommonRequestHeader('1')); ml.api.assertResponseStatusCode(200, status, body); - expect(body.info).to.eql(0); - expect(body.warning).to.eql(0); - expect(body.error).to.eql(0); + expect(body.info).to.eql(0, `Expecting info count to be 0, got ${body.info}`); + expect(body.warning).to.eql(0, `Expecting warning count to be 0, got ${body.warning}`); + expect(body.error).to.eql(0, `Expecting error count to be 0, got ${body.error}`); }); }); @@ -36,10 +40,11 @@ export default ({ getService }: FtrProviderContext) => { await ml.api.initSavedObjects(); await ml.testResources.setKibanaTimeZoneToUTC(); - const adJobConfig = ml.commonConfig.getADFqSingleMetricJobConfig('fq_job'); + const jobId = `fq_job_${Date.now()}`; + const adJobConfig = ml.commonConfig.getADFqSingleMetricJobConfig(jobId); await ml.api.createAnomalyDetectionJob(adJobConfig); - await ml.api.waitForJobNotificationsToIndex('fq_job'); + await ml.api.waitForJobNotificationsToIndex(jobId); }); after(async () => { @@ -50,14 +55,14 @@ export default ({ getService }: FtrProviderContext) => { it('return notifications count by level', async () => { const { body, status } = await supertest .get(`/internal/ml/notifications/count`) - .query({ lastCheckedAt: moment().subtract(7, 'd').valueOf() }) + .query({ lastCheckedAt: testStart }) .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) .set(getCommonRequestHeader('1')); ml.api.assertResponseStatusCode(200, status, body); - expect(body.info).to.eql(1); - expect(body.warning).to.eql(0); - expect(body.error).to.eql(0); + expect(body.info).to.eql(1, `Expecting info count to be 1, got ${body.info}`); + expect(body.warning).to.eql(0, `Expecting warning count to be 0, got ${body.warning}`); + expect(body.error).to.eql(0, `Expecting error count to be 0, got ${body.error}`); }); it('returns an error for unauthorized user', async () => { diff --git a/x-pack/test/api_integration/apis/ml/notifications/get_notifications.ts b/x-pack/test/api_integration/apis/ml/notifications/get_notifications.ts index e13f1869e665..5e5f2c584c46 100644 --- a/x-pack/test/api_integration/apis/ml/notifications/get_notifications.ts +++ b/x-pack/test/api_integration/apis/ml/notifications/get_notifications.ts @@ -18,9 +18,11 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertestWithoutAuth'); const esArchiver = getService('esArchiver'); const ml = getService('ml'); + let testStart: number; describe('GET notifications', () => { before(async () => { + testStart = Date.now(); await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification'); await ml.api.initSavedObjects(); await ml.testResources.setKibanaTimeZoneToUTC(); @@ -45,7 +47,7 @@ export default ({ getService }: FtrProviderContext) => { it('return all notifications ', async () => { const { body, status } = await supertest .get(`/internal/ml/notifications`) - .query({ earliest: 'now-1d', latest: 'now' }) + .query({ earliest: testStart, latest: 'now' }) .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) .set(getCommonRequestHeader('1')); ml.api.assertResponseStatusCode(200, status, body); @@ -56,7 +58,7 @@ export default ({ getService }: FtrProviderContext) => { it('return notifications based on the query string', async () => { const { body, status } = await supertest .get(`/internal/ml/notifications`) - .query({ earliest: 'now-1d', latest: 'now', queryString: 'job_type:anomaly_detector' }) + .query({ earliest: testStart, latest: 'now', queryString: 'job_type:anomaly_detector' }) .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER)) .set(getCommonRequestHeader('1')); ml.api.assertResponseStatusCode(200, status, body); @@ -72,7 +74,7 @@ export default ({ getService }: FtrProviderContext) => { it('supports sorting asc sorting by field', async () => { const { body, status } = await supertest .get(`/internal/ml/notifications`) - .query({ earliest: 'now-1d', latest: 'now', sortField: 'job_id', sortDirection: 'asc' }) + .query({ earliest: testStart, latest: 'now', sortField: 'job_id', sortDirection: 'asc' }) .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) .set(getCommonRequestHeader('1')); ml.api.assertResponseStatusCode(200, status, body); @@ -83,7 +85,7 @@ export default ({ getService }: FtrProviderContext) => { it('supports sorting desc sorting by field', async () => { const { body, status } = await supertest .get(`/internal/ml/notifications`) - .query({ earliest: 'now-1h', latest: 'now', sortField: 'job_id', sortDirection: 'desc' }) + .query({ earliest: testStart, latest: 'now', sortField: 'job_id', sortDirection: 'desc' }) .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) .set(getCommonRequestHeader('1')); ml.api.assertResponseStatusCode(200, status, body); diff --git a/x-pack/test/api_integration/apis/ml/trained_models/get_models.ts b/x-pack/test/api_integration/apis/ml/trained_models/get_models.ts index 78f4347cd091..a3953a87b82b 100644 --- a/x-pack/test/api_integration/apis/ml/trained_models/get_models.ts +++ b/x-pack/test/api_integration/apis/ml/trained_models/get_models.ts @@ -32,7 +32,6 @@ export default ({ getService }: FtrProviderContext) => { }); after(async () => { - await ml.api.cleanMlIndices(); await esDeleteAllIndices('user-index_dfa*'); // delete created ingest pipelines @@ -42,6 +41,7 @@ export default ({ getService }: FtrProviderContext) => { ) ); await ml.testResources.cleanMLSavedObjects(); + await ml.api.cleanMlIndices(); }); it('returns all trained models with associated pipelines including aliases', async () => { diff --git a/x-pack/test/functional/services/ml/api.ts b/x-pack/test/functional/services/ml/api.ts index 0a3d988fd750..3d2d1004528d 100644 --- a/x-pack/test/functional/services/ml/api.ts +++ b/x-pack/test/functional/services/ml/api.ts @@ -85,7 +85,6 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { const retry = getService('retry'); const esSupertest = getService('esSupertest'); const kbnSupertest = getService('supertest'); - const esDeleteAllIndices = getService('esDeleteAllIndices'); return { assertResponseStatusCode(expectedStatus: number, actualStatus: number, responseBody: object) { @@ -310,8 +309,37 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> Indices deleted.'); }, + async deleteExpiredAnomalyDetectionData() { + log.debug('Deleting expired data ...'); + const { body, status } = await esSupertest.delete('/_ml/_delete_expired_data'); + this.assertResponseStatusCode(200, status, body); + log.debug('> Expired data deleted.'); + }, + + async cleanAnomalyDetection() { + await this.deleteAllAnomalyDetectionJobs(); + await this.deleteAllCalendars(); + await this.deleteAllFilters(); + await this.deleteAllAnnotations(); + await this.deleteExpiredAnomalyDetectionData(); + await this.syncSavedObjects(); + }, + + async cleanDataFrameAnalytics() { + await this.deleteAllDataFrameAnalyticsJobs(); + await this.syncSavedObjects(); + }, + + async cleanTrainedModels() { + await this.deleteAllTrainedModelsIngestPipelines(); + await this.deleteAllTrainedModelsES(); + await this.syncSavedObjects(); + }, + async cleanMlIndices() { - await esDeleteAllIndices('.ml-*'); + await this.cleanAnomalyDetection(); + await this.cleanDataFrameAnalytics(); + await this.cleanTrainedModels(); }, async getJobState(jobId: string): Promise { @@ -537,6 +565,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { return response; }, + async getAllCalendars(expectedCode = 200) { + const response = await esSupertest.get('/_ml/calendars/_all'); + this.assertResponseStatusCode(expectedCode, response.status, response.body); + return response; + }, + async createCalendar( calendarId: string, requestBody: Partial = { description: '', job_ids: [] } @@ -559,6 +593,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> Calendar deleted.'); }, + async deleteAllCalendars() { + log.debug('Deleting all calendars'); + const getAllCalendarsRsp = await this.getAllCalendars(); + for (const calendar of getAllCalendarsRsp.body.calendars) { + await this.deleteCalendar(calendar.calendar_id); + } + }, + async waitForCalendarToExist(calendarId: string, errorMsg?: string) { await retry.waitForWithTimeout(`'${calendarId}' to exist`, 5 * 1000, async () => { if (await this.getCalendar(calendarId, 200)) { @@ -660,6 +702,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { return response; }, + async getAllAnomalyDetectionJobs() { + const response = await esSupertest.get('/_ml/anomaly_detectors/_all'); + this.assertResponseStatusCode(200, response.status, response.body); + return response; + }, + async getAnomalyDetectionJobsKibana(jobId?: string, space?: string) { const { body, status } = await kbnSupertest .get( @@ -831,6 +879,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> AD job deleted.'); }, + async deleteAllAnomalyDetectionJobs() { + log.debug('Deleting all anomaly detection jobs'); + const getAllAdJobsResp = await this.getAllAnomalyDetectionJobs(); + for (const job of getAllAdJobsResp.body.jobs) { + await this.deleteAnomalyDetectionJobES(job.job_id); + } + }, + async getDatafeed(datafeedId: string) { const response = await esSupertest.get(`/_ml/datafeeds/${datafeedId}`); this.assertResponseStatusCode(200, response.status, response.body); @@ -1034,6 +1090,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> DFA job created.'); }, + async getAllDataFrameAnalyticsJobs(expectedCode = 200) { + const response = await esSupertest.get('/_ml/data_frame/analytics/_all'); + this.assertResponseStatusCode(expectedCode, response.status, response.body); + return response; + }, + async createDataFrameAnalyticsJobES(jobConfig: DataFrameAnalyticsConfig) { const { id: analyticsId, ...analyticsConfig } = jobConfig; log.debug(`Creating data frame analytic job with id '${analyticsId}' via ES API...`); @@ -1064,6 +1126,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> DFA job deleted.'); }, + async deleteAllDataFrameAnalyticsJobs() { + log.debug('Deleting all data frame analytics jobs'); + const getAllDfaJobsResp = await this.getAllDataFrameAnalyticsJobs(); + for (const job of getAllDfaJobsResp.body.data_frame_analytics) { + await this.deleteDataFrameAnalyticsJobES(job.id); + } + }, + async getADJobRecordCount(jobId: string): Promise { const jobStats = await this.getADJobStats(jobId); @@ -1114,12 +1184,24 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { ); }, + async filterExists(filterId: string): Promise { + const { status } = await esSupertest.get(`/_ml/filters/${filterId}`); + if (status !== 200) return false; + return true; + }, + async getFilter(filterId: string, expectedCode = 200) { const response = await esSupertest.get(`/_ml/filters/${filterId}`); this.assertResponseStatusCode(expectedCode, response.status, response.body); return response; }, + async getAllFilters(expectedCode = 200) { + const response = await esSupertest.get(`/_ml/filters`); + this.assertResponseStatusCode(expectedCode, response.status, response.body); + return response; + }, + async createFilter(filterId: string, requestBody: object) { log.debug(`Creating filter with id '${filterId}'...`); const { body, status } = await esSupertest.put(`/_ml/filters/${filterId}`).send(requestBody); @@ -1131,12 +1213,27 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { async deleteFilter(filterId: string) { log.debug(`Deleting filter with id '${filterId}'...`); - await esSupertest.delete(`/_ml/filters/${filterId}`); + + if ((await this.filterExists(filterId)) === false) { + log.debug('> Filter does not exist, nothing to delete'); + return; + } + + const { body, status } = await esSupertest.delete(`/_ml/filters/${filterId}`); + this.assertResponseStatusCode(200, status, body); await this.waitForFilterToNotExist(filterId, `expected filter '${filterId}' to be deleted`); log.debug('> Filter deleted.'); }, + async deleteAllFilters() { + log.debug('Deleting all filters'); + const getAllFiltersRsp = await this.getAllFilters(); + for (const filter of getAllFiltersRsp.body.filters) { + await this.deleteFilter(filter.filter_id); + } + }, + async assertModelMemoryLimitForJob(jobId: string, expectedMml: string) { const { body: { @@ -1198,6 +1295,25 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { return body.hits.hits; }, + async getAllAnnotations() { + log.debug('Fetching all annotations ...'); + + if ( + (await es.indices.exists({ + index: ML_ANNOTATIONS_INDEX_ALIAS_READ, + allow_no_indices: false, + })) === false + ) { + return []; + } + + const body = await es.search({ index: ML_ANNOTATIONS_INDEX_ALIAS_READ }); + expect(body).to.not.be(undefined); + expect(body).to.have.property('hits'); + log.debug('> All annotations fetched.'); + return body.hits.hits; + }, + async getAnnotationById(annotationId: string): Promise { log.debug(`Fetching annotation '${annotationId}'...`); @@ -1264,6 +1380,24 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { ); }, + async deleteAnnotation(annotationId: string) { + log.debug(`Deleting annotation with id "${annotationId}"`); + const { body, status } = await kbnSupertest + .delete(`/internal/ml/annotations/delete/${annotationId}`) + .set(getCommonRequestHeader('1')); + this.assertResponseStatusCode(200, status, body); + + log.debug('> Annotation deleted'); + }, + + async deleteAllAnnotations() { + log.debug('Deleting all annotations.'); + const allAnnotations = await this.getAllAnnotations(); + for (const annotation of allAnnotations) { + await this.deleteAnnotation(annotation._id!); + } + }, + async runDFAJob(dfaId: string) { log.debug(`Starting data frame analytics job '${dfaId}'...`); const { body: startResponse, status } = await esSupertest @@ -1647,6 +1781,18 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> Ingest pipeline deleted'); }, + async deleteAllTrainedModelsIngestPipelines() { + log.debug(`Deleting all trained models ingest pipelines`); + const getModelsRsp = await this.getTrainedModelsES(); + for (const model of getModelsRsp.trained_model_configs) { + if (this.isInternalModelId(model.model_id)) { + log.debug(`> Skipping internal ${model.model_id}`); + continue; + } + await this.deleteIngestPipeline(model.model_id); + } + }, + async assureMlStatsIndexExists(timeout: number = 60 * 1000) { const params = { index: '.ml-stats-000001', diff --git a/x-pack/test/functional_basic/apps/ml/permissions/full_ml_access.ts b/x-pack/test/functional_basic/apps/ml/permissions/full_ml_access.ts index 010bae3ba4bb..ab4dc572517c 100644 --- a/x-pack/test/functional_basic/apps/ml/permissions/full_ml_access.ts +++ b/x-pack/test/functional_basic/apps/ml/permissions/full_ml_access.ts @@ -32,8 +32,6 @@ export default function ({ getService }: FtrProviderContext) { const expectedUploadFileTitle = 'artificial_server_log'; before(async () => { - await ml.api.cleanMlIndices(); - await esArchiver.loadIfNeeded( 'x-pack/test/functional/es_archives/ml/module_sample_ecommerce' ); diff --git a/x-pack/test/functional_basic/apps/ml/permissions/read_ml_access.ts b/x-pack/test/functional_basic/apps/ml/permissions/read_ml_access.ts index 96aec074ec55..14a4be1ac8fd 100644 --- a/x-pack/test/functional_basic/apps/ml/permissions/read_ml_access.ts +++ b/x-pack/test/functional_basic/apps/ml/permissions/read_ml_access.ts @@ -32,8 +32,6 @@ export default function ({ getService }: FtrProviderContext) { const expectedUploadFileTitle = 'artificial_server_log'; before(async () => { - await ml.api.cleanMlIndices(); - await esArchiver.loadIfNeeded( 'x-pack/test/functional/es_archives/ml/module_sample_ecommerce' ); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts index 25bbeb183a3b..8965504aafc3 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts @@ -30,7 +30,6 @@ export async function createKnowledgeBaseModel(ml: ReturnType) { await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true); await ml.api.deleteTrainedModelES(TINY_ELSER.id); - await ml.api.cleanMlIndices(); await ml.testResources.cleanMLSavedObjects(); } diff --git a/x-pack/test_serverless/functional/test_suites/observability/ml/anomaly_detection_jobs_list.ts b/x-pack/test_serverless/functional/test_suites/observability/ml/anomaly_detection_jobs_list.ts index bdd5d443b359..8073a7c5fcc7 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/ml/anomaly_detection_jobs_list.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/ml/anomaly_detection_jobs_list.ts @@ -31,7 +31,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); after(async () => { - await ml.api.cleanMlIndices(); + await ml.api.cleanAnomalyDetection(); await ml.testResources.cleanMLSavedObjects(); await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); await kibanaServer.savedObjects.cleanStandardList(); diff --git a/x-pack/test_serverless/functional/test_suites/security/ml/anomaly_detection_jobs_list.ts b/x-pack/test_serverless/functional/test_suites/security/ml/anomaly_detection_jobs_list.ts index 9e1154ea09bb..e8f3f5e1796f 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ml/anomaly_detection_jobs_list.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ml/anomaly_detection_jobs_list.ts @@ -29,7 +29,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); after(async () => { - await ml.api.cleanMlIndices(); + await ml.api.cleanAnomalyDetection(); await ml.testResources.cleanMLSavedObjects(); await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); await kibanaServer.savedObjects.cleanStandardList(); diff --git a/x-pack/test_serverless/functional/test_suites/security/ml/data_frame_analytics_jobs_list.ts b/x-pack/test_serverless/functional/test_suites/security/ml/data_frame_analytics_jobs_list.ts index d3caa3425f75..110cf64e07a1 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ml/data_frame_analytics_jobs_list.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ml/data_frame_analytics_jobs_list.ts @@ -29,7 +29,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); after(async () => { - await ml.api.cleanMlIndices(); + await ml.api.cleanAnomalyDetection(); await ml.testResources.cleanMLSavedObjects(); });