[ML] Functional tests - cleanMlIndices without system index access (e…

…lastic#199653)

## Summary

This PR updates the `cleanMlIndices` service method to no longer run
with `esDeleteAllIndices` and thus no longer requires system index
superuser privileges.

### Details / other changes

- Not all ML items can be cleaned up through APIs (e.g. notifications),
so tests have been adjusted to deal with pre-existing data
- Some cleanup steps had to be re-ordered
- Basic license tests didn't need the `cleanMlIndices` in their `before`
so it was removed there
- Observability serverless tests can't use `cleanMlIndices` as the APIs
for DFA are not available for that project type, so the cleanup is
changed to `cleanAnomalyDetection` for the AD tests and the
`cleanMlIndices` is removed from the AI assistant helpers as the
existing cleanup there should be enough

(cherry picked from commit 93dac54)

x-pack/test/api_integration/apis/ml/notifications/count_notifications.ts

@@ -6,28 +6,32 @@
  */
 
 import expect from '@kbn/expect';
-import moment from 'moment';
 import type { FtrProviderContext } from '../../../ftr_provider_context';
 import { USER } from '../../../../functional/services/ml/security_common';
 import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api';
 
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertestWithoutAuth');
   const ml = getService('ml');
+  let testStart: number;
 
   describe('GET notifications count', () => {
+    before(async () => {
+      testStart = Date.now();
+    });
+
     describe('when no ML entities present', () => {
       it('return a default response', async () => {
         const { body, status } = await supertest
           .get(`/internal/ml/notifications/count`)
-          .query({ lastCheckedAt: moment().subtract(7, 'd').valueOf() })
+          .query({ lastCheckedAt: testStart })
           .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
           .set(getCommonRequestHeader('1'));
         ml.api.assertResponseStatusCode(200, status, body);
 
-        expect(body.info).to.eql(0);
-        expect(body.warning).to.eql(0);
-        expect(body.error).to.eql(0);
+        expect(body.info).to.eql(0, `Expecting info count to be 0, got ${body.info}`);
+        expect(body.warning).to.eql(0, `Expecting warning count to be 0, got ${body.warning}`);
+        expect(body.error).to.eql(0, `Expecting error count to be 0, got ${body.error}`);
       });
     });
 
@@ -36,10 +40,11 @@ export default ({ getService }: FtrProviderContext) => {
         await ml.api.initSavedObjects();
         await ml.testResources.setKibanaTimeZoneToUTC();
 
-        const adJobConfig = ml.commonConfig.getADFqSingleMetricJobConfig('fq_job');
+        const jobId = `fq_job_${Date.now()}`;
+        const adJobConfig = ml.commonConfig.getADFqSingleMetricJobConfig(jobId);
         await ml.api.createAnomalyDetectionJob(adJobConfig);
 
-        await ml.api.waitForJobNotificationsToIndex('fq_job');
+        await ml.api.waitForJobNotificationsToIndex(jobId);
       });
 
       after(async () => {
@@ -50,14 +55,14 @@ export default ({ getService }: FtrProviderContext) => {
       it('return notifications count by level', async () => {
         const { body, status } = await supertest
           .get(`/internal/ml/notifications/count`)
-          .query({ lastCheckedAt: moment().subtract(7, 'd').valueOf() })
+          .query({ lastCheckedAt: testStart })
           .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
           .set(getCommonRequestHeader('1'));
         ml.api.assertResponseStatusCode(200, status, body);
 
-        expect(body.info).to.eql(1);
-        expect(body.warning).to.eql(0);
-        expect(body.error).to.eql(0);
+        expect(body.info).to.eql(1, `Expecting info count to be 1, got ${body.info}`);
+        expect(body.warning).to.eql(0, `Expecting warning count to be 0, got ${body.warning}`);
+        expect(body.error).to.eql(0, `Expecting error count to be 0, got ${body.error}`);
       });
 
       it('returns an error for unauthorized user', async () => {

x-pack/test/api_integration/apis/ml/notifications/get_notifications.ts

@@ -18,9 +18,11 @@ export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertestWithoutAuth');
   const esArchiver = getService('esArchiver');
   const ml = getService('ml');
+  let testStart: number;
 
   describe('GET notifications', () => {
     before(async () => {
+      testStart = Date.now();
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification');
       await ml.api.initSavedObjects();
       await ml.testResources.setKibanaTimeZoneToUTC();
@@ -45,7 +47,7 @@ export default ({ getService }: FtrProviderContext) => {
     it('return all notifications ', async () => {
       const { body, status } = await supertest
         .get(`/internal/ml/notifications`)
-        .query({ earliest: 'now-1d', latest: 'now' })
+        .query({ earliest: testStart, latest: 'now' })
         .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
         .set(getCommonRequestHeader('1'));
       ml.api.assertResponseStatusCode(200, status, body);
@@ -56,7 +58,7 @@ export default ({ getService }: FtrProviderContext) => {
     it('return notifications based on the query string', async () => {
       const { body, status } = await supertest
         .get(`/internal/ml/notifications`)
-        .query({ earliest: 'now-1d', latest: 'now', queryString: 'job_type:anomaly_detector' })
+        .query({ earliest: testStart, latest: 'now', queryString: 'job_type:anomaly_detector' })
         .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER))
         .set(getCommonRequestHeader('1'));
       ml.api.assertResponseStatusCode(200, status, body);
@@ -72,7 +74,7 @@ export default ({ getService }: FtrProviderContext) => {
     it('supports sorting asc sorting by field', async () => {
       const { body, status } = await supertest
         .get(`/internal/ml/notifications`)
-        .query({ earliest: 'now-1d', latest: 'now', sortField: 'job_id', sortDirection: 'asc' })
+        .query({ earliest: testStart, latest: 'now', sortField: 'job_id', sortDirection: 'asc' })
         .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
         .set(getCommonRequestHeader('1'));
       ml.api.assertResponseStatusCode(200, status, body);
@@ -83,7 +85,7 @@ export default ({ getService }: FtrProviderContext) => {
     it('supports sorting desc sorting by field', async () => {
       const { body, status } = await supertest
         .get(`/internal/ml/notifications`)
-        .query({ earliest: 'now-1h', latest: 'now', sortField: 'job_id', sortDirection: 'desc' })
+        .query({ earliest: testStart, latest: 'now', sortField: 'job_id', sortDirection: 'desc' })
         .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
         .set(getCommonRequestHeader('1'));
       ml.api.assertResponseStatusCode(200, status, body);

x-pack/test/api_integration/apis/ml/trained_models/get_models.ts

@@ -32,7 +32,6 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     after(async () => {
-      await ml.api.cleanMlIndices();
       await esDeleteAllIndices('user-index_dfa*');
 
       // delete created ingest pipelines
@@ -42,6 +41,7 @@ export default ({ getService }: FtrProviderContext) => {
         )
       );
       await ml.testResources.cleanMLSavedObjects();
+      await ml.api.cleanMlIndices();
     });
 
     it('returns all trained models with associated pipelines including aliases', async () => {

x-pack/test/functional/services/ml/api.ts

@@ -85,7 +85,6 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
   const retry = getService('retry');
   const esSupertest = getService('esSupertest');
   const kbnSupertest = getService('supertest');
-  const esDeleteAllIndices = getService('esDeleteAllIndices');
 
   return {
     assertResponseStatusCode(expectedStatus: number, actualStatus: number, responseBody: object) {
@@ -310,8 +309,37 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       log.debug('> Indices deleted.');
     },
 
+    async deleteExpiredAnomalyDetectionData() {
+      log.debug('Deleting expired data ...');
+      const { body, status } = await esSupertest.delete('/_ml/_delete_expired_data');
+      this.assertResponseStatusCode(200, status, body);
+      log.debug('> Expired data deleted.');
+    },
+
+    async cleanAnomalyDetection() {
+      await this.deleteAllAnomalyDetectionJobs();
+      await this.deleteAllCalendars();
+      await this.deleteAllFilters();
+      await this.deleteAllAnnotations();
+      await this.deleteExpiredAnomalyDetectionData();
+      await this.syncSavedObjects();
+    },
+
+    async cleanDataFrameAnalytics() {
+      await this.deleteAllDataFrameAnalyticsJobs();
+      await this.syncSavedObjects();
+    },
+
+    async cleanTrainedModels() {
+      await this.deleteAllTrainedModelsIngestPipelines();
+      await this.deleteAllTrainedModelsES();
+      await this.syncSavedObjects();
+    },
+
     async cleanMlIndices() {
-      await esDeleteAllIndices('.ml-*');
+      await this.cleanAnomalyDetection();
+      await this.cleanDataFrameAnalytics();
+      await this.cleanTrainedModels();
     },
 
     async getJobState(jobId: string): Promise<JOB_STATE> {
@@ -537,6 +565,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       return response;
     },
 
+    async getAllCalendars(expectedCode = 200) {
+      const response = await esSupertest.get('/_ml/calendars/_all');
+      this.assertResponseStatusCode(expectedCode, response.status, response.body);
+      return response;
+    },
+
     async createCalendar(
       calendarId: string,
       requestBody: Partial<Calendar> = { description: '', job_ids: [] }
@@ -559,6 +593,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       log.debug('> Calendar deleted.');
     },
 
+    async deleteAllCalendars() {
+      log.debug('Deleting all calendars');
+      const getAllCalendarsRsp = await this.getAllCalendars();
+      for (const calendar of getAllCalendarsRsp.body.calendars) {
+        await this.deleteCalendar(calendar.calendar_id);
+      }
+    },
+
     async waitForCalendarToExist(calendarId: string, errorMsg?: string) {
       await retry.waitForWithTimeout(`'${calendarId}' to exist`, 5 * 1000, async () => {
         if (await this.getCalendar(calendarId, 200)) {
@@ -660,6 +702,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       return response;
     },
 
+    async getAllAnomalyDetectionJobs() {
+      const response = await esSupertest.get('/_ml/anomaly_detectors/_all');
+      this.assertResponseStatusCode(200, response.status, response.body);
+      return response;
+    },
+
     async getAnomalyDetectionJobsKibana(jobId?: string, space?: string) {
       const { body, status } = await kbnSupertest
         .get(
@@ -831,6 +879,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       log.debug('> AD job deleted.');
     },
 
+    async deleteAllAnomalyDetectionJobs() {
+      log.debug('Deleting all anomaly detection jobs');
+      const getAllAdJobsResp = await this.getAllAnomalyDetectionJobs();
+      for (const job of getAllAdJobsResp.body.jobs) {
+        await this.deleteAnomalyDetectionJobES(job.job_id);
+      }
+    },
+
     async getDatafeed(datafeedId: string) {
       const response = await esSupertest.get(`/_ml/datafeeds/${datafeedId}`);
       this.assertResponseStatusCode(200, response.status, response.body);
@@ -1034,6 +1090,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       log.debug('> DFA job created.');
     },
 
+    async getAllDataFrameAnalyticsJobs(expectedCode = 200) {
+      const response = await esSupertest.get('/_ml/data_frame/analytics/_all');
+      this.assertResponseStatusCode(expectedCode, response.status, response.body);
+      return response;
+    },
+
     async createDataFrameAnalyticsJobES(jobConfig: DataFrameAnalyticsConfig) {
       const { id: analyticsId, ...analyticsConfig } = jobConfig;
       log.debug(`Creating data frame analytic job with id '${analyticsId}' via ES API...`);
@@ -1064,6 +1126,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       log.debug('> DFA job deleted.');
     },
 
+    async deleteAllDataFrameAnalyticsJobs() {
+      log.debug('Deleting all data frame analytics jobs');
+      const getAllDfaJobsResp = await this.getAllDataFrameAnalyticsJobs();
+      for (const job of getAllDfaJobsResp.body.data_frame_analytics) {
+        await this.deleteDataFrameAnalyticsJobES(job.id);
+      }
+    },
+
     async getADJobRecordCount(jobId: string): Promise<number> {
       const jobStats = await this.getADJobStats(jobId);
 
@@ -1114,12 +1184,24 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       );
     },
 
+    async filterExists(filterId: string): Promise<boolean> {
+      const { status } = await esSupertest.get(`/_ml/filters/${filterId}`);
+      if (status !== 200) return false;
+      return true;
+    },
+
     async getFilter(filterId: string, expectedCode = 200) {
       const response = await esSupertest.get(`/_ml/filters/${filterId}`);
       this.assertResponseStatusCode(expectedCode, response.status, response.body);
       return response;
     },
 
+    async getAllFilters(expectedCode = 200) {
+      const response = await esSupertest.get(`/_ml/filters`);
+      this.assertResponseStatusCode(expectedCode, response.status, response.body);
+      return response;
+    },
+
     async createFilter(filterId: string, requestBody: object) {
       log.debug(`Creating filter with id '${filterId}'...`);
       const { body, status } = await esSupertest.put(`/_ml/filters/${filterId}`).send(requestBody);
@@ -1131,12 +1213,27 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
 
     async deleteFilter(filterId: string) {
       log.debug(`Deleting filter with id '${filterId}'...`);
-      await esSupertest.delete(`/_ml/filters/${filterId}`);
+
+      if ((await this.filterExists(filterId)) === false) {
+        log.debug('> Filter does not exist, nothing to delete');
+        return;
+      }
+
+      const { body, status } = await esSupertest.delete(`/_ml/filters/${filterId}`);
+      this.assertResponseStatusCode(200, status, body);
 
       await this.waitForFilterToNotExist(filterId, `expected filter '${filterId}' to be deleted`);
       log.debug('> Filter deleted.');
     },
 
+    async deleteAllFilters() {
+      log.debug('Deleting all filters');
+      const getAllFiltersRsp = await this.getAllFilters();
+      for (const filter of getAllFiltersRsp.body.filters) {
+        await this.deleteFilter(filter.filter_id);
+      }
+    },
+
     async assertModelMemoryLimitForJob(jobId: string, expectedMml: string) {
       const {
         body: {
@@ -1198,6 +1295,25 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       return body.hits.hits;
     },
 
+    async getAllAnnotations() {
+      log.debug('Fetching all annotations ...');
+
+      if (
+        (await es.indices.exists({
+          index: ML_ANNOTATIONS_INDEX_ALIAS_READ,
+          allow_no_indices: false,
+        })) === false
+      ) {
+        return [];
+      }
+
+      const body = await es.search<Annotation>({ index: ML_ANNOTATIONS_INDEX_ALIAS_READ });
+      expect(body).to.not.be(undefined);
+      expect(body).to.have.property('hits');
+      log.debug('> All annotations fetched.');
+      return body.hits.hits;
+    },
+
     async getAnnotationById(annotationId: string): Promise<Annotation | undefined> {
       log.debug(`Fetching annotation '${annotationId}'...`);
 
@@ -1264,6 +1380,24 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       );
     },
 
+    async deleteAnnotation(annotationId: string) {
+      log.debug(`Deleting annotation with id "${annotationId}"`);
+      const { body, status } = await kbnSupertest
+        .delete(`/internal/ml/annotations/delete/${annotationId}`)
+        .set(getCommonRequestHeader('1'));
+      this.assertResponseStatusCode(200, status, body);
+
+      log.debug('> Annotation deleted');
+    },
+
+    async deleteAllAnnotations() {
+      log.debug('Deleting all annotations.');
+      const allAnnotations = await this.getAllAnnotations();
+      for (const annotation of allAnnotations) {
+        await this.deleteAnnotation(annotation._id!);
+      }
+    },
+
     async runDFAJob(dfaId: string) {
       log.debug(`Starting data frame analytics job '${dfaId}'...`);
       const { body: startResponse, status } = await esSupertest
@@ -1647,6 +1781,18 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
       log.debug('> Ingest pipeline deleted');
     },
 
+    async deleteAllTrainedModelsIngestPipelines() {
+      log.debug(`Deleting all trained models ingest pipelines`);
+      const getModelsRsp = await this.getTrainedModelsES();
+      for (const model of getModelsRsp.trained_model_configs) {
+        if (this.isInternalModelId(model.model_id)) {
+          log.debug(`> Skipping internal ${model.model_id}`);
+          continue;
+        }
+        await this.deleteIngestPipeline(model.model_id);
+      }
+    },
+
     async assureMlStatsIndexExists(timeout: number = 60 * 1000) {
       const params = {
         index: '.ml-stats-000001',

x-pack/test/functional_basic/apps/ml/permissions/full_ml_access.ts

@@ -32,8 +32,6 @@ export default function ({ getService }: FtrProviderContext) {
         const expectedUploadFileTitle = 'artificial_server_log';
 
         before(async () => {
-          await ml.api.cleanMlIndices();
-
           await esArchiver.loadIfNeeded(
             'x-pack/test/functional/es_archives/ml/module_sample_ecommerce'
           );