Skip to content

Commit

Permalink
[ML] Functional tests - cleanMlIndices without system index access (#…
Browse files Browse the repository at this point in the history 
…199653)

## Summary

This PR updates the `cleanMlIndices` service method to no longer run
with `esDeleteAllIndices` and thus no longer requires system index
superuser privileges.

### Details / other changes

- Not all ML items can be cleaned up through APIs (e.g. notifications),
so tests have been adjusted to deal with pre-existing data
- Some cleanup steps had to be re-ordered
- Basic license tests didn't need the `cleanMlIndices` in their `before`
so it was removed there
- Observability serverless tests can't use `cleanMlIndices` as the APIs
for DFA are not available for that project type, so the cleanup is
changed to `cleanAnomalyDetection` for the AD tests and the
`cleanMlIndices` is removed from the AI assistant helpers as the
existing cleanup there should be enough
  • Loading branch information
@pheyos
pheyos authored Nov 22, 2024
1 parent 3890bde commit 93dac54
Show file tree
Hide file tree
Showing 10 changed files with 175 additions and 27 deletions.
27 changes: 16 additions & 11 deletions x-pack/test/api_integration/apis/ml/notifications/count_notifications.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,32 @@
*/

import expect from '@kbn/expect';
import moment from 'moment';
import type { FtrProviderContext } from '../../../ftr_provider_context';
import { USER } from '../../../../functional/services/ml/security_common';
import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api';

export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertestWithoutAuth');
const ml = getService('ml');
let testStart: number;

describe('GET notifications count', () => {
before(async () => {
testStart = Date.now();
});

describe('when no ML entities present', () => {
it('return a default response', async () => {
const { body, status } = await supertest
.get(`/internal/ml/notifications/count`)
.query({ lastCheckedAt: moment().subtract(7, 'd').valueOf() })
.query({ lastCheckedAt: testStart })
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);

expect(body.info).to.eql(0);
expect(body.warning).to.eql(0);
expect(body.error).to.eql(0);
expect(body.info).to.eql(0, `Expecting info count to be 0, got ${body.info}`);
expect(body.warning).to.eql(0, `Expecting warning count to be 0, got ${body.warning}`);
expect(body.error).to.eql(0, `Expecting error count to be 0, got ${body.error}`);
});
});

Expand All @@ -36,10 +40,11 @@ export default ({ getService }: FtrProviderContext) => {
await ml.api.initSavedObjects();
await ml.testResources.setKibanaTimeZoneToUTC();

const adJobConfig = ml.commonConfig.getADFqSingleMetricJobConfig('fq_job');
const jobId = `fq_job_${Date.now()}`;
const adJobConfig = ml.commonConfig.getADFqSingleMetricJobConfig(jobId);
await ml.api.createAnomalyDetectionJob(adJobConfig);

await ml.api.waitForJobNotificationsToIndex('fq_job');
await ml.api.waitForJobNotificationsToIndex(jobId);
});

after(async () => {
Expand All @@ -50,14 +55,14 @@ export default ({ getService }: FtrProviderContext) => {
it('return notifications count by level', async () => {
const { body, status } = await supertest
.get(`/internal/ml/notifications/count`)
.query({ lastCheckedAt: moment().subtract(7, 'd').valueOf() })
.query({ lastCheckedAt: testStart })
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);

expect(body.info).to.eql(1);
expect(body.warning).to.eql(0);
expect(body.error).to.eql(0);
expect(body.info).to.eql(1, `Expecting info count to be 1, got ${body.info}`);
expect(body.warning).to.eql(0, `Expecting warning count to be 0, got ${body.warning}`);
expect(body.error).to.eql(0, `Expecting error count to be 0, got ${body.error}`);
});

it('returns an error for unauthorized user', async () => {
Expand Down
10 changes: 6 additions & 4 deletions x-pack/test/api_integration/apis/ml/notifications/get_notifications.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertestWithoutAuth');
const esArchiver = getService('esArchiver');
const ml = getService('ml');
let testStart: number;

describe('GET notifications', () => {
before(async () => {
testStart = Date.now();
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/bm_classification');
await ml.api.initSavedObjects();
await ml.testResources.setKibanaTimeZoneToUTC();
Expand All @@ -45,7 +47,7 @@ export default ({ getService }: FtrProviderContext) => {
it('return all notifications ', async () => {
const { body, status } = await supertest
.get(`/internal/ml/notifications`)
.query({ earliest: 'now-1d', latest: 'now' })
.query({ earliest: testStart, latest: 'now' })
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);
Expand All @@ -56,7 +58,7 @@ export default ({ getService }: FtrProviderContext) => {
it('return notifications based on the query string', async () => {
const { body, status } = await supertest
.get(`/internal/ml/notifications`)
.query({ earliest: 'now-1d', latest: 'now', queryString: 'job_type:anomaly_detector' })
.query({ earliest: testStart, latest: 'now', queryString: 'job_type:anomaly_detector' })
.auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);
Expand All @@ -72,7 +74,7 @@ export default ({ getService }: FtrProviderContext) => {
it('supports sorting asc sorting by field', async () => {
const { body, status } = await supertest
.get(`/internal/ml/notifications`)
.query({ earliest: 'now-1d', latest: 'now', sortField: 'job_id', sortDirection: 'asc' })
.query({ earliest: testStart, latest: 'now', sortField: 'job_id', sortDirection: 'asc' })
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);
Expand All @@ -83,7 +85,7 @@ export default ({ getService }: FtrProviderContext) => {
it('supports sorting desc sorting by field', async () => {
const { body, status } = await supertest
.get(`/internal/ml/notifications`)
.query({ earliest: 'now-1h', latest: 'now', sortField: 'job_id', sortDirection: 'desc' })
.query({ earliest: testStart, latest: 'now', sortField: 'job_id', sortDirection: 'desc' })
.auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER))
.set(getCommonRequestHeader('1'));
ml.api.assertResponseStatusCode(200, status, body);
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/api_integration/apis/ml/trained_models/get_models.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ export default ({ getService }: FtrProviderContext) => {
});

after(async () => {
await ml.api.cleanMlIndices();
await esDeleteAllIndices('user-index_dfa*');

// delete created ingest pipelines
Expand All @@ -42,6 +41,7 @@ export default ({ getService }: FtrProviderContext) => {
)
);
await ml.testResources.cleanMLSavedObjects();
await ml.api.cleanMlIndices();
});

it('returns all trained models with associated pipelines including aliases', async () => {
Expand Down
152 changes: 149 additions & 3 deletions x-pack/test/functional/services/ml/api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
const retry = getService('retry');
const esSupertest = getService('esSupertest');
const kbnSupertest = getService('supertest');
const esDeleteAllIndices = getService('esDeleteAllIndices');

return {
assertResponseStatusCode(expectedStatus: number, actualStatus: number, responseBody: object) {
Expand Down Expand Up @@ -310,8 +309,37 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
log.debug('> Indices deleted.');
},

async deleteExpiredAnomalyDetectionData() {
log.debug('Deleting expired data ...');
const { body, status } = await esSupertest.delete('/_ml/_delete_expired_data');
this.assertResponseStatusCode(200, status, body);
log.debug('> Expired data deleted.');
},

async cleanAnomalyDetection() {
await this.deleteAllAnomalyDetectionJobs();
await this.deleteAllCalendars();
await this.deleteAllFilters();
await this.deleteAllAnnotations();
await this.deleteExpiredAnomalyDetectionData();
await this.syncSavedObjects();
},

async cleanDataFrameAnalytics() {
await this.deleteAllDataFrameAnalyticsJobs();
await this.syncSavedObjects();
},

async cleanTrainedModels() {
await this.deleteAllTrainedModelsIngestPipelines();
await this.deleteAllTrainedModelsES();
await this.syncSavedObjects();
},

async cleanMlIndices() {
await esDeleteAllIndices('.ml-*');
await this.cleanAnomalyDetection();
await this.cleanDataFrameAnalytics();
await this.cleanTrainedModels();
},

async getJobState(jobId: string): Promise<JOB_STATE> {
Expand Down Expand Up @@ -537,6 +565,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
return response;
},

async getAllCalendars(expectedCode = 200) {
const response = await esSupertest.get('/_ml/calendars/_all');
this.assertResponseStatusCode(expectedCode, response.status, response.body);
return response;
},

async createCalendar(
calendarId: string,
requestBody: Partial<Calendar> = { description: '', job_ids: [] }
Expand All @@ -559,6 +593,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
log.debug('> Calendar deleted.');
},

async deleteAllCalendars() {
log.debug('Deleting all calendars');
const getAllCalendarsRsp = await this.getAllCalendars();
for (const calendar of getAllCalendarsRsp.body.calendars) {
await this.deleteCalendar(calendar.calendar_id);
}
},

async waitForCalendarToExist(calendarId: string, errorMsg?: string) {
await retry.waitForWithTimeout(`'${calendarId}' to exist`, 5 * 1000, async () => {
if (await this.getCalendar(calendarId, 200)) {
Expand Down Expand Up @@ -660,6 +702,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
return response;
},

async getAllAnomalyDetectionJobs() {
const response = await esSupertest.get('/_ml/anomaly_detectors/_all');
this.assertResponseStatusCode(200, response.status, response.body);
return response;
},

async getAnomalyDetectionJobsKibana(jobId?: string, space?: string) {
const { body, status } = await kbnSupertest
.get(
Expand Down Expand Up @@ -831,6 +879,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
log.debug('> AD job deleted.');
},

async deleteAllAnomalyDetectionJobs() {
log.debug('Deleting all anomaly detection jobs');
const getAllAdJobsResp = await this.getAllAnomalyDetectionJobs();
for (const job of getAllAdJobsResp.body.jobs) {
await this.deleteAnomalyDetectionJobES(job.job_id);
}
},

async getDatafeed(datafeedId: string) {
const response = await esSupertest.get(`/_ml/datafeeds/${datafeedId}`);
this.assertResponseStatusCode(200, response.status, response.body);
Expand Down Expand Up @@ -1034,6 +1090,12 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
log.debug('> DFA job created.');
},

async getAllDataFrameAnalyticsJobs(expectedCode = 200) {
const response = await esSupertest.get('/_ml/data_frame/analytics/_all');
this.assertResponseStatusCode(expectedCode, response.status, response.body);
return response;
},

async createDataFrameAnalyticsJobES(jobConfig: DataFrameAnalyticsConfig) {
const { id: analyticsId, ...analyticsConfig } = jobConfig;
log.debug(`Creating data frame analytic job with id '${analyticsId}' via ES API...`);
Expand Down Expand Up @@ -1064,6 +1126,14 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
log.debug('> DFA job deleted.');
},

async deleteAllDataFrameAnalyticsJobs() {
log.debug('Deleting all data frame analytics jobs');
const getAllDfaJobsResp = await this.getAllDataFrameAnalyticsJobs();
for (const job of getAllDfaJobsResp.body.data_frame_analytics) {
await this.deleteDataFrameAnalyticsJobES(job.id);
}
},

async getADJobRecordCount(jobId: string): Promise<number> {
const jobStats = await this.getADJobStats(jobId);

Expand Down Expand Up @@ -1114,12 +1184,24 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
);
},

async filterExists(filterId: string): Promise<boolean> {
const { status } = await esSupertest.get(`/_ml/filters/${filterId}`);
if (status !== 200) return false;
return true;
},

async getFilter(filterId: string, expectedCode = 200) {
const response = await esSupertest.get(`/_ml/filters/${filterId}`);
this.assertResponseStatusCode(expectedCode, response.status, response.body);
return response;
},

async getAllFilters(expectedCode = 200) {
const response = await esSupertest.get(`/_ml/filters`);
this.assertResponseStatusCode(expectedCode, response.status, response.body);
return response;
},

async createFilter(filterId: string, requestBody: object) {
log.debug(`Creating filter with id '${filterId}'...`);
const { body, status } = await esSupertest.put(`/_ml/filters/${filterId}`).send(requestBody);
Expand All @@ -1131,12 +1213,27 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {

async deleteFilter(filterId: string) {
log.debug(`Deleting filter with id '${filterId}'...`);
await esSupertest.delete(`/_ml/filters/${filterId}`);

if ((await this.filterExists(filterId)) === false) {
log.debug('> Filter does not exist, nothing to delete');
return;
}

const { body, status } = await esSupertest.delete(`/_ml/filters/${filterId}`);
this.assertResponseStatusCode(200, status, body);

await this.waitForFilterToNotExist(filterId, `expected filter '${filterId}' to be deleted`);
log.debug('> Filter deleted.');
},

async deleteAllFilters() {
log.debug('Deleting all filters');
const getAllFiltersRsp = await this.getAllFilters();
for (const filter of getAllFiltersRsp.body.filters) {
await this.deleteFilter(filter.filter_id);
}
},

async assertModelMemoryLimitForJob(jobId: string, expectedMml: string) {
const {
body: {
Expand Down Expand Up @@ -1198,6 +1295,25 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
return body.hits.hits;
},

async getAllAnnotations() {
log.debug('Fetching all annotations ...');

if (
(await es.indices.exists({
index: ML_ANNOTATIONS_INDEX_ALIAS_READ,
allow_no_indices: false,
})) === false
) {
return [];
}

const body = await es.search<Annotation>({ index: ML_ANNOTATIONS_INDEX_ALIAS_READ });
expect(body).to.not.be(undefined);
expect(body).to.have.property('hits');
log.debug('> All annotations fetched.');
return body.hits.hits;
},

async getAnnotationById(annotationId: string): Promise<Annotation | undefined> {
log.debug(`Fetching annotation '${annotationId}'...`);

Expand Down Expand Up @@ -1264,6 +1380,24 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
);
},

async deleteAnnotation(annotationId: string) {
log.debug(`Deleting annotation with id "${annotationId}"`);
const { body, status } = await kbnSupertest
.delete(`/internal/ml/annotations/delete/${annotationId}`)
.set(getCommonRequestHeader('1'));
this.assertResponseStatusCode(200, status, body);

log.debug('> Annotation deleted');
},

async deleteAllAnnotations() {
log.debug('Deleting all annotations.');
const allAnnotations = await this.getAllAnnotations();
for (const annotation of allAnnotations) {
await this.deleteAnnotation(annotation._id!);
}
},

async runDFAJob(dfaId: string) {
log.debug(`Starting data frame analytics job '${dfaId}'...`);
const { body: startResponse, status } = await esSupertest
Expand Down Expand Up @@ -1647,6 +1781,18 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
log.debug('> Ingest pipeline deleted');
},

async deleteAllTrainedModelsIngestPipelines() {
log.debug(`Deleting all trained models ingest pipelines`);
const getModelsRsp = await this.getTrainedModelsES();
for (const model of getModelsRsp.trained_model_configs) {
if (this.isInternalModelId(model.model_id)) {
log.debug(`> Skipping internal ${model.model_id}`);
continue;
}
await this.deleteIngestPipeline(model.model_id);
}
},

async assureMlStatsIndexExists(timeout: number = 60 * 1000) {
const params = {
index: '.ml-stats-000001',
Expand Down
2 changes: 0 additions & 2 deletions x-pack/test/functional_basic/apps/ml/permissions/full_ml_access.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ export default function ({ getService }: FtrProviderContext) {
const expectedUploadFileTitle = 'artificial_server_log';

before(async () => {
await ml.api.cleanMlIndices();

await esArchiver.loadIfNeeded(
'x-pack/test/functional/es_archives/ml/module_sample_ecommerce'
);
Expand Down
Loading

0 comments on commit 93dac54

Please sign in to comment.