From f0ad217b34cb67b184b194d48f39b2e9b9ca2ae4 Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Tue, 12 Mar 2024 15:25:53 +0100 Subject: [PATCH 1/6] Improve logging, use retryService instead of pRetry, fix flakyness in rate reason message --- .../alerting_api_integration/common/retry.ts | 92 +++++++++++++++++++ .../custom_threshold_rule/avg_pct_fired.ts | 11 +++ .../custom_threshold_rule/avg_pct_no_data.ts | 12 +++ .../custom_threshold_rule/avg_us_fired.ts | 12 +++ .../custom_eq_avg_bytes_fired.ts | 20 +++- .../documents_count_fired.ts | 13 +++ .../custom_threshold_rule/group_by_fired.ts | 13 +++ .../custom_threshold_rule/p99_pct_fired.ts | 11 +++ .../custom_threshold_rule/rate_bytes_fired.ts | 29 ++++-- .../custom_threshold_rule_data_view.ts | 4 + .../helpers/alerting_api_helper.ts | 19 ++-- .../helpers/alerting_wait_for_helpers.ts | 66 ++++++++++--- .../observability/helpers/data_view.ts | 15 ++- .../observability/metric_threshold_rule.ts | 9 ++ 14 files changed, 293 insertions(+), 33 deletions(-) create mode 100644 x-pack/test/alerting_api_integration/common/retry.ts diff --git a/x-pack/test/alerting_api_integration/common/retry.ts b/x-pack/test/alerting_api_integration/common/retry.ts new file mode 100644 index 0000000000000..cfe265acaf604 --- /dev/null +++ b/x-pack/test/alerting_api_integration/common/retry.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RetryService } from '@kbn/ftr-common-functional-services'; +import type { ToolingLog } from '@kbn/tooling-log'; + +/** + * Copied from x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts + * + * Retry wrapper for async supertests, with a maximum number of retries. + * You can pass in a function that executes a supertest test, and make assertions + * on the response. If the test fails, it will retry the test the number of retries + * that are passed in. + * + * Example usage: + * ```ts + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(`/api/fleet/epm/packages/security_detection_engine`) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); + * ``` + * @param test The function containing a test to run + * @param retryService The retry service to use + * @param retries The maximum number of retries + * @param timeout The timeout for each retry + * @param retryDelay The delay between each retry + * @returns The response from the test + */ +export const retry = async ({ + test, + retryService, + utilityName, + retries = 3, + timeout = 30000, + retryDelay = 200, + logger, +}: { + test: () => Promise; + utilityName: string; + retryService: RetryService; + retries?: number; + timeout?: number; + retryDelay?: number; + logger: ToolingLog; +}): Promise => { + let retryAttempt = 0; + const response = await retryService.tryForTime( + timeout, + async () => { + if (retryAttempt > retries) { + // Log error message if we reached the maximum number of retries + // but don't throw an error, return it to break the retry loop. + const errorMessage = `Reached maximum number of retries for test ${utilityName}: ${ + retryAttempt - 1 + }/${retries}`; + logger.error(errorMessage); + return new Error(JSON.stringify(errorMessage)); + } + + retryAttempt = retryAttempt + 1; + + return await test(); + }, + undefined, + retryDelay + ); + + // Now throw the error in order to fail the test. + if (response instanceof Error) { + throw response; + } + + return response; +}; diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts index c566c077bc1ca..7ce03992d04f2 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts @@ -31,6 +31,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - AVG - PCT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -81,6 +82,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: DATA_VIEW_TITLE, docCountTarget: 270, + retryService, + logger, }); }); @@ -105,10 +108,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -168,6 +173,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -177,6 +184,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -232,6 +241,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts index c495b38919913..4261cf3e26743 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts @@ -31,6 +31,8 @@ export default function ({ getService }: FtrProviderContext) { const esClient = getService('es'); const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); + const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - AVG - PCT - NoData', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -47,6 +49,7 @@ export default function ({ getService }: FtrProviderContext) { name: DATA_VIEW, id: DATA_VIEW_ID, title: DATA_VIEW, + logger, }); }); @@ -64,6 +67,7 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); await esDeleteAllIndices([ALERT_ACTION_INDEX]); }); @@ -74,10 +78,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -136,6 +142,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -145,6 +153,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -200,6 +210,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts index 6f2f43721bb82..0fabe97e3263f 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts @@ -36,6 +36,8 @@ export default function ({ getService }: FtrProviderContext) { const kibanaServerConfig = config.get('servers.kibana'); const kibanaUrl = format(kibanaServerConfig); const supertest = getService('supertest'); + const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - AVG - US - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -57,6 +59,7 @@ export default function ({ getService }: FtrProviderContext) { name: DATA_VIEW_NAME, id: DATA_VIEW_ID, title: DATA_VIEW, + logger, }); }); @@ -76,6 +79,7 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); }); @@ -85,10 +89,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -147,6 +153,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -156,6 +164,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -211,6 +221,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts index 963b0a86acf73..22ac985ad9eaa 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts @@ -29,6 +29,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -67,12 +68,15 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: DATA_VIEW, docCountTarget: 75, + retryService, + logger, }); await createDataView({ supertest, name: DATA_VIEW, id: DATA_VIEW_ID, title: DATA_VIEW, + logger, }); }); @@ -90,22 +94,24 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]); await cleanup({ client: esClient, config: dataForgeConfig, logger }); }); - // FLAKY: https://github.com/elastic/kibana/issues/175360 - describe.skip('Rule creation', () => { + describe('Rule creation', () => { it('creates rule successfully', async () => { actionId = await createIndexConnector({ supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -165,6 +171,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -174,6 +182,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -230,6 +240,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); @@ -237,9 +249,9 @@ export default function ({ getService }: FtrProviderContext) { `https://localhost:5601/app/observability/alerts/${alertId}` ); expect(resp.hits.hits[0]._source?.reason).eql( - `Custom equation is 1, above the threshold of 0.9. (duration: 1 min, data view: ${DATA_VIEW})` + `Custom equation is 1 B, above the threshold of 0.9 B. (duration: 1 min, data view: ${DATA_VIEW})` ); - expect(resp.hits.hits[0]._source?.value).eql('1'); + expect(resp.hits.hits[0]._source?.value).eql('1 B'); }); }); }); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts index 6438af0af51a6..23f602253252f 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts @@ -32,6 +32,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - DOCUMENTS_COUNT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -71,12 +72,15 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: dataForgeIndices.join(','), docCountTarget: 45, + retryService, + logger, }); await createDataView({ supertest, name: DATA_VIEW_NAME, id: DATA_VIEW_ID, title: DATA_VIEW, + logger, }); }); @@ -94,6 +98,7 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]); await cleanup({ client: esClient, config: dataForgeConfig, logger }); @@ -105,10 +110,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -165,6 +172,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -174,6 +183,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -231,6 +242,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts index e41b77caf0727..b4b0843b84b11 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts @@ -29,6 +29,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - GROUP_BY - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -67,12 +68,15 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: dataForgeIndices.join(','), docCountTarget: 270, + retryService, + logger, }); await createDataView({ supertest, name: DATA_VIEW, id: DATA_VIEW_ID, title: DATA_VIEW, + logger, }); }); @@ -90,6 +94,7 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]); await cleanup({ client: esClient, config: dataForgeConfig, logger }); @@ -101,10 +106,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -165,6 +172,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -174,6 +183,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -251,6 +262,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts index f826a72fae4ea..c2fe32c19a214 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts @@ -31,6 +31,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule - P99 - PCT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -78,6 +79,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: DATE_VIEW_TITLE, docCountTarget: 270, + retryService, + logger, }); }); @@ -102,10 +105,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -163,6 +168,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -172,6 +179,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -227,6 +236,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts index 4ab583c47df62..ee63ed3793cc2 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts @@ -29,6 +29,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const logger = getService('log'); + const retryService = getService('retry'); describe('Custom Threshold rule RATE - GROUP_BY - BYTES - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; @@ -46,9 +47,11 @@ export default function ({ getService }: FtrProviderContext) { schedule: [ { template: 'good', - start: 'now-15m', - end: 'now', - metrics: [{ name: 'system.network.in.bytes', method: 'exp', start: 10, end: 100 }], + start: 'now-10m', + end: 'now+5m', + metrics: [ + { name: 'system.network.in.bytes', method: 'linear', start: 0, end: 54000000 }, + ], }, ], indexing: { @@ -63,12 +66,15 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: dataForgeIndices.join(','), docCountTarget: 270, + retryService, + logger, }); await createDataView({ supertest, name: DATE_VIEW, id: DATA_VIEW_ID, title: DATE_VIEW, + logger, }); }); @@ -86,6 +92,7 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]); await cleanup({ client: esClient, config: dataForgeConfig, logger }); @@ -97,10 +104,12 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', @@ -109,7 +118,7 @@ export default function ({ getService }: FtrProviderContext) { criteria: [ { comparator: Comparator.GT_OR_EQ, - threshold: [0.2], + threshold: [50_000], timeSize: 1, timeUnit: 'm', metrics: [ @@ -161,6 +170,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -170,6 +181,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; @@ -230,7 +243,7 @@ export default function ({ getService }: FtrProviderContext) { criteria: [ { comparator: '>=', - threshold: [0.2], + threshold: [50_000], timeSize: 1, timeUnit: 'm', metrics: [{ name: 'A', field: 'system.network.in.bytes', aggType: 'rate' }], @@ -247,6 +260,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); @@ -254,9 +269,9 @@ export default function ({ getService }: FtrProviderContext) { `https://localhost:5601/app/observability/alerts/${alertId}` ); expect(resp.hits.hits[0]._source?.reason).eql( - `Rate of system.network.in.bytes is 0.3 B/s, above or equal the threshold of 0.2 B/s. (duration: 1 min, data view: kbn-data-forge-fake_hosts.fake_hosts-*, group: host-0,container-0)` + `Rate of system.network.in.bytes is 60 kB/s, above or equal the threshold of 50 kB/s. (duration: 1 min, data view: kbn-data-forge-fake_hosts.fake_hosts-*, group: host-0,container-0)` ); - expect(resp.hits.hits[0]._source?.value).eql('0.3 B/s'); + expect(resp.hits.hits[0]._source?.value).eql('60 kB/s'); expect(resp.hits.hits[0]._source?.host).eql( '{"name":"host-0","mac":["00-00-5E-00-53-23","00-00-5E-00-53-24"]}' ); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts index 6071a67023433..18960248d5321 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts @@ -22,6 +22,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const objectRemover = new ObjectRemover(supertest); const es = getService('es'); + const logger = getService('log'); describe('Custom Threshold rule data view >', () => { const DATA_VIEW_ID = 'data-view-id'; @@ -51,6 +52,7 @@ export default function ({ getService }: FtrProviderContext) { name: 'test-data-view', id: DATA_VIEW_ID, title: 'random-index*', + logger, }); }); @@ -59,6 +61,7 @@ export default function ({ getService }: FtrProviderContext) { await deleteDataView({ supertest, id: DATA_VIEW_ID, + logger, }); }); @@ -66,6 +69,7 @@ export default function ({ getService }: FtrProviderContext) { it('create a threshold rule', async () => { const createdRule = await createRule({ supertest, + logger, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts index 57b5721701a40..8981796ac66d1 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts @@ -6,17 +6,19 @@ */ import type { SuperTest, Test } from 'supertest'; -import expect from '@kbn/expect'; +import { ToolingLog } from '@kbn/tooling-log'; import { ThresholdParams } from '@kbn/observability-plugin/common/custom_threshold_rule/types'; export async function createIndexConnector({ supertest, name, indexName, + logger, }: { supertest: SuperTest; name: string; indexName: string; + logger: ToolingLog; }) { const { body } = await supertest .post(`/api/actions/connector`) @@ -28,7 +30,10 @@ export async function createIndexConnector({ refresh: true, }, connector_type_id: '.index', - }); + }) + .expect(200); + + logger.debug(`Created index connector id: ${body.id}`); return body.id as string; } @@ -41,6 +46,7 @@ export async function createRule({ tags = [], schedule, consumer, + logger, }: { supertest: SuperTest; ruleTypeId: string; @@ -50,6 +56,7 @@ export async function createRule({ tags?: any[]; schedule?: { interval: string }; consumer: string; + logger: ToolingLog; }) { const { body } = await supertest .post(`/api/alerting/rule`) @@ -64,9 +71,9 @@ export async function createRule({ name, rule_type_id: ruleTypeId, actions, - }); - if (body.statusCode) { - expect(body.statusCode).eql(200, body.message); - } + }) + .expect(200); + + logger.debug(`Created rule id: ${body.id}`); return body; } diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts index 77806c70ece1a..bef8dae9ed33a 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts @@ -5,7 +5,7 @@ * 2.0. */ -import pRetry from 'p-retry'; +import { ToolingLog } from '@kbn/tooling-log'; import type SuperTest from 'supertest'; import type { Client } from '@elastic/elasticsearch'; @@ -13,18 +13,28 @@ import type { AggregationsAggregate, SearchResponse, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { RetryService } from '@kbn/ftr-common-functional-services'; +import { retry } from '../../common/retry'; + +const TIMEOUT = 60000; +const RETRIES = 10; +const RETRY_DELAY = 3000; export async function waitForRuleStatus({ id, expectedStatus, supertest, + retryService, + logger, }: { id: string; expectedStatus: string; supertest: SuperTest.SuperTest; + retryService: RetryService; + logger: ToolingLog; }): Promise> { - return pRetry( - async () => { + const ruleResponse = await retry>({ + test: async () => { const response = await supertest.get(`/api/alerting/rule/${id}`); const { execution_status: executionStatus } = response.body || {}; const { status } = executionStatus || {}; @@ -33,42 +43,65 @@ export async function waitForRuleStatus({ } return executionStatus; }, - { retries: 10 } - ); + utilityName: 'fetching rule', + logger, + retryService, + timeout: TIMEOUT, + retries: RETRIES, + retryDelay: RETRY_DELAY, + }); + + return ruleResponse; } export async function waitForDocumentInIndex({ esClient, indexName, docCountTarget = 1, + retryService, + logger, }: { esClient: Client; indexName: string; docCountTarget?: number; + retryService: RetryService; + logger: ToolingLog; }): Promise>> { - return pRetry( - async () => { + return await retry>>({ + test: async () => { const response = await esClient.search({ index: indexName, rest_total_hits_as_int: true }); if (!response.hits.total || response.hits.total < docCountTarget) { - throw new Error('No hits found'); + throw new Error( + `Number of hits does not match expectation (total: ${response.hits.total}, target: ${docCountTarget})` + ); } + logger.debug(`Returned document: ${JSON.stringify(response.hits.hits[0])}`); return response; }, - { retries: 10 } - ); + utilityName: 'fetching rule', + logger, + retryService, + timeout: TIMEOUT, + retries: RETRIES, + retryDelay: RETRY_DELAY, + }); } export async function waitForAlertInIndex({ esClient, indexName, ruleId, + retryService, + logger, }: { esClient: Client; indexName: string; ruleId: string; + retryService: RetryService; + logger: ToolingLog; }): Promise>> { - return pRetry( - async () => { + return await retry>>({ + test: async () => { const response = await esClient.search({ index: indexName, body: { @@ -84,6 +117,11 @@ export async function waitForAlertInIndex({ } return response; }, - { retries: 10 } - ); + utilityName: 'fetching rule', + logger, + retryService, + timeout: TIMEOUT, + retries: RETRIES, + retryDelay: RETRY_DELAY, + }); } diff --git a/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts b/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts index 0b0e85b104962..1f84b2556ab70 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts @@ -6,17 +6,20 @@ */ import { SuperTest, Test } from 'supertest'; +import { ToolingLog } from '@kbn/tooling-log'; export const createDataView = async ({ supertest, id, name, title, + logger, }: { supertest: SuperTest; id: string; name: string; title: string; + logger: ToolingLog; }) => { const { body } = await supertest .post(`/api/content_management/rpc/create`) @@ -36,15 +39,20 @@ export const createDataView = async ({ }, options: { id }, version: 1, - }); + }) + .expect(200); + + logger.debug(`Created data view: ${JSON.stringify(body)}`); return body; }; export const deleteDataView = async ({ supertest, id, + logger, }: { supertest: SuperTest; id: string; + logger: ToolingLog; }) => { const { body } = await supertest .post(`/api/content_management/rpc/delete`) @@ -54,6 +62,9 @@ export const deleteDataView = async ({ id, options: { force: true }, version: 1, - }); + }) + .expect(200); + + logger.debug(`Deleted data view id: ${id}`); return body; }; diff --git a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts index 4c4c5306d708a..043e4f2a30a05 100644 --- a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts +++ b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts @@ -28,6 +28,7 @@ export default function ({ getService }: FtrProviderContext) { const esDeleteAllIndices = getService('esDeleteAllIndices'); const supertest = getService('supertest'); const logger = getService('log'); + const retryService = getService('retry'); describe('Metric threshold rule >', () => { let ruleId: string; @@ -57,9 +58,11 @@ export default function ({ getService }: FtrProviderContext) { supertest, name: 'Index Connector: Metric threshold API test', indexName: ALERT_ACTION_INDEX, + logger, }); const createdRule = await createRule({ supertest, + logger, ruleTypeId: InfraRuleType.MetricThreshold, consumer: 'infrastructure', tags: ['infrastructure'], @@ -125,6 +128,8 @@ export default function ({ getService }: FtrProviderContext) { id: ruleId, expectedStatus: 'active', supertest, + retryService, + logger, }); expect(executionStatus.status).to.be('active'); }); @@ -134,6 +139,8 @@ export default function ({ getService }: FtrProviderContext) { esClient, indexName: METRICS_ALERTS_INDEX, ruleId, + retryService, + logger, }); alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; startedAt = (resp.hits.hits[0]._source as any)['kibana.alert.start']; @@ -191,6 +198,8 @@ export default function ({ getService }: FtrProviderContext) { const resp = await waitForDocumentInIndex<{ ruleType: string; alertDetailsUrl: string }>({ esClient, indexName: ALERT_ACTION_INDEX, + retryService, + logger, }); expect(resp.hits.hits[0]._source?.ruleType).eql('metrics.alert.threshold'); From a03e1bcbcf48cba8a7de7d689a07b01ad0876faa Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Tue, 12 Mar 2024 15:52:20 +0100 Subject: [PATCH 2/6] Adjust timeout --- .../observability/helpers/alerting_wait_for_helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts index bef8dae9ed33a..29c1fb196730b 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts @@ -16,9 +16,9 @@ import type { import type { RetryService } from '@kbn/ftr-common-functional-services'; import { retry } from '../../common/retry'; -const TIMEOUT = 60000; +const TIMEOUT = 10000; const RETRIES = 10; -const RETRY_DELAY = 3000; +const RETRY_DELAY = 500; export async function waitForRuleStatus({ id, From aceaf5f634a58e494ee19fff1251f117bb6b61f1 Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Wed, 13 Mar 2024 11:05:36 +0100 Subject: [PATCH 3/6] Generate data after now in test and wait for documents in metric threshold test. --- .../custom_threshold_rule/avg_pct_fired.ts | 4 ++-- .../documents_count_fired.ts | 4 ++-- .../custom_threshold_rule/group_by_fired.ts | 4 ++-- .../custom_threshold_rule/p99_pct_fired.ts | 4 ++-- .../helpers/alerting_wait_for_helpers.ts | 4 ++-- .../observability/metric_threshold_rule.ts | 20 ++++++++++++++++++- 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts index 7ce03992d04f2..65c8e2cb0e57b 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts @@ -61,8 +61,8 @@ export default function ({ getService }: FtrProviderContext) { schedule: [ { template: 'good', - start: 'now-15m', - end: 'now', + start: 'now-10m', + end: 'now+5m', metrics: [ { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }, { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 }, diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts index 23f602253252f..7828226182072 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts @@ -51,8 +51,8 @@ export default function ({ getService }: FtrProviderContext) { schedule: [ { template: 'good', - start: 'now-15m', - end: 'now', + start: 'now-10m', + end: 'now+5m', metrics: [ { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }, { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 }, diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts index b4b0843b84b11..35ef47a6f26bd 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts @@ -47,8 +47,8 @@ export default function ({ getService }: FtrProviderContext) { schedule: [ { template: 'good', - start: 'now-15m', - end: 'now', + start: 'now-10m', + end: 'now+5m', metrics: [ { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }, { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 }, diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts index c2fe32c19a214..f24478c30c9ab 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts @@ -61,8 +61,8 @@ export default function ({ getService }: FtrProviderContext) { schedule: [ { template: 'good', - start: 'now-15m', - end: 'now', + start: 'now-10m', + end: 'now+5m', metrics: [{ name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }], }, ], diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts index 29c1fb196730b..ebe51849154a8 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts @@ -78,7 +78,7 @@ export async function waitForDocumentInIndex({ logger.debug(`Returned document: ${JSON.stringify(response.hits.hits[0])}`); return response; }, - utilityName: 'fetching rule', + utilityName: `waiting for documents in ${indexName} index`, logger, retryService, timeout: TIMEOUT, @@ -117,7 +117,7 @@ export async function waitForAlertInIndex({ } return response; }, - utilityName: 'fetching rule', + utilityName: `waiting for alerting document in the alerting index (${indexName})`, logger, retryService, timeout: TIMEOUT, diff --git a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts index 043e4f2a30a05..b5ab8fbd12820 100644 --- a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts +++ b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts @@ -50,10 +50,24 @@ export default function ({ getService }: FtrProviderContext) { name: 'Default', }); dataForgeConfig = { - schedule: [{ template: 'good', start: 'now-15m', end: 'now' }], + schedule: [ + { + template: 'good', + start: 'now-10m', + end: 'now+5m', + metrics: [{ name: 'system.cpu.user.pct', method: 'linear', start: 0.9, end: 0.9 }], + }, + ], indexing: { dataset: 'fake_hosts' as Dataset }, }; dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger }); + await waitForDocumentInIndex({ + esClient, + indexName: dataForgeIndices.join(','), + docCountTarget: 45, + retryService, + logger, + }); actionId = await createIndexConnector({ supertest, name: 'Index Connector: Metric threshold API test', @@ -91,6 +105,7 @@ export default function ({ getService }: FtrProviderContext) { { ruleType: '{{rule.type}}', alertDetailsUrl: '{{context.alertDetailsUrl}}', + reason: '{{context.reason}}', }, ], }, @@ -206,6 +221,9 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( `https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)` ); + expect(resp.hits.hits[0]._source?.reason).eql( + `system.cpu.user.pct is 90% in the last 5 mins. Alert when > 50%.` + ); }); }); }); From 86ab6137ebaecc8e8b5cff74f1df38a0425b7921 Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Wed, 13 Mar 2024 15:27:47 +0100 Subject: [PATCH 4/6] Add refreshSavedObjectIndices and increase timeout --- .../custom_threshold_rule/avg_pct_fired.ts | 1 + .../custom_threshold_rule/avg_pct_no_data.ts | 1 + .../custom_threshold_rule/avg_us_fired.ts | 1 + .../custom_eq_avg_bytes_fired.ts | 1 + .../documents_count_fired.ts | 1 + .../custom_threshold_rule/group_by_fired.ts | 1 + .../custom_threshold_rule/p99_pct_fired.ts | 1 + .../custom_threshold_rule/rate_bytes_fired.ts | 1 + .../custom_threshold_rule_data_view.ts | 5 +-- .../helpers/alerting_api_helper.ts | 5 +++ .../helpers/alerting_wait_for_helpers.ts | 4 +-- .../observability/helpers/refresh_index.ts | 36 +++++++++++++++++++ .../observability/metric_threshold_rule.ts | 1 + 13 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts index 65c8e2cb0e57b..65a14577f06ea 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts @@ -114,6 +114,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts index 4261cf3e26743..8892fee26a28e 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts @@ -84,6 +84,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts index 0fabe97e3263f..fac69cd18a9bb 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts @@ -95,6 +95,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts index 22ac985ad9eaa..b4c39683fade9 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts @@ -112,6 +112,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts index 7828226182072..34238d5149388 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts @@ -116,6 +116,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts index 35ef47a6f26bd..dc7db5117c662 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts @@ -112,6 +112,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts index f24478c30c9ab..e04242f4ae1b9 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts @@ -111,6 +111,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts index ee63ed3793cc2..da613a8a4fb7c 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts @@ -110,6 +110,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts index 18960248d5321..9ecccffbd596e 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts @@ -21,7 +21,7 @@ import { createDataView, deleteDataView } from './helpers/data_view'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const objectRemover = new ObjectRemover(supertest); - const es = getService('es'); + const esClient = getService('es'); const logger = getService('log'); describe('Custom Threshold rule data view >', () => { @@ -30,7 +30,7 @@ export default function ({ getService }: FtrProviderContext) { let ruleId: string; const searchRule = () => - es.search<{ references: unknown; alert: { params: any } }>({ + esClient.search<{ references: unknown; alert: { params: any } }>({ index: '.kibana*', query: { bool: { @@ -70,6 +70,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, tags: ['observability'], consumer: 'logs', name: 'Threshold rule', diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts index 8981796ac66d1..2999efc556c9b 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts @@ -5,9 +5,11 @@ * 2.0. */ +import type { Client } from '@elastic/elasticsearch'; import type { SuperTest, Test } from 'supertest'; import { ToolingLog } from '@kbn/tooling-log'; import { ThresholdParams } from '@kbn/observability-plugin/common/custom_threshold_rule/types'; +import { refreshSavedObjectIndices } from '../../../security_solution_api_integration/test_suites/detections_response/utils'; export async function createIndexConnector({ supertest, @@ -47,6 +49,7 @@ export async function createRule({ schedule, consumer, logger, + esClient, }: { supertest: SuperTest; ruleTypeId: string; @@ -57,6 +60,7 @@ export async function createRule({ schedule?: { interval: string }; consumer: string; logger: ToolingLog; + esClient: Client; }) { const { body } = await supertest .post(`/api/alerting/rule`) @@ -74,6 +78,7 @@ export async function createRule({ }) .expect(200); + await refreshSavedObjectIndices(esClient); logger.debug(`Created rule id: ${body.id}`); return body; } diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts index ebe51849154a8..965fb4a1df006 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts @@ -16,8 +16,8 @@ import type { import type { RetryService } from '@kbn/ftr-common-functional-services'; import { retry } from '../../common/retry'; -const TIMEOUT = 10000; -const RETRIES = 10; +const TIMEOUT = 60_000; +const RETRIES = 120; const RETRY_DELAY = 500; export async function waitForRuleStatus({ diff --git a/x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts b/x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts new file mode 100644 index 0000000000000..4808a07ba4b1d --- /dev/null +++ b/x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; + +/** + * Copied from x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts + * + * Refresh an index, making changes available to search. + * Reusable utility which refreshes all saved object indices, to make them available for search, especially + * useful when needing to perform a search on an index that has just been written to. + * + * An example of this when installing the prebuilt detection rules SO of type 'security-rule': + * the savedObjectsClient does this with a call with explicit `refresh: false`. + * So, despite of the fact that the endpoint waits until the prebuilt rule will be + * successfully indexed, it doesn't wait until they become "visible" for subsequent read + * operations. + * + * Additionally, this method clears the cache for all saved object indices. This helps in cases in which + * saved object is read, then written to, and then read again, and the second read returns stale data. + * @param es The Elasticsearch client + */ +export const refreshSavedObjectIndices = async (es: Client) => { + // Refresh indices to prevent a race condition between a write and subsequent read operation. To + // fix it deterministically we have to refresh saved object indices and wait until it's done. + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + + // Additionally, we need to clear the cache to ensure that the next read operation will + // not return stale data. + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); +}; diff --git a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts index b5ab8fbd12820..19d9086d04af5 100644 --- a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts +++ b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts @@ -77,6 +77,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await createRule({ supertest, logger, + esClient, ruleTypeId: InfraRuleType.MetricThreshold, consumer: 'infrastructure', tags: ['infrastructure'], From 95711e0b1fcc99e281698d11e9cc3a9107ddac06 Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Thu, 14 Mar 2024 09:55:33 +0100 Subject: [PATCH 5/6] Fix type --- .../observability/helpers/alerting_api_helper.ts | 2 +- .../observability/metric_threshold_rule.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts index 2999efc556c9b..bd6c7761a5fcd 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts @@ -9,7 +9,7 @@ import type { Client } from '@elastic/elasticsearch'; import type { SuperTest, Test } from 'supertest'; import { ToolingLog } from '@kbn/tooling-log'; import { ThresholdParams } from '@kbn/observability-plugin/common/custom_threshold_rule/types'; -import { refreshSavedObjectIndices } from '../../../security_solution_api_integration/test_suites/detections_response/utils'; +import { refreshSavedObjectIndices } from './refresh_index'; export async function createIndexConnector({ supertest, diff --git a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts index 19d9086d04af5..808df288b094e 100644 --- a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts +++ b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts @@ -211,7 +211,11 @@ export default function ({ getService }: FtrProviderContext) { it('should set correct action parameter: ruleType', async () => { const rangeFrom = moment(startedAt).subtract('5', 'minute').toISOString(); - const resp = await waitForDocumentInIndex<{ ruleType: string; alertDetailsUrl: string }>({ + const resp = await waitForDocumentInIndex<{ + ruleType: string; + alertDetailsUrl: string; + reason: string; + }>({ esClient, indexName: ALERT_ACTION_INDEX, retryService, From 535dab2efbce6e840db11a921a59823daf540b4f Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Tue, 19 Mar 2024 12:26:37 +0100 Subject: [PATCH 6/6] Remove unnecessary JSON.stringify and adjust timeout --- x-pack/test/alerting_api_integration/common/retry.ts | 2 +- .../observability/helpers/alerting_wait_for_helpers.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/alerting_api_integration/common/retry.ts b/x-pack/test/alerting_api_integration/common/retry.ts index cfe265acaf604..6a32db757bb82 100644 --- a/x-pack/test/alerting_api_integration/common/retry.ts +++ b/x-pack/test/alerting_api_integration/common/retry.ts @@ -72,7 +72,7 @@ export const retry = async ({ retryAttempt - 1 }/${retries}`; logger.error(errorMessage); - return new Error(JSON.stringify(errorMessage)); + return new Error(errorMessage); } retryAttempt = retryAttempt + 1; diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts index 965fb4a1df006..63960b222cede 100644 --- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts +++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts @@ -16,7 +16,7 @@ import type { import type { RetryService } from '@kbn/ftr-common-functional-services'; import { retry } from '../../common/retry'; -const TIMEOUT = 60_000; +const TIMEOUT = 70_000; const RETRIES = 120; const RETRY_DELAY = 500;