[test_serverless/api_integration] convert alerting_wait_for_helpers i…

…nto FTR service (#162623) ## Summary This PR converts helpers into FTR services that should provide better test capabilities and improve stability/error output. Before change: ``` └-> should be active └-> "before each" hook: global before each for "should be active" └- ✖ fail: serverless observability API Threshold rule - AVG - PCT - FIRED Rule creation should be active │ Error: Timeout of 360000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. (/Users/dmle/github/kibana/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts) │ at listOnTimeout (node:internal/timers:559:17) │ at processTimers (node:internal/timers:502:7) ``` The test is not handling its state properly and fails on mocha timeout (6 minutes) without proper error message. It leads the test suite usually taking ~**5** min keep running for over [1 hour](https://buildkite.com/elastic/appex-qa-kibana-serverless-ftr-tests/builds/30#01898d63-a0e1-4cb0-b460-b4824ce2d5b1) After change: ``` 2) serverless observability API │ Threshold rule - AVG - PCT - FIRED │ Rule creation │ should be active: │ │ Error: 'ruleId' is undefined │ at Object.waitForRuleStatus (alerting_api.ts:31:15) │ at Context.apply (avg_pct_fired.ts:122:51) │ at Object.apply (wrap_function.js:73:30) ``` Since the follow-up tests depend on the first test to success, we can either move Rule creation to before hook or check for `ruleId` to be defined. If it is defined, `retry` service will pull for status up to 90 sec (important to keep below Mocha timeout so you can get error) and fail if it is not matching expected one.
elastic · Aug 2, 2023 · c4692b3 · c4692b3
1 parent d0d3127
commit c4692b3
Show file tree

Hide file tree

Showing 7 changed files with 118 additions and 41 deletions.
diff --git a/x-pack/test_serverless/api_integration/services/alerting_api.ts b/x-pack/test_serverless/api_integration/services/alerting_api.ts
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  AggregationsAggregate,
+  SearchResponse,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export function AlertingApiProvider({ getService }: FtrProviderContext) {
+  const retry = getService('retry');
+  const supertest = getService('supertest');
+  const es = getService('es');
+  const requestTimeout = 30 * 1000;
+  const retryTimeout = 120 * 1000;
+
+  return {
+    async waitForRuleStatus({
+      ruleId,
+      expectedStatus,
+    }: {
+      ruleId: string;
+      expectedStatus: string;
+    }) {
+      if (!ruleId) {
+        throw new Error(`'ruleId' is undefined`);
+      }
+      return await retry.tryForTime(retryTimeout, async () => {
+        const response = await supertest
+          .get(`/api/alerting/rule/${ruleId}`)
+          .set('kbn-xsrf', 'foo')
+          .set('x-elastic-internal-origin', 'foo')
+          .timeout(requestTimeout);
+        const { execution_status: executionStatus } = response.body || {};
+        const { status } = executionStatus || {};
+        if (status !== expectedStatus) {
+          throw new Error(`waitForStatus(${expectedStatus}): got ${status}`);
+        }
+        return executionStatus?.status;
+      });
+    },
+
+    async waitForDocumentInIndex<T>({
+      indexName,
+    }: {
+      indexName: string;
+    }): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
+      return await retry.tryForTime(retryTimeout, async () => {
+        const response = await es.search<T>({ index: indexName });
+        if (response.hits.hits.length === 0) {
+          throw new Error('No hits found');
+        }
+        return response;
+      });
+    },
+
+    async waitForAlertInIndex<T>({
+      indexName,
+      ruleId,
+    }: {
+      indexName: string;
+      ruleId: string;
+    }): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
+      if (!ruleId) {
+        throw new Error(`'ruleId' is undefined`);
+      }
+      return await retry.tryForTime(retryTimeout, async () => {
+        const response = await es.search<T>({
+          index: indexName,
+          body: {
+            query: {
+              term: {
+                'kibana.alert.rule.uuid': ruleId,
+              },
+            },
+          },
+        });
+        if (response.hits.hits.length === 0) {
+          throw new Error('No hits found');
+        }
+        return response;
+      });
+    },
+  };
+}
diff --git a/x-pack/test_serverless/api_integration/services/index.ts b/x-pack/test_serverless/api_integration/services/index.ts
@@ -11,12 +11,14 @@ import { services as xpackApiIntegrationServices } from '../../../test/api_integ
 import { services as svlSharedServices } from '../../shared/services';
 
 import { SvlCommonApiServiceProvider } from './svl_common_api';
+import { AlertingApiProvider } from './alerting_api';
 
 export const services = {
   ...xpackApiIntegrationServices,
   ...svlSharedServices,
 
   svlCommonApi: SvlCommonApiServiceProvider,
+  alertingApi: AlertingApiProvider,
 };
 
 export type InheritedFtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

diff --git a/...test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts b/...test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts
@@ -13,12 +13,12 @@ import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
 import { createDataView, deleteDataView } from '../helpers/data_view';
-import { waitForAlertInIndex, waitForRuleStatus } from '../helpers/alerting_wait_for_helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
+  const alertingApi = getService('alertingApi');
   const logger = getService('log');
 
   describe('Threshold rule - AVG - PCT - FIRED', () => {
@@ -125,17 +125,15 @@ export default function ({ getService }: FtrProviderContext) {
       });
 
       it('should be active', async () => {
-        const executionStatus = await waitForRuleStatus({
-          id: ruleId,
+        const executionStatus = await alertingApi.waitForRuleStatus({
+          ruleId,
           expectedStatus: 'active',
-          supertest,
         });
-        expect(executionStatus.status).to.be('active');
+        expect(executionStatus).to.be('active');
       });
 
       it('should set correct information in the alert document', async () => {
-        const resp = await waitForAlertInIndex({
-          esClient,
+        const resp = await alertingApi.waitForAlertInIndex({
           indexName: THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });

diff --git a/...st_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts b/...st_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts
@@ -12,11 +12,11 @@ import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
 import { createDataView, deleteDataView } from '../helpers/data_view';
-import { waitForAlertInIndex, waitForRuleStatus } from '../helpers/alerting_wait_for_helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
+  const alertingApi = getService('alertingApi');
 
   describe('Threshold rule - AVG - PCT - NoData', () => {
     const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -118,17 +118,15 @@ export default function ({ getService }: FtrProviderContext) {
       });
 
       it('should be active', async () => {
-        const executionStatus = await waitForRuleStatus({
-          id: ruleId,
+        const executionStatus = await alertingApi.waitForRuleStatus({
+          ruleId,
           expectedStatus: 'active',
-          supertest,
         });
-        expect(executionStatus.status).to.be('active');
+        expect(executionStatus).to.be('active');
       });
 
       it('should set correct information in the alert document', async () => {
-        const resp = await waitForAlertInIndex({
-          esClient,
+        const resp = await alertingApi.waitForAlertInIndex({
           indexName: THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });

diff --git a/...ess/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts b/...ess/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts
@@ -19,13 +19,13 @@ import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
 import { createDataView, deleteDataView } from '../helpers/data_view';
-import { waitForAlertInIndex, waitForRuleStatus } from '../helpers/alerting_wait_for_helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const alertingApi = getService('alertingApi');
 
   describe('Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => {
     const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -133,17 +133,15 @@ export default function ({ getService }: FtrProviderContext) {
       });
 
       it('should be active', async () => {
-        const executionStatus = await waitForRuleStatus({
-          id: ruleId,
+        const executionStatus = await alertingApi.waitForRuleStatus({
+          ruleId,
           expectedStatus: 'active',
-          supertest,
         });
-        expect(executionStatus.status).to.be('active');
+        expect(executionStatus).to.be('active');
       });
 
       it('should set correct information in the alert document', async () => {
-        const resp = await waitForAlertInIndex({
-          esClient,
+        const resp = await alertingApi.waitForAlertInIndex({
           indexName: THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });

diff --git a/...verless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts b/...verless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts
@@ -13,13 +13,13 @@ import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
 import { createDataView, deleteDataView } from '../helpers/data_view';
-import { waitForAlertInIndex, waitForRuleStatus } from '../helpers/alerting_wait_for_helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const alertingApi = getService('alertingApi');
 
   describe('Threshold rule - DOCUMENTS_COUNT - FIRED', () => {
     const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -123,17 +123,15 @@ export default function ({ getService }: FtrProviderContext) {
       });
 
       it('should be active', async () => {
-        const executionStatus = await waitForRuleStatus({
-          id: ruleId,
+        const executionStatus = await alertingApi.waitForRuleStatus({
+          ruleId,
           expectedStatus: 'active',
-          supertest,
         });
-        expect(executionStatus.status).to.be('active');
+        expect(executionStatus).to.be('active');
       });
 
       it('should set correct information in the alert document', async () => {
-        const resp = await waitForAlertInIndex({
-          esClient,
+        const resp = await alertingApi.waitForAlertInIndex({
           indexName: THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });

diff --git a/...est_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts b/...est_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts
@@ -20,17 +20,13 @@ import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
 import { createDataView, deleteDataView } from '../helpers/data_view';
-import {
-  waitForAlertInIndex,
-  waitForDocumentInIndex,
-  waitForRuleStatus,
-} from '../helpers/alerting_wait_for_helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const alertingApi = getService('alertingApi');
   let alertId: string;
   let startedAt: string;
 
@@ -143,17 +139,15 @@ export default function ({ getService }: FtrProviderContext) {
       });
 
       it('should be active', async () => {
-        const executionStatus = await waitForRuleStatus({
-          id: ruleId,
+        const executionStatus = await alertingApi.waitForRuleStatus({
+          ruleId,
           expectedStatus: 'active',
-          supertest,
         });
-        expect(executionStatus.status).to.be('active');
+        expect(executionStatus).to.be('active');
       });
 
       it('should set correct information in the alert document', async () => {
-        const resp = await waitForAlertInIndex({
-          esClient,
+        const resp = await alertingApi.waitForAlertInIndex({
           indexName: THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });
@@ -214,14 +208,13 @@ export default function ({ getService }: FtrProviderContext) {
 
       it('should set correct action variables', async () => {
         const rangeFrom = moment(startedAt).subtract('5', 'minute').toISOString();
-        const resp = await waitForDocumentInIndex<{
+        const resp = await alertingApi.waitForDocumentInIndex<{
           ruleType: string;
           alertDetailsUrl: string;
           reason: string;
           value: string;
           host: string;
         }>({
-          esClient,
           indexName: ALERT_ACTION_INDEX,
         });