diff --git a/x-pack/test/alerting_api_integration/common/retry.ts b/x-pack/test/alerting_api_integration/common/retry.ts
new file mode 100644
index 0000000000000..6a32db757bb82
--- /dev/null
+++ b/x-pack/test/alerting_api_integration/common/retry.ts
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RetryService } from '@kbn/ftr-common-functional-services';
+import type { ToolingLog } from '@kbn/tooling-log';
+
+/**
+ * Copied from x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts
+ *
+ * Retry wrapper for async supertests, with a maximum number of retries.
+ * You can pass in a function that executes a supertest test, and make assertions
+ * on the response. If the test fails, it will retry the test the number of retries
+ * that are passed in.
+ *
+ * Example usage:
+ * ```ts
+  const fleetResponse = await retry<InstallPackageResponse>({
+    test: async () => {
+      const testResponse = await supertest
+        .post(`/api/fleet/epm/packages/security_detection_engine`)
+        .set('kbn-xsrf', 'xxxx')
+        .set('elastic-api-version', '2023-10-31')
+        .type('application/json')
+        .send({ force: true })
+        .expect(200);
+      expect((testResponse.body as InstallPackageResponse).items).toBeDefined();
+      expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0);
+
+      return testResponse.body;
+    },
+    retryService,
+    retries: MAX_RETRIES,
+    timeout: ATTEMPT_TIMEOUT,
+  });
+ * ```
+ * @param test The function containing a test to run
+ * @param retryService The retry service to use
+ * @param retries The maximum number of retries
+ * @param timeout The timeout for each retry
+ * @param retryDelay The delay between each retry
+ * @returns The response from the test
+ */
+export const retry = async <T>({
+  test,
+  retryService,
+  utilityName,
+  retries = 3,
+  timeout = 30000,
+  retryDelay = 200,
+  logger,
+}: {
+  test: () => Promise<T>;
+  utilityName: string;
+  retryService: RetryService;
+  retries?: number;
+  timeout?: number;
+  retryDelay?: number;
+  logger: ToolingLog;
+}): Promise<T> => {
+  let retryAttempt = 0;
+  const response = await retryService.tryForTime(
+    timeout,
+    async () => {
+      if (retryAttempt > retries) {
+        // Log error message if we reached the maximum number of retries
+        // but don't throw an error, return it to break the retry loop.
+        const errorMessage = `Reached maximum number of retries for test ${utilityName}: ${
+          retryAttempt - 1
+        }/${retries}`;
+        logger.error(errorMessage);
+        return new Error(errorMessage);
+      }
+
+      retryAttempt = retryAttempt + 1;
+
+      return await test();
+    },
+    undefined,
+    retryDelay
+  );
+
+  // Now throw the error in order to fail the test.
+  if (response instanceof Error) {
+    throw response;
+  }
+
+  return response;
+};
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
index c566c077bc1ca..65a14577f06ea 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
@@ -31,6 +31,7 @@ export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom Threshold rule - AVG - PCT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -60,8 +61,8 @@ export default function ({ getService }: FtrProviderContext) {
         schedule: [
           {
             template: 'good',
-            start: 'now-15m',
-            end: 'now',
+            start: 'now-10m',
+            end: 'now+5m',
             metrics: [
               { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 },
               { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 },
@@ -81,6 +82,8 @@ export default function ({ getService }: FtrProviderContext) {
         esClient,
         indexName: DATA_VIEW_TITLE,
         docCountTarget: 270,
+        retryService,
+        logger,
       });
     });
 
@@ -105,10 +108,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -168,6 +174,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -177,6 +185,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -232,6 +242,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts
index c495b38919913..8892fee26a28e 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_no_data.ts
@@ -31,6 +31,8 @@ export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
+  const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom Threshold rule - AVG - PCT - NoData', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -47,6 +49,7 @@ export default function ({ getService }: FtrProviderContext) {
         name: DATA_VIEW,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
+        logger,
       });
     });
 
@@ -64,6 +67,7 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
       await esDeleteAllIndices([ALERT_ACTION_INDEX]);
     });
@@ -74,10 +78,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -136,6 +143,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -145,6 +154,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -200,6 +211,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts
index 6f2f43721bb82..fac69cd18a9bb 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts
@@ -36,6 +36,8 @@ export default function ({ getService }: FtrProviderContext) {
   const kibanaServerConfig = config.get('servers.kibana');
   const kibanaUrl = format(kibanaServerConfig);
   const supertest = getService('supertest');
+  const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom Threshold rule - AVG - US - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -57,6 +59,7 @@ export default function ({ getService }: FtrProviderContext) {
         name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
+        logger,
       });
     });
 
@@ -76,6 +79,7 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
     });
 
@@ -85,10 +89,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -147,6 +154,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -156,6 +165,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -211,6 +222,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts
index 963b0a86acf73..b4c39683fade9 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts
@@ -29,6 +29,7 @@ export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -67,12 +68,15 @@ export default function ({ getService }: FtrProviderContext) {
         esClient,
         indexName: DATA_VIEW,
         docCountTarget: 75,
+        retryService,
+        logger,
       });
       await createDataView({
         supertest,
         name: DATA_VIEW,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
+        logger,
       });
     });
 
@@ -90,22 +94,25 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
       await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]);
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/175360
-    describe.skip('Rule creation', () => {
+    describe('Rule creation', () => {
       it('creates rule successfully', async () => {
         actionId = await createIndexConnector({
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -165,6 +172,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -174,6 +183,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -230,6 +241,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
@@ -237,9 +250,9 @@ export default function ({ getService }: FtrProviderContext) {
           `https://localhost:5601/app/observability/alerts/${alertId}`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Custom equation is 1, above the threshold of 0.9. (duration: 1 min, data view: ${DATA_VIEW})`
+          `Custom equation is 1 B, above the threshold of 0.9 B. (duration: 1 min, data view: ${DATA_VIEW})`
         );
-        expect(resp.hits.hits[0]._source?.value).eql('1');
+        expect(resp.hits.hits[0]._source?.value).eql('1 B');
       });
     });
   });
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts
index 6438af0af51a6..34238d5149388 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts
@@ -32,6 +32,7 @@ export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom  Threshold rule - DOCUMENTS_COUNT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -50,8 +51,8 @@ export default function ({ getService }: FtrProviderContext) {
         schedule: [
           {
             template: 'good',
-            start: 'now-15m',
-            end: 'now',
+            start: 'now-10m',
+            end: 'now+5m',
             metrics: [
               { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 },
               { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 },
@@ -71,12 +72,15 @@ export default function ({ getService }: FtrProviderContext) {
         esClient,
         indexName: dataForgeIndices.join(','),
         docCountTarget: 45,
+        retryService,
+        logger,
       });
       await createDataView({
         supertest,
         name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
+        logger,
       });
     });
 
@@ -94,6 +98,7 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
       await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]);
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
@@ -105,10 +110,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -165,6 +173,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -174,6 +184,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -231,6 +243,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts
index e41b77caf0727..dc7db5117c662 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts
@@ -29,6 +29,7 @@ export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom  Threshold rule - GROUP_BY - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -46,8 +47,8 @@ export default function ({ getService }: FtrProviderContext) {
         schedule: [
           {
             template: 'good',
-            start: 'now-15m',
-            end: 'now',
+            start: 'now-10m',
+            end: 'now+5m',
             metrics: [
               { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 },
               { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 },
@@ -67,12 +68,15 @@ export default function ({ getService }: FtrProviderContext) {
         esClient,
         indexName: dataForgeIndices.join(','),
         docCountTarget: 270,
+        retryService,
+        logger,
       });
       await createDataView({
         supertest,
         name: DATA_VIEW,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
+        logger,
       });
     });
 
@@ -90,6 +94,7 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
       await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]);
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
@@ -101,10 +106,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -165,6 +173,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -174,6 +184,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -251,6 +263,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts
index f826a72fae4ea..e04242f4ae1b9 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts
@@ -31,6 +31,7 @@ export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom Threshold rule - P99 - PCT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -60,8 +61,8 @@ export default function ({ getService }: FtrProviderContext) {
         schedule: [
           {
             template: 'good',
-            start: 'now-15m',
-            end: 'now',
+            start: 'now-10m',
+            end: 'now+5m',
             metrics: [{ name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }],
           },
         ],
@@ -78,6 +79,8 @@ export default function ({ getService }: FtrProviderContext) {
         esClient,
         indexName: DATE_VIEW_TITLE,
         docCountTarget: 270,
+        retryService,
+        logger,
       });
     });
 
@@ -102,10 +105,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -163,6 +169,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -172,6 +180,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -227,6 +237,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts
index 4ab583c47df62..da613a8a4fb7c 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/rate_bytes_fired.ts
@@ -29,6 +29,7 @@ export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Custom Threshold rule RATE - GROUP_BY - BYTES - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@@ -46,9 +47,11 @@ export default function ({ getService }: FtrProviderContext) {
         schedule: [
           {
             template: 'good',
-            start: 'now-15m',
-            end: 'now',
-            metrics: [{ name: 'system.network.in.bytes', method: 'exp', start: 10, end: 100 }],
+            start: 'now-10m',
+            end: 'now+5m',
+            metrics: [
+              { name: 'system.network.in.bytes', method: 'linear', start: 0, end: 54000000 },
+            ],
           },
         ],
         indexing: {
@@ -63,12 +66,15 @@ export default function ({ getService }: FtrProviderContext) {
         esClient,
         indexName: dataForgeIndices.join(','),
         docCountTarget: 270,
+        retryService,
+        logger,
       });
       await createDataView({
         supertest,
         name: DATE_VIEW,
         id: DATA_VIEW_ID,
         title: DATE_VIEW,
+        logger,
       });
     });
 
@@ -86,6 +92,7 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
       await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]);
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
@@ -97,10 +104,13 @@ export default function ({ getService }: FtrProviderContext) {
           supertest,
           name: 'Index Connector: Threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
 
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
@@ -109,7 +119,7 @@ export default function ({ getService }: FtrProviderContext) {
             criteria: [
               {
                 comparator: Comparator.GT_OR_EQ,
-                threshold: [0.2],
+                threshold: [50_000],
                 timeSize: 1,
                 timeUnit: 'm',
                 metrics: [
@@ -161,6 +171,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -170,6 +182,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
@@ -230,7 +244,7 @@ export default function ({ getService }: FtrProviderContext) {
             criteria: [
               {
                 comparator: '>=',
-                threshold: [0.2],
+                threshold: [50_000],
                 timeSize: 1,
                 timeUnit: 'm',
                 metrics: [{ name: 'A', field: 'system.network.in.bytes', aggType: 'rate' }],
@@ -247,6 +261,8 @@ export default function ({ getService }: FtrProviderContext) {
         const resp = await waitForDocumentInIndex<ActionDocument>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
@@ -254,9 +270,9 @@ export default function ({ getService }: FtrProviderContext) {
           `https://localhost:5601/app/observability/alerts/${alertId}`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Rate of system.network.in.bytes is 0.3 B/s, above or equal the threshold of 0.2 B/s. (duration: 1 min, data view: kbn-data-forge-fake_hosts.fake_hosts-*, group: host-0,container-0)`
+          `Rate of system.network.in.bytes is 60 kB/s, above or equal the threshold of 50 kB/s. (duration: 1 min, data view: kbn-data-forge-fake_hosts.fake_hosts-*, group: host-0,container-0)`
         );
-        expect(resp.hits.hits[0]._source?.value).eql('0.3 B/s');
+        expect(resp.hits.hits[0]._source?.value).eql('60 kB/s');
         expect(resp.hits.hits[0]._source?.host).eql(
           '{"name":"host-0","mac":["00-00-5E-00-53-23","00-00-5E-00-53-24"]}'
         );
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts
index 6071a67023433..9ecccffbd596e 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule_data_view.ts
@@ -21,7 +21,8 @@ import { createDataView, deleteDataView } from './helpers/data_view';
 export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const objectRemover = new ObjectRemover(supertest);
-  const es = getService('es');
+  const esClient = getService('es');
+  const logger = getService('log');
 
   describe('Custom Threshold rule data view >', () => {
     const DATA_VIEW_ID = 'data-view-id';
@@ -29,7 +30,7 @@ export default function ({ getService }: FtrProviderContext) {
     let ruleId: string;
 
     const searchRule = () =>
-      es.search<{ references: unknown; alert: { params: any } }>({
+      esClient.search<{ references: unknown; alert: { params: any } }>({
         index: '.kibana*',
         query: {
           bool: {
@@ -51,6 +52,7 @@ export default function ({ getService }: FtrProviderContext) {
         name: 'test-data-view',
         id: DATA_VIEW_ID,
         title: 'random-index*',
+        logger,
       });
     });
 
@@ -59,6 +61,7 @@ export default function ({ getService }: FtrProviderContext) {
       await deleteDataView({
         supertest,
         id: DATA_VIEW_ID,
+        logger,
       });
     });
 
@@ -66,6 +69,8 @@ export default function ({ getService }: FtrProviderContext) {
       it('create a threshold rule', async () => {
         const createdRule = await createRule({
           supertest,
+          logger,
+          esClient,
           tags: ['observability'],
           consumer: 'logs',
           name: 'Threshold rule',
diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts
index 57b5721701a40..bd6c7761a5fcd 100644
--- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts
+++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts
@@ -5,18 +5,22 @@
  * 2.0.
  */
 
+import type { Client } from '@elastic/elasticsearch';
 import type { SuperTest, Test } from 'supertest';
-import expect from '@kbn/expect';
+import { ToolingLog } from '@kbn/tooling-log';
 import { ThresholdParams } from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+import { refreshSavedObjectIndices } from './refresh_index';
 
 export async function createIndexConnector({
   supertest,
   name,
   indexName,
+  logger,
 }: {
   supertest: SuperTest<Test>;
   name: string;
   indexName: string;
+  logger: ToolingLog;
 }) {
   const { body } = await supertest
     .post(`/api/actions/connector`)
@@ -28,7 +32,10 @@ export async function createIndexConnector({
         refresh: true,
       },
       connector_type_id: '.index',
-    });
+    })
+    .expect(200);
+
+  logger.debug(`Created index connector id: ${body.id}`);
   return body.id as string;
 }
 
@@ -41,6 +48,8 @@ export async function createRule<Params = ThresholdParams>({
   tags = [],
   schedule,
   consumer,
+  logger,
+  esClient,
 }: {
   supertest: SuperTest<Test>;
   ruleTypeId: string;
@@ -50,6 +59,8 @@ export async function createRule<Params = ThresholdParams>({
   tags?: any[];
   schedule?: { interval: string };
   consumer: string;
+  logger: ToolingLog;
+  esClient: Client;
 }) {
   const { body } = await supertest
     .post(`/api/alerting/rule`)
@@ -64,9 +75,10 @@ export async function createRule<Params = ThresholdParams>({
       name,
       rule_type_id: ruleTypeId,
       actions,
-    });
-  if (body.statusCode) {
-    expect(body.statusCode).eql(200, body.message);
-  }
+    })
+    .expect(200);
+
+  await refreshSavedObjectIndices(esClient);
+  logger.debug(`Created rule id: ${body.id}`);
   return body;
 }
diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts
index 77806c70ece1a..63960b222cede 100644
--- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts
+++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_wait_for_helpers.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import pRetry from 'p-retry';
+import { ToolingLog } from '@kbn/tooling-log';
 
 import type SuperTest from 'supertest';
 import type { Client } from '@elastic/elasticsearch';
@@ -13,18 +13,28 @@ import type {
   AggregationsAggregate,
   SearchResponse,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { RetryService } from '@kbn/ftr-common-functional-services';
+import { retry } from '../../common/retry';
+
+const TIMEOUT = 70_000;
+const RETRIES = 120;
+const RETRY_DELAY = 500;
 
 export async function waitForRuleStatus({
   id,
   expectedStatus,
   supertest,
+  retryService,
+  logger,
 }: {
   id: string;
   expectedStatus: string;
   supertest: SuperTest.SuperTest<SuperTest.Test>;
+  retryService: RetryService;
+  logger: ToolingLog;
 }): Promise<Record<string, any>> {
-  return pRetry(
-    async () => {
+  const ruleResponse = await retry<Record<string, any>>({
+    test: async () => {
       const response = await supertest.get(`/api/alerting/rule/${id}`);
       const { execution_status: executionStatus } = response.body || {};
       const { status } = executionStatus || {};
@@ -33,42 +43,65 @@ export async function waitForRuleStatus({
       }
       return executionStatus;
     },
-    { retries: 10 }
-  );
+    utilityName: 'fetching rule',
+    logger,
+    retryService,
+    timeout: TIMEOUT,
+    retries: RETRIES,
+    retryDelay: RETRY_DELAY,
+  });
+
+  return ruleResponse;
 }
 
 export async function waitForDocumentInIndex<T>({
   esClient,
   indexName,
   docCountTarget = 1,
+  retryService,
+  logger,
 }: {
   esClient: Client;
   indexName: string;
   docCountTarget?: number;
+  retryService: RetryService;
+  logger: ToolingLog;
 }): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
-  return pRetry(
-    async () => {
+  return await retry<SearchResponse<T, Record<string, AggregationsAggregate>>>({
+    test: async () => {
       const response = await esClient.search<T>({ index: indexName, rest_total_hits_as_int: true });
       if (!response.hits.total || response.hits.total < docCountTarget) {
-        throw new Error('No hits found');
+        throw new Error(
+          `Number of hits does not match expectation (total: ${response.hits.total}, target: ${docCountTarget})`
+        );
       }
+      logger.debug(`Returned document: ${JSON.stringify(response.hits.hits[0])}`);
       return response;
     },
-    { retries: 10 }
-  );
+    utilityName: `waiting for documents in ${indexName} index`,
+    logger,
+    retryService,
+    timeout: TIMEOUT,
+    retries: RETRIES,
+    retryDelay: RETRY_DELAY,
+  });
 }
 
 export async function waitForAlertInIndex<T>({
   esClient,
   indexName,
   ruleId,
+  retryService,
+  logger,
 }: {
   esClient: Client;
   indexName: string;
   ruleId: string;
+  retryService: RetryService;
+  logger: ToolingLog;
 }): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
-  return pRetry(
-    async () => {
+  return await retry<SearchResponse<T, Record<string, AggregationsAggregate>>>({
+    test: async () => {
       const response = await esClient.search<T>({
         index: indexName,
         body: {
@@ -84,6 +117,11 @@ export async function waitForAlertInIndex<T>({
       }
       return response;
     },
-    { retries: 10 }
-  );
+    utilityName: `waiting for alerting document in the alerting index (${indexName})`,
+    logger,
+    retryService,
+    timeout: TIMEOUT,
+    retries: RETRIES,
+    retryDelay: RETRY_DELAY,
+  });
 }
diff --git a/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts b/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts
index 0b0e85b104962..1f84b2556ab70 100644
--- a/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts
+++ b/x-pack/test/alerting_api_integration/observability/helpers/data_view.ts
@@ -6,17 +6,20 @@
  */
 
 import { SuperTest, Test } from 'supertest';
+import { ToolingLog } from '@kbn/tooling-log';
 
 export const createDataView = async ({
   supertest,
   id,
   name,
   title,
+  logger,
 }: {
   supertest: SuperTest<Test>;
   id: string;
   name: string;
   title: string;
+  logger: ToolingLog;
 }) => {
   const { body } = await supertest
     .post(`/api/content_management/rpc/create`)
@@ -36,15 +39,20 @@ export const createDataView = async ({
       },
       options: { id },
       version: 1,
-    });
+    })
+    .expect(200);
+
+  logger.debug(`Created data view: ${JSON.stringify(body)}`);
   return body;
 };
 export const deleteDataView = async ({
   supertest,
   id,
+  logger,
 }: {
   supertest: SuperTest<Test>;
   id: string;
+  logger: ToolingLog;
 }) => {
   const { body } = await supertest
     .post(`/api/content_management/rpc/delete`)
@@ -54,6 +62,9 @@ export const deleteDataView = async ({
       id,
       options: { force: true },
       version: 1,
-    });
+    })
+    .expect(200);
+
+  logger.debug(`Deleted data view id: ${id}`);
   return body;
 };
diff --git a/x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts b/x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts
new file mode 100644
index 0000000000000..4808a07ba4b1d
--- /dev/null
+++ b/x-pack/test/alerting_api_integration/observability/helpers/refresh_index.ts
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
+
+/**
+ * Copied from x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts
+ *
+ * Refresh an index, making changes available to search.
+ * Reusable utility which refreshes all saved object indices, to make them available for search, especially
+ * useful when needing to perform a search on an index that has just been written to.
+ *
+ * An example of this when installing the prebuilt detection rules SO of type 'security-rule':
+ * the savedObjectsClient does this with a call with explicit `refresh: false`.
+ * So, despite of the fact that the endpoint waits until the prebuilt rule will be
+ * successfully indexed, it doesn't wait until they become "visible" for subsequent read
+ * operations.
+ *
+ * Additionally, this method clears the cache for all saved object indices. This helps in cases in which
+ * saved object is read, then written to, and then read again, and the second read returns stale data.
+ * @param es The Elasticsearch client
+ */
+export const refreshSavedObjectIndices = async (es: Client) => {
+  // Refresh indices to prevent a race condition between a write and subsequent read operation. To
+  // fix it deterministically we have to refresh saved object indices and wait until it's done.
+  await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });
+
+  // Additionally, we need to clear the cache to ensure that the next read operation will
+  // not return stale data.
+  await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES });
+};
diff --git a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts
index 4c4c5306d708a..808df288b094e 100644
--- a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts
+++ b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts
@@ -28,6 +28,7 @@ export default function ({ getService }: FtrProviderContext) {
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const supertest = getService('supertest');
   const logger = getService('log');
+  const retryService = getService('retry');
 
   describe('Metric threshold rule >', () => {
     let ruleId: string;
@@ -49,17 +50,34 @@ export default function ({ getService }: FtrProviderContext) {
           name: 'Default',
         });
         dataForgeConfig = {
-          schedule: [{ template: 'good', start: 'now-15m', end: 'now' }],
+          schedule: [
+            {
+              template: 'good',
+              start: 'now-10m',
+              end: 'now+5m',
+              metrics: [{ name: 'system.cpu.user.pct', method: 'linear', start: 0.9, end: 0.9 }],
+            },
+          ],
           indexing: { dataset: 'fake_hosts' as Dataset },
         };
         dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
+        await waitForDocumentInIndex({
+          esClient,
+          indexName: dataForgeIndices.join(','),
+          docCountTarget: 45,
+          retryService,
+          logger,
+        });
         actionId = await createIndexConnector({
           supertest,
           name: 'Index Connector: Metric threshold API test',
           indexName: ALERT_ACTION_INDEX,
+          logger,
         });
         const createdRule = await createRule<MetricThresholdParams>({
           supertest,
+          logger,
+          esClient,
           ruleTypeId: InfraRuleType.MetricThreshold,
           consumer: 'infrastructure',
           tags: ['infrastructure'],
@@ -88,6 +106,7 @@ export default function ({ getService }: FtrProviderContext) {
                   {
                     ruleType: '{{rule.type}}',
                     alertDetailsUrl: '{{context.alertDetailsUrl}}',
+                    reason: '{{context.reason}}',
                   },
                 ],
               },
@@ -125,6 +144,8 @@ export default function ({ getService }: FtrProviderContext) {
           id: ruleId,
           expectedStatus: 'active',
           supertest,
+          retryService,
+          logger,
         });
         expect(executionStatus.status).to.be('active');
       });
@@ -134,6 +155,8 @@ export default function ({ getService }: FtrProviderContext) {
           esClient,
           indexName: METRICS_ALERTS_INDEX,
           ruleId,
+          retryService,
+          logger,
         });
         alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
         startedAt = (resp.hits.hits[0]._source as any)['kibana.alert.start'];
@@ -188,15 +211,24 @@ export default function ({ getService }: FtrProviderContext) {
 
       it('should set correct action parameter: ruleType', async () => {
         const rangeFrom = moment(startedAt).subtract('5', 'minute').toISOString();
-        const resp = await waitForDocumentInIndex<{ ruleType: string; alertDetailsUrl: string }>({
+        const resp = await waitForDocumentInIndex<{
+          ruleType: string;
+          alertDetailsUrl: string;
+          reason: string;
+        }>({
           esClient,
           indexName: ALERT_ACTION_INDEX,
+          retryService,
+          logger,
         });
 
         expect(resp.hits.hits[0]._source?.ruleType).eql('metrics.alert.threshold');
         expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
           `https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
         );
+        expect(resp.hits.hits[0]._source?.reason).eql(
+          `system.cpu.user.pct is 90% in the last 5 mins. Alert when > 50%.`
+        );
       });
     });
   });