[Security Solution] Add retryIfConflict util for 409 conflicts in Integration tests

x-pack/test/security_solution_api_integration/package.json

@@ -83,7 +83,7 @@
     "prebuilt_rules_update_prebuilt_rules_package:runner:serverless": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package serverless serverlessEnv",
     "prebuilt_rules_update_prebuilt_rules_package:qa:serverless": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package serverless qaEnv",
     "prebuilt_rules_update_prebuilt_rules_package:server:ess": "npm run initialize-server:dr:default prebuilt_rules/update_prebuilt_rules_package ess",
-    "prebuilt_rules_update_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package ess essEnvs",
+    "prebuilt_rules_update_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package ess essEnv",
 
     "rule_execution_logic:server:serverless": "npm run initialize-server:dr:default rule_execution_logic serverless",
     "rule_execution_logic:runner:serverless": "npm run run-tests:dr:default rule_execution_logic serverless serverlessEnv",

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts

@@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext): void => {
   describe.skip('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllPrebuiltRuleAssets(es, log);
     });
 
     it('should list `security_detection_engine` as a bundled fleet package in the `fleet_package.json` file', async () => {

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts

@@ -33,7 +33,7 @@ export default ({ getService }: FtrProviderContext): void => {
   describe('@ess @serverless @skipInQA prerelease_packages', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllPrebuiltRuleAssets(es, log);
       await deletePrebuiltRulesFleetPackage(supertest);
     });
 

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts

@@ -21,12 +21,12 @@ export default ({ getService }: FtrProviderContext): void => {
   describe('@ess @serverless @skipInQA install_large_prebuilt_rules_package', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllPrebuiltRuleAssets(es, log);
     });
 
     afterEach(async () => {
       await deleteAllRules(supertest, log);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllPrebuiltRuleAssets(es, log);
     });
 
     it('should install a package containing 15000 prebuilt rules without crashing', async () => {

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts

@@ -25,8 +25,8 @@ export default ({ getService }: FtrProviderContext): void => {
     beforeEach(async () => {
       await deletePrebuiltRulesFleetPackage(supertest);
       await deleteAllRules(supertest, log);
-      await deleteAllTimelines(es);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllTimelines(es, log);
+      await deleteAllPrebuiltRuleAssets(es, log);
     });
 
     /**

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts

@@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => {
   describe('@ess @serverless @skipInQA Prebuilt Rules status', () => {
     describe('get_prebuilt_rules_status', () => {
       beforeEach(async () => {
-        await deleteAllPrebuiltRuleAssets(es);
+        await deleteAllPrebuiltRuleAssets(es, log);
         await deleteAllRules(supertest, log);
       });
 
@@ -110,7 +110,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Increment the version of one of the installed rules and create the new rule assets
           ruleAssetSavedObjects[0]['security-rule'].version += 1;
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
@@ -130,7 +130,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Increment the version of one of the installed rules and create the new rule assets
           ruleAssetSavedObjects[0]['security-rule'].version += 1;
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
@@ -152,7 +152,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Recreate the rules without bumping any versions
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
 
@@ -238,7 +238,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Delete the previous versions of rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
 
           // Add a new rule version
           await createHistoricalPrebuiltRuleAssetSavedObjects(es, [
@@ -261,7 +261,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Delete the previous versions of rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
 
           // Add a new rule version
           await createHistoricalPrebuiltRuleAssetSavedObjects(es, [
@@ -286,7 +286,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
     describe('get_prebuilt_rules_status - legacy', () => {
       beforeEach(async () => {
-        await deleteAllPrebuiltRuleAssets(es);
+        await deleteAllPrebuiltRuleAssets(es, log);
         await deleteAllRules(supertest, log);
       });
 
@@ -367,7 +367,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRulesAndTimelines(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Increment the version of one of the installed rules and create the new rule assets
           ruleAssetSavedObjects[0]['security-rule'].version += 1;
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
@@ -387,7 +387,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRulesAndTimelines(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Recreate the rules without bumping any versions
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
 
@@ -473,7 +473,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRulesAndTimelines(es, supertest);
 
           // Delete the previous versions of rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
 
           // Add a new rule version
           await createHistoricalPrebuiltRuleAssetSavedObjects(es, [

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts

@@ -16,10 +16,11 @@ import {
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
   const es = getService('es');
+  const log = getService('log');
 
   describe('@ess @serverless @skipInQA get_prebuilt_timelines_status', () => {
     beforeEach(async () => {
-      await deleteAllTimelines(es);
+      await deleteAllTimelines(es, log);
     });
 
     it('should return the number of timeline templates available to install', async () => {

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts

@@ -30,8 +30,8 @@ export default ({ getService }: FtrProviderContext): void => {
   describe('@ess @serverless @skipInQA install and upgrade prebuilt rules with mock rule assets', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
-      await deleteAllTimelines(es);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllTimelines(es, log);
+      await deleteAllPrebuiltRuleAssets(es, log);
     });
 
     describe(`rule package without historical versions`, () => {
@@ -96,7 +96,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRulesAndTimelines(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Increment the version of one of the installed rules and create the new rule assets
           ruleAssetSavedObjects[0]['security-rule'].version += 1;
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
@@ -177,7 +177,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
           // Increment the version of one of the installed rules and create the new rule assets
           ruleAssetSavedObjects[0]['security-rule'].version += 1;
           await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects);
@@ -315,7 +315,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRulesAndTimelines(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
 
           // Add a new rule version
           await createHistoricalPrebuiltRuleAssetSavedObjects(es, [
@@ -423,7 +423,7 @@ export default ({ getService }: FtrProviderContext): void => {
           await installPrebuiltRules(es, supertest);
 
           // Clear previous rule assets
-          await deleteAllPrebuiltRuleAssets(es);
+          await deleteAllPrebuiltRuleAssets(es, log);
 
           // Add a new rule version
           await createHistoricalPrebuiltRuleAssetSavedObjects(es, [

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts

@@ -89,7 +89,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
-      await deleteAllPrebuiltRuleAssets(es);
+      await deleteAllPrebuiltRuleAssets(es, log);
     });
 
     it('should allow user to install prebuilt rules from scratch, then install new rules and upgrade existing rules from the new package', async () => {

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts

@@ -89,7 +89,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
     describe('when there are installed prebuilt rules', () => {
       beforeEach(async () => {
-        await deleteAllPrebuiltRuleAssets(es);
+        await deleteAllPrebuiltRuleAssets(es, log);
         await installMockPrebuiltRules(supertest, es);
       });
 

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry_delete_by_query_conflicts.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeleteByQueryResponse } from '@elastic/elasticsearch/lib/api/types';
+import { ToolingLog } from '@kbn/tooling-log';
+
+// Number of times to retry when conflicts occur
+export const RetryForConflictsAttempts = 2;
+
+// Delay between retries when conflicts occur
+export const RetryForConflictsDelay = 200;
+
+/*
+ * Retry an Elasticsearch deleteByQuery operation if it runs into 409 Conflicts,
+ * up to a maximum number of attempts.
+ */
+export async function retryIfDeleteByQueryConflicts<T>(
+  logger: ToolingLog,
+  name: string,
+  operation: () => Promise<DeleteByQueryResponse>,
+  retries: number = RetryForConflictsAttempts,
+  retryDelay: number = RetryForConflictsDelay
+): Promise<DeleteByQueryResponse> {
+  const operationResult = await operation();
+  if (!operationResult.failures || operationResult.failures?.length === 0) {
+    return operationResult;
+  }
+
+  for (const failure of operationResult.failures) {
+    if (failure.status === 409) {
+      // if no retries left, throw it
+      if (retries <= 0) {
+        logger.error(`${name} conflict, exceeded retries`);
+        throw new Error(`${name} conflict, exceeded retries`);
+      }
+
+      // Otherwise, delay a bit before retrying
+      logger.debug(`${name} conflict, retrying ...`);
+      await waitBeforeNextRetry(retryDelay);
+      return await retryIfDeleteByQueryConflicts(logger, name, operation, retries - 1);
+    }
+  }
+
+  return operationResult;
+}
+
+async function waitBeforeNextRetry(retryDelay: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, retryDelay));
+}

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets.ts

@@ -4,20 +4,26 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import type { ToolingLog } from '@kbn/tooling-log';
 import type { Client } from '@elastic/elasticsearch';
 import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
+import { retryIfDeleteByQueryConflicts } from '../../retry_delete_by_query_conflicts';
 
 /**
  * Remove all prebuilt rule assets from the security solution savedObjects index
  * @param es The ElasticSearch handle
  */
-export const deleteAllPrebuiltRuleAssets = async (es: Client): Promise<void> => {
-  await es.deleteByQuery({
-    index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
-    q: 'type:security-rule',
-    wait_for_completion: true,
-    refresh: true,
-    body: {},
+export const deleteAllPrebuiltRuleAssets = async (
+  es: Client,
+  logger: ToolingLog
+): Promise<void> => {
+  await retryIfDeleteByQueryConflicts(logger, deleteAllPrebuiltRuleAssets.name, async () => {
+    return await es.deleteByQuery({
+      index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
+      q: 'type:security-rule',
+      wait_for_completion: true,
+      refresh: true,
+      body: {},
+    });
   });
 };

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_timelines.ts

@@ -4,20 +4,23 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import type { ToolingLog } from '@kbn/tooling-log';
 import type { Client } from '@elastic/elasticsearch';
 import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
+import { retryIfDeleteByQueryConflicts } from '../../retry_delete_by_query_conflicts';
 
 /**
  * Remove all timelines from the security solution savedObjects index
  * @param es The ElasticSearch handle
  */
-export const deleteAllTimelines = async (es: Client): Promise<void> => {
-  await es.deleteByQuery({
-    index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
-    q: 'type:siem-ui-timeline',
-    wait_for_completion: true,
-    refresh: true,
-    body: {},
+export const deleteAllTimelines = async (es: Client, logger: ToolingLog): Promise<void> => {
+  await retryIfDeleteByQueryConflicts(logger, deleteAllTimelines.name, async () => {
+    return await es.deleteByQuery({
+      index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX,
+      q: 'type:siem-ui-timeline',
+      wait_for_completion: true,
+      refresh: true,
+      body: {},
+    });
   });
 };