diff --git a/x-pack/test/security_solution_api_integration/package.json b/x-pack/test/security_solution_api_integration/package.json index 9dd84db6cb924..dae58653831ef 100644 --- a/x-pack/test/security_solution_api_integration/package.json +++ b/x-pack/test/security_solution_api_integration/package.json @@ -87,7 +87,8 @@ "prebuilt_rules_update_prebuilt_rules_package:runner:serverless": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package serverless serverlessEnv", "prebuilt_rules_update_prebuilt_rules_package:qa:serverless": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package serverless qaEnv", "prebuilt_rules_update_prebuilt_rules_package:server:ess": "npm run initialize-server:dr:default prebuilt_rules/update_prebuilt_rules_package ess", - "prebuilt_rules_update_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package ess essEnvs", + "prebuilt_rules_update_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package ess essEnv", + "rule_execution_logic:server:serverless": "npm run initialize-server:dr:default rule_execution_logic serverless", "rule_execution_logic:runner:serverless": "npm run run-tests:dr:default rule_execution_logic serverless serverlessEnv", "rule_execution_logic:qa:serverless": "npm run run-tests:dr:default rule_execution_logic serverless qaEnv", diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index d2908f148690f..33f79b1bff8d7 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext): void => { describe('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => { beforeEach(async () => { await deleteAllRules(supertest, log); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); }); it('should list `security_detection_engine` as a bundled fleet package in the `fleet_package.json` file', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts index f9450929cf53e..83faf92bf7c84 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts @@ -34,7 +34,7 @@ export default ({ getService }: FtrProviderContext): void => { describe('@ess @serverless @skipInQA prerelease_packages', () => { beforeEach(async () => { await deleteAllRules(supertest, log); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); await deletePrebuiltRulesFleetPackage(supertest); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts index 20f0e214c1056..17a6723a53b88 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts @@ -21,12 +21,12 @@ export default ({ getService }: FtrProviderContext): void => { describe('@ess @serverless @skipInQA install_large_prebuilt_rules_package', () => { beforeEach(async () => { await deleteAllRules(supertest, log); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); }); afterEach(async () => { await deleteAllRules(supertest, log); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); }); it('should install a package containing 15000 prebuilt rules without crashing', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index b8f29010d5a0c..a9653d7593209 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -26,8 +26,8 @@ export default ({ getService }: FtrProviderContext): void => { beforeEach(async () => { await deletePrebuiltRulesFleetPackage(supertest); await deleteAllRules(supertest, log); - await deleteAllTimelines(es); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllTimelines(es, log); + await deleteAllPrebuiltRuleAssets(es, log); }); /** diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts index bfbe4f00cb32a..eca4e51bcc6f8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => { describe('@ess @serverless @skipInQA Prebuilt Rules status', () => { describe('get_prebuilt_rules_status', () => { beforeEach(async () => { - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); await deleteAllRules(supertest, log); }); @@ -110,7 +110,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Increment the version of one of the installed rules and create the new rule assets ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -130,7 +130,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Increment the version of one of the installed rules and create the new rule assets ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -152,7 +152,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Recreate the rules without bumping any versions await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -238,7 +238,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Delete the previous versions of rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Add a new rule version await createHistoricalPrebuiltRuleAssetSavedObjects(es, [ @@ -261,7 +261,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Delete the previous versions of rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Add a new rule version await createHistoricalPrebuiltRuleAssetSavedObjects(es, [ @@ -286,7 +286,7 @@ export default ({ getService }: FtrProviderContext): void => { describe('get_prebuilt_rules_status - legacy', () => { beforeEach(async () => { - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); await deleteAllRules(supertest, log); }); @@ -367,7 +367,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Increment the version of one of the installed rules and create the new rule assets ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -387,7 +387,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Recreate the rules without bumping any versions await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -473,7 +473,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Delete the previous versions of rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Add a new rule version await createHistoricalPrebuiltRuleAssetSavedObjects(es, [ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts index cf7a2c961e063..60b1e5e1ba526 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts @@ -16,10 +16,11 @@ import { export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); + const log = getService('log'); describe('@ess @serverless @skipInQA get_prebuilt_timelines_status', () => { beforeEach(async () => { - await deleteAllTimelines(es); + await deleteAllTimelines(es, log); }); it('should return the number of timeline templates available to install', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts index 4a85fe68fcba3..4e7ea364f0968 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts @@ -30,8 +30,8 @@ export default ({ getService }: FtrProviderContext): void => { describe('@ess @serverless @skipInQA install and upgrade prebuilt rules with mock rule assets', () => { beforeEach(async () => { await deleteAllRules(supertest, log); - await deleteAllTimelines(es); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllTimelines(es, log); + await deleteAllPrebuiltRuleAssets(es, log); }); describe(`rule package without historical versions`, () => { @@ -96,7 +96,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Increment the version of one of the installed rules and create the new rule assets ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -177,7 +177,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Increment the version of one of the installed rules and create the new rule assets ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); @@ -315,7 +315,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Add a new rule version await createHistoricalPrebuiltRuleAssetSavedObjects(es, [ @@ -423,7 +423,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Clear previous rule assets - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); // Add a new rule version await createHistoricalPrebuiltRuleAssetSavedObjects(es, [ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts index 2711f28ce88e4..688816569c181 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts @@ -90,7 +90,7 @@ export default ({ getService }: FtrProviderContext): void => { beforeEach(async () => { await deleteAllRules(supertest, log); - await deleteAllPrebuiltRuleAssets(es); + await deleteAllPrebuiltRuleAssets(es, log); }); it('should allow user to install prebuilt rules from scratch, then install new rules and upgrade existing rules from the new package', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry_delete_by_query_conflicts.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry_delete_by_query_conflicts.ts new file mode 100644 index 0000000000000..1d2cd8b24239c --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry_delete_by_query_conflicts.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeleteByQueryResponse } from '@elastic/elasticsearch/lib/api/types'; +import { ToolingLog } from '@kbn/tooling-log'; + +// Number of times to retry when conflicts occur +const RETRY_ATTEMPTS = 2; + +// Delay between retries when conflicts occur +const RETRY_DELAY = 200; + +/* + * Retry an Elasticsearch deleteByQuery operation if it runs into 409 Conflicts, + * up to a maximum number of attempts. + */ +export async function retryIfDeleteByQueryConflicts( + logger: ToolingLog, + name: string, + operation: () => Promise, + retries: number = RETRY_ATTEMPTS, + retryDelay: number = RETRY_DELAY +): Promise { + const operationResult = await operation(); + if (!operationResult.failures || operationResult.failures?.length === 0) { + return operationResult; + } + + for (const failure of operationResult.failures) { + if (failure.status === 409) { + // if no retries left, throw it + if (retries <= 0) { + logger.error(`${name} conflict, exceeded retries`); + throw new Error(`${name} conflict, exceeded retries`); + } + + // Otherwise, delay a bit before retrying + logger.debug(`${name} conflict, retrying ...`); + await waitBeforeNextRetry(retryDelay); + return await retryIfDeleteByQueryConflicts(logger, name, operation, retries - 1); + } + } + + return operationResult; +} + +async function waitBeforeNextRetry(retryDelay: number): Promise { + await new Promise((resolve) => setTimeout(resolve, retryDelay)); +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets.ts index 899d5ddd7f83f..179840ee608fc 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets.ts @@ -4,20 +4,26 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import type { ToolingLog } from '@kbn/tooling-log'; import type { Client } from '@elastic/elasticsearch'; import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { retryIfDeleteByQueryConflicts } from '../../retry_delete_by_query_conflicts'; /** * Remove all prebuilt rule assets from the security solution savedObjects index * @param es The ElasticSearch handle */ -export const deleteAllPrebuiltRuleAssets = async (es: Client): Promise => { - await es.deleteByQuery({ - index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, - q: 'type:security-rule', - wait_for_completion: true, - refresh: true, - body: {}, +export const deleteAllPrebuiltRuleAssets = async ( + es: Client, + logger: ToolingLog +): Promise => { + await retryIfDeleteByQueryConflicts(logger, deleteAllPrebuiltRuleAssets.name, async () => { + return await es.deleteByQuery({ + index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, + q: 'type:security-rule', + wait_for_completion: true, + refresh: true, + body: {}, + }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_timelines.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_timelines.ts index 291cd269580b0..df677656fbd94 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_timelines.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_all_timelines.ts @@ -4,20 +4,23 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import type { ToolingLog } from '@kbn/tooling-log'; import type { Client } from '@elastic/elasticsearch'; import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { retryIfDeleteByQueryConflicts } from '../../retry_delete_by_query_conflicts'; /** * Remove all timelines from the security solution savedObjects index * @param es The ElasticSearch handle */ -export const deleteAllTimelines = async (es: Client): Promise => { - await es.deleteByQuery({ - index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, - q: 'type:siem-ui-timeline', - wait_for_completion: true, - refresh: true, - body: {}, +export const deleteAllTimelines = async (es: Client, logger: ToolingLog): Promise => { + await retryIfDeleteByQueryConflicts(logger, deleteAllTimelines.name, async () => { + return await es.deleteByQuery({ + index: SECURITY_SOLUTION_SAVED_OBJECT_INDEX, + q: 'type:siem-ui-timeline', + wait_for_completion: true, + refresh: true, + body: {}, + }); }); };