From 9119133467845751ab699e9e4ded20b426f9d8d0 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 16 Jul 2024 14:21:13 -0500 Subject: [PATCH] [Detection Engine] Fix flake in ML Rule Cypress tests (#188164) This API call was found to be sporadically failing in #182183. This applies the same changes made in #188155, but for Cypress tests instead of FTR. Since none of the cypress tests are currently skipped, this PR just serves to add robustness to the suite, which performs nearly identical setup to that of the FTR tests. I think the biggest difference is how often these tests are run vs FTRs. Combined with the low failure rate for the underlying issue, cypress's auto-retrying may smooth over many of these failures when they occur. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] [Detection Engine Cypress - ESS x 200](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6530) - [ ] [Detection Engine Cypress - Serverless x 200](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6531) (cherry picked from commit ed934e3253b47a6902904633530ec181037d4946) --- .../machine_learning_rule_suppression.cy.ts | 4 ++-- .../rule_edit/machine_learning_rule.cy.ts | 4 ++-- .../cypress/support/machine_learning.ts | 20 ++++++++++++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts index befa75fce93ff..3052c6dbc95c2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts @@ -19,9 +19,9 @@ import { SUPPRESS_MISSING_FIELD, } from '../../../../screens/rule_details'; import { - executeSetupModuleRequest, forceStartDatafeeds, forceStopAndCloseJob, + setupMlModulesWithRetry, } from '../../../../support/machine_learning'; import { continueFromDefineStep, @@ -94,7 +94,7 @@ describe( describe('when ML jobs have run', () => { before(() => { cy.task('esArchiverLoad', { archiveName: '../auditbeat/hosts', type: 'ftr' }); - executeSetupModuleRequest({ moduleName: 'security_linux_v3' }); + setupMlModulesWithRetry({ moduleName: 'security_linux_v3' }); forceStartDatafeeds({ jobIds: [jobId] }); cy.task('esArchiverLoad', { archiveName: 'anomalies', type: 'ftr' }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts index 5e6cd673070ba..abaa3c567b690 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts @@ -19,9 +19,9 @@ import { SUPPRESS_MISSING_FIELD, } from '../../../../screens/rule_details'; import { - executeSetupModuleRequest, forceStartDatafeeds, forceStopAndCloseJob, + setupMlModulesWithRetry, } from '../../../../support/machine_learning'; import { editFirstRule } from '../../../../tasks/alerts_detection_rules'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; @@ -71,7 +71,7 @@ describe( login(); deleteAlertsAndRules(); cy.task('esArchiverLoad', { archiveName: '../auditbeat/hosts', type: 'ftr' }); - executeSetupModuleRequest({ moduleName: 'security_linux_v3' }); + setupMlModulesWithRetry({ moduleName: 'security_linux_v3' }); forceStartDatafeeds({ jobIds: [jobId] }); cy.task('esArchiverLoad', { archiveName: 'anomalies', type: 'ftr' }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts b/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts index 5fb869cebc29f..b10b4d827445a 100644 --- a/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts +++ b/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { recurse } from 'cypress-recurse'; import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants'; import { rootRequest } from '../tasks/api_calls/common'; @@ -16,7 +17,7 @@ import { rootRequest } from '../tasks/api_calls/common'; * @returns the response from the setup module request */ export const executeSetupModuleRequest = ({ moduleName }: { moduleName: string }) => - rootRequest({ + rootRequest<{ jobs: Array<{ success: boolean; error?: { status: number } }> }>({ headers: { 'elastic-api-version': 1, }, @@ -33,6 +34,23 @@ export const executeSetupModuleRequest = ({ moduleName }: { moduleName: string } }, }); +/** + * + * Calls {@link executeSetupModuleRequest} until all jobs in the module are + * successfully set up. + * @param moduleName the name of the ML module to set up + * @returns the response from the setup module request + */ +export const setupMlModulesWithRetry = ({ moduleName }: { moduleName: string }) => + recurse( + () => executeSetupModuleRequest({ moduleName }), + (response) => + response.body.jobs.every( + (job) => job.success || (job.error?.status && job.error.status < 500) + ), + { delay: 1000 } + ); + /** * * Calls the internal ML Jobs API to force start the datafeeds for the given job IDs. Necessary to get them in the "started" state for the purposes of the detection engine