[Detection Engine] Fix flake in ML Rule Cypress tests (elastic#188164)

This API call was found to be sporadically failing in elastic#182183. This
applies the same changes made in elastic#188155, but for Cypress tests instead
of FTR.

Since none of the cypress tests are currently skipped, this PR just
serves to add robustness to the suite, which performs nearly identical
setup to that of the FTR tests. I think the biggest difference is how
often these tests are run vs FTRs. Combined with the low failure rate
for the underlying issue, cypress's auto-retrying may smooth over many
of these failures when they occur.


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] [Detection Engine Cypress - ESS x
200](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6530)
- [ ] [Detection Engine Cypress - Serverless x
200](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6531)

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts

@@ -19,9 +19,9 @@ import {
   SUPPRESS_MISSING_FIELD,
 } from '../../../../screens/rule_details';
 import {
-  executeSetupModuleRequest,
   forceStartDatafeeds,
   forceStopAndCloseJob,
+  setupMlModulesWithRetry,
 } from '../../../../support/machine_learning';
 import {
   continueFromDefineStep,
@@ -94,7 +94,7 @@ describe(
       describe('when ML jobs have run', () => {
         before(() => {
           cy.task('esArchiverLoad', { archiveName: '../auditbeat/hosts', type: 'ftr' });
-          executeSetupModuleRequest({ moduleName: 'security_linux_v3' });
+          setupMlModulesWithRetry({ moduleName: 'security_linux_v3' });
           forceStartDatafeeds({ jobIds: [jobId] });
           cy.task('esArchiverLoad', { archiveName: 'anomalies', type: 'ftr' });
         });

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts

@@ -19,9 +19,9 @@ import {
   SUPPRESS_MISSING_FIELD,
 } from '../../../../screens/rule_details';
 import {
-  executeSetupModuleRequest,
   forceStartDatafeeds,
   forceStopAndCloseJob,
+  setupMlModulesWithRetry,
 } from '../../../../support/machine_learning';
 import { editFirstRule } from '../../../../tasks/alerts_detection_rules';
 import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common';
@@ -71,7 +71,7 @@ describe(
       login();
       deleteAlertsAndRules();
       cy.task('esArchiverLoad', { archiveName: '../auditbeat/hosts', type: 'ftr' });
-      executeSetupModuleRequest({ moduleName: 'security_linux_v3' });
+      setupMlModulesWithRetry({ moduleName: 'security_linux_v3' });
       forceStartDatafeeds({ jobIds: [jobId] });
       cy.task('esArchiverLoad', { archiveName: 'anomalies', type: 'ftr' });
     });

x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { recurse } from 'cypress-recurse';
 import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants';
 import { rootRequest } from '../tasks/api_calls/common';
 
@@ -16,7 +17,7 @@ import { rootRequest } from '../tasks/api_calls/common';
  * @returns the response from the setup module request
  */
 export const executeSetupModuleRequest = ({ moduleName }: { moduleName: string }) =>
-  rootRequest({
+  rootRequest<{ jobs: Array<{ success: boolean; error?: { status: number } }> }>({
     headers: {
       'elastic-api-version': 1,
     },
@@ -33,6 +34,23 @@ export const executeSetupModuleRequest = ({ moduleName }: { moduleName: string }
     },
   });
 
+/**
+ *
+ * Calls {@link executeSetupModuleRequest} until all jobs in the module are
+ * successfully set up.
+ * @param moduleName the name of the ML module to set up
+ * @returns the response from the setup module request
+ */
+export const setupMlModulesWithRetry = ({ moduleName }: { moduleName: string }) =>
+  recurse(
+    () => executeSetupModuleRequest({ moduleName }),
+    (response) =>
+      response.body.jobs.every(
+        (job) => job.success || (job.error?.status && job.error.status < 500)
+      ),
+    { delay: 1000 }
+  );
+
 /**
  *
  * Calls the internal ML Jobs API to force start the datafeeds for the given job IDs. Necessary to get them in the "started" state for the purposes of the detection engine