[security_solution] Cypress flaky tests catcher (#162376)

## Summary

Inspired by https://glebbahmutov.com/blog/burning-tests/

Implements the idea presented here
https://glebbahmutov.com/blog/burning-tests/#bonus-3-burning-new-or-changed-specs

In short, on PR that is changing/adding new Cypress spec files we will
try to "burn" them, it means we will try to run each `it` `2` times to
make sure tests are written in a way that gives Cypress a chance to
recover from the failed test.
Also adding a command that allows to "burn" tests locally
```
yarn cypress:burn --spec "<>"
```

Right now the job is set to `soft_fail`, so it is not going to block the
PR from merging, but hopefully will help the Team to recognize potential
flakiness before it is merged to `main`

.buildkite/pipelines/pull_request/security_solution.yml

@@ -12,3 +12,14 @@ steps:
           limit: 1
     artifact_paths:
       - "target/kibana-security-solution/**/*"
+
+  - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
+    label: 'Security Solution Cypress tests, burning changed specs'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 1
+    soft_fail: true
+    artifact_paths:
+      - "target/kibana-security-solution/**/*"

.buildkite/scripts/steps/functional/security_solution_burn.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false'
+
+echo "--- Security Solution Cypress tests, burning changed specs (Chrome)"
+
+yarn --cwd x-pack/plugins/security_solution cypress:changed-specs-only

package.json

@@ -1444,6 +1444,7 @@
     "faker": "^5.1.0",
     "fetch-mock": "^7.3.9",
     "file-loader": "^4.2.0",
+    "find-cypress-specs": "^1.35.1",
     "form-data": "^4.0.0",
     "geckodriver": "^4.0.0",
     "gulp-brotli": "^3.0.0",
@@ -1526,6 +1527,7 @@
     "sinon": "^7.4.2",
     "sort-package-json": "^1.53.1",
     "source-map": "^0.7.4",
+    "spec-change": "^1.7.1",
     "string-replace-loader": "^2.2.0",
     "style-loader": "^1.1.3",
     "stylelint": "^14.9.1",

typings/index.d.ts

@@ -20,3 +20,5 @@ declare module 'react-syntax-highlighter/dist/cjs/prism-light';
 declare module 'monaco-editor/esm/vs/basic-languages/markdown/markdown';
 declare module 'monaco-editor/esm/vs/basic-languages/css/css';
 declare module 'monaco-editor/esm/vs/basic-languages/yaml/yaml';
+
+declare module 'find-cypress-specs';

x-pack/plugins/security_solution/cypress/support/e2e.js

@@ -23,6 +23,9 @@
 // Import commands.js using ES2015 syntax:
 import './commands';
 import 'cypress-real-events/support';
+import registerCypressGrep from '@cypress/grep';
+
+registerCypressGrep();
 
 Cypress.on('uncaught:exception', () => {
   return false;

x-pack/plugins/security_solution/package.json

@@ -8,17 +8,19 @@
     "extract-mitre-attacks": "node scripts/extract_tactics_techniques_mitre.js && node ../../../scripts/eslint ./public/detections/mitre/mitre_tactics_techniques.ts --fix",
     "build-beat-doc": "node scripts/beat_docs/build.js && node ../../../scripts/eslint ../timelines/server/utils/beat_schema/fields.ts --fix",
     "cypress": "../../../node_modules/.bin/cypress",
+    "cypress:burn": "yarn cypress:run:reporter --env burn=2 --concurrency=1 --headed",
+    "cypress:changed-specs-only": "yarn cypress:run:reporter --changed-specs-only --env burn=2",
     "cypress:open": "TZ=UTC node ./scripts/start_cypress_parallel open --spec './cypress/e2e/**/*.cy.ts' --config-file ./cypress/cypress.config.ts --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config",
-    "cypress:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/{,!(investigations,explore)/**/}*.cy.ts'; status=$?; yarn junit:merge && exit $status",
-    "cypress:run:cases": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/explore/cases/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
-    "cypress:run:reporter": "TZ=UTC node ./scripts/start_cypress_parallel run --config-file ./cypress/cypress_ci.config.ts --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json",
-    "cypress:run:respops": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/(detection_alerts|detection_rules|exceptions)/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
+    "cypress:run": "yarn cypress:run:reporter --spec './cypress/e2e/{,!(investigations,explore)/**/}*.cy.ts'; status=$?; yarn junit:merge && exit $status",
+    "cypress:run:cases": "yarn cypress:run:reporter --spec './cypress/e2e/explore/cases/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
+    "cypress:run:reporter": "TZ=UTC node ./scripts/start_cypress_parallel run --browser chrome --config-file ./cypress/cypress_ci.config.ts --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json",
+    "cypress:run:respops": "yarn cypress:run:reporter --spec './cypress/e2e/(detection_alerts|detection_rules|exceptions)/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
     "cypress:dw:open": "node ./scripts/start_cypress_parallel open --config-file ./public/management/cypress.config.ts ts --ftr-config-file ../../../../../../x-pack/test/defend_workflows_cypress/cli_config",
     "cypress:dw:run": "node ./scripts/start_cypress_parallel run --config-file ./public/management/cypress.config.ts --ftr-config-file ../../../../../../x-pack/test/defend_workflows_cypress/cli_config --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json; status=$?; yarn junit:merge && exit $status",
     "cypress:dw:endpoint:run": "node ./scripts/start_cypress_parallel run --config-file ./public/management/cypress_endpoint.config.ts --ftr-config-file ../../../../../../x-pack/test/defend_workflows_cypress/endpoint_config --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json --concurrency 1; status=$?; yarn junit:merge && exit $status",
     "cypress:dw:endpoint:open": "node ./scripts/start_cypress_parallel open --config-file ./public/management/cypress_endpoint.config.ts ts --ftr-config-file ../../../../../../x-pack/test/defend_workflows_cypress/endpoint_config",
-    "cypress:investigations:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/investigations/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
-    "cypress:explore:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/explore/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
+    "cypress:investigations:run": "yarn cypress:run:reporter --spec './cypress/e2e/investigations/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
+    "cypress:explore:run": "yarn cypress:run:reporter --spec './cypress/e2e/explore/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
     "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results && yarn junit:transform && mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/",
     "test:generate": "node scripts/endpoint/resolver_generator",
     "mappings:generate": "node scripts/mappings/mappings_generator",

x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts

@@ -13,6 +13,9 @@ import pMap from 'p-map';
 import { ToolingLog } from '@kbn/tooling-log';
 import { withProcRunner } from '@kbn/dev-proc-runner';
 import cypress from 'cypress';
+import { findChangedFiles } from 'find-cypress-specs';
+import minimatch from 'minimatch';
+import path from 'path';
 
 import {
   EsVersion,
@@ -68,13 +71,39 @@ const retrieveIntegrations = (
 export const cli = () => {
   run(
     async () => {
-      const { argv } = yargs(process.argv.slice(2));
+      const { argv } = yargs(process.argv.slice(2)).coerce('env', (arg: string) =>
+        arg.split(',').reduce((acc, curr) => {
+          const [key, value] = curr.split('=');
+          if (key === 'burn') {
+            acc[key] = parseInt(value, 10);
+          } else {
+            acc[key] = value;
+          }
+          return acc;
+        }, {} as Record<string, string | number>)
+      );
 
       const isOpen = argv._[0] === 'open';
       const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string;
       const cypressConfigFile = await import(require.resolve(`../../${argv.configFile}`));
       const spec: string | undefined = argv?.spec as string;
-      const files = retrieveIntegrations(spec ? [spec] : cypressConfigFile?.e2e?.specPattern);
+      let files = retrieveIntegrations(spec ? [spec] : cypressConfigFile?.e2e?.specPattern);
+
+      if (argv.changedSpecsOnly) {
+        const basePath = process.cwd().split('kibana/')[1];
+        files = findChangedFiles('main', false)
+          .filter(
+            minimatch.filter(path.join(basePath, cypressConfigFile?.e2e?.specPattern), {
+              matchBase: true,
+            })
+          )
+          .map((filePath: string) => filePath.replace(basePath, '.'));
+
+        if (!files?.length) {
+          // eslint-disable-next-line no-process-exit
+          return process.exit(0);
+        }
+      }
 
       if (!files?.length) {
         throw new Error('No files found');
@@ -323,8 +352,8 @@ ${JSON.stringify(config.getAll(), null, 2)}
               type: 'elasticsearch' | 'kibana' | 'fleetserver',
               withAuth: boolean = false
             ): string => {
-              const getKeyPath = (path: string = ''): string => {
-                return `servers.${type}${path ? `.${path}` : ''}`;
+              const getKeyPath = (keyPath: string = ''): string => {
+                return `servers.${type}${keyPath ? `.${keyPath}` : ''}`;
               };
 
               if (!config.get(getKeyPath())) {
@@ -361,7 +390,7 @@ ${JSON.stringify(config.getAll(), null, 2)}
               ...ftrEnv,
 
               // NOTE:
-              // ELASTICSEARCH_URL needs to be crated here with auth because SIEM cypress setup depends on it. At some
+              // ELASTICSEARCH_URL needs to be created here with auth because SIEM cypress setup depends on it. At some
               // points we should probably try to refactor that code to use `ELASTICSEARCH_URL_WITH_AUTH` instead
               ELASTICSEARCH_URL:
                 ftrEnv.ELASTICSEARCH_URL ?? createUrlFromFtrConfig('elasticsearch', true),
@@ -377,6 +406,8 @@ ${JSON.stringify(config.getAll(), null, 2)}
               KIBANA_URL_WITH_AUTH: createUrlFromFtrConfig('kibana', true),
               KIBANA_USERNAME: config.get('servers.kibana.username'),
               KIBANA_PASSWORD: config.get('servers.kibana.password'),
+
+              ...argv.env,
             };
 
             log.info(`
@@ -407,6 +438,7 @@ ${JSON.stringify(cyCustomEnv, null, 2)}
                   configFile: cypressConfigFilePath,
                   reporter: argv.reporter as string,
                   reporterOptions: argv.reporterOptions,
+                  headed: argv.headed as boolean,
                   config: {
                     e2e: {
                       baseUrl,