[Defend Workflows] POC2 for reusing tests for serverless - using tags

x-pack/plugins/security_solution/cypress/cypress.config.ts

@@ -22,5 +22,6 @@ export default defineCypressConfig({
   e2e: {
     experimentalRunAllSpecs: true,
     experimentalMemoryManagement: true,
+    specPattern: ['./e2e/**/*.cy.ts', '../../../../../../security_solution/cypress/e2e/**/*.cy.ts'],
   },
 });

x-pack/plugins/security_solution/cypress/e2e/explore/users/user_details.cy.ts

@@ -20,7 +20,7 @@ import {
 } from '../../../tasks/alerts';
 import { USER_COLUMN } from '../../../screens/alerts';
 
-describe('user details flyout', () => {
+describe('user details flyout', { tags: '@serverless' }, () => {
   beforeEach(() => {
     cleanKibana();
     login();

x-pack/plugins/security_solution/cypress/tasks/common.ts

@@ -272,8 +272,10 @@ export const deleteDataView = (dataSource: string) => {
 export const scrollToBottom = () => cy.scrollTo('bottom');
 
 export const waitForPageToBeLoaded = () => {
-  cy.get(LOADING_INDICATOR_HIDDEN).should('exist');
-  cy.get(LOADING_INDICATOR).should('not.exist');
+  if (Cypress.env('grepTags') !== '@serverless') {
+    cy.get(LOADING_INDICATOR_HIDDEN).should('exist');
+    cy.get(LOADING_INDICATOR).should('not.exist');
+  }
 };
 
 export const waitForWelcomePanelToBeLoaded = () => {

x-pack/plugins/security_solution/package.json

@@ -25,9 +25,10 @@
     "cypress:dw:endpoint:open": "node ./scripts/start_cypress_parallel open --config-file ./public/management/cypress_endpoint.config.ts ts --ftr-config-file ../../../../../../x-pack/test/defend_workflows_cypress/endpoint_config",
     "cypress:investigations:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/investigations/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
     "cypress:explore:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/explore/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
+    "cypress:dw:serverless:open": "../../../node_modules/.bin/cypress open --config-file ./public/management/cypress_serverless.config.ts",
     "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results && mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/",
     "test:generate": "node scripts/endpoint/resolver_generator",
     "mappings:generate": "node scripts/mappings/mappings_generator",
     "mappings:load": "node scripts/mappings/mappings_loader"
   }
-}
+}

x-pack/plugins/security_solution/public/management/cypress/e2e/mocked_data/policy_details.cy.ts

@@ -17,7 +17,7 @@ import type { CyIndexEndpointHosts } from '../../tasks/index_endpoint_hosts';
 import { indexEndpointHosts } from '../../tasks/index_endpoint_hosts';
 import { login } from '../../tasks/login';
 
-describe('Policy Details', () => {
+describe('Policy Details', { tags: '@serverless' }, () => {
   const packagePolicyBackupHelper = new PackagePolicyBackupHelper();
   let indexedHostsData: CyIndexEndpointHosts;
 

x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts

@@ -29,6 +29,10 @@ export {};
 
 import 'cypress-react-selector';
 
+// @ts-ignore
+import registerCypressGrep from '@cypress/grep';
+registerCypressGrep();
+
 Cypress.Commands.addQuery<'getByTestSubj'>(
   'getByTestSubj',
   function getByTestSubj(selector, options) {

x-pack/plugins/security_solution/public/management/cypress_serverless.config.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineCypressConfig } from '@kbn/cypress-config';
+
+// eslint-disable-next-line import/no-default-export
+export default defineCypressConfig({
+  defaultCommandTimeout: 60000,
+  execTimeout: 120000,
+  pageLoadTimeout: 12000,
+
+  retries: {
+    runMode: 1,
+    openMode: 0,
+  },
+
+  screenshotsFolder:
+    '../../../target/kibana-security-solution/public/management/cypress/screenshots',
+  trashAssetsBeforeRuns: false,
+  video: false,
+  viewportHeight: 900,
+  viewportWidth: 1440,
+  experimentalStudio: true,
+
+  env: {
+    grepTags: '@serverless',
+    grepFilterSpecs: true,
+    KIBANA_URL: 'http://localhost:5620',
+    ELASTICSEARCH_URL: 'http://localhost:9200',
+    FLEET_SERVER_URL: 'https://localhost:8220',
+    // Username/password used for both elastic and kibana
+    ELASTICSEARCH_USERNAME: 'elastic',
+    ELASTICSEARCH_PASSWORD: 'changeme',
+  },
+
+  e2e: {
+    // baseUrl: To override, set Env. variable `CYPRESS_BASE_URL`
+    baseUrl: 'http://localhost:5620',
+    supportFile: 'public/management/cypress/support/e2e.ts',
+    specPattern: 'public/management/cypress/e2e/mocked_data/',
+    experimentalRunAllSpecs: true,
+    experimentalMemoryManagement: true,
+    setupNodeEvents: (on, config) => {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires, import/no-extraneous-dependencies
+      require('@cypress/grep/src/plugin')(config);
+
+      // todo: dataLoaders fail on serverless env, needs to be adapted to be able to use e.g. `indexEndpointHosts()`
+      // return dataLoaders(on, config);
+
+      return config;
+    },
+  },
+});

x-pack/plugins/security_solution/tsconfig.json

@@ -161,6 +161,7 @@
     "@kbn/field-formats-plugin",
     "@kbn/dev-proc-runner",
     "@kbn/cloud-chat-plugin",
-    "@kbn/alerts-ui-shared"
+    "@kbn/alerts-ui-shared",
+    "@kbn/cypress-config"
   ]
 }

x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts

@@ -8,20 +8,32 @@
 import { defineCypressConfig } from '@kbn/cypress-config';
 
 export default defineCypressConfig({
-  defaultCommandTimeout: 60000,
-  execTimeout: 60000,
-  pageLoadTimeout: 60000,
+  defaultCommandTimeout: 150000,
+  env: {
+    grepFilterSpecs: true,
+  },
+  execTimeout: 150000,
+  pageLoadTimeout: 150000,
   responseTimeout: 60000,
+  screenshotsFolder: '../../../../../../target/kibana-security-solution/cypress/screenshots',
   trashAssetsBeforeRuns: false,
   video: false,
   viewportHeight: 946,
   viewportWidth: 1680,
   numTestsKeptInMemory: 10,
   e2e: {
+    setupNodeEvents(on, config) {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      require('@cypress/grep/src/plugin')(config);
+      return config;
+    },
     baseUrl: 'http://localhost:5620/app/security/get_started',
     experimentalRunAllSpecs: true,
     experimentalMemoryManagement: true,
     supportFile: './support/e2e.js',
-    specPattern: './e2e/**/*.cy.ts',
+    specPattern: [
+      './e2e/**/*.cy.ts',
+      '../../../../../plugins/security_solution/cypress/e2e/**/*.cy.ts',
+    ],
   },
 });

x-pack/test_serverless/functional/test_suites/security/cypress/e2e/serverless.cy.ts

@@ -8,7 +8,7 @@
 import { LEFT_NAVIGATION } from '../screens/landing_page';
 import { navigatesToLandingPage } from '../tasks/navigation';
 
-describe('Serverless', () => {
+describe('Serverless', { tags: '@serverless' }, () => {
   it('Should navigate to the landing page', () => {
     navigatesToLandingPage();
     cy.get(LEFT_NAVIGATION).should('exist');

x-pack/test_serverless/functional/test_suites/security/cypress/package.json

@@ -5,8 +5,8 @@
   "private": true,
   "license": "Elastic License 2.0",
   "scripts": {
-    "cypress:open": "../../../../../../node_modules/.bin/cypress open --config-file ./cypress.config.ts",
-    "cypress:run": "../../../../../../node_modules/.bin/cypress run --browser chrome --config-file ./cypress.config.ts",
+    "cypress:open": "../../../../../../node_modules/.bin/cypress open --config-file ./cypress.config.ts --env grepTags=@serverless",
+    "cypress:run": "../../../../../../node_modules/.bin/cypress run --browser chrome --config-file ./cypress.config.ts --env grepTags=@serverless",
     "cypress:serverless:open": "node ../../../../../../scripts/functional_tests --config ./config_visual.ts",
     "cypress:serverless:run":  "node ../../../../../../scripts/functional_tests --config ./config_headless.ts"
   }

x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts

@@ -7,6 +7,7 @@
 
 import { resolve } from 'path';
 import { withProcRunner } from '@kbn/dev-proc-runner';
+import Url from 'url';
 
 import { FtrProviderContext } from '../../../ftr_provider_context';
 
@@ -17,6 +18,10 @@ export async function SecuritySolutionCypressTestRunner(
   command: string
 ) {
   const log = getService('log');
+  const config = getService('config');
+  const esArchiver = getService('esArchiver');
+
+  await esArchiver.load('x-pack/test/security_solution_cypress/es_archives/auditbeat');
 
   await withProcRunner(log, async (procs) => {
     await procs.run('cypress', {
@@ -25,6 +30,11 @@ export async function SecuritySolutionCypressTestRunner(
       cwd: resolve(__dirname),
       env: {
         ...process.env,
+        FORCE_COLOR: '1',
+        CYPRESS_BASE_URL: Url.format(config.get('servers.kibana')),
+        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
+        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
+        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
       },
       wait: true,
     });

x-pack/test_serverless/functional/test_suites/security/cypress/security_config.base.ts

@@ -28,6 +28,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         ...svlSharedConfig.get('kbnTestServer.serverArgs'),
         '--csp.strict=false',
         '--csp.warnLegacyBrowsers=false',
+        '--xpack.encryptedSavedObjects.encryptionKey="abcdefghijklmnopqrstuvwxyz123456"',
         '--serverless=security',
       ],
     },

x-pack/test_serverless/functional/test_suites/security/cypress/support/commands.js

@@ -30,3 +30,57 @@
 //
 // -- This is will overwrite an existing command --
 // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })
+
+import 'cypress-file-upload';
+import 'cypress-recurse/commands';
+
+Cypress.Commands.add(
+  'attachFile',
+  {
+    prevSubject: 'element',
+  },
+  (input, fileName, fileType = 'text/plain') => {
+    cy.fixture(fileName).then((content) => {
+      const blob = Cypress.Blob.base64StringToBlob(btoa(content), fileType);
+      const testFile = new File([blob], fileName, { type: fileType });
+      const dataTransfer = new DataTransfer();
+
+      dataTransfer.items.add(testFile);
+      input[0].files = dataTransfer.files;
+      return input;
+    });
+  }
+);
+
+const waitUntil = (subject, fn, options = {}) => {
+  const { interval = 200, timeout = 5000 } = options;
+  let attempts = Math.floor(timeout / interval);
+
+  const completeOrRetry = (result) => {
+    if (result) {
+      return result;
+    }
+    if (attempts < 1) {
+      throw new Error(`Timed out while retrying, last result was: {${result}}`);
+    }
+    cy.wait(interval, { log: false }).then(() => {
+      attempts--;
+      // eslint-disable-next-line no-use-before-define
+      return evaluate();
+    });
+  };
+
+  const evaluate = () => {
+    const result = fn(subject);
+
+    if (result && result.then) {
+      return result.then(completeOrRetry);
+    } else {
+      return completeOrRetry(result);
+    }
+  };
+
+  return evaluate();
+};
+
+Cypress.Commands.add('waitUntil', { prevSubject: 'optional' }, waitUntil);

x-pack/test_serverless/functional/test_suites/security/cypress/support/e2e.js

@@ -23,6 +23,9 @@
 // Import commands.js using ES2015 syntax:
 import './commands';
 import 'cypress-real-events/support';
+import registerCypressGrep from '@cypress/grep';
+
+registerCypressGrep();
 
 Cypress.on('uncaught:exception', () => {
   return false;