Skip to content

Commit

Permalink
[Defend Workflows] Add tags for mocked and real Endpoint Cypress test…
Browse files Browse the repository at this point in the history
…s for Serverless (#165094)

## Summary

- introduces tags for [Defend Workflows] cypress tests (similarly to
#162698)
- adds scripts to Security Solution:
  - `cypress:dw:serverless:open` and `:run`
  - `cypress:dw:endpoint:serverless:open` and `:run`
- adds CI jobs to run these scripts
- so far most of the expected tests got both `@serverless` and
`@brokenInServerless` tests, because of other issues to be solved,
- one test is able to run against serverless:
`x-pack/plugins/security_solution/public/management/cypress/e2e/mocked_data/policy_details.cy.ts`
  • Loading branch information
gergoabraham authored Sep 20, 2023
1 parent 4270c5e commit aa36fe6
Show file tree
Hide file tree
Showing 44 changed files with 682 additions and 408 deletions.
1 change: 1 addition & 0 deletions .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ disabled:
- x-pack/test/defend_workflows_cypress/cli_config.ts
- x-pack/test/defend_workflows_cypress/config.ts
- x-pack/test/defend_workflows_cypress/endpoint_config.ts
- x-pack/test/defend_workflows_cypress/endpoint_serverless_config.ts
- x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts
- x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts
- x-pack/plugins/observability_onboarding/e2e/ftr_config.ts
Expand Down
25 changes: 25 additions & 0 deletions .buildkite/pipelines/pull_request/defend_workflows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,28 @@ steps:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
label: 'Defend Workflows Cypress Tests on Serverless'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
parallelism: 2
retry:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/functional/defend_workflows_vagrant_serverless.sh
label: 'Defend Workflows Endpoint Cypress Tests on Serverless'
agents:
queue: n2-4-virt
depends_on: build
timeout_in_minutes: 120
parallelism: 5
retry:
automatic:
- exit_status: '*'
limit: 1

15 changes: 15 additions & 0 deletions .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/steps/functional/common.sh
source .buildkite/scripts/steps/functional/common_cypress.sh

export JOB=kibana-defend-workflows-serverless-cypress
export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

echo "--- Defend Workflows Cypress tests on Serverless"

cd x-pack/plugins/security_solution

yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/steps/functional/common.sh
source .buildkite/scripts/steps/functional/common_cypress.sh

export JOB=kibana-defend-workflows-endpoint-serverless-cypress
export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

echo "--- Defend Workflows Endpoint Cypress tests on Serverless"

cd x-pack/plugins/security_solution

yarn cypress:dw:endpoint:serverless:run; status=$?; yarn junit:merge || :; exit $status
14 changes: 10 additions & 4 deletions x-pack/plugins/security_solution/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,18 @@
"cypress": "NODE_OPTIONS=--openssl-legacy-provider ../../../node_modules/.bin/cypress",
"cypress:burn": "yarn cypress:dw run --env burn=2 --headed",
"cypress:changed-specs-only": "yarn cypress:dw run --changed-specs-only --env burn=2",
"cypress:dw": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress.config.ts ts --ftr-config-file ../../test/defend_workflows_cypress/cli_config",
"cypress:dw": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress.config.ts --ftr-config-file ../../test/defend_workflows_cypress/cli_config",
"cypress:dw:open": "yarn cypress:dw open",
"cypress:dw:run": "yarn cypress:dw run",
"cypress:dw:endpoint": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress_endpoint.config.ts --ftr-config-file ../../test/defend_workflows_cypress/endpoint_config",
"cypress:dw:endpoint": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_endpoint.config.ts --ftr-config-file ../../test/defend_workflows_cypress/endpoint_config",
"cypress:dw:endpoint:run": "yarn cypress:dw:endpoint run",
"cypress:dw:endpoint:open": "yarn cypress:dw:endpoint open ",
"cypress:dw:endpoint:open": "yarn cypress:dw:endpoint open",
"cypress:dw:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_serverless.config.ts --ftr-config-file ../../../x-pack/test_serverless/functional/test_suites/security/cypress/security_config",
"cypress:dw:serverless:open": "yarn cypress:dw:serverless open",
"cypress:dw:serverless:run": "yarn cypress:dw:serverless run",
"cypress:dw:endpoint:serverless": "NODE_OPTIONS=--openssl-legacy-provider node ./scripts/start_cypress_parallel --config-file ./public/management/cypress/cypress_endpoint_serverless.config.ts --ftr-config-file ../../test/defend_workflows_cypress/endpoint_serverless_config",
"cypress:dw:endpoint:serverless:open": "yarn cypress:dw:endpoint:serverless open",
"cypress:dw:endpoint:serverless:run": "yarn cypress:dw:endpoint:serverless run",
"junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results && yarn junit:transform && mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/",
"test:generate": "node scripts/endpoint/resolver_generator",
"mappings:generate": "node scripts/mappings/mappings_generator",
Expand All @@ -24,4 +30,4 @@
"openapi:generate": "node scripts/openapi/generate",
"openapi:generate:debug": "node --inspect-brk scripts/openapi/generate"
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"plugins": ["cypress"],
"extends": [
"plugin:cypress/recommended"
],
"env": {
"cypress/globals": true
},
"rules": {
"cypress/no-force": "warn",
"import/no-extraneous-dependencies": "off"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,16 @@ If you also want to run the tests against real endpoints as on the CI pipeline,
See [running interactive tests on real endpoint with vagrant](#cypress-interactive-with-real-endpoints-using-vagrant)
for more information.

## Adding new tests - tagging for ESS vs Serverless

Similarly to Security Solution cypress tests, we use tags in order to select which tests we want to execute on which environment:

- `@serverless` includes a test in the Serverless test suite. You need to explicitly add this tag to any test you want to run against a Serverless environment.
- `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment.
- `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken.

Important: if you don't provide any tag, your test won't be executed.

## Running the tests

There are currently three ways to run the tests, comprised of two execution modes and two target environments, which
Expand Down Expand Up @@ -102,14 +112,11 @@ failures locally, etc.
# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
yarn kbn bootstrap && node scripts/build_kibana_platform_plugins

# launch the cypress test runner
cd x-pack/plugins/security_solution
yarn cypress:dw:run-as-ci

# or
# launch the cypress test runner against ESS
yarn --cwd x-pack/plugins/security_solution cypress:dw:run

# launch without changing directory from kibana/
yarn --cwd x-pack/plugins/security_solution cypress:dw:run-as-ci
# or against Serverless
yarn --cwd x-pack/plugins/security_solution cypress:dw:serverless:run
```

#### Cypress
Expand All @@ -120,14 +127,11 @@ This is the preferred mode for developing new tests against mocked data
# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
yarn kbn bootstrap && node scripts/build_kibana_platform_plugins

# launch the cypress test runner
cd x-pack/plugins/security_solution
yarn cypress:dw:open

# or

# launch without changing directory from kibana/
# launch the cypress test runner against ESS
yarn --cwd x-pack/plugins/security_solution cypress:dw:open

# or against Serverless
yarn --cwd x-pack/plugins/security_solution cypress:dw:serverless:open
```

For developing/debugging tests against real endpoint please use:
Expand All @@ -138,14 +142,11 @@ Endpoint tests require [Multipass](https://multipass.run/) to be installed on yo
# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
yarn kbn bootstrap && node scripts/build_kibana_platform_plugins

# launch the cypress test runner with real endpoint
cd x-pack/plugins/security_solution
yarn cypress:dw:endpoint:open

# or

# launch without changing directory from kibana/
# launch the cypress test runner against ESS
yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:open

# or against Serverless
yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:serverless:open
```

#### Cypress (interactive) with real Endpoints using Vagrant
Expand All @@ -172,14 +173,11 @@ Endpoint tests require [Multipass](https://multipass.run/) to be installed on yo
# bootstrap kibana from the project root and build the plugins/assets that cypress will execute against
yarn kbn bootstrap && node scripts/build_kibana_platform_plugins
# launch the cypress test runner with real endpoint
cd x-pack/plugins/security_solution
yarn cypress:dw:endpoint:run
# or
# launch without changing directory from kibana/
# launch the cypress test runner with real endpoint against ESS
yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:run
# or against Serverless
yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:serverless:run
```

## Folder Structure
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { defineCypressConfig } from '@kbn/cypress-config';
import { getCypressBaseConfig } from './cypress_base.config';

import { dataLoaders } from './support/data_loaders';

export default defineCypressConfig({
...getCypressBaseConfig(),

env: {
...getCypressBaseConfig().env,

grepTags: '@ess',
},

e2e: {
...getCypressBaseConfig().e2e,

specPattern: 'public/management/cypress/e2e/mocked_data/',
setupNodeEvents: (on, config) => {
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('@cypress/grep/src/plugin')(config);

return dataLoaders(on, config);
},
},
});
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,7 @@
* 2.0.
*/

import { defineCypressConfig } from '@kbn/cypress-config';
// eslint-disable-next-line @kbn/imports/no_boundary_crossing
import { dataLoaders } from './cypress/support/data_loaders';

export default defineCypressConfig({
export const getCypressBaseConfig = () => ({
reporter: '../../../../node_modules/cypress-multi-reporters',
reporterOptions: {
configFile: './public/management/reporter_config.json',
Expand All @@ -36,21 +32,23 @@ export default defineCypressConfig({
KIBANA_URL: 'http://localhost:5601',
ELASTICSEARCH_URL: 'http://localhost:9200',
FLEET_SERVER_URL: 'https://localhost:8220',

// Username/password used for both elastic and kibana
KIBANA_USERNAME: 'system_indices_superuser',
KIBANA_PASSWORD: 'changeme',
ELASTICSEARCH_USERNAME: 'system_indices_superuser',
ELASTICSEARCH_PASSWORD: 'changeme',

// grep related configs
grepFilterSpecs: true,
grepOmitFiltered: true,
},

e2e: {
// baseUrl: To override, set Env. variable `CYPRESS_BASE_URL`
baseUrl: 'http://localhost:5601',

supportFile: 'public/management/cypress/support/e2e.ts',
specPattern: 'public/management/cypress/e2e/mocked_data/',
experimentalRunAllSpecs: true,
setupNodeEvents: (on, config) => {
return dataLoaders(on, config);
},
},
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { defineCypressConfig } from '@kbn/cypress-config';
import { getCypressBaseConfig } from './cypress_base.config';

import { dataLoaders, dataLoadersForRealEndpoints } from './support/data_loaders';

import { responseActionTasks } from './support/response_actions';

// eslint-disable-next-line import/no-default-export
export default defineCypressConfig({
...getCypressBaseConfig(),

env: {
...getCypressBaseConfig().env,

'cypress-react-selector': {
root: '#security-solution-app',
},

grepTags: '@ess',
},

e2e: {
...getCypressBaseConfig().e2e,

experimentalMemoryManagement: true,
experimentalInteractiveRunEvents: true,
specPattern: 'public/management/cypress/e2e/endpoint/*.cy.{js,jsx,ts,tsx}',
setupNodeEvents: (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => {
dataLoaders(on, config);
// Data loaders specific to "real" Endpoint testing
dataLoadersForRealEndpoints(on, config);
responseActionTasks(on, config);

// eslint-disable-next-line @typescript-eslint/no-var-requires
require('@cypress/grep/src/plugin')(config);

return config;
},
},
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { defineCypressConfig } from '@kbn/cypress-config';
import { getCypressBaseConfig } from './cypress_base.config';
import { dataLoaders } from './support/data_loaders';
import { responseActionTasks } from './support/response_actions';

// eslint-disable-next-line import/no-default-export
export default defineCypressConfig({
...getCypressBaseConfig(),

env: {
...getCypressBaseConfig().env,

IS_SERVERLESS: true,
grepTags: '@serverless --@brokenInServerless',

'cypress-react-selector': {
root: '#security-solution-app',
},
},

e2e: {
...getCypressBaseConfig().e2e,

experimentalMemoryManagement: true,
experimentalInteractiveRunEvents: true,

specPattern: 'public/management/cypress/e2e/endpoint/*.cy.{js,jsx,ts,tsx}',

setupNodeEvents: (on, config) => {
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('@cypress/grep/src/plugin')(config);

dataLoaders(on, config);

// skip dataLoadersForRealEndpoints()
// https://github.com/elastic/security-team/issues/7467
// Data loaders specific to "real" Endpoint testing
// dataLoadersForRealEndpoints(on, config);

responseActionTasks(on, config);
},
},
});
Loading

0 comments on commit aa36fe6

Please sign in to comment.