From ed4ef2a3ff54d05c925cd6f8b8453607e6e778e7 Mon Sep 17 00:00:00 2001 From: dkirchan <55240027+dkirchan@users.noreply.github.com> Date: Mon, 6 Nov 2023 19:21:01 +0200 Subject: [PATCH] Fixed parallel script for cypress tests in QA and buildkite (#169311) Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Gloria Hornero Co-authored-by: Maxim Palenov --- .../pipelines/security_solution/base.yml | 36 ++ .../mki_security_solution_cypress.sh | 24 + .../pipeline.sh | 18 +- .../pipeline.ts | 38 ++ .../scripts/run_cypress/parallel.ts | 31 +- .../run_cypress/parallel_serverless.ts | 480 ++++++++++++++++++ .../scripts/run_cypress/utils.ts | 29 ++ .../start_cypress_parallel_serverless.js | 9 + .../plugins/security_solution/tsconfig.json | 2 +- .../cypress_ci_serverless_qa.config.ts | 13 +- .../missing_privileges_callout.cy.ts | 148 +++--- .../install_update_authorization.cy.ts | 2 +- .../install_update_error_handling.cy.ts | 3 +- .../prebuilt_rules/install_workflow.cy.ts | 2 +- .../prebuilt_rules/management.cy.ts | 2 +- .../prebuilt_rules/notifications.cy.ts | 180 ++++--- .../prebuilt_rules/update_workflow.ts | 2 +- .../rule_creation/common_flows.cy.ts | 2 +- .../event_correlation_rule.cy.ts | 6 +- .../rule_creation/machine_learning_rule.cy.ts | 126 ++--- .../rule_details/common_flows.cy.ts | 19 +- .../rule_edit/custom_query_rule.cy.ts | 2 +- .../related_integrations.cy.ts | 2 +- .../bulk_actions/bulk_edit_rules.cy.ts | 74 +-- .../bulk_edit_rules_actions.cy.ts | 332 ++++++------ .../import_export/export_rule.cy.ts | 26 +- .../rule_actions/snoozing/rule_snoozing.cy.ts | 2 +- .../rules_table/rules_table_links.cy.ts | 6 +- .../rules_table/rules_table_selection.cy.ts | 98 ++-- .../e2e/exceptions/entry/use_value_list.cy.ts | 16 +- .../e2e/explore/cases/connector_options.cy.ts | 6 - .../cypress/e2e/explore/cases/creation.cy.ts | 2 - .../explore/dashboards/entity_analytics.cy.ts | 4 +- .../investigations/alerts/alerts_charts.cy.ts | 76 +-- .../alerts/alerts_details.cy.ts | 2 +- ...s_left_panel_threat_intelligence_tab.cy.ts | 2 - .../alerts/ransomware_prevention.cy.ts | 2 - .../timeline_templates/export.cy.ts | 3 - .../e2e/investigations/timelines/export.cy.ts | 6 +- .../timelines/open_timeline.cy.ts | 11 +- .../timelines/row_renderers.cy.ts | 6 +- .../cypress/objects/timeline.ts | 8 +- .../cypress/tasks/api_calls/timelines.ts | 6 +- .../security_solution_cypress/package.json | 7 +- 44 files changed, 1227 insertions(+), 644 deletions(-) create mode 100644 .buildkite/pipelines/security_solution/base.yml create mode 100755 .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh create mode 100644 .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts create mode 100644 x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts create mode 100644 x-pack/plugins/security_solution/scripts/start_cypress_parallel_serverless.js diff --git a/.buildkite/pipelines/security_solution/base.yml b/.buildkite/pipelines/security_solution/base.yml new file mode 100644 index 0000000000000..337c44ccdcc7e --- /dev/null +++ b/.buildkite/pipelines/security_solution/base.yml @@ -0,0 +1,36 @@ +steps: + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless + label: 'Serverless MKI QA Security Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore + label: 'Serverless MKI QA Explore - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 4 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations + label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 \ No newline at end of file diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh new file mode 100755 index 0000000000000..8d56cedd13d5f --- /dev/null +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +if [ -z "$1" ] + then + echo "No target script from the package.json file, is supplied" + exit 1 +fi + +source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/functional/common_cypress.sh +.buildkite/scripts/bootstrap.sh + +export JOB=kibana-security-solution-chrome + +buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" + +cd x-pack/test/security_solution_cypress +set +e + +QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key) + +CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status \ No newline at end of file diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh index 3d61d70cf6828..3f4b2093b807e 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh @@ -1,19 +1,5 @@ #!/bin/bash -set -euo pipefail - -source .buildkite/scripts/common/util.sh -source .buildkite/scripts/steps/functional/common_cypress.sh -.buildkite/scripts/bootstrap.sh - -export JOB=kibana-security-solution-chrome -buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" - -echo "--- Serverless Security Second Quality Gate" -cd x-pack/test/security_solution_cypress -set +e - -VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key) -ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY) +set -euo pipefail -CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status \ No newline at end of file +ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts b/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts new file mode 100644 index 0000000000000..4084696d5c21c --- /dev/null +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { execSync } from 'child_process'; +import fs from 'fs'; + +const getPipeline = (filename: string, removeSteps = true) => { + const str = fs.readFileSync(filename).toString(); + return removeSteps ? str.replace(/^steps:/, '') : str; +}; + +const uploadPipeline = (pipelineContent: string | object) => { + const str = + typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent); + + execSync('buildkite-agent pipeline upload', { + input: str, + stdio: ['pipe', 'inherit', 'inherit'], + }); +}; + +(async () => { + try { + const pipeline = []; + + pipeline.push(getPipeline('.buildkite/pipelines/security_solution/base.yml', false)); + // remove duplicated steps + uploadPipeline([...new Set(pipeline)].join('\n')); + } catch (ex) { + console.error('PR pipeline generation error', ex.message); + process.exit(1); + } +})(); diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 19eeafaecfcdc..b7cfabb2bc937 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -28,38 +28,9 @@ import { import { createFailError } from '@kbn/dev-cli-errors'; import pRetry from 'p-retry'; import { renderSummaryTable } from './print_run'; -import { isSkipped, parseTestFileConfig } from './utils'; +import { parseTestFileConfig, retrieveIntegrations } from './utils'; import { getFTRConfig } from './get_ftr_config'; -/** - * Retrieve test files using a glob pattern. - * If process.env.RUN_ALL_TESTS is true, returns all matching files, otherwise, return files that should be run by this job based on process.env.BUILDKITE_PARALLEL_JOB_COUNT and process.env.BUILDKITE_PARALLEL_JOB - */ -const retrieveIntegrations = (integrationsPaths: string[]) => { - const nonSkippedSpecs = integrationsPaths.filter((filePath) => !isSkipped(filePath)); - - if (process.env.RUN_ALL_TESTS === 'true') { - return nonSkippedSpecs; - } else { - // The number of instances of this job were created - const chunksTotal: number = process.env.BUILDKITE_PARALLEL_JOB_COUNT - ? parseInt(process.env.BUILDKITE_PARALLEL_JOB_COUNT, 10) - : 1; - // An index which uniquely identifies this instance of the job - const chunkIndex: number = process.env.BUILDKITE_PARALLEL_JOB - ? parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) - : 0; - - const nonSkippedSpecsForChunk: string[] = []; - - for (let i = chunkIndex; i < nonSkippedSpecs.length; i += chunksTotal) { - nonSkippedSpecsForChunk.push(nonSkippedSpecs[i]); - } - - return nonSkippedSpecsForChunk; - } -}; - export const cli = () => { run( async () => { diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts new file mode 100644 index 0000000000000..56740845126a8 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -0,0 +1,480 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { run } from '@kbn/dev-cli-runner'; +import yargs from 'yargs'; +import _ from 'lodash'; +import globby from 'globby'; +import pMap from 'p-map'; +import { ToolingLog } from '@kbn/tooling-log'; +import { withProcRunner } from '@kbn/dev-proc-runner'; +import cypress from 'cypress'; +import grep from '@cypress/grep/src/plugin'; +import crypto from 'crypto'; +import fs from 'fs'; +import { createFailError } from '@kbn/dev-cli-errors'; +import axios, { AxiosError } from 'axios'; +import path from 'path'; +import os from 'os'; +import pRetry from 'p-retry'; + +import { renderSummaryTable } from './print_run'; +import type { SecuritySolutionDescribeBlockFtrConfig } from './utils'; +import { parseTestFileConfig, retrieveIntegrations } from './utils'; + +interface ProductType { + product_line: string; + product_tier: string; +} + +interface CreateEnvironmentRequestBody { + name: string; + region_id: string; + product_types?: ProductType[]; +} + +interface Environment { + name: string; + id: string; + region: string; + es_url: string; + kb_url: string; + product: string; +} + +interface Credentials { + username: string; + password: string; +} + +const DEFAULT_REGION = 'aws-eu-west-1'; +const PROJECT_NAME_PREFIX = 'kibana-cypress-security-solution-ephemeral'; +const BASE_ENV_URL = 'https://global.qa.cld.elstc.co'; +let log: ToolingLog; + +const delay = async (timeout: number) => { + await new Promise((r) => setTimeout(r, timeout)); +}; + +const getApiKeyFromElasticCloudJsonFile = (): string | undefined => { + const userHomeDir = os.homedir(); + try { + const jsonString = fs.readFileSync(path.join(userHomeDir, '.elastic/cloud.json'), 'utf-8'); + const jsonData = JSON.parse(jsonString); + return jsonData.api_key.qa; + } catch (e) { + log.info('API KEY could not be found in .elastic/cloud.json'); + } +}; + +// Method to invoke the create environment API for serverless. +async function createEnvironment( + projectName: string, + apiKey: string, + ftrConfig: SecuritySolutionDescribeBlockFtrConfig +): Promise { + const body: CreateEnvironmentRequestBody = { + name: projectName, + region_id: DEFAULT_REGION, + }; + + const productTypes: ProductType[] = []; + ftrConfig?.productTypes?.forEach((t) => { + productTypes.push(t as ProductType); + }); + if (productTypes.length > 0) body.product_types = productTypes; + + try { + const response = await axios.post(`${BASE_ENV_URL}/api/v1/serverless/projects/security`, body, { + headers: { + Authorization: `ApiKey ${apiKey}`, + }, + }); + return { + name: response.data.name, + id: response.data.id, + region: response.data.region_id, + es_url: `${response.data.endpoints.elasticsearch}:443`, + kb_url: `${response.data.endpoints.kibana}:443`, + product: response.data.type, + }; + } catch (error) { + if (error instanceof AxiosError) { + log.error(`${error.response?.status}:${error.response?.data}`); + } else { + log.error(`${error.message}`); + } + } +} + +// Method to invoke the delete environment API for serverless. +async function deleteEnvironment( + projectId: string, + projectName: string, + apiKey: string +): Promise { + try { + await axios.delete(`${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}`, { + headers: { + Authorization: `ApiKey ${apiKey}`, + }, + }); + log.info(`Environment ${projectName} was successfully deleted!`); + } catch (error) { + if (error instanceof AxiosError) { + log.error(`${error.response?.status}:${error.response?.data}`); + } else { + log.error(`${error.message}`); + } + } +} + +// Method to reset the credentials for the created environment. +async function resetCredentials( + environmentId: string, + runnerId: string, + apiKey: string +): Promise { + log.info(`${runnerId} : Reseting credentials`); + try { + const response = await axios.post( + `${BASE_ENV_URL}/api/v1/serverless/projects/security/${environmentId}/_reset-credentials`, + {}, + { + headers: { + Authorization: `ApiKey ${apiKey}`, + }, + } + ); + return { + password: response.data.password, + username: response.data.username, + }; + } catch (error) { + throw new Error(`${error.message}`); + } +} + +// Wait until elasticsearch status goes green +function waitForEsStatusGreen(esUrl: string, auth: string, runnerId: string): Promise { + const fetchHealthStatusAttempt = async (attemptNum: number) => { + log.info(`Retry number ${attemptNum} to check if es is green.`); + + const response = await axios.get(`${esUrl}/_cluster/health?wait_for_status=green&timeout=50s`, { + headers: { + Authorization: `Basic ${auth}`, + }, + }); + + log.info(`${runnerId}: Elasticsearch is ready with status ${response.data.status}.`); + }; + const retryOptions = { + onFailedAttempt: (error: Error | AxiosError) => { + if (error instanceof AxiosError && error.code === 'ENOTFOUND') { + log.info( + `${runnerId}: The elasticsearch url is not yet reachable. A retry will be triggered soon...` + ); + } + }, + retries: 50, + factor: 2, + maxTimeout: 20000, + }; + + return pRetry(fetchHealthStatusAttempt, retryOptions); +} + +// Wait until Kibana is available +function waitForKibanaAvailable(kbUrl: string, auth: string, runnerId: string): Promise { + const fetchKibanaStatusAttempt = async (attemptNum: number) => { + log.info(`Retry number ${attemptNum} to check if kibana is available.`); + const response = await axios.get(`${kbUrl}/api/status`, { + headers: { + Authorization: `Basic ${auth}`, + }, + }); + if (response.data.status.overall.level !== 'available') { + throw new Error(`${runnerId}: Kibana is not available. Retrying in 20s...`); + } else { + log.info(`${runnerId}: Kibana status overall is ${response.data.status.overall.level}.`); + } + }; + const retryOptions = { + onFailedAttempt: (error: Error | AxiosError) => { + if (error instanceof AxiosError && error.code === 'ENOTFOUND') { + log.info(`${runnerId}: The kibana url is not yet reachable. Retrying in 20s...`); + } else { + log.info(`${runnerId}: ${error}`); + } + }, + retries: 50, + factor: 2, + maxTimeout: 20000, + }; + return pRetry(fetchKibanaStatusAttempt, retryOptions); +} + +export const cli = () => { + run( + async () => { + log = new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }); + + // Checking if API key is either provided via env variable or in ~/.elastic.cloud.json + if (!process.env.CLOUD_QA_API_KEY && !getApiKeyFromElasticCloudJsonFile()) { + log.error('The api key for the environment needs to be provided with the env var API_KEY.'); + log.error( + 'If running locally, ~/.elastic/cloud.json is attempted to be read which contains the api key.' + ); + // eslint-disable-next-line no-process-exit + return process.exit(1); + } + + const API_KEY = process.env.CLOUD_QA_API_KEY + ? process.env.CLOUD_QA_API_KEY + : getApiKeyFromElasticCloudJsonFile(); + + const PARALLEL_COUNT = process.env.PARALLEL_COUNT ? Number(process.env.PARALLEL_COUNT) : 1; + + if (!process.env.CLOUD_ENV) { + log.warning( + 'The cloud environment to be provided with the env var CLOUD_ENV. Currently working only for QA so the script can proceed.' + ); + // Abort when more environments will be integrated + + // return process.exit(0); + } + + const { argv } = yargs(process.argv.slice(2)) + .coerce('configFile', (arg) => (_.isArray(arg) ? _.last(arg) : arg)) + .coerce('spec', (arg) => (_.isArray(arg) ? _.last(arg) : arg)) + .coerce('env', (arg: string) => + arg.split(',').reduce((acc, curr) => { + const [key, value] = curr.split('='); + if (key === 'burn') { + acc[key] = parseInt(value, 10); + } else { + acc[key] = value; + } + return acc; + }, {} as Record) + ); + + log.info(` +---------------------------------------------- +Script arguments: +---------------------------------------------- + +${JSON.stringify(argv, null, 2)} + +---------------------------------------------- +`); + + const isOpen = argv._.includes('open'); + + const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; + const cypressConfigFile = await import(cypressConfigFilePath); + + log.info(` +---------------------------------------------- +Cypress config for file: ${cypressConfigFilePath}: +---------------------------------------------- + +${JSON.stringify(cypressConfigFile, null, 2)} + +---------------------------------------------- +`); + + const specConfig = cypressConfigFile.e2e.specPattern; + const specArg = argv.spec; + const specPattern = specArg ?? specConfig; + + log.info('Config spec pattern:', specConfig); + log.info('Arguments spec pattern:', specArg); + log.info('Resulting spec pattern:', specPattern); + + // The grep function will filter Cypress specs by tags: it will include and exclude + // spec files according to the tags configuration. + const grepSpecPattern = grep({ + ...cypressConfigFile, + specPattern, + excludeSpecPattern: [], + }).specPattern; + + log.info('Resolved spec files or pattern after grep:', grepSpecPattern); + const isGrepReturnedFilePaths = _.isArray(grepSpecPattern); + const isGrepReturnedSpecPattern = !isGrepReturnedFilePaths && grepSpecPattern === specPattern; + const grepFilterSpecs = cypressConfigFile.env?.grepFilterSpecs; + + // IMPORTANT! + // When grep returns the same spec pattern as it gets in its arguments, we treat it as + // it couldn't find any concrete specs to execute (maybe because all of them are skipped). + // In this case, we do an early return - it's important to do that. + // If we don't return early, these specs will start executing, and Cypress will be skipping + // tests at runtime: those that should be excluded according to the tags passed in the config. + // This can take so much time that the job can fail by timeout in CI. + if (grepFilterSpecs && isGrepReturnedSpecPattern) { + log.info('No tests found - all tests could have been skipped via Cypress tags'); + // eslint-disable-next-line no-process-exit + return process.exit(0); + } + + const concreteFilePaths = isGrepReturnedFilePaths + ? grepSpecPattern // use the returned concrete file paths + : globby.sync(specPattern); // convert the glob pattern to concrete file paths + + const files = retrieveIntegrations(concreteFilePaths); + + log.info('Resolved spec files after retrieveIntegrations:', files); + + if (!files?.length) { + log.info('No tests found'); + // eslint-disable-next-line no-process-exit + return process.exit(0); + } + + const results = await pMap( + files, + async (filePath) => { + let result: + | CypressCommandLine.CypressRunResult + | CypressCommandLine.CypressFailedRunResult + | undefined; + await withProcRunner(log, async (procs) => { + const id = crypto.randomBytes(8).toString('hex'); + const PROJECT_NAME = `${PROJECT_NAME_PREFIX}-${id}`; + const specFileFTRConfig = parseTestFileConfig(filePath); + + if (!API_KEY) { + log.info('API KEY to create environment could not be retrieved.'); + // eslint-disable-next-line no-process-exit + return process.exit(1); + } + + log.info(`${id}: Creating environment ${PROJECT_NAME}...`); + // Creating environment for the test to run + const environment = await createEnvironment(PROJECT_NAME, API_KEY, specFileFTRConfig); + + if (!environment) { + log.info('Failed to create environment.'); + // eslint-disable-next-line no-process-exit + return process.exit(1); + } + + // Reset credentials for elastic user + const credentials = await resetCredentials(environment.id, id, API_KEY); + + if (!credentials) { + log.info('Credentials could not be reset.'); + // eslint-disable-next-line no-process-exit + return process.exit(1); + } + + // Wait for 8 minutes in order for the environment to be ready + delay(480000); + + // Base64 encode the credentials in order to invoke ES and KB APIs + const auth = btoa(`${credentials.username}:${credentials.password}`); + + // Wait for elasticsearch status to go green. + await waitForEsStatusGreen(environment.es_url, auth, id); + + // Wait until Kibana is available + await waitForKibanaAvailable(environment.kb_url, auth, id); + + // Normalized the set of available env vars in cypress + const cyCustomEnv = { + CYPRESS_BASE_URL: environment.kb_url, + + ELASTICSEARCH_URL: environment.es_url, + ELASTICSEARCH_USERNAME: credentials.username, + ELASTICSEARCH_PASSWORD: credentials.password, + + KIBANA_URL: environment.kb_url, + KIBANA_USERNAME: credentials.username, + KIBANA_PASSWORD: credentials.password, + + CLOUD_SERVERLESS: true, + }; + + if (process.env.DEBUG && !process.env.CI) { + log.info(` + ---------------------------------------------- + Cypress run ENV for file: ${filePath}: + ---------------------------------------------- + ${JSON.stringify(cyCustomEnv, null, 2)} + ---------------------------------------------- + `); + } + + if (isOpen) { + await cypress.open({ + configFile: cypressConfigFilePath, + config: { + e2e: { + baseUrl: environment.kb_url, + }, + env: cyCustomEnv, + }, + }); + } else { + try { + result = await cypress.run({ + browser: 'electron', + spec: filePath, + configFile: cypressConfigFilePath, + reporter: argv.reporter as string, + reporterOptions: argv.reporterOptions, + headed: argv.headed as boolean, + config: { + e2e: { + baseUrl: environment.kb_url, + }, + numTestsKeptInMemory: 0, + env: cyCustomEnv, + }, + }); + } catch (error) { + result = error; + } + } + + // Delete serverless environment + log.info(`${id} : Deleting Environment ${PROJECT_NAME}...`); + await deleteEnvironment(environment.id, PROJECT_NAME, API_KEY); + + return result; + }); + return result; + }, + { + concurrency: PARALLEL_COUNT, + } + ); + + if (results) { + renderSummaryTable(results as CypressCommandLine.CypressRunResult[]); + const hasFailedTests = _.some( + results, + (result) => + (result as CypressCommandLine.CypressFailedRunResult)?.status === 'failed' || + (result as CypressCommandLine.CypressRunResult)?.totalFailed + ); + if (hasFailedTests) { + throw createFailError('Not all tests passed'); + } + } + }, + { + flags: { + allowUnexpected: true, + }, + } + ); +}; diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/utils.ts b/x-pack/plugins/security_solution/scripts/run_cypress/utils.ts index fdaf5aeac1288..37ef1d58baf42 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/utils.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/utils.ts @@ -13,6 +13,35 @@ import type { ExpressionStatement, ObjectExpression, ObjectProperty } from '@bab import { schema, type TypeOf } from '@kbn/config-schema'; import { getExperimentalAllowedValues } from '../../common/experimental_features'; +/** + * Retrieve test files using a glob pattern. + * If process.env.RUN_ALL_TESTS is true, returns all matching files, otherwise, return files that should be run by this job based on process.env.BUILDKITE_PARALLEL_JOB_COUNT and process.env.BUILDKITE_PARALLEL_JOB + */ +export const retrieveIntegrations = (integrationsPaths: string[]) => { + const nonSkippedSpecs = integrationsPaths.filter((filePath) => !isSkipped(filePath)); + + if (process.env.RUN_ALL_TESTS === 'true') { + return nonSkippedSpecs; + } else { + // The number of instances of this job were created + const chunksTotal: number = process.env.BUILDKITE_PARALLEL_JOB_COUNT + ? parseInt(process.env.BUILDKITE_PARALLEL_JOB_COUNT, 10) + : 1; + // An index which uniquely identifies this instance of the job + const chunkIndex: number = process.env.BUILDKITE_PARALLEL_JOB + ? parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + : 0; + + const nonSkippedSpecsForChunk: string[] = []; + + for (let i = chunkIndex; i < nonSkippedSpecs.length; i += chunksTotal) { + nonSkippedSpecsForChunk.push(nonSkippedSpecs[i]); + } + + return nonSkippedSpecsForChunk; + } +}; + export const isSkipped = (filePath: string): boolean => { const testFile = fs.readFileSync(filePath, { encoding: 'utf8' }); diff --git a/x-pack/plugins/security_solution/scripts/start_cypress_parallel_serverless.js b/x-pack/plugins/security_solution/scripts/start_cypress_parallel_serverless.js new file mode 100644 index 0000000000000..ab8850323536d --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/start_cypress_parallel_serverless.js @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +require('../../../../src/setup_node_env'); +require('./run_cypress/parallel_serverless').cli(); diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 16ad154a95d83..1766051325f5f 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -177,6 +177,6 @@ "@kbn/react-kibana-mount", "@kbn/unified-doc-viewer-plugin", "@kbn/shared-ux-error-boundary", - "@kbn/zod-helpers" + "@kbn/zod-helpers", ] } diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts index e76893eceea36..4688cbe343e1c 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts @@ -14,18 +14,17 @@ export default defineCypressConfig({ reporterOptions: { configFile: './cypress/reporter_config.json', }, - defaultCommandTimeout: 150000, + defaultCommandTimeout: 300000, env: { grepFilterSpecs: true, grepOmitFiltered: true, - grepTags: '@serverlessQA --@brokenInServerless --@skipInServerless', - // Grep plugin is working taking under consideration the directory where cypress lives. - // https://github.com/elastic/kibana/pull/167494#discussion_r1340567022 for more context. - grepIntegrationFolder: '../', + grepTags: '@serverless --@brokenInServerless --@skipInServerless --@brokenInServerlessQA', }, - execTimeout: 150000, - pageLoadTimeout: 150000, + execTimeout: 300000, + pageLoadTimeout: 300000, numTestsKeptInMemory: 0, + requestTimeout: 300000, + responseTimeout: 300000, retries: { runMode: 1, }, diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/missing_privileges_callout.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/missing_privileges_callout.cy.ts index d6b4aec5bf3ea..3e75f8b4a28bb 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/missing_privileges_callout.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/missing_privileges_callout.cy.ts @@ -44,101 +44,105 @@ const waitForPageTitleToBeShown = () => { cy.get(PAGE_TITLE).should('be.visible'); }; -describe('Detections > Callouts', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - // First, we have to open the app on behalf of a privileged user in order to initialize it. - // Otherwise the app will be disabled and show a "welcome"-like page. - login(); - visit(ALERTS_URL); - waitForPageTitleToBeShown(); - }); - - context('indicating read-only access to resources', () => { - context('On Detections home page', () => { - beforeEach(() => { - loadPageAsReadOnlyUser(ALERTS_URL); - }); +describe( + 'Detections > Callouts', + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + () => { + before(() => { + // First, we have to open the app on behalf of a privileged user in order to initialize it. + // Otherwise the app will be disabled and show a "welcome"-like page. + login(); + visit(ALERTS_URL); + waitForPageTitleToBeShown(); + }); - it('We show one primary callout', () => { - waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary'); - }); + context('indicating read-only access to resources', () => { + context('On Detections home page', () => { + beforeEach(() => { + loadPageAsReadOnlyUser(ALERTS_URL); + }); - context('When a user clicks Dismiss on the callout', () => { - it('We hide it and persist the dismissal', () => { + it('We show one primary callout', () => { waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary'); - dismissCallOut(MISSING_PRIVILEGES_CALLOUT); - reloadPage(); - getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); }); - }); - }); - // FYI: Rules Management check moved to ../detection_rules/all_rules_read_only.spec.ts - - context('On Rule Details page', () => { - beforeEach(() => { - createRule(getNewRule()).then((rule) => - loadPageAsReadOnlyUser(ruleDetailsUrl(rule.body.id)) - ); + context('When a user clicks Dismiss on the callout', () => { + it('We hide it and persist the dismissal', () => { + waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary'); + dismissCallOut(MISSING_PRIVILEGES_CALLOUT); + reloadPage(); + getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + }); + }); }); - afterEach(() => { - deleteCustomRule(); - }); + // FYI: Rules Management check moved to ../detection_rules/all_rules_read_only.spec.ts - it('We show one primary callout', () => { - waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary'); - }); + context('On Rule Details page', () => { + beforeEach(() => { + createRule(getNewRule()).then((rule) => + loadPageAsReadOnlyUser(ruleDetailsUrl(rule.body.id)) + ); + }); + + afterEach(() => { + deleteCustomRule(); + }); - context('When a user clicks Dismiss on the callouts', () => { - it('We hide them and persist the dismissal', () => { + it('We show one primary callout', () => { waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary'); + }); - dismissCallOut(MISSING_PRIVILEGES_CALLOUT); - reloadPage(); + context('When a user clicks Dismiss on the callouts', () => { + it('We hide them and persist the dismissal', () => { + waitForCallOutToBeShown(MISSING_PRIVILEGES_CALLOUT, 'primary'); - getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + dismissCallOut(MISSING_PRIVILEGES_CALLOUT); + reloadPage(); + + getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + }); }); }); }); - }); - context('indicating read-write access to resources', () => { - context('On Detections home page', () => { - beforeEach(() => { - loadPageAsPlatformEngineer(ALERTS_URL); - }); + context('indicating read-write access to resources', () => { + context('On Detections home page', () => { + beforeEach(() => { + loadPageAsPlatformEngineer(ALERTS_URL); + }); - it('We show no callout', () => { - getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + it('We show no callout', () => { + getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + }); }); - }); - context('On Rules Management page', () => { - beforeEach(() => { - login(ROLES.platform_engineer); - loadPageAsPlatformEngineer(RULES_MANAGEMENT_URL); - }); + context('On Rules Management page', () => { + beforeEach(() => { + login(ROLES.platform_engineer); + loadPageAsPlatformEngineer(RULES_MANAGEMENT_URL); + }); - it('We show no callout', () => { - getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + it('We show no callout', () => { + getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + }); }); - }); - context('On Rule Details page', () => { - beforeEach(() => { - createRule(getNewRule()).then((rule) => - loadPageAsPlatformEngineer(ruleDetailsUrl(rule.body.id)) - ); - }); + context('On Rule Details page', () => { + beforeEach(() => { + createRule(getNewRule()).then((rule) => + loadPageAsPlatformEngineer(ruleDetailsUrl(rule.body.id)) + ); + }); - afterEach(() => { - deleteCustomRule(); - }); + afterEach(() => { + deleteCustomRule(); + }); - it('We show no callouts', () => { - getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + it('We show no callouts', () => { + getCallOut(MISSING_PRIVILEGES_CALLOUT).should('not.exist'); + }); }); }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts index 052306817a87d..d40aca2054372 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts @@ -68,7 +68,7 @@ const loginPageAsWriteAuthorizedUser = (url: string) => { describe( 'Detection rules, Prebuilt Rules Installation and Update - Authorization/RBAC', - { tags: ['@ess', '@serverless'] }, + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { beforeEach(() => { preventPrebuiltRulesPackageInstallation(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts index eedb4aad15881..a02486e580f20 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts @@ -39,13 +39,12 @@ import { visitRulesManagementTable } from '../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update - Error handling', - { tags: ['@ess', '@serverless'] }, + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { beforeEach(() => { preventPrebuiltRulesPackageInstallation(); cleanKibana(); login(); - visitRulesManagementTable(); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts index 5d91deef2524e..15e77fad28d03 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts @@ -31,7 +31,7 @@ import { visitRulesManagementTable } from '../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', - { tags: ['@ess', '@serverless'] }, + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { describe('Installation of prebuilt rules', () => { const RULE_1 = createRuleAssetSavedObject({ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts index 7fef8e4abbea2..b0ad9b9da100a 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts @@ -51,7 +51,7 @@ const rules = Array.from(Array(5)).map((_, i) => { }); }); -describe('Prebuilt rules', { tags: ['@ess', '@serverless'] }, () => { +describe('Prebuilt rules', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { before(() => { cleanKibana(); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts index eef35c79d3624..180c435c10213 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts @@ -54,16 +54,20 @@ describe( cy.get(RULES_UPDATES_TAB).should('not.exist'); }); - it('should NOT display install or update notifications when latest rules are installed', () => { - visitRulesManagementTable(); - createAndInstallMockedPrebuiltRules([RULE_1]); + it( + 'should NOT display install or update notifications when latest rules are installed', + { tags: ['@brokenInServerlessQA'] }, + () => { + visitRulesManagementTable(); + createAndInstallMockedPrebuiltRules([RULE_1]); - /* Assert that there are no installation or update notifications */ - /* Add Elastic Rules button should not contain a number badge */ - /* and Rule Upgrade tab should not be displayed */ - cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', 'Add Elastic rules'); - cy.get(RULES_UPDATES_TAB).should('not.exist'); - }); + /* Assert that there are no installation or update notifications */ + /* Add Elastic Rules button should not contain a number badge */ + /* and Rule Upgrade tab should not be displayed */ + cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', 'Add Elastic rules'); + cy.get(RULES_UPDATES_TAB).should('not.exist'); + } + ); }); describe('Notifications', () => { @@ -71,60 +75,68 @@ describe( installPrebuiltRuleAssets([RULE_1]); }); - describe('Rules installation notification when no rules have been installed', () => { - beforeEach(() => { - visitRulesManagementTable(); - }); - - it('should notify user about prebuilt rules available for installation', () => { - cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible'); - cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`); - cy.get(RULES_UPDATES_TAB).should('not.exist'); - }); - }); - - describe('Rule installation notification when at least one rule already installed', () => { - beforeEach(() => { - installAllPrebuiltRulesRequest().then(() => { - /* Create new rule assets with a different rule_id as the one that was */ - /* installed before in order to trigger the installation notification */ - const RULE_2 = createRuleAssetSavedObject({ - name: 'Test rule 2', - rule_id: 'rule_2', - }); - const RULE_3 = createRuleAssetSavedObject({ - name: 'Test rule 3', - rule_id: 'rule_3', - }); - + describe( + 'Rules installation notification when no rules have been installed', + { tags: ['@brokenInServerlessQA'] }, + () => { + beforeEach(() => { visitRulesManagementTable(); - installPrebuiltRuleAssets([RULE_2, RULE_3]); }); - }); - it('should notify user about prebuilt rules available for installation', () => { - const numberOfAvailableRules = 2; - cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible'); - cy.get(ADD_ELASTIC_RULES_BTN).should( - 'have.text', - `Add Elastic rules${numberOfAvailableRules}` - ); - cy.get(RULES_UPDATES_TAB).should('not.exist'); - }); - - it('should notify user a rule is again available for installation if it is deleted', () => { - /* Install available rules, assert that the notification is gone */ - /* then delete one rule and assert that the notification is back */ - installAllPrebuiltRulesRequest().then(() => { - cy.reload(); - deleteFirstRule(); + it('should notify user about prebuilt rules available for installation', () => { cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible'); cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`); + cy.get(RULES_UPDATES_TAB).should('not.exist'); + }); + } + ); + + describe( + 'Rule installation notification when at least one rule already installed', + { tags: ['@brokenInServerlessQA'] }, + () => { + beforeEach(() => { + installAllPrebuiltRulesRequest().then(() => { + /* Create new rule assets with a different rule_id as the one that was */ + /* installed before in order to trigger the installation notification */ + const RULE_2 = createRuleAssetSavedObject({ + name: 'Test rule 2', + rule_id: 'rule_2', + }); + const RULE_3 = createRuleAssetSavedObject({ + name: 'Test rule 3', + rule_id: 'rule_3', + }); + + visitRulesManagementTable(); + installPrebuiltRuleAssets([RULE_2, RULE_3]); + }); }); - }); - }); - describe('Rule update notification', () => { + it('should notify user about prebuilt rules available for installation', () => { + const numberOfAvailableRules = 2; + cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible'); + cy.get(ADD_ELASTIC_RULES_BTN).should( + 'have.text', + `Add Elastic rules${numberOfAvailableRules}` + ); + cy.get(RULES_UPDATES_TAB).should('not.exist'); + }); + + it('should notify user a rule is again available for installation if it is deleted', () => { + /* Install available rules, assert that the notification is gone */ + /* then delete one rule and assert that the notification is back */ + installAllPrebuiltRulesRequest().then(() => { + cy.reload(); + deleteFirstRule(); + cy.get(ADD_ELASTIC_RULES_BTN).should('be.visible'); + cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`); + }); + }); + } + ); + + describe('Rule update notification', { tags: ['@brokenInServerlessQA'] }, () => { beforeEach(() => { installAllPrebuiltRulesRequest().then(() => { /* Create new rule asset with the same rule_id as the one that was installed */ @@ -148,35 +160,39 @@ describe( }); }); - describe('Rule installation available and rule update available notifications', () => { - beforeEach(() => { - installAllPrebuiltRulesRequest().then(() => { - /* Create new rule assets with a different rule_id as the one that was */ - /* installed before in order to trigger the installation notification */ - const RULE_2 = createRuleAssetSavedObject({ - name: 'Test rule 2', - rule_id: 'rule_2', + describe( + 'Rule installation available and rule update available notifications', + { tags: ['@brokenInServerlessQA'] }, + () => { + beforeEach(() => { + installAllPrebuiltRulesRequest().then(() => { + /* Create new rule assets with a different rule_id as the one that was */ + /* installed before in order to trigger the installation notification */ + const RULE_2 = createRuleAssetSavedObject({ + name: 'Test rule 2', + rule_id: 'rule_2', + }); + /* Create new rule asset with the same rule_id as the one that was installed */ + /* but with a higher version, in order to trigger the update notification */ + const UPDATED_RULE = createRuleAssetSavedObject({ + name: 'Test rule 1.1 (updated)', + rule_id: 'rule_1', + version: 2, + }); + installPrebuiltRuleAssets([RULE_2, UPDATED_RULE]); + visitRulesManagementTable(); }); - /* Create new rule asset with the same rule_id as the one that was installed */ - /* but with a higher version, in order to trigger the update notification */ - const UPDATED_RULE = createRuleAssetSavedObject({ - name: 'Test rule 1.1 (updated)', - rule_id: 'rule_1', - version: 2, - }); - installPrebuiltRuleAssets([RULE_2, UPDATED_RULE]); - visitRulesManagementTable(); }); - }); - it('should notify user about prebuilt rules available for installation and for upgrade', () => { - // 1 rule available for installation - cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`); - // 1 rule available for update - cy.get(RULES_UPDATES_TAB).should('be.visible'); - cy.get(RULES_UPDATES_TAB).should('have.text', `Rule Updates${1}`); - }); - }); + it('should notify user about prebuilt rules available for installation and for upgrade', () => { + // 1 rule available for installation + cy.get(ADD_ELASTIC_RULES_BTN).should('have.text', `Add Elastic rules${1}`); + // 1 rule available for update + cy.get(RULES_UPDATES_TAB).should('be.visible'); + cy.get(RULES_UPDATES_TAB).should('have.text', `Rule Updates${1}`); + }); + } + ); }); } ); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts index d9ad5c4e0716c..2e38b4782b433 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts @@ -31,7 +31,7 @@ import { visitRulesManagementTable } from '../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', - { tags: ['@ess', '@serverless'] }, + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { describe('Upgrade of prebuilt rules', () => { const RULE_1_ID = 'rule_1'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/common_flows.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/common_flows.cy.ts index dffb50ab6dae6..e8780d8696d29 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/common_flows.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/common_flows.cy.ts @@ -48,13 +48,13 @@ import { visit } from '../../../tasks/navigation'; // them in the relevant /rule_creation/[RULE_TYPE].cy.ts test. describe('Common rule creation flows', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { + login(); deleteAlertsAndRules(); createTimeline(getTimeline()) .then((response) => { return response.body.data.persistTimeline.timeline.savedObjectId; }) .as('timelineId'); - login(); visit(CREATE_RULE_URL); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/event_correlation_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/event_correlation_rule.cy.ts index 36fa6cd484154..eeca0d06bcc57 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/event_correlation_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/event_correlation_rule.cy.ts @@ -43,7 +43,7 @@ import { import { getDetails, waitForTheRuleToBeExecuted } from '../../../tasks/rule_details'; import { expectNumberOfRules, goToRuleDetailsOf } from '../../../tasks/alerts_detection_rules'; -import { cleanKibana, deleteAlertsAndRules } from '../../../tasks/common'; +import { deleteAlertsAndRules } from '../../../tasks/common'; import { createAndEnableRule, fillAboutRuleAndContinue, @@ -58,10 +58,6 @@ import { openRuleManagementPageViaBreadcrumbs } from '../../../tasks/rules_manag import { CREATE_RULE_URL } from '../../../urls/navigation'; describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - }); - beforeEach(() => { login(); deleteAlertsAndRules(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/machine_learning_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/machine_learning_rule.cy.ts index 684a44d37d214..3929f57d0b6f5 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/machine_learning_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/machine_learning_rule.cy.ts @@ -41,7 +41,6 @@ import { import { getDetails } from '../../../tasks/rule_details'; import { expectNumberOfRules, goToRuleDetailsOf } from '../../../tasks/alerts_detection_rules'; -import { cleanKibana } from '../../../tasks/common'; import { createAndEnableRule, fillAboutRuleAndContinue, @@ -54,74 +53,77 @@ import { visit } from '../../../tasks/navigation'; import { openRuleManagementPageViaBreadcrumbs } from '../../../tasks/rules_management'; import { CREATE_RULE_URL } from '../../../urls/navigation'; -describe('Machine Learning rules', { tags: ['@ess', '@serverless'] }, () => { - const expectedUrls = (getMachineLearningRule().references ?? []).join(''); - const expectedFalsePositives = (getMachineLearningRule().false_positives ?? []).join(''); - const expectedTags = (getMachineLearningRule().tags ?? []).join(''); - const expectedMitre = formatMitreAttackDescription(getMachineLearningRule().threat ?? []); - const expectedNumberOfRules = 1; +describe( + 'Machine Learning rules', + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + () => { + const expectedUrls = (getMachineLearningRule().references ?? []).join(''); + const expectedFalsePositives = (getMachineLearningRule().false_positives ?? []).join(''); + const expectedTags = (getMachineLearningRule().tags ?? []).join(''); + const expectedMitre = formatMitreAttackDescription(getMachineLearningRule().threat ?? []); + const expectedNumberOfRules = 1; - beforeEach(() => { - cleanKibana(); - login(); - visit(CREATE_RULE_URL); - }); + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + }); - it('Creates and enables a new ml rule', () => { - const mlRule = getMachineLearningRule(); - selectMachineLearningRuleType(); - fillDefineMachineLearningRuleAndContinue(mlRule); - fillAboutRuleAndContinue(mlRule); - fillScheduleRuleAndContinue(mlRule); - createAndEnableRule(); - openRuleManagementPageViaBreadcrumbs(); + it('Creates and enables a new ml rule', () => { + const mlRule = getMachineLearningRule(); + selectMachineLearningRuleType(); + fillDefineMachineLearningRuleAndContinue(mlRule); + fillAboutRuleAndContinue(mlRule); + fillScheduleRuleAndContinue(mlRule); + createAndEnableRule(); + openRuleManagementPageViaBreadcrumbs(); - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); + cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); + expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); - cy.get(RULE_NAME).should('have.text', mlRule.name); - cy.get(RISK_SCORE).should('have.text', mlRule.risk_score); - cy.get(SEVERITY).should('have.text', 'Critical'); - cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); + cy.get(RULE_NAME).should('have.text', mlRule.name); + cy.get(RISK_SCORE).should('have.text', mlRule.risk_score); + cy.get(SEVERITY).should('have.text', 'Critical'); + cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); - goToRuleDetailsOf(mlRule.name); + goToRuleDetailsOf(mlRule.name); - cy.get(RULE_NAME_HEADER).should('contain', `${mlRule.name}`); - cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', mlRule.description); - cy.get(ABOUT_DETAILS).within(() => { - getDetails(SEVERITY_DETAILS).should('have.text', 'Critical'); - getDetails(RISK_SCORE_DETAILS).should('have.text', mlRule.risk_score); - getDetails(REFERENCE_URLS_DETAILS).should((details) => { - expect(removeExternalLinkText(details.text())).equal(expectedUrls); + cy.get(RULE_NAME_HEADER).should('contain', `${mlRule.name}`); + cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', mlRule.description); + cy.get(ABOUT_DETAILS).within(() => { + getDetails(SEVERITY_DETAILS).should('have.text', 'Critical'); + getDetails(RISK_SCORE_DETAILS).should('have.text', mlRule.risk_score); + getDetails(REFERENCE_URLS_DETAILS).should((details) => { + expect(removeExternalLinkText(details.text())).equal(expectedUrls); + }); + getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); + getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { + expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + }); + getDetails(TAGS_DETAILS).should('have.text', expectedTags); }); - getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); - getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { - expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(ANOMALY_SCORE_DETAILS).should('have.text', mlRule.anomaly_threshold); + getDetails(RULE_TYPE_DETAILS).should('have.text', 'Machine Learning'); + getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); + const machineLearningJobsArray = isArray(mlRule.machine_learning_job_id) + ? mlRule.machine_learning_job_id + : [mlRule.machine_learning_job_id]; + // With the #1912 ML rule improvement changes we enable jobs on rule creation. + // Though, in cypress jobs enabling does not work reliably and job can be started or stopped. + // Thus, we disable next check till we fix the issue with enabling jobs in cypress. + // Relevant ticket: https://github.com/elastic/security-team/issues/5389 + // cy.get(MACHINE_LEARNING_JOB_STATUS).should('have.text', 'StoppedStopped'); + cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', machineLearningJobsArray.join('')); + }); + cy.get(SCHEDULE_DETAILS).within(() => { + getDetails(RUNS_EVERY_DETAILS).should('have.text', `${mlRule.interval}`); + const humanizedDuration = getHumanizedDuration( + mlRule.from ?? 'now-6m', + mlRule.interval ?? '5m' + ); + getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should('have.text', `${humanizedDuration}`); }); - getDetails(TAGS_DETAILS).should('have.text', expectedTags); - }); - cy.get(DEFINITION_DETAILS).within(() => { - getDetails(ANOMALY_SCORE_DETAILS).should('have.text', mlRule.anomaly_threshold); - getDetails(RULE_TYPE_DETAILS).should('have.text', 'Machine Learning'); - getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - const machineLearningJobsArray = isArray(mlRule.machine_learning_job_id) - ? mlRule.machine_learning_job_id - : [mlRule.machine_learning_job_id]; - // With the #1912 ML rule improvement changes we enable jobs on rule creation. - // Though, in cypress jobs enabling does not work reliably and job can be started or stopped. - // Thus, we disable next check till we fix the issue with enabling jobs in cypress. - // Relevant ticket: https://github.com/elastic/security-team/issues/5389 - // cy.get(MACHINE_LEARNING_JOB_STATUS).should('have.text', 'StoppedStopped'); - cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', machineLearningJobsArray.join('')); - }); - cy.get(SCHEDULE_DETAILS).within(() => { - getDetails(RUNS_EVERY_DETAILS).should('have.text', `${mlRule.interval}`); - const humanizedDuration = getHumanizedDuration( - mlRule.from ?? 'now-6m', - mlRule.interval ?? '5m' - ); - getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should('have.text', `${humanizedDuration}`); }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts index a5e404f9d706b..5e5af0e6ad4a7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts @@ -45,7 +45,7 @@ import { } from '../../../screens/rule_details'; import { createTimeline } from '../../../tasks/api_calls/timelines'; -import { cleanKibana, deleteAlertsAndRules, deleteConnectors } from '../../../tasks/common'; +import { deleteAlertsAndRules, deleteConnectors } from '../../../tasks/common'; import { login } from '../../../tasks/login'; import { visit } from '../../../tasks/navigation'; import { ruleDetailsUrl } from '../../../urls/rule_details'; @@ -53,15 +53,11 @@ import { ruleDetailsUrl } from '../../../urls/rule_details'; // This test is meant to test all common aspects of the rule details page that should function // the same regardless of rule type. For any rule type specific functionalities, please include // them in the relevant /rule_details/[RULE_TYPE].cy.ts test. -describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - }); - +describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, function () { beforeEach(() => { + login(); deleteAlertsAndRules(); deleteConnectors(); - login(); createTimeline(getTimeline()).then((response) => { createRule({ ...getNewRule({ @@ -93,12 +89,13 @@ describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => { ], }), }).then((rule) => { - visit(ruleDetailsUrl(rule.body.id)); + cy.wrap(rule.body.id).as('ruleId'); }); }); }); - it('Only modifies rule active status on enable/disable', () => { + it('Only modifies rule active status on enable/disable', function () { + visit(ruleDetailsUrl(this.ruleId)); cy.get(RULE_NAME_HEADER).should('contain', ruleFields.ruleName); cy.intercept('POST', '/api/detection_engine/rules/_bulk_action?dry_run=false').as( @@ -117,6 +114,7 @@ describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => { }); it('Displays rule details', function () { + visit(ruleDetailsUrl(this.ruleId)); cy.get(RULE_NAME_HEADER).should('contain', ruleFields.ruleName); cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', ruleFields.ruleDescription); cy.get(ABOUT_DETAILS).within(() => { @@ -161,7 +159,8 @@ describe('Common rule detail flows', { tags: ['@ess', '@serverless'] }, () => { }); }); - it('Deletes one rule from detail page', () => { + it('Deletes one rule from detail page', function () { + visit(ruleDetailsUrl(this.ruleId)); cy.intercept('POST', '/api/detection_engine/rules/_bulk_delete').as('deleteRule'); deleteRuleFromDetailsPage(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/custom_query_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/custom_query_rule.cy.ts index 22f7cfbf759f3..ca6d6c56adcf7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/custom_query_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit/custom_query_rule.cy.ts @@ -54,7 +54,7 @@ import { saveEditedRule, visitEditRulePage } from '../../../tasks/edit_rule'; import { login } from '../../../tasks/login'; import { getDetails } from '../../../tasks/rule_details'; -describe('Custom query rules', { tags: ['@ess', '@serverless'] }, () => { +describe('Custom query rules', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { const rule = getEditedRule(); const expectedEditedtags = rule.tags?.join(''); const expectedEditedIndexPatterns = rule.index; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts index 6f5434772e69a..1376e486790b7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts @@ -45,7 +45,7 @@ import { waitForPageToBeLoaded, } from '../../../../tasks/rule_details'; -describe('Related integrations', { tags: ['@ess', '@serverless'] }, () => { +describe('Related integrations', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { const DATA_STREAM_NAME = 'logs-related-integrations-test'; const PREBUILT_RULE_NAME = 'Prebuilt rule with related integrations'; const RULE_RELATED_INTEGRATIONS: IntegrationDefinition[] = [ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts index 9a7eb4547cee0..ef3754c898ecf 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts @@ -185,7 +185,7 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () => cy.get(APPLY_TIMELINE_RULE_BULK_MENU_ITEM).should('be.disabled'); }); - it('Only prebuilt rules selected', () => { + it('Only prebuilt rules selected', { tags: ['@brokenInServerlessQA'] }, () => { createAndInstallMockedPrebuiltRules(PREBUILT_RULES); // select Elastic(prebuilt) rules, check if we can't proceed further, as Elastic rules are not editable @@ -203,47 +203,55 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () => }); }); - it('Prebuilt and custom rules selected: user proceeds with custom rules editing', () => { - getRulesManagementTableRows().then((existedRulesRows) => { - createAndInstallMockedPrebuiltRules(PREBUILT_RULES); + it( + 'Prebuilt and custom rules selected: user proceeds with custom rules editing', + { tags: ['@brokenInServerlessQA'] }, + () => { + getRulesManagementTableRows().then((existedRulesRows) => { + createAndInstallMockedPrebuiltRules(PREBUILT_RULES); - // modal window should show how many rules can be edit, how many not - selectAllRules(); - clickAddTagsMenuItem(); + // modal window should show how many rules can be edit, how many not + selectAllRules(); + clickAddTagsMenuItem(); - waitForMixedRulesBulkEditModal(existedRulesRows.length); + waitForMixedRulesBulkEditModal(existedRulesRows.length); - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - checkPrebuiltRulesCannotBeModified(availablePrebuiltRulesCount); - }); + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + checkPrebuiltRulesCannotBeModified(availablePrebuiltRulesCount); + }); - // user can proceed with custom rule editing - cy.get(MODAL_CONFIRMATION_BTN) - .should('have.text', `Edit ${existedRulesRows.length} custom rules`) - .click(); + // user can proceed with custom rule editing + cy.get(MODAL_CONFIRMATION_BTN) + .should('have.text', `Edit ${existedRulesRows.length} custom rules`) + .click(); - // action should finish - typeTags(['test-tag']); - submitBulkEditForm(); - waitForBulkEditActionToFinish({ updatedCount: existedRulesRows.length }); - }); - }); + // action should finish + typeTags(['test-tag']); + submitBulkEditForm(); + waitForBulkEditActionToFinish({ updatedCount: existedRulesRows.length }); + }); + } + ); - it('Prebuilt and custom rules selected: user cancels action', () => { - createAndInstallMockedPrebuiltRules(PREBUILT_RULES); + it( + 'Prebuilt and custom rules selected: user cancels action', + { tags: ['@brokenInServerlessQA'] }, + () => { + createAndInstallMockedPrebuiltRules(PREBUILT_RULES); - getRulesManagementTableRows().then((rows) => { - // modal window should show how many rules can be edit, how many not - selectAllRules(); - clickAddTagsMenuItem(); - waitForMixedRulesBulkEditModal(rows.length); + getRulesManagementTableRows().then((rows) => { + // modal window should show how many rules can be edit, how many not + selectAllRules(); + clickAddTagsMenuItem(); + waitForMixedRulesBulkEditModal(rows.length); - checkPrebuiltRulesCannotBeModified(PREBUILT_RULES.length); + checkPrebuiltRulesCannotBeModified(PREBUILT_RULES.length); - // user cancels action and modal disappears - cancelConfirmationModal(); - }); - }); + // user cancels action and modal disappears + cancelConfirmationModal(); + }); + } + ); it('should not lose rules selection after edit action', () => { const rulesToUpdate = [RULE_NAME, 'New EQL Rule', 'New Terms Rule'] as const; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts index 3631a429880d5..6f110207a3d6e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts @@ -74,78 +74,80 @@ const ruleNameToAssert = 'Custom rule name with actions'; const expectedExistingSlackMessage = 'Existing slack action'; const expectedSlackMessage = 'Slack action test message'; -describe('Detection rules, bulk edit of rule actions', { tags: ['@ess', '@serverless'] }, () => { - beforeEach(() => { - cleanKibana(); - login(); - deleteAlertsAndRules(); - deleteConnectors(); - - createSlackConnector().then(({ body }) => { - const actions: RuleActionArray = [ - { - id: body.id, - action_type_id: '.slack', - group: 'default', - params: { - message: expectedExistingSlackMessage, - }, - frequency: { - summary: true, - throttle: null, - notifyWhen: 'onActiveAlert', +describe( + 'Detection rules, bulk edit of rule actions', + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + () => { + beforeEach(() => { + cleanKibana(); + login(); + deleteAlertsAndRules(); + deleteConnectors(); + + createSlackConnector().then(({ body }) => { + const actions: RuleActionArray = [ + { + id: body.id, + action_type_id: '.slack', + group: 'default', + params: { + message: expectedExistingSlackMessage, + }, + frequency: { + summary: true, + throttle: null, + notifyWhen: 'onActiveAlert', + }, }, - }, - ]; + ]; + + createRule( + getNewRule({ + rule_id: '1', + name: ruleNameToAssert, + max_signals: 500, + actions, + enabled: false, + }) + ); + }); + createRule(getEqlRule({ rule_id: '2', name: 'New EQL Rule', enabled: false })); createRule( - getNewRule({ - rule_id: '1', - name: ruleNameToAssert, - max_signals: 500, - actions, + getMachineLearningRule({ rule_id: '3', name: 'New ML Rule Test', enabled: false }) + ); + createRule( + getNewThreatIndicatorRule({ + rule_id: '4', + name: 'Threat Indicator Rule Test', enabled: false, }) ); - }); + createRule(getNewThresholdRule({ rule_id: '5', name: 'Threshold Rule', enabled: false })); + createRule(getNewTermsRule({ rule_id: '6', name: 'New Terms Rule', enabled: false })); + createRule( + getNewRule({ saved_id: 'mocked', rule_id: '7', name: 'New Rule Test', enabled: false }) + ); - createRule(getEqlRule({ rule_id: '2', name: 'New EQL Rule', enabled: false })); - createRule(getMachineLearningRule({ rule_id: '3', name: 'New ML Rule Test', enabled: false })); - createRule( - getNewThreatIndicatorRule({ - rule_id: '4', - name: 'Threat Indicator Rule Test', - enabled: false, - }) - ); - createRule(getNewThresholdRule({ rule_id: '5', name: 'Threshold Rule', enabled: false })); - createRule(getNewTermsRule({ rule_id: '6', name: 'New Terms Rule', enabled: false })); - createRule( - getNewRule({ saved_id: 'mocked', rule_id: '7', name: 'New Rule Test', enabled: false }) - ); - - createSlackConnector(); - - // Prevent prebuilt rules package installation and mock two prebuilt rules - preventPrebuiltRulesPackageInstallation(); - - const RULE_1 = createRuleAssetSavedObject({ - name: 'Test rule 1', - rule_id: 'rule_1', - }); - const RULE_2 = createRuleAssetSavedObject({ - name: 'Test rule 2', - rule_id: 'rule_2', - }); + createSlackConnector(); + + // Prevent prebuilt rules package installation and mock two prebuilt rules + preventPrebuiltRulesPackageInstallation(); - createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]); - }); + const RULE_1 = createRuleAssetSavedObject({ + name: 'Test rule 1', + rule_id: 'rule_1', + }); + const RULE_2 = createRuleAssetSavedObject({ + name: 'Test rule 2', + rule_id: 'rule_2', + }); + + createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]); + }); - context('Restricted action privileges', () => { - it( - "User with no privileges can't add rule actions", - { tags: ['@ess', '@skipInServerless'] }, - () => { + context('Restricted action privileges', () => { + it("User with no privileges can't add rule actions", () => { login(ROLES.hunter_no_actions); visitRulesManagementTable(ROLES.hunter_no_actions); @@ -167,129 +169,129 @@ describe('Detection rules, bulk edit of rule actions', { tags: ['@ess', '@server openBulkActionsMenu(); cy.get(ADD_RULE_ACTIONS_MENU_ITEM).should('be.disabled'); - } - ); - }); - - context('All actions privileges', () => { - beforeEach(() => { - login(); - visitRulesManagementTable(); - disableAutoRefresh(); - - expectManagementTableRules([ - ruleNameToAssert, - 'New EQL Rule', - 'New ML Rule Test', - 'Threat Indicator Rule Test', - 'Threshold Rule', - 'New Terms Rule', - 'New Rule Test', - 'Test rule 1', - 'Test rule 2', - ]); + }); }); - it('Add a rule action to rules (existing connector)', () => { - const expectedActionFrequency: RuleActionCustomFrequency = { - throttle: 1, - throttleUnit: 'd', - }; + context('All actions privileges', () => { + beforeEach(() => { + login(); + visitRulesManagementTable(); + disableAutoRefresh(); + + expectManagementTableRules([ + ruleNameToAssert, + 'New EQL Rule', + 'New ML Rule Test', + 'Threat Indicator Rule Test', + 'Threshold Rule', + 'New Terms Rule', + 'New Rule Test', + 'Test rule 1', + 'Test rule 2', + ]); + }); + + it('Add a rule action to rules (existing connector)', () => { + const expectedActionFrequency: RuleActionCustomFrequency = { + throttle: 1, + throttleUnit: 'd', + }; - excessivelyInstallAllPrebuiltRules(); + excessivelyInstallAllPrebuiltRules(); - getRulesManagementTableRows().then((rows) => { - // select both custom and prebuilt rules - selectAllRules(); - openBulkEditRuleActionsForm(); + getRulesManagementTableRows().then((rows) => { + // select both custom and prebuilt rules + selectAllRules(); + openBulkEditRuleActionsForm(); - // ensure rule actions info callout displayed on the form - cy.get(RULES_BULK_EDIT_ACTIONS_INFO).should('be.visible'); + // ensure rule actions info callout displayed on the form + cy.get(RULES_BULK_EDIT_ACTIONS_INFO).should('be.visible'); - addSlackRuleAction(expectedSlackMessage); - pickSummaryOfAlertsOption(); - pickCustomFrequencyOption(expectedActionFrequency); + addSlackRuleAction(expectedSlackMessage); + pickSummaryOfAlertsOption(); + pickCustomFrequencyOption(expectedActionFrequency); - submitBulkEditForm(); - waitForBulkEditActionToFinish({ updatedCount: rows.length }); + submitBulkEditForm(); + waitForBulkEditActionToFinish({ updatedCount: rows.length }); - // check if rule has been updated - goToEditRuleActionsSettingsOf(ruleNameToAssert); + // check if rule has been updated + goToEditRuleActionsSettingsOf(ruleNameToAssert); - assertSelectedSummaryOfAlertsOption(); - assertSelectedCustomFrequencyOption(expectedActionFrequency, 1); - assertSlackRuleAction(expectedExistingSlackMessage, 0); - assertSlackRuleAction(expectedSlackMessage, 1); - // ensure there is no third action - cy.get(actionFormSelector(2)).should('not.exist'); + assertSelectedSummaryOfAlertsOption(); + assertSelectedCustomFrequencyOption(expectedActionFrequency, 1); + assertSlackRuleAction(expectedExistingSlackMessage, 0); + assertSlackRuleAction(expectedSlackMessage, 1); + // ensure there is no third action + cy.get(actionFormSelector(2)).should('not.exist'); + }); }); - }); - it('Overwrite rule actions in rules', () => { - excessivelyInstallAllPrebuiltRules(); + it('Overwrite rule actions in rules', () => { + excessivelyInstallAllPrebuiltRules(); - getRulesManagementTableRows().then((rows) => { - // select both custom and prebuilt rules - selectAllRules(); + getRulesManagementTableRows().then((rows) => { + // select both custom and prebuilt rules + selectAllRules(); + openBulkEditRuleActionsForm(); + + addSlackRuleAction(expectedSlackMessage); + pickSummaryOfAlertsOption(); + pickPerRuleRunFrequencyOption(); + + // check overwrite box, ensure warning is displayed + checkOverwriteRuleActionsCheckbox(); + cy.get(RULES_BULK_EDIT_ACTIONS_WARNING).contains( + `You're about to overwrite rule actions for ${rows.length} selected rules` + ); + + submitBulkEditForm(); + waitForBulkEditActionToFinish({ updatedCount: rows.length }); + + // check if rule has been updated + goToEditRuleActionsSettingsOf(ruleNameToAssert); + + assertSelectedSummaryOfAlertsOption(); + assertSelectedPerRuleRunFrequencyOption(); + assertSlackRuleAction(expectedSlackMessage); + // ensure existing action was overwritten + cy.get(actionFormSelector(1)).should('not.exist'); + }); + }); + + it('Add a rule action to rules (new connector)', () => { + const rulesToSelect = [ + ruleNameToAssert, + 'New EQL Rule', + 'New ML Rule Test', + 'Threat Indicator Rule Test', + 'Threshold Rule', + 'New Terms Rule', + 'New Rule Test', + ] as const; + const expectedActionFrequency: RuleActionCustomFrequency = { + throttle: 2, + throttleUnit: 'h', + }; + const expectedEmail = 'test@example.com'; + const expectedSubject = 'Subject'; + + selectRulesByName(rulesToSelect); openBulkEditRuleActionsForm(); - addSlackRuleAction(expectedSlackMessage); + addEmailConnectorAndRuleAction(expectedEmail, expectedSubject); pickSummaryOfAlertsOption(); - pickPerRuleRunFrequencyOption(); - - // check overwrite box, ensure warning is displayed - checkOverwriteRuleActionsCheckbox(); - cy.get(RULES_BULK_EDIT_ACTIONS_WARNING).contains( - `You're about to overwrite rule actions for ${rows.length} selected rules` - ); + pickCustomFrequencyOption(expectedActionFrequency); submitBulkEditForm(); - waitForBulkEditActionToFinish({ updatedCount: rows.length }); + waitForBulkEditActionToFinish({ updatedCount: rulesToSelect.length }); // check if rule has been updated goToEditRuleActionsSettingsOf(ruleNameToAssert); assertSelectedSummaryOfAlertsOption(); - assertSelectedPerRuleRunFrequencyOption(); - assertSlackRuleAction(expectedSlackMessage); - // ensure existing action was overwritten - cy.get(actionFormSelector(1)).should('not.exist'); + assertSelectedCustomFrequencyOption(expectedActionFrequency, 1); + assertEmailRuleAction(expectedEmail, expectedSubject); }); }); - - it('Add a rule action to rules (new connector)', () => { - const rulesToSelect = [ - ruleNameToAssert, - 'New EQL Rule', - 'New ML Rule Test', - 'Threat Indicator Rule Test', - 'Threshold Rule', - 'New Terms Rule', - 'New Rule Test', - ] as const; - const expectedActionFrequency: RuleActionCustomFrequency = { - throttle: 2, - throttleUnit: 'h', - }; - const expectedEmail = 'test@example.com'; - const expectedSubject = 'Subject'; - - selectRulesByName(rulesToSelect); - openBulkEditRuleActionsForm(); - - addEmailConnectorAndRuleAction(expectedEmail, expectedSubject); - pickSummaryOfAlertsOption(); - pickCustomFrequencyOption(expectedActionFrequency); - - submitBulkEditForm(); - waitForBulkEditActionToFinish({ updatedCount: rulesToSelect.length }); - - // check if rule has been updated - goToEditRuleActionsSettingsOf(ruleNameToAssert); - - assertSelectedSummaryOfAlertsOption(); - assertSelectedCustomFrequencyOption(expectedActionFrequency, 1); - assertEmailRuleAction(expectedEmail, expectedSubject); - }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts index 275160d1f04a0..3983b5e8f534e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts @@ -100,19 +100,23 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { expectManagementTableRules(['Enabled rule to export']); }); - it('shows a modal saying that no rules can be exported if all the selected rules are prebuilt', function () { - createAndInstallMockedPrebuiltRules(prebuiltRules); + it( + 'shows a modal saying that no rules can be exported if all the selected rules are prebuilt', + { tags: ['@brokenInServerlessQA'] }, + function () { + createAndInstallMockedPrebuiltRules(prebuiltRules); - filterByElasticRules(); - selectAllRules(); - bulkExportRules(); + filterByElasticRules(); + selectAllRules(); + bulkExportRules(); - cy.get(MODAL_CONFIRMATION_BODY).contains( - `${prebuiltRules.length} prebuilt Elastic rules (exporting prebuilt rules is not supported)` - ); - }); + cy.get(MODAL_CONFIRMATION_BODY).contains( + `${prebuiltRules.length} prebuilt Elastic rules (exporting prebuilt rules is not supported)` + ); + } + ); - it('exports only custom rules', function () { + it('exports only custom rules', { tags: ['@brokenInServerlessQA'] }, function () { const expectedNumberCustomRulesToBeExported = 1; createAndInstallMockedPrebuiltRules(prebuiltRules); @@ -141,7 +145,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { }); }); - context('rules with exceptions', () => { + context('rules with exceptions', { tags: ['@brokenInServerlessQA'] }, () => { beforeEach(() => { deleteExceptionList(exceptionList.list_id, exceptionList.namespace_type); // create rule with exceptions diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/snoozing/rule_snoozing.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/snoozing/rule_snoozing.cy.ts index cbac9328fb4aa..fd300c04bfe4f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/snoozing/rule_snoozing.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/snoozing/rule_snoozing.cy.ts @@ -181,7 +181,7 @@ describe('rule snoozing', { tags: ['@ess', '@serverless'] }, () => { deleteConnectors(); }); - it('adds an action to a snoozed rule', () => { + it('adds an action to a snoozed rule', { tags: ['@brokenInServerlessQA'] }, () => { createSnoozedRule(getNewRule({ name: 'Snoozed rule' })).then(({ body: rule }) => { visitEditRulePage(rule.id); goToActionsStepTab(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_links.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_links.cy.ts index ad7813991cad7..cf81271c1ad33 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_links.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_links.cy.ts @@ -8,16 +8,12 @@ import { getNewRule } from '../../../../objects/rule'; import { RULES_MONITORING_TAB, RULE_NAME } from '../../../../screens/alerts_detection_rules'; import { createRule } from '../../../../tasks/api_calls/rules'; -import { cleanKibana, deleteAlertsAndRules } from '../../../../tasks/common'; +import { deleteAlertsAndRules } from '../../../../tasks/common'; import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; describe('Rules table: links', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - }); - beforeEach(() => { login(); deleteAlertsAndRules(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts index d07186c8252f9..ab957879885e8 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts @@ -34,68 +34,72 @@ const RULE_2 = createRuleAssetSavedObject({ rule_id: 'rule_2', }); -describe('Rules table: selection', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - }); +describe( + 'Rules table: selection', + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + () => { + before(() => { + cleanKibana(); + }); - beforeEach(() => { - login(); - /* Create and install two mock rules */ - createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]); - visit(RULES_MANAGEMENT_URL); - waitForPrebuiltDetectionRulesToBeLoaded(); - }); + beforeEach(() => { + login(); + /* Create and install two mock rules */ + createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]); + visit(RULES_MANAGEMENT_URL); + waitForPrebuiltDetectionRulesToBeLoaded(); + }); - it('should correctly update the selection label when rules are individually selected and unselected', () => { - waitForPrebuiltDetectionRulesToBeLoaded(); + it('should correctly update the selection label when rules are individually selected and unselected', () => { + waitForPrebuiltDetectionRulesToBeLoaded(); - selectRulesByName(['Test rule 1', 'Test rule 2']); + selectRulesByName(['Test rule 1', 'Test rule 2']); - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '2'); + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '2'); - unselectRulesByName(['Test rule 1', 'Test rule 2']); + unselectRulesByName(['Test rule 1', 'Test rule 2']); - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); - }); + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); + }); - it('should correctly update the selection label when rules are bulk selected and then bulk un-selected', () => { - waitForPrebuiltDetectionRulesToBeLoaded(); + it('should correctly update the selection label when rules are bulk selected and then bulk un-selected', () => { + waitForPrebuiltDetectionRulesToBeLoaded(); - cy.get(SELECT_ALL_RULES_BTN).click(); + cy.get(SELECT_ALL_RULES_BTN).click(); - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); - }); + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); + }); - // Un-select all rules via the Bulk Selection button from the Utility bar - cy.get(SELECT_ALL_RULES_BTN).click(); + // Un-select all rules via the Bulk Selection button from the Utility bar + cy.get(SELECT_ALL_RULES_BTN).click(); - // Current selection should be 0 rules - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); - // Bulk selection button should be back to displaying all rules - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); + // Current selection should be 0 rules + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); + // Bulk selection button should be back to displaying all rules + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); + }); }); - }); - it('should correctly update the selection label when rules are bulk selected and then unselected via the table select all checkbox', () => { - waitForPrebuiltDetectionRulesToBeLoaded(); + it('should correctly update the selection label when rules are bulk selected and then unselected via the table select all checkbox', () => { + waitForPrebuiltDetectionRulesToBeLoaded(); - cy.get(SELECT_ALL_RULES_BTN).click(); + cy.get(SELECT_ALL_RULES_BTN).click(); - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); - }); + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); + }); - // Un-select all rules via the Un-select All checkbox from the table - cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click(); + // Un-select all rules via the Un-select All checkbox from the table + cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click(); - // Current selection should be 0 rules - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); - // Bulk selection button should be back to displaying all rules - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); + // Current selection should be 0 rules + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); + // Bulk selection button should be back to displaying all rules + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); + }); }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/exceptions/entry/use_value_list.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/exceptions/entry/use_value_list.cy.ts index 7ecf112d08e91..530114585fa1c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/exceptions/entry/use_value_list.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/exceptions/entry/use_value_list.cy.ts @@ -17,7 +17,6 @@ import { visitRuleDetailsPage, } from '../../../tasks/rule_details'; import { getNewRule } from '../../../objects/rule'; -import { cleanKibana } from '../../../tasks/common'; import { login } from '../../../tasks/login'; import { visit } from '../../../tasks/navigation'; import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; @@ -29,6 +28,7 @@ import { deleteValueListsFile, importValueList, KNOWN_VALUE_LIST_FILES, + deleteValueLists, } from '../../../tasks/lists'; import { createRule } from '../../../tasks/api_calls/rules'; import { @@ -48,11 +48,17 @@ const goToRulesAndOpenValueListModal = () => { }; describe('Use Value list in exception entry', { tags: ['@ess', '@serverless'] }, () => { + before(() => { + cy.task('esArchiverLoad', { archiveName: 'exceptions' }); + }); + + after(() => { + cy.task('esArchiverUnload', 'exceptions'); + }); beforeEach(() => { - cleanKibana(); login(); + deleteValueLists([KNOWN_VALUE_LIST_FILES.TEXT]); createListsIndex(); - cy.task('esArchiverLoad', { archiveName: 'exceptions' }); importValueList(KNOWN_VALUE_LIST_FILES.TEXT, 'keyword'); createRule( @@ -66,10 +72,6 @@ describe('Use Value list in exception entry', { tags: ['@ess', '@serverless'] }, ).then((rule) => visitRuleDetailsPage(rule.body.id, { tab: 'rule_exceptions' })); }); - afterEach(() => { - cy.task('esArchiverUnload', 'exceptions'); - }); - it('Should use value list in exception entry, and validate deleting value list prompt', () => { const ITEM_NAME = 'Exception item with value list'; const ITEM_FIELD = 'agent.name'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/connector_options.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/connector_options.cy.ts index 8bc8d67d9e1be..9bff9b341c68a 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/connector_options.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/connector_options.cy.ts @@ -27,15 +27,9 @@ import { import { goToCreateNewCase } from '../../../tasks/all_cases'; import { CASES_URL } from '../../../urls/navigation'; import { CONNECTOR_CARD_DETAILS, CONNECTOR_TITLE } from '../../../screens/case_details'; -import { cleanKibana } from '../../../tasks/common'; // FLAKY: https://github.com/elastic/kibana/issues/165712 describe('Cases connector incident fields', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - login(); - }); - beforeEach(() => { login(); cy.intercept('GET', '/api/cases/configure/connectors/_find', getMockConnectorsResponse()); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts index 6e66299f5d42a..ef24d84ee7624 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts @@ -41,7 +41,6 @@ import { OVERVIEW_CASE_DESCRIPTION, OVERVIEW_CASE_NAME } from '../../../screens/ import { goToCaseDetails, goToCreateNewCase } from '../../../tasks/all_cases'; import { createTimeline } from '../../../tasks/api_calls/timelines'; import { openCaseTimeline } from '../../../tasks/case_details'; -import { cleanKibana } from '../../../tasks/common'; import { attachTimeline, backToCases, @@ -58,7 +57,6 @@ import { ELASTICSEARCH_USERNAME } from '../../../env_var_names_constants'; // Tracked by https://github.com/elastic/security-team/issues/7696 describe('Cases', { tags: ['@ess', '@serverless'] }, () => { before(() => { - cleanKibana(); createTimeline(getCase1().timeline).then((response) => cy .wrap({ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts index e5ef69188a8ba..db07fcc84061d 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts @@ -11,7 +11,7 @@ import { visitWithTimeRange } from '../../../tasks/navigation'; import { ALERTS_URL, ENTITY_ANALYTICS_URL } from '../../../urls/navigation'; -import { cleanKibana, deleteAlertsAndRules } from '../../../tasks/common'; +import { deleteAlertsAndRules } from '../../../tasks/common'; import { ANOMALIES_TABLE, @@ -67,8 +67,6 @@ const OLDEST_DATE = moment('2019-01-19T16:22:56.217Z').format(DATE_FORMAT); describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () => { before(() => { - cleanKibana(); - login(); deleteRiskEngineConfiguration(); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_charts.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_charts.cy.ts index 91e68ac4d5c8b..45eccfc68fa5c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_charts.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_charts.cy.ts @@ -25,46 +25,50 @@ import { } from '../../../screens/search_bar'; import { TOASTER } from '../../../screens/alerts_detection_rules'; -describe('Histogram legend hover actions', { tags: ['@ess', '@serverless'] }, () => { - const ruleConfigs = getNewRule(); +describe( + 'Histogram legend hover actions', + { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + () => { + const ruleConfigs = getNewRule(); - before(() => { - cleanKibana(); - }); + before(() => { + cleanKibana(); + }); - beforeEach(() => { - login(); - createRule(getNewRule({ rule_id: 'new custom rule' })); - visitWithTimeRange(ALERTS_URL); - selectAlertsHistogram(); - }); + beforeEach(() => { + login(); + createRule(getNewRule({ rule_id: 'new custom rule' })); + visitWithTimeRange(ALERTS_URL); + selectAlertsHistogram(); + }); - it('Filter in/out should add a filter to KQL bar', function () { - const expectedNumberOfAlerts = 2; - clickAlertsHistogramLegend(); - clickAlertsHistogramLegendFilterFor(ruleConfigs.name); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( - 'have.text', - `kibana.alert.rule.name: ${ruleConfigs.name}` - ); - cy.get(ALERTS_COUNT).should('have.text', `${expectedNumberOfAlerts} alerts`); + it('Filter in/out should add a filter to KQL bar', function () { + const expectedNumberOfAlerts = 2; + clickAlertsHistogramLegend(); + clickAlertsHistogramLegendFilterFor(ruleConfigs.name); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( + 'have.text', + `kibana.alert.rule.name: ${ruleConfigs.name}` + ); + cy.get(ALERTS_COUNT).should('have.text', `${expectedNumberOfAlerts} alerts`); - clickAlertsHistogramLegend(); - clickAlertsHistogramLegendFilterOut(ruleConfigs.name); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( - 'have.text', - `NOT kibana.alert.rule.name: ${ruleConfigs.name}` - ); - cy.get(ALERTS_COUNT).should('not.exist'); + clickAlertsHistogramLegend(); + clickAlertsHistogramLegendFilterOut(ruleConfigs.name); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( + 'have.text', + `NOT kibana.alert.rule.name: ${ruleConfigs.name}` + ); + cy.get(ALERTS_COUNT).should('not.exist'); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM_DELETE).click(); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should('not.exist'); - }); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM_DELETE).click(); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should('not.exist'); + }); - it('Add To Timeline', function () { - clickAlertsHistogramLegend(); - clickAlertsHistogramLegendAddToTimeline(ruleConfigs.name); + it('Add To Timeline', function () { + clickAlertsHistogramLegend(); + clickAlertsHistogramLegendAddToTimeline(ruleConfigs.name); - cy.get(TOASTER).should('have.text', `Added ${ruleConfigs.name} to timeline`); - }); -}); + cy.get(TOASTER).should('have.text', `Added ${ruleConfigs.name} to timeline`); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts index 3f89a68441cdb..50e953cef0309 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts @@ -182,7 +182,7 @@ describe('Alert details flyout', { tags: ['@ess', '@serverless'] }, () => { }); }); - describe('Localstorage management', () => { + describe('Localstorage management', { tags: ['@brokenInServerlessQA'] }, () => { const ARCHIVED_RULE_ID = '7015a3e2-e4ea-11ed-8c11-49608884878f'; const ARCHIVED_RULE_NAME = 'Endpoint Security'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_threat_intelligence_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_threat_intelligence_tab.cy.ts index a8c97d8beb9d7..85338d85ed732 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_threat_intelligence_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_threat_intelligence_tab.cy.ts @@ -10,7 +10,6 @@ import { getNewRule } from '../../../../objects/rule'; import { expandDocumentDetailsExpandableFlyoutLeftSection } from '../../../../tasks/expandable_flyout/alert_details_right_panel'; import { expandFirstAlertExpandableFlyout } from '../../../../tasks/expandable_flyout/common'; import { INDICATOR_MATCH_ENRICHMENT_SECTION } from '../../../../screens/alerts_details'; -import { cleanKibana } from '../../../../tasks/common'; import { waitForAlertsToPopulate } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; @@ -28,7 +27,6 @@ describe( { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { - cleanKibana(); login(); createRule(getNewRule()); visit(ALERTS_URL); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/ransomware_prevention.cy.ts index f3fc88f6518ac..31290ce9e17b0 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/ransomware_prevention.cy.ts @@ -14,11 +14,9 @@ import { ALERTS_HISTOGRAM_SERIES, ALERT_RULE_NAME, MESSAGE } from '../../../scre import { TIMELINE_QUERY, TIMELINE_VIEW_IN_ANALYZER } from '../../../screens/timeline'; import { selectAlertsHistogram } from '../../../tasks/alerts'; import { createTimeline } from '../../../tasks/timelines'; -import { cleanKibana } from '../../../tasks/common'; describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () => { before(() => { - cleanKibana(); cy.task('esArchiverLoad', { archiveName: 'ransomware_prevention', useCreate: true, diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/export.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/export.cy.ts index 2fd83d61a2927..ad3fdcc6fb973 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/export.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timeline_templates/export.cy.ts @@ -15,15 +15,12 @@ import { import { TIMELINE_TEMPLATES_URL } from '../../../urls/navigation'; import { createTimelineTemplate } from '../../../tasks/api_calls/timelines'; -import { cleanKibana } from '../../../tasks/common'; import { searchByTitle } from '../../../tasks/table_pagination'; // FLAKY: https://github.com/elastic/kibana/issues/165760 // FLAKY: https://github.com/elastic/kibana/issues/165645 describe('Export timelines', { tags: ['@ess', '@serverless'] }, () => { before(() => { - cleanKibana(); - createTimelineTemplate(getTimelineTemplate()).then((response) => { cy.wrap(response).as('templateResponse'); cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('templateId'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/export.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/export.cy.ts index ca80b427f9c16..294ecf5c9127f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/export.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/export.cy.ts @@ -19,13 +19,12 @@ import { TOASTER } from '../../../screens/alerts_detection_rules'; import { TIMELINE_CHECKBOX } from '../../../screens/timelines'; import { createTimeline } from '../../../tasks/api_calls/timelines'; import { expectedExportedTimeline, getTimeline } from '../../../objects/timeline'; -import { cleanKibana } from '../../../tasks/common'; +import { deleteTimelines } from '../../../tasks/common'; // FLAKY: https://github.com/elastic/kibana/issues/165744 describe('Export timelines', { tags: ['@ess', '@serverless'] }, () => { before(() => { - cleanKibana(); - login(); + deleteTimelines(); cy.intercept({ method: 'POST', path: '/api/timeline/_export?file_name=timelines_export.ndjson', @@ -38,7 +37,6 @@ describe('Export timelines', { tags: ['@ess', '@serverless'] }, () => { cy.wrap(response).as('timelineResponse2'); cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('timelineId2'); }); - visit(TIMELINES_URL); }); beforeEach(() => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/open_timeline.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/open_timeline.cy.ts index 7f9083acb0b9f..372a0a3178975 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/open_timeline.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/open_timeline.cy.ts @@ -22,8 +22,6 @@ import { addNoteToTimeline } from '../../../tasks/api_calls/notes'; import { createTimeline } from '../../../tasks/api_calls/timelines'; -import { cleanKibana } from '../../../tasks/common'; - import { login } from '../../../tasks/login'; import { visit } from '../../../tasks/navigation'; import { @@ -39,10 +37,8 @@ import { TIMELINES_URL } from '../../../urls/navigation'; describe('Open timeline', { tags: ['@serverless', '@ess'] }, () => { describe('Open timeline modal', () => { before(function () { - cleanKibana(); login(); visit(TIMELINES_URL); - createTimeline(getTimeline()) .then((response) => response.body.data.persistTimeline.timeline.savedObjectId) .then((timelineId: string) => { @@ -63,14 +59,9 @@ describe('Open timeline', { tags: ['@serverless', '@ess'] }, () => { }); }); - beforeEach(function () { - login(); - visit(TIMELINES_URL); + it('should display timeline info', function () { openTimelineFromSettings(); openTimelineById(this.timelineId); - }); - - it('should display timeline info', () => { cy.get(OPEN_TIMELINE_MODAL).should('be.visible'); cy.contains(getTimeline().title).should('exist'); cy.get(TIMELINES_DESCRIPTION).last().should('have.text', getTimeline().description); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/row_renderers.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/row_renderers.cy.ts index 60af66a577b23..960989b848977 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/row_renderers.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/row_renderers.cy.ts @@ -15,7 +15,7 @@ import { TIMELINE_ROW_RENDERERS_SURICATA_SIGNATURE_TOOLTIP, TIMELINE_ROW_RENDERERS_SURICATA_LINK_TOOLTIP, } from '../../../screens/timeline'; -import { cleanKibana, deleteTimelines, waitForWelcomePanelToBeLoaded } from '../../../tasks/common'; +import { deleteTimelines, waitForWelcomePanelToBeLoaded } from '../../../tasks/common'; import { waitForAllHostsToBeLoaded } from '../../../tasks/hosts/all_hosts'; import { login } from '../../../tasks/login'; @@ -26,10 +26,6 @@ import { populateTimeline } from '../../../tasks/timeline'; import { hostsUrl } from '../../../urls/navigation'; describe('Row renderers', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - }); - beforeEach(() => { deleteTimelines(); login(); diff --git a/x-pack/test/security_solution_cypress/cypress/objects/timeline.ts b/x-pack/test/security_solution_cypress/cypress/objects/timeline.ts index 5ad542ac49530..d57f389a04fc4 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/timeline.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/timeline.ts @@ -108,9 +108,9 @@ export const expectedExportedTimelineTemplate = ( templateTimelineVersion: 1, timelineType: 'template', created: timelineTemplateBody.created, - createdBy: 'system_indices_superuser', + createdBy: Cypress.env('ELASTICSEARCH_USERNAME'), updated: timelineTemplateBody.updated, - updatedBy: 'system_indices_superuser', + updatedBy: Cypress.env('ELASTICSEARCH_USERNAME'), sort: [], eventNotes: [], globalNotes: [], @@ -143,9 +143,9 @@ export const expectedExportedTimeline = (timelineResponse: Cypress.Response