[Security Solution] Adding serverlessQA tag

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
-
 set -euo pipefail
 
-echo "In the entrypoint for the quality gate"
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-security-solution-chrome
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+echo "--- Serverless Security Second Quality Gate"
+cd x-pack/test/security_solution_cypress
+set +e
+
+VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
+ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
+
+CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status

x-pack/test/security_solution_cypress/cypress/README.md

@@ -42,7 +42,9 @@ Please, before opening a PR with the new test, please make sure that the test fa
 
 Note that we use tags in order to select which tests we want to execute:
 
-- `@serverless` includes a test in the Serverless test suite. You need to explicitly add this tag to any test you want to run against a Serverless environment.
+- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate). You need to explicitly add this tag to any test you want to run in CI for open PRs. These tests will run against a local, "simulated" serverless environment.
+- `@serverlessQA` includes a test in the Serverless test suite for QA (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in the CD pipeline against real serverless projects deployed in the Serverless QA environment.
+  - **NOTE:** We are adding this tag temporarily until we check the behavior of our tests in the second quality gate.
 - `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment.
 - `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken.
 - `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environemnt", "the test has been temporarily excluded, see the comment above why".

x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { defineCypressConfig } from '@kbn/cypress-config';
+import { esArchiver } from './support/es_archiver';
+
+// eslint-disable-next-line import/no-default-export
+export default defineCypressConfig({
+  reporter: '../../../node_modules/cypress-multi-reporters',
+  reporterOptions: {
+    configFile: './cypress/reporter_config.json',
+  },
+  defaultCommandTimeout: 150000,
+  env: {
+    grepFilterSpecs: true,
+    grepOmitFiltered: true,
+    grepTags: '@serverlessQA --@brokenInServerless --@skipInServerless',
+    // Grep plugin is working taking under consideration the directory where cypress lives.
+    // https://github.com/elastic/kibana/pull/167494#discussion_r1340567022 for more context.
+    grepIntegrationFolder: '../',
+  },
+  execTimeout: 150000,
+  pageLoadTimeout: 150000,
+  numTestsKeptInMemory: 0,
+  retries: {
+    runMode: 1,
+  },
+  screenshotsFolder: '../../../target/kibana-security-solution/cypress/screenshots',
+  trashAssetsBeforeRuns: false,
+  video: false,
+  videosFolder: '../../../../target/kibana-security-solution/cypress/videos',
+  viewportHeight: 946,
+  viewportWidth: 1680,
+  e2e: {
+    baseUrl: 'http://localhost:5601',
+    experimentalCspAllowList: ['default-src', 'script-src', 'script-src-elem'],
+    experimentalMemoryManagement: true,
+    specPattern: './cypress/e2e/**/*.cy.ts',
+    setupNodeEvents(on, config) {
+      esArchiver(on, config);
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      require('@cypress/grep/src/plugin')(config);
+      return config;
+    },
+  },
+});

x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts

@@ -17,7 +17,7 @@ import { cleanKibana } from '../../../tasks/common';
 import { createTimeline, favoriteTimeline } from '../../../tasks/api_calls/timelines';
 import { getTimeline } from '../../../objects/timeline';
 
-describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => {
+describe('Overview Page', { tags: ['@ess', '@serverless', '@serverlessQA'] }, () => {
   before(() => {
     cleanKibana();
     cy.task('esArchiverLoad', { archiveName: 'overview' });

x-pack/test/security_solution_cypress/package.json

@@ -23,6 +23,7 @@
       "cypress:open:serverless": "yarn cypress:serverless open --config-file ../../test/security_solution_cypress/cypress/cypress_serverless.config.ts --spec './cypress/e2e/**/*.cy.ts'",
       "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'",
       "cypress:run:cloud:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless.config.ts --env CLOUD_SERVERLESS=true",
+      "cypress:run:qa:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless_qa.config.ts --env CLOUD_SERVERLESS=true",
       "cypress:investigations:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'",
       "cypress:explore:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/explore/**/*.cy.ts'",
       "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5",