[Security Solution][Detection Engine] Running API tests in Serverless & ESS using Mocha Tagging

.buildkite/ftr_configs.yml

@@ -13,6 +13,9 @@ disabled:
   - x-pack/test/functional_with_es_ssl/config.base.ts
   - x-pack/test/api_integration/config.ts
   - x-pack/test/fleet_api_integration/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
+
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -438,3 +441,9 @@ enabled:
   - x-pack/performance/journeys/apm_service_inventory.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/ess.config.ts
+
+
+
+

.github/CODEOWNERS

@@ -1301,6 +1301,7 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 /x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics @elastic/security-detection-engine
 /x-pack/test/security_solution_cypress/cypress/e2e/exceptions @elastic/security-detection-engine
 /x-pack/test/security_solution_cypress/cypress/e2e/overview @elastic/security-detection-engine
+x-pack/test/security_solution_api_integration/test_suites/detections_response/exceptions  @elastic/security-detection-engine
 
 ## Security Threat Intelligence - Under Security Platform
 /x-pack/plugins/security_solution/public/common/components/threat_match @elastic/security-detection-engine

x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts

@@ -19,11 +19,9 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
     loadTestFile(require.resolve('./update_actions'));
     loadTestFile(require.resolve('./check_privileges'));
     loadTestFile(require.resolve('./create_index'));
-    loadTestFile(require.resolve('./create_rules'));
     loadTestFile(require.resolve('./preview_rules'));
     loadTestFile(require.resolve('./create_rules_bulk'));
     loadTestFile(require.resolve('./create_new_terms'));
-    loadTestFile(require.resolve('./create_rule_exceptions'));
     loadTestFile(require.resolve('./delete_rules'));
     loadTestFile(require.resolve('./delete_rules_bulk'));
     loadTestFile(require.resolve('./export_rules'));

x-pack/test/security_solution_api_integration/README.md

@@ -0,0 +1,47 @@
+# security_solution_api_integration
+
+This directory
+ serves as a centralized location to place the security solution tests that run in Serverless and ESS environments.
+
+## Subdirectories
+
+1. `config` stores base configurations specific to both the Serverless and ESS environments, These configurations build upon the base configuration provided by `xpack/test_serverless` and `x-pack-api_integrations`, incorporating additional settings such as environment variables and tagging options.
+
+
+2. `test_suites` directory now houses all the tests along with their utility functions. As an initial step,
+we have introduced the `detection_response` directory to consolidate all the integration tests related to detection and response APIs.
+
+
+## Overview
+
+- In this directory, Mocha tagging is utilized to assign tags to specific test suites and individual test cases. This tagging system enables the ability to selectively apply tags to test suites and test cases, facilitating the exclusion of specific test cases within a test suite as needed.
+
+- There are three primary tags that have been defined: @ess, @serverless, and @brokenInServerless
+
+- Test suites and cases are prefixed with specific tags to determine their execution in particular environments or to exclude them from specific environments. 
+
+ex:
+```
+ describe('@serverless @ess create_rules', () => { ==> tests in this suite will run in both Ess and Serverless
+   describe('creating rules', () => {}); 
+
+   describe('@brokenInServerless missing timestamps', () => {}); ==> tests in this suite will be excluded in Serverless
+
+ ```
+
+## Adding new security area's tests
+
+1. Within the `test_suites` directory, create a new area folder.
+2. Introduce `ess.config` and `serverless.config` files to reference the new test files and incorporate any additional custom properties defined in the `CreateTestConfigOptions` interface.
+3. In these new configuration files, include references to the base configurations located under the config directory to inherit CI configurations, environment variables, and other settings.
+4. Append a new entry in the `ftr_configs.yml` file to enable the execution of the newly added tests within the CI pipeline.
+
+
+## Testing locally 
+
+In the `package.json` file, you'll find commands to configure the server for each environment and to run tests against that specific environment. These commands adhere to the Mocha tagging system, allowing for the inclusion and exclusion of tags, mirroring the setup of the CI pipeline.
+
+
+
+
+

x-pack/test/security_solution_api_integration/config/ess/config.base.ts

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { FtrConfigProviderContext, kbnTestConfig, kibanaTestUser } from '@kbn/test';
+import { services } from '../../../api_integration/services';
+
+// TODO combine it with
+interface CreateTestConfigOptions {
+  license: string;
+  ssl?: boolean;
+}
+
+// test.not-enabled is specifically not enabled
+const enabledActionTypes = [
+  '.email',
+  '.index',
+  '.pagerduty',
+  '.swimlane',
+  '.server-log',
+  '.servicenow',
+  '.slack',
+  '.webhook',
+  'test.authorization',
+  'test.failing',
+  'test.index-record',
+  'test.noop',
+  'test.rate-limit',
+];
+
+export function createTestConfig(options: CreateTestConfigOptions, testFiles?: string[]) {
+  const { license = 'trial', ssl = false } = options;
+
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const xPackApiIntegrationTestsConfig = await readConfigFile(
+      require.resolve('../../../api_integration/config.ts')
+    );
+    const servers = {
+      ...xPackApiIntegrationTestsConfig.get('servers'),
+      elasticsearch: {
+        ...xPackApiIntegrationTestsConfig.get('servers.elasticsearch'),
+        protocol: ssl ? 'https' : 'http',
+      },
+    };
+
+    return {
+      testFiles,
+      servers,
+      services,
+      junit: {
+        reportName: 'X-Pack Detection Engine API Integration Tests',
+      },
+      esTestCluster: {
+        ...xPackApiIntegrationTestsConfig.get('esTestCluster'),
+        license,
+        ssl,
+        serverArgs: [`xpack.license.self_generated.type=${license}`],
+      },
+      kbnTestServer: {
+        ...xPackApiIntegrationTestsConfig.get('kbnTestServer'),
+        env: {
+          ELASTICSEARCH_USERNAME: kbnTestConfig.getUrlParts(kibanaTestUser).username,
+        },
+        serverArgs: [
+          ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'),
+          `--xpack.actions.allowedHosts=${JSON.stringify(['localhost', 'some.non.existent.com'])}`,
+          `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`,
+          '--xpack.eventLog.logEntries=true',
+          `--xpack.securitySolution.alertIgnoreFields=${JSON.stringify([
+            'testing_ignored.constant',
+            '/testing_regex*/',
+          ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields"
+          '--xpack.ruleRegistry.write.enabled=true',
+          '--xpack.ruleRegistry.write.cache.enabled=false',
+          '--xpack.ruleRegistry.unsafe.indexUpgrade.enabled=true',
+          '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true',
+          `--xpack.securitySolution.enableExperimental=${JSON.stringify([
+            'previewTelemetryUrlEnabled',
+            'riskScoringPersistence',
+            'riskScoringRoutesEnabled',
+          ])}`,
+          '--xpack.task_manager.poll_interval=1000',
+          `--xpack.actions.preconfigured=${JSON.stringify({
+            'my-test-email': {
+              actionTypeId: '.email',
+              name: 'TestEmail#xyz',
+              config: {
+                from: '[email protected]',
+                service: '__json',
+              },
+              secrets: {
+                user: 'user',
+                password: 'password',
+              },
+            },
+          })}`,
+          ...(ssl
+            ? [
+                `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`,
+                `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+              ]
+            : []),
+        ],
+      },
+      mochaOpts: {
+        grep: '/^(?!.*@brokenInEss).*@ess.*/',
+      },
+    };
+  };
+}

x-pack/test/security_solution_api_integration/config/ess/config.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from './config.base';
+
+// eslint-disable-next-line import/no-default-export
+export default createTestConfig({
+  license: 'trial',
+  ssl: true,
+});

x-pack/test/security_solution_api_integration/config/serverless/config.base.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext, kbnTestConfig, kibanaTestSuperuserServerless } from '@kbn/test';
+
+import { services } from '../../../../test_serverless/api_integration/services';
+import type { CreateTestConfigOptions } from '../../../../test_serverless/shared/types';
+
+export function createTestConfig(options: Partial<CreateTestConfigOptions>) {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const svlSharedConfig = await readConfigFile(
+      require.resolve('../../../../test_serverless/shared/config.base.ts')
+    );
+
+    return {
+      ...svlSharedConfig.getAll(),
+
+      services: {
+        ...services,
+        ...options.services,
+      },
+      kbnTestServer: {
+        ...svlSharedConfig.get('kbnTestServer'),
+
+        serverArgs: [
+          ...svlSharedConfig.get('kbnTestServer.serverArgs'),
+          '--serverless=security',
+          ...(options.kbnServerArgs || []),
+        ],
+        env: {
+          ...svlSharedConfig.get('kbnTestServer').env,
+          ELASTICSEARCH_USERNAME: kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).username,
+        },
+      },
+      testFiles: options.testFiles,
+      junit: options.junit,
+
+      mochaOpts: {
+        ...svlSharedConfig.get('mochaOpts'),
+        grep: '/^(?!.*@brokenInServerless).*@serverless.*/',
+      },
+    };
+  };
+}