[Observability solution] [SLO] Run burn rate api tests in serverless & ess using mocha tagging

.buildkite/ftr_configs.yml

@@ -18,6 +18,8 @@ disabled:
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
   - x-pack/test/security_solution_endpoint/config.base.ts
   - x-pack/test/security_solution_endpoint_api_int/config.base.ts
+  - x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts
+  - x-pack/test/observability_solution_api_integration/config/ess/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -564,3 +566,5 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts
+  - x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts

.eslintrc.js

@@ -613,6 +613,8 @@ module.exports = {
         'x-pack/test/profiling_api_integration/**/*.ts',
         'x-pack/test/security_solution_api_integration/*/test_suites/**/*',
         'x-pack/test/security_solution_api_integration/**/config*.ts',
+        'x-pack/test/observability_solution_api_integration/*/test_suites/**/*',
+        'x-pack/test/observability_solution_api_integration/**/config*.ts',
       ],
       rules: {
         'import/no-default-export': 'off',

x-pack/test/observability_solution_api_integration/README.md

@@ -0,0 +1,78 @@
+# observability_solution_api_integration
+
+This directory serves as a centralized location to place the observability solution tests that run in Serverless and ESS environments
+
+## Subdirectories
+
+1. `config` stores base configurations specific to both the Serverless and ESS environments. These configurations build upon the base configuration provided by `x-pack/test_serverless` and `x-pack/test/api_integration`, incorporating additional settings such as environmental variables and tagging options
+
+2. `test_suites` directory houses all the tests along with the utility functions.
+
+## Overview
+
+- In this directory Mocha tagging is utilized to assign tags to specific test suites and individual test cases. This tagging system enables the ability to selectively apply tags to test suites and test cases, facilitating the exclusion of specific test cases within a test suite as needed.
+
+- Test suites and cases are prefixed with specific tags to determine their execution in particular environments or to exclude them from specific environments.
+
+- We are using the following tags:
+    * `@ess`: Runs in an ESS environment (on-prem installation) as part of the CI validation on PRs.
+
+    * `@serverless`: Runs in the first quality gate and in the periodic pipeline.
+
+    * `@skipInEss`: Skipped for ESS environment.
+
+    * `@skipInServerless`: Skipped for all quality gates, CI and periodic pipeline.
+
+ex:
+```
+ describe('@serverless @ess create_rules', () => { ==> tests in this suite will run in both Ess and Serverless
+   describe('creating rules', () => {}); 
+
+   describe('@skipInServerless missing timestamps', () => {}); ==> tests in this suite will be excluded in Serverless
+
+```
+
+# Adding new observabiluty area's tests
+
+1. Within the `test_suites` directory, create a new area folder, for example slos, rules, apm etc
+2. Introduce `ess.config` and `serverless.config` files to reference the new test files and incorporate any additional custom properties defined in the `CreateTestConfigOptions` interface.
+3. In these new configuration files, include references to the base configurations located under the config directory to inherit CI configurations, environment variables, and other settings.
+4. Append a new entry in the `ftr_configs.yml` file to enable the execution of the newly added tests within the CI pipeline.
+
+
+# Testing locally
+
+In the `package.json` file, you'll find commands to configure the server for each environment and to run tests against that specific environment. These commands adhere to the Mocha tagging system, allowing for the inclusion and exclusion of tags, mirroring the setup of the CI pipeline.
+
+# How to run
+You can run various commands with different parameters for the different test worflows.
+
+The command structure follows this pattern:
+
+- `<test>`: The test workflow you want to run.
+- `<type>`: The type of operation, either "server" or "runner."
+- `<environment>`: The testing environment, such as "serverless," or "ess", specifies the correct configuration file for the tests.
+
+Run the server for "alerting_burn_rate" in the "serverless" environment:
+
+```shell
+npm run alerting_burn_rate:server:serverless
+```
+
+Run tests for "alerting_burn_rate" in the "serverless" environment:
+
+```shell
+npm run alerting_burn_rate:runner:serverless
+```
+
+Run the server for "alerting_burn_rate" in the "ess" environment:
+
+```shell
+npm run alerting_burn_rate:server:ess
+```
+
+Run tests for "alerting_burn_rate" in the "ess" environment:
+
+```shell
+npm run alerting_burn_rate:runner:ess
+```

x-pack/test/observability_solution_api_integration/config/ess/config.base.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from '../../../api_integration/services';
+
+export interface CreateTestConfigOptions {
+  testFiles: string[];
+  junit: { reportName: string };
+}
+
+export function createTestConfig(options: CreateTestConfigOptions) {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const xPackApiIntegrationTestsConfig = await readConfigFile(
+      require.resolve('../../../api_integration/config.ts')
+    );
+
+    const svlSharedConfig = await readConfigFile(
+      require.resolve('../../../../test_serverless/shared/config.base.ts')
+    );
+    const svlSharedServices = svlSharedConfig.get('services');
+
+    return {
+      ...xPackApiIntegrationTestsConfig.getAll(),
+      testFiles: options.testFiles,
+      services: {
+        ...services,
+        sloApi: svlSharedServices.sloApi,
+        alertingApi: svlSharedServices.alertingApi,
+      },
+      junit: {
+        reportName: 'X-Pack Οbservability Solution API Integration Tests',
+      },
+      mochaOpts: {
+        grep: '/^(?!.*@skipInEss).*@ess.*/',
+      },
+    };
+  };
+}

x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from '../../../../test_serverless/api_integration/services';
+
+export interface CreateTestConfigOptions {
+  testFiles: string[];
+  junit: { reportName: string };
+}
+
+export function createTestConfig(options: CreateTestConfigOptions) {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const svlSharedConfig = await readConfigFile(
+      require.resolve('../../../../test_serverless/shared/config.base.ts')
+    );
+
+    return {
+      ...svlSharedConfig.getAll(),
+      services: {
+        ...services,
+      },
+      kbnTestServer: {
+        ...svlSharedConfig.get('kbnTestServer'),
+        serverArgs: [...svlSharedConfig.get('kbnTestServer.serverArgs'), `--serverless=oblt`],
+      },
+      testFiles: options.testFiles,
+      junit: options.junit,
+      mochaOpts: {
+        ...svlSharedConfig.get('mochaOpts'),
+        grep: '/^(?!.*@skipInServerless).*@serverless.*/',
+      },
+    };
+  };
+}

x-pack/test/observability_solution_api_integration/ftr_provider_context.d.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { FtrProviderContext } from '../../test_serverless/api_integration/ftr_provider_context';
+
+export type { FtrProviderContext };

x-pack/test/observability_solution_api_integration/package.json

@@ -0,0 +1,13 @@
+{
+  "author": "Elastic",
+  "name": "@kbn/observability_solution_api_integration",
+  "version": "1.0.0",
+  "private": true,
+  "license": "Elastic License 2.0",
+  "scripts": {
+    "alerting_burn_rate:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/burn_rate/configs/serverless.config.ts",
+    "alerting_burn_rate:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/burn_rate/configs/serverless.config.ts --grep @serverless --grep  @skipInServerless --invert",
+    "alerting_burn_rate:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/burn_rate/configs/ess.config.ts",
+    "alerting_burn_rate:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/burn_rate/configs/ess.config.ts --grep @ess --grep  @skipInEss --invert"
+  }
+}

x-pack/test_serverless/api_integration/test_suites/observability/burn_rate_rule/burn_rate_rule.ts → x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/burn_rate_rule.ts

@@ -4,12 +4,6 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
 
 import { cleanup, Dataset, generate, PartialConfig } from '@kbn/data-forge';
 import expect from '@kbn/expect';
@@ -23,8 +17,11 @@ export default function ({ getService }: FtrProviderContext) {
   const alertingApi = getService('alertingApi');
   const dataViewApi = getService('dataViewApi');
   const sloApi = getService('sloApi');
+  const config = getService('config');
+  const isServerless = config.get('serverless');
+  const expectedConsumer = isServerless ? 'observability' : 'slo';
 
-  describe('Burn rate rule', () => {
+  describe('@serverless @ess Burn rate rule', () => {
     const RULE_TYPE_ID = 'slo.rules.burnRate';
     const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const RULE_ALERT_INDEX = '.alerts-observability.slo.alerts-default';
@@ -120,7 +117,7 @@ export default function ({ getService }: FtrProviderContext) {
 
         const dependencyRule = await alertingApi.createRule({
           tags: ['observability'],
-          consumer: 'observability',
+          consumer: expectedConsumer,
           name: 'SLO Burn Rate rule - Dependency',
           ruleTypeId: RULE_TYPE_ID,
           schedule: {
@@ -192,7 +189,7 @@ export default function ({ getService }: FtrProviderContext) {
 
         const createdRule = await alertingApi.createRule({
           tags: ['observability'],
-          consumer: 'observability',
+          consumer: expectedConsumer,
           name: 'SLO Burn Rate rule',
           ruleTypeId: RULE_TYPE_ID,
           schedule: {
@@ -294,7 +291,7 @@ export default function ({ getService }: FtrProviderContext) {
       it('should find the created rule with correct information about the consumer', async () => {
         const match = await alertingApi.findRule(ruleId);
         expect(match).not.to.be(undefined);
-        expect(match.consumer).to.be('observability');
+        expect(match.consumer).to.be(expectedConsumer);
       });
     });
   });

x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../../../../config/ess/config.base';
+
+export default createTestConfig({
+  testFiles: [require.resolve('..')],
+  junit: {
+    reportName: 'SLO - Burn rate Integration Tests - ESS Env',
+  },
+});

x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../../../../config/serverless/config.base';
+
+export default createTestConfig({
+  testFiles: [require.resolve('..')],
+  junit: {
+    reportName: 'SLO - Integration Tests - Serverless Env',
+  },
+});

x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Slo - Burn rate rule', function () {
+    loadTestFile(require.resolve('./burn_rate_rule'));
+  });
+}

x-pack/test/observability_solution_api_integration/tsconfig.json

@@ -0,0 +1,48 @@
+{
+    "extends": "../../../tsconfig.base.json",
+    "compilerOptions": {
+      "outDir": "target/types",
+      "types": ["node", "jest","@kbn/ambient-ftr-types"]
+    },
+    "include": [
+      "**/*",
+      "../../../typings/**/*",
+      "../../../packages/kbn-test/types/ftr_globals/**/*",
+    ], 
+    "exclude": [
+      "target/**/*"
+    ],
+    "kbn_references": [
+      { "path": "../../test_serverless/tsconfig.json" },
+      { "path": "../../test_serverless/api_integration/**/*" },
+      { "path": "../../test_serverless/shared/**/*" },
+      { "path": "../../api_integration/services/**/*" },
+      "@kbn/dev-utils",
+      "@kbn/test",
+      "@kbn/expect",
+      "@kbn/security-solution-plugin",
+      "@kbn/securitysolution-io-ts-list-types",
+      "@kbn/lists-plugin",
+      "@kbn/securitysolution-io-ts-alerting-types",
+      "@kbn/tooling-log",
+      "@kbn/rule-data-utils",
+      "@kbn/securitysolution-list-constants",
+      "@kbn/core-saved-objects-server",
+      "@kbn/core",
+      "@kbn/alerting-plugin",
+      "@kbn/securitysolution-ecs",
+      "@kbn/securitysolution-rules",
+      "@kbn/core-http-common",
+      "@kbn/securitysolution-ecs",
+      "@kbn/fleet-plugin",
+      "@kbn/repo-info",
+      "@kbn/securitysolution-es-utils",
+      "@kbn/datemath",
+      "@kbn/safer-lodash-set",
+      "@kbn/stack-connectors-plugin",
+      "@kbn/ftr-common-functional-services",
+      "@kbn/actions-plugin",
+      "@kbn/task-manager-plugin",
+    ]
+  }
+

x-pack/test_serverless/api_integration/services/index.ts

@@ -8,11 +8,8 @@
 import { GenericFtrProviderContext } from '@kbn/test';
 import { services as deploymentAgnosticSharedServices } from '../../shared/services/deployment_agnostic_services';
 import { services as svlSharedServices } from '../../shared/services';
-
-import { AlertingApiProvider } from './alerting_api';
 import { SamlToolsProvider } from './saml_tools';
 import { SvlCasesServiceProvider } from './svl_cases';
-import { SloApiProvider } from './slo_api';
 import { TransformProvider } from './transform';
 
 export const services = {
@@ -21,10 +18,8 @@ export const services = {
 
   // serverless FTR services
   ...svlSharedServices,
-  alertingApi: AlertingApiProvider,
   samlTools: SamlToolsProvider,
   svlCases: SvlCasesServiceProvider,
-  sloApi: SloApiProvider,
   transform: TransformProvider,
 };
 

x-pack/test_serverless/api_integration/test_suites/observability/index.ts

@@ -15,7 +15,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./telemetry/telemetry_config'));
     loadTestFile(require.resolve('./apm_api_integration/feature_flags.ts'));
     loadTestFile(require.resolve('./cases'));
-    loadTestFile(require.resolve('./burn_rate_rule/burn_rate_rule'));
     loadTestFile(require.resolve('./es_query_rule/es_query_rule'));
     loadTestFile(require.resolve('./slos'));
     loadTestFile(require.resolve('./synthetics'));