diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 052f6623dc1a6..25e4a9d0be3d1 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -18,6 +18,8 @@ disabled: - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts - x-pack/test/security_solution_endpoint/config.base.ts - x-pack/test/security_solution_endpoint_api_int/config.base.ts + - x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts + - x-pack/test/observability_solution_api_integration/config/ess/config.base.ts # QA suites that are run out-of-band - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js @@ -568,3 +570,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts + - x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts + - x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/serverless.config.ts + - x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/ess.config.ts diff --git a/.eslintrc.js b/.eslintrc.js index 52807d4b45e62..9c2a4a808e604 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -614,6 +614,8 @@ module.exports = { 'x-pack/test/profiling_api_integration/**/*.ts', 'x-pack/test/security_solution_api_integration/*/test_suites/**/*', 'x-pack/test/security_solution_api_integration/**/config*.ts', + 'x-pack/test/observability_solution_api_integration/*/test_suites/**/*', + 'x-pack/test/observability_solution_api_integration/**/config*.ts', ], rules: { 'import/no-default-export': 'off', @@ -1847,6 +1849,14 @@ module.exports = { }, }, + /** Observerability Solution API Integration tests + * Ensures appropriate mocha tagging for tests + */ + { + files: ['x-pack/test/observability_solution_api_integration/**/*.{ts,tsx}'], + rules: { '@kbn/eslint/require_mocha_tagging': 'error' }, + }, + /** * Code inside .buildkite runs separately from everything else in CI, before bootstrap, with ts-node. It needs a few tweaks because of this. */ diff --git a/packages/kbn-eslint-plugin-eslint/index.js b/packages/kbn-eslint-plugin-eslint/index.js index 3d161ec33dda1..d421f489c0cdf 100644 --- a/packages/kbn-eslint-plugin-eslint/index.js +++ b/packages/kbn-eslint-plugin-eslint/index.js @@ -18,5 +18,6 @@ module.exports = { no_constructor_args_in_property_initializers: require('./rules/no_constructor_args_in_property_initializers'), no_this_in_property_initializers: require('./rules/no_this_in_property_initializers'), no_unsafe_console: require('./rules/no_unsafe_console'), + require_mocha_tagging: require('./rules/require_mocha_tagging'), }, }; diff --git a/packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.js b/packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.js new file mode 100644 index 0000000000000..2d4ff9267b876 --- /dev/null +++ b/packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.js @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +const tsEstree = require('@typescript-eslint/typescript-estree'); +const esTypes = tsEstree.AST_NODE_TYPES; + +/** @typedef {import("eslint").Rule.RuleModule} Rule */ +/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.Node} Node */ +/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.CallExpression} CallExpression */ +/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.FunctionExpression} FunctionExpression */ +/** @typedef {import("@typescript-eslint/typescript-estree").TSESTree.ArrowFunctionExpression} ArrowFunctionExpression */ +/** @typedef {import("eslint").Rule.RuleFixer} Fixer */ + +const ERROR_MSG = `Describe blocks must be tagged with either @ess or @serverless to specify the test execution environment. Ex: describe('@ess @serverless API Integration test', () => {})`; + +/** + * @param {any} context + * @param {CallExpression} node + */ +const isDescribeBlockWithoutTagging = (node) => { + const isDescribeBlock = + node.type === esTypes.CallExpression && + node.callee.type === esTypes.Identifier && + node.callee.name === 'describe' && + node.arguments.length >= 1; + + if (!isDescribeBlock) { + return false; + } + const title = node.arguments[0].value; + const hasTags = /^(@ess|@serverless)/.test(title); + if (hasTags) { + return false; + } else { + return true; + } +}; + +/** @type {Rule} */ +module.exports = { + meta: { + fixable: 'code', + schema: [], + }, + create: (context) => ({ + CallExpression(_) { + const node = /** @type {CallExpression} */ (_); + + if (isDescribeBlockWithoutTagging(node)) { + context.report({ + message: ERROR_MSG, + loc: node.arguments[0].loc, + }); + } + }, + }), +}; diff --git a/packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.test.js b/packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.test.js new file mode 100644 index 0000000000000..3aa67a569c537 --- /dev/null +++ b/packages/kbn-eslint-plugin-eslint/rules/require_mocha_tagging.test.js @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +const { RuleTester } = require('eslint'); +const rule = require('./require_mocha_tagging'); +const dedent = require('dedent'); + +const ruleTester = new RuleTester({ + parser: require.resolve('@typescript-eslint/parser'), + parserOptions: { + sourceType: 'module', + ecmaVersion: 2018, + ecmaFeatures: { + jsx: true, + }, + }, +}); + +ruleTester.run('@kbn/eslint/require_mocha_tagging', rule, { + valid: [ + { + code: dedent` + describe('@ess @serverless API Integration test', () => {}) + `, + }, + { + code: dedent` + describe('@ess API Integration test', () => {}) + `, + }, + { + code: dedent` + describe('@serverless API Integration test', () => {}) + `, + }, + ], + + invalid: [ + { + code: dedent` + describe('API Integration test', () => {}) + `, + errors: [ + { + line: 1, + message: + 'Passing an async function to .forEach() prevents promise rejections from being handled. Use asyncForEach() or similar helper from "@kbn/std" instead.', + }, + ], + }, + ], +}); diff --git a/x-pack/test_serverless/api_integration/services/alerting_api.ts b/x-pack/test/api_integration/services/alerting_api.ts similarity index 100% rename from x-pack/test_serverless/api_integration/services/alerting_api.ts rename to x-pack/test/api_integration/services/alerting_api.ts diff --git a/x-pack/test/api_integration/services/index.ts b/x-pack/test/api_integration/services/index.ts index 5d2508ee627c7..ba41810960ecf 100644 --- a/x-pack/test/api_integration/services/index.ts +++ b/x-pack/test/api_integration/services/index.ts @@ -24,6 +24,8 @@ import { IngestPipelinesProvider } from './ingest_pipelines'; import { IndexManagementProvider } from './index_management'; import { DataViewApiProvider } from './data_view_api'; import { SloApiProvider } from './slo'; +import { SloApiProvider as SloApiProviderNew } from './slo_api'; +import { AlertingApiProvider } from './alerting_api'; import { SecuritySolutionApiProvider } from './security_solution_api.gen'; export const services = { @@ -45,4 +47,6 @@ export const services = { indexManagement: IndexManagementProvider, slo: SloApiProvider, securitySolutionApi: SecuritySolutionApiProvider, + alertingApi: AlertingApiProvider, + sloApi: SloApiProviderNew, // TODO: Need to unify SloApiProvider and SloApiProviderNew, there was already an slo service here, part of the slo API migration issue https://github.com/elastic/kibana/issues/183397 }; diff --git a/x-pack/test_serverless/api_integration/services/slo_api.ts b/x-pack/test/api_integration/services/slo_api.ts similarity index 98% rename from x-pack/test_serverless/api_integration/services/slo_api.ts rename to x-pack/test/api_integration/services/slo_api.ts index 7312905589cb7..b63d2cd29bb7b 100644 --- a/x-pack/test_serverless/api_integration/services/slo_api.ts +++ b/x-pack/test/api_integration/services/slo_api.ts @@ -10,7 +10,7 @@ import { FetchHistoricalSummaryResponse, } from '@kbn/slo-schema'; import * as t from 'io-ts'; -import { FtrProviderContext } from '../ftr_provider_context'; +import { FtrProviderContext } from '../../functional/ftr_provider_context'; type DurationUnit = 'm' | 'h' | 'd' | 'w' | 'M'; diff --git a/x-pack/test/observability_solution_api_integration/README.md b/x-pack/test/observability_solution_api_integration/README.md new file mode 100644 index 0000000000000..2a9808181cf7f --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/README.md @@ -0,0 +1,78 @@ +# observability_solution_api_integration + +This directory serves as a centralized location to place the observability solution tests that run in Serverless and ESS environments + +## Subdirectories + +1. `config` stores base configurations specific to both the Serverless and ESS environments. These configurations build upon the base configuration provided by `x-pack/test_serverless` and `x-pack/test/api_integration`, incorporating additional settings such as environmental variables and tagging options + +2. `test_suites` directory houses all the tests along with the utility functions. + +## Overview + +- In this directory Mocha tagging is utilized to assign tags to specific test suites and individual test cases. This tagging system enables the ability to selectively apply tags to test suites and test cases, facilitating the exclusion of specific test cases within a test suite as needed. + +- Test suites and cases are prefixed with specific tags to determine their execution in particular environments or to exclude them from specific environments. + +- We are using the following tags: + * `@ess`: Runs in an ESS environment (on-prem installation) as part of the CI validation on PRs. + + * `@serverless`: Runs in the first quality gate and in the periodic pipeline. + + * `@skipInEss`: Skipped for ESS environment. + + * `@skipInServerless`: Skipped for all quality gates, CI and periodic pipeline. + +ex: +``` + describe('@serverless @ess create_rules', () => { ==> tests in this suite will run in both Ess and Serverless + describe('creating rules', () => {}); + + describe('@skipInServerless missing timestamps', () => {}); ==> tests in this suite will be excluded in Serverless + +``` + +# Adding new observabiluty area's tests + +1. Within the `test_suites` directory, create a new area folder, for example slos, rules, apm etc +2. Introduce `ess.config` and `serverless.config` files to reference the new test files and incorporate any additional custom properties defined in the `CreateTestConfigOptions` interface. +3. In these new configuration files, include references to the base configurations located under the config directory to inherit CI configurations, environment variables, and other settings. +4. Append a new entry in the `ftr_configs.yml` file to enable the execution of the newly added tests within the CI pipeline. + + +# Testing locally + +In the `package.json` file, you'll find commands to configure the server for each environment and to run tests against that specific environment. These commands adhere to the Mocha tagging system, allowing for the inclusion and exclusion of tags, mirroring the setup of the CI pipeline. + +# How to run +You can run various commands with different parameters for the different test worflows. + +The command structure follows this pattern: + +- ``: The test workflow you want to run. +- ``: The type of operation, either "server" or "runner." +- ``: The testing environment, such as "serverless," or "ess", specifies the correct configuration file for the tests. + +Run the server for "alerting_burn_rate" in the "serverless" environment: + +```shell +npm run alerting_burn_rate:server:serverless +``` + +Run tests for "alerting_burn_rate" in the "serverless" environment: + +```shell +npm run alerting_burn_rate:runner:serverless +``` + +Run the server for "alerting_burn_rate" in the "ess" environment: + +```shell +npm run alerting_burn_rate:server:ess +``` + +Run tests for "alerting_burn_rate" in the "ess" environment: + +```shell +npm run alerting_burn_rate:runner:ess +``` \ No newline at end of file diff --git a/x-pack/test/observability_solution_api_integration/config/ess/config.base.ts b/x-pack/test/observability_solution_api_integration/config/ess/config.base.ts new file mode 100644 index 0000000000000..cf1edd2cd3c9e --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/config/ess/config.base.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { FtrConfigProviderContext } from '@kbn/test'; +import { services } from '../../../api_integration/services'; + +export interface CreateTestConfigOptions { + testFiles: string[]; + junit: { reportName: string }; + publicBaseUrl?: boolean; +} + +export function createTestConfig(options: CreateTestConfigOptions) { + return async ({ readConfigFile }: FtrConfigProviderContext) => { + const xPackApiIntegrationTestsConfig = await readConfigFile( + require.resolve('../../../api_integration/config.ts') + ); + + return { + ...xPackApiIntegrationTestsConfig.getAll(), + testFiles: options.testFiles, + services: { + ...services, + }, + junit: { + reportName: 'X-Pack Οbservability Solution API Integration Tests', + }, + mochaOpts: { + grep: '/^(?!.*@skipInEss).*@ess.*/', + }, + kbnTestServer: { + ...xPackApiIntegrationTestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'), + ...(options.publicBaseUrl ? ['--server.publicBaseUrl=http://localhost:5620'] : []), + ], + }, + }; + }; +} diff --git a/x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts b/x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts new file mode 100644 index 0000000000000..d332fc3f7ae18 --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/config/serverless/config.base.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import { services } from '../../../../test_serverless/api_integration/services'; + +export interface CreateTestConfigOptions { + testFiles: string[]; + junit: { reportName: string }; +} + +export function createTestConfig(options: CreateTestConfigOptions) { + return async ({ readConfigFile }: FtrConfigProviderContext) => { + const svlSharedConfig = await readConfigFile( + require.resolve('../../../../test_serverless/shared/config.base.ts') + ); + + return { + ...svlSharedConfig.getAll(), + services: { + ...services, + }, + kbnTestServer: { + ...svlSharedConfig.get('kbnTestServer'), + serverArgs: [...svlSharedConfig.get('kbnTestServer.serverArgs'), `--serverless=oblt`], + }, + testFiles: options.testFiles, + junit: options.junit, + mochaOpts: { + ...svlSharedConfig.get('mochaOpts'), + grep: '/^(?!.*@skipInServerless).*@serverless.*/', + }, + }; + }; +} diff --git a/x-pack/test/observability_solution_api_integration/ftr_provider_context.d.ts b/x-pack/test/observability_solution_api_integration/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..ed7c5216c803c --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/ftr_provider_context.d.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { FtrProviderContext } from '../api_integration/ftr_provider_context'; + +export type { FtrProviderContext }; diff --git a/x-pack/test/observability_solution_api_integration/package.json b/x-pack/test/observability_solution_api_integration/package.json new file mode 100644 index 0000000000000..c67dafb3da6c4 --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/package.json @@ -0,0 +1,17 @@ +{ + "author": "Elastic", + "name": "@kbn/observability_solution_api_integration", + "version": "1.0.0", + "private": true, + "license": "Elastic License 2.0", + "scripts": { + "alerting_burn_rate:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/burn_rate/configs/serverless.config.ts", + "alerting_burn_rate:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/burn_rate/configs/serverless.config.ts --grep @serverless --grep @skipInServerless --invert", + "alerting_burn_rate:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/burn_rate/configs/ess.config.ts", + "alerting_burn_rate:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/burn_rate/configs/ess.config.ts --grep @ess --grep @skipInEss --invert", + "alerting_custom_threshold:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/custom_threshold/configs/serverless.config.ts", + "alerting_custom_threshold:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/custom_threshold/configs/serverless.config.ts --grep @serverless --grep @skipInServerless --invert", + "alerting_custom_threshold:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/alerting/custom_threshold/configs/ess.config.ts", + "alerting_custom_threshold:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/alerting/custom_threshold/configs/ess.config.ts --grep @ess --grep @skipInEss --invert" + } +} \ No newline at end of file diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/burn_rate_rule/burn_rate_rule.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/burn_rate_rule.ts similarity index 95% rename from x-pack/test_serverless/api_integration/test_suites/observability/burn_rate_rule/burn_rate_rule.ts rename to x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/burn_rate_rule.ts index a651dfc773b62..32da5d58a8af8 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/burn_rate_rule/burn_rate_rule.ts +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/burn_rate_rule.ts @@ -4,12 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ import { cleanup, Dataset, generate, PartialConfig } from '@kbn/data-forge'; import expect from '@kbn/expect'; @@ -23,8 +17,11 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); const sloApi = getService('sloApi'); + const config = getService('config'); + const isServerless = config.get('serverless'); + const expectedConsumer = isServerless ? 'observability' : 'slo'; - describe('Burn rate rule', () => { + describe('@serverless @ess Burn rate rule', () => { const RULE_TYPE_ID = 'slo.rules.burnRate'; const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*'; const RULE_ALERT_INDEX = '.alerts-observability.slo.alerts-default'; @@ -120,7 +117,7 @@ export default function ({ getService }: FtrProviderContext) { const dependencyRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'observability', + consumer: expectedConsumer, name: 'SLO Burn Rate rule - Dependency', ruleTypeId: RULE_TYPE_ID, schedule: { @@ -192,7 +189,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'observability', + consumer: expectedConsumer, name: 'SLO Burn Rate rule', ruleTypeId: RULE_TYPE_ID, schedule: { @@ -294,7 +291,7 @@ export default function ({ getService }: FtrProviderContext) { it('should find the created rule with correct information about the consumer', async () => { const match = await alertingApi.findRule(ruleId); expect(match).not.to.be(undefined); - expect(match.consumer).to.be('observability'); + expect(match.consumer).to.be(expectedConsumer); }); }); }); diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts new file mode 100644 index 0000000000000..faeda1a68fc2e --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/ess.config.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../config/ess/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: 'SLO - Burn rate Integration Tests - ESS Env', + }, +}); diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts new file mode 100644 index 0000000000000..b8cf174c8b69f --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/configs/serverless.config.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../config/serverless/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: 'SLO - Integration Tests - Serverless Env', + }, +}); diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/index.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/index.ts new file mode 100644 index 0000000000000..97b0669cacd3c --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/burn_rate/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Slo - Burn rate rule', function () { + loadTestFile(require.resolve('./burn_rate_rule')); + }); +} diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/avg_pct_fired.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/avg_pct_fired.ts new file mode 100644 index 0000000000000..d519f166a050d --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/avg_pct_fired.ts @@ -0,0 +1,258 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cleanup, generate, Dataset, PartialConfig } from '@kbn/data-forge'; +import { + Aggregators, + Comparator, +} from '@kbn/observability-plugin/common/custom_threshold_rule/types'; +import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/custom_threshold/constants'; +import expect from '@kbn/expect'; +import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils'; +import { parseSearchParams } from '@kbn/share-plugin/common/url_service'; +import { omit } from 'lodash'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { ISO_DATE_REGEX } from './constants'; +import { ActionDocument, LogsExplorerLocatorParsedParams } from './typings'; + +export default function ({ getService }: FtrProviderContext) { + const esClient = getService('es'); + const supertest = getService('supertest'); + const esDeleteAllIndices = getService('esDeleteAllIndices'); + const alertingApi = getService('alertingApi'); + const dataViewApi = getService('dataViewApi'); + const logger = getService('log'); + const config = getService('config'); + const isServerless = config.get('serverless'); + const expectedConsumer = isServerless ? 'observability' : 'logs'; + + describe('@ess @serverless Custom Threshold rule - AVG - PCT - FIRED', () => { + const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; + const ALERT_ACTION_INDEX = 'alert-action-threshold'; + const DATA_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*'; + const DATA_VIEW_NAME = 'data-view-name'; + const DATA_VIEW_ID = 'data-view-id'; + let dataForgeConfig: PartialConfig; + let dataForgeIndices: string[]; + let actionId: string; + let ruleId: string; + let alertId: string; + + before(async () => { + dataForgeConfig = { + schedule: [ + { + template: 'good', + start: 'now-15m', + end: 'now+5m', + metrics: [ + { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }, + { name: 'system.cpu.total.pct', method: 'linear', start: 0.5, end: 0.5 }, + ], + }, + ], + indexing: { + dataset: 'fake_hosts' as Dataset, + eventsPerCycle: 1, + interval: 10000, + alignEventsToInterval: true, + }, + }; + dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger }); + await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW_TITLE, docCountTarget: 360 }); + await dataViewApi.create({ + name: DATA_VIEW_NAME, + id: DATA_VIEW_ID, + title: DATA_VIEW_TITLE, + }); + }); + + after(async () => { + await supertest + .delete(`/api/alerting/rule/${ruleId}`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + await supertest + .delete(`/api/actions/connector/${actionId}`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo'); + await esClient.deleteByQuery({ + index: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, + query: { term: { 'kibana.alert.rule.uuid': ruleId } }, + conflicts: 'proceed', + }); + await esClient.deleteByQuery({ + index: '.kibana-event-log-*', + query: { term: { 'rule.id': ruleId } }, + conflicts: 'proceed', + }); + await dataViewApi.delete({ + id: DATA_VIEW_ID, + }); + await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]); + await cleanup({ client: esClient, config: dataForgeConfig, logger }); + }); + + describe('Rule creation', () => { + it('creates rule successfully', async () => { + actionId = await alertingApi.createIndexConnector({ + name: 'Index Connector: Threshold API test', + indexName: ALERT_ACTION_INDEX, + }); + + const createdRule = await alertingApi.createRule({ + tags: ['observability'], + consumer: expectedConsumer, + name: 'Threshold rule', + ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, + params: { + criteria: [ + { + comparator: Comparator.GT, + threshold: [0.5], + timeSize: 5, + timeUnit: 'm', + metrics: [ + { name: 'A', field: 'system.cpu.user.pct', aggType: Aggregators.AVERAGE }, + ], + }, + ], + alertOnNoData: true, + alertOnGroupDisappear: true, + searchConfiguration: { + query: { + query: '', + language: 'kuery', + }, + index: DATA_VIEW_ID, + }, + }, + actions: [ + { + group: FIRED_ACTIONS_ID, + id: actionId, + params: { + documents: [ + { + ruleType: '{{rule.type}}', + alertDetailsUrl: '{{context.alertDetailsUrl}}', + reason: '{{context.reason}}', + value: '{{context.value}}', + host: '{{context.host}}', + viewInAppUrl: '{{context.viewInAppUrl}}', + }, + ], + }, + frequency: { + notify_when: 'onActionGroupChange', + throttle: null, + summary: false, + }, + }, + ], + }); + ruleId = createdRule.id; + expect(ruleId).not.to.be(undefined); + }); + + it('should be active', async () => { + const executionStatus = await alertingApi.waitForRuleStatus({ + ruleId, + expectedStatus: 'active', + }); + expect(executionStatus).to.be('active'); + }); + + it('should find the created rule with correct information about the consumer', async () => { + const match = await alertingApi.findRule(ruleId); + expect(match).not.to.be(undefined); + expect(match.consumer).to.be(expectedConsumer); + }); + + it('should set correct information in the alert document', async () => { + const resp = await alertingApi.waitForAlertInIndex({ + indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX, + ruleId, + }); + alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid']; + + expect(resp.hits.hits[0]._source).property( + 'kibana.alert.rule.category', + 'Custom threshold' + ); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', expectedConsumer); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.name', 'Threshold rule'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'observability'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0); + expect(resp.hits.hits[0]._source).property( + 'kibana.alert.rule.rule_type_id', + 'observability.rules.custom_threshold' + ); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.uuid', ruleId); + expect(resp.hits.hits[0]._source).property('kibana.space_ids').contain('default'); + expect(resp.hits.hits[0]._source) + .property('kibana.alert.rule.tags') + .contain('observability'); + expect(resp.hits.hits[0]._source).property( + 'kibana.alert.action_group', + 'custom_threshold.fired' + ); + expect(resp.hits.hits[0]._source).property('tags').contain('observability'); + expect(resp.hits.hits[0]._source).property('kibana.alert.instance.id', '*'); + expect(resp.hits.hits[0]._source).property('kibana.alert.workflow_status', 'open'); + expect(resp.hits.hits[0]._source).property('event.kind', 'signal'); + expect(resp.hits.hits[0]._source).property('event.action', 'open'); + expect(resp.hits.hits[0]._source).property('kibana.alert.evaluation.threshold').eql([0.5]); + expect(resp.hits.hits[0]._source) + .property('kibana.alert.rule.parameters') + .eql({ + criteria: [ + { + comparator: '>', + threshold: [0.5], + timeSize: 5, + timeUnit: 'm', + metrics: [{ name: 'A', field: 'system.cpu.user.pct', aggType: 'avg' }], + }, + ], + alertOnNoData: true, + alertOnGroupDisappear: true, + searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } }, + }); + }); + + it('should set correct action variables', async () => { + const resp = await alertingApi.waitForDocumentInIndex({ + indexName: ALERT_ACTION_INDEX, + docCountTarget: 1, + }); + + expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); + expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( + `http://localhost:5620/app/observability/alerts/${alertId}` + ); + expect(resp.hits.hits[0]._source?.reason).eql( + `Average system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATA_VIEW_NAME})` + ); + expect(resp.hits.hits[0]._source?.value).eql('250%'); + + const parsedViewInAppUrl = parseSearchParams( + new URL(resp.hits.hits[0]._source?.viewInAppUrl || '').search + ); + + expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOGS_EXPLORER_LOCATOR'); + expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({ + dataset: DATA_VIEW_ID, + timeRange: { to: 'now' }, + query: { query: '', language: 'kuery' }, + filters: [], + }); + expect(parsedViewInAppUrl.params.timeRange.from).match(ISO_DATE_REGEX); + }); + }); + }); +} diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/ess.config.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/ess.config.ts new file mode 100644 index 0000000000000..5847aa180dd22 --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/ess.config.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../config/ess/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: 'Custom threshold - Integration Tests - ESS Env', + }, + publicBaseUrl: true, +}); diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/serverless.config.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/serverless.config.ts new file mode 100644 index 0000000000000..407df673fbb7e --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/configs/serverless.config.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../config/serverless/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: 'Custom threshold - Integration Tests - Serverless Env', + }, +}); diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/constants.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/constants.ts new file mode 100644 index 0000000000000..5cf1e0b4d6614 --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/index.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/index.ts new file mode 100644 index 0000000000000..4d8e470a4af9a --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Custom threshold rule', function () { + loadTestFile(require.resolve('./avg_pct_fired')); + }); +} diff --git a/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/typings.ts b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/typings.ts new file mode 100644 index 0000000000000..9002e9991292f --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/test_suites/alerting/custom_threshold/typings.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Query, TimeRange } from '@kbn/es-query'; +import { SerializableRecord } from '@kbn/utility-types'; + +export interface ActionDocument { + ruleType: string; + alertDetailsUrl: string; + reason: string; + value: string; + viewInAppUrl: string; + host?: string; + group?: string; +} + +export interface LogsExplorerLocatorParsedParams extends SerializableRecord { + dataset: string; + timeRange: TimeRange; + query: Query; +} diff --git a/x-pack/test/observability_solution_api_integration/tsconfig.json b/x-pack/test/observability_solution_api_integration/tsconfig.json new file mode 100644 index 0000000000000..dac8664e336e9 --- /dev/null +++ b/x-pack/test/observability_solution_api_integration/tsconfig.json @@ -0,0 +1,29 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": ["node", "jest","@kbn/ambient-ftr-types"] + }, + "include": [ + "**/*", + "../../../typings/**/*", + "../../../packages/kbn-test/types/ftr_globals/**/*", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + { "path": "../../test_serverless/tsconfig.json" }, + { "path": "../../test_serverless/api_integration/**/*" }, + { "path": "../../test_serverless/shared/**/*" }, + { "path": "../../api_integration/services/**/*" }, + "@kbn/test", + "@kbn/expect", + "@kbn/data-forge", + "@kbn/observability-plugin", + "@kbn/rule-data-utils", + "@kbn/share-plugin", + "@kbn/es-query", + "@kbn/utility-types", + ] +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 2df784cd0c8ba..fa727ac25deb7 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -27,7 +27,8 @@ "*/plugins/**/*", "*/packages/**/*", "*/*/packages/**/*", - "security_solution_api_integration/**/*" + "security_solution_api_integration/**/*", + "observability_solution_api_integration/**/*" ], "kbn_references": [ { diff --git a/x-pack/test_serverless/api_integration/services/index.ts b/x-pack/test_serverless/api_integration/services/index.ts index 844c9339cf8ac..5ff5a2942a4f4 100644 --- a/x-pack/test_serverless/api_integration/services/index.ts +++ b/x-pack/test_serverless/api_integration/services/index.ts @@ -8,11 +8,8 @@ import { GenericFtrProviderContext } from '@kbn/test'; import { services as deploymentAgnosticSharedServices } from '../../shared/services/deployment_agnostic_services'; import { services as svlSharedServices } from '../../shared/services'; - -import { AlertingApiProvider } from './alerting_api'; import { SamlToolsProvider } from './saml_tools'; import { SvlCasesServiceProvider } from './svl_cases'; -import { SloApiProvider } from './slo_api'; import { TransformProvider } from './transform'; export const services = { @@ -21,10 +18,8 @@ export const services = { // serverless FTR services ...svlSharedServices, - alertingApi: AlertingApiProvider, samlTools: SamlToolsProvider, svlCases: SvlCasesServiceProvider, - sloApi: SloApiProvider, transform: TransformProvider, }; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/index.ts index 0d4f4ffe52814..2779fa68a2d33 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/index.ts @@ -15,7 +15,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./telemetry/telemetry_config')); loadTestFile(require.resolve('./apm_api_integration/feature_flags.ts')); loadTestFile(require.resolve('./cases')); - loadTestFile(require.resolve('./burn_rate_rule/burn_rate_rule')); loadTestFile(require.resolve('./es_query_rule/es_query_rule')); loadTestFile(require.resolve('./slos')); loadTestFile(require.resolve('./synthetics')); diff --git a/x-pack/test_serverless/shared/services/deployment_agnostic_services.ts b/x-pack/test_serverless/shared/services/deployment_agnostic_services.ts index 34e6b7d5facd5..f846d6bf2ede5 100644 --- a/x-pack/test_serverless/shared/services/deployment_agnostic_services.ts +++ b/x-pack/test_serverless/shared/services/deployment_agnostic_services.ts @@ -30,6 +30,8 @@ const deploymentAgnosticApiIntegrationServices = _.pick(apiIntegrationServices, 'usageAPI', 'console', 'securitySolutionApi', + 'alertingApi', + 'sloApi', ]); export const services = { diff --git a/x-pack/test_serverless/shared/services/index.ts b/x-pack/test_serverless/shared/services/index.ts index 1241c0c9aea37..8d0f6420ee818 100644 --- a/x-pack/test_serverless/shared/services/index.ts +++ b/x-pack/test_serverless/shared/services/index.ts @@ -10,7 +10,6 @@ import { SvlCommonApiServiceProvider } from './svl_common_api'; import { SvlReportingServiceProvider } from './svl_reporting'; import { SvlUserManagerProvider } from './svl_user_manager'; import { DataViewApiProvider } from './data_view_api'; - export type { RoleCredentials } from './svl_user_manager'; export type { InternalRequestHeader } from './svl_common_api'; export type { SupertestWithoutAuthType } from './supertest';