From 3d7fed0f92c953ff5196e8bbb78afcfaea98339b Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Wed, 25 Aug 2021 19:54:25 +0200 Subject: [PATCH] [RAC][Rule Registry] Implement versioning and backing indices (#109276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Ticket:** https://github.com/elastic/kibana/issues/109293 🚨 **This PR is critical for Observability 7.15** 🚨 ## Summary This PR fixes the indexing implementation in `rule_registry`. It implements the suggestions for backwards compatibility described in the ticket: - changes the naming scheme and introduces the concept of "backing indices", so that names of the concrete ("backing") indices != names of their aliases - adds versioning based on the current Kibana version TODO: - [x] Change index naming (implement the concept of backing indices) - [x] Include Kibana version into the index template metadata - [x] Include Kibana version into the document fields - [x] Remove `version` from `IndexOptions` (parameters provided by solutions/plugins when initializing alerts-as-data indices) - [x] Fix CI ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../apm/server/lib/alerts/test_utils/index.ts | 20 +- x-pack/plugins/apm/server/plugin.ts | 4 - .../server/services/rules/rule_data_client.ts | 4 - .../field_maps/technical_rule_field_map.ts | 2 +- .../common/field_map/es_field_type_map.ts | 23 -- .../field_map/runtime_type_from_fieldmap.ts | 1 + x-pack/plugins/rule_registry/server/config.ts | 1 + x-pack/plugins/rule_registry/server/mocks.ts | 2 + x-pack/plugins/rule_registry/server/plugin.ts | 8 +- .../rule_data_client/rule_data_client.mock.ts | 12 +- .../rule_data_client/rule_data_client.ts | 4 + .../server/rule_data_client/types.ts | 1 + .../rule_data_plugin_service/index_info.ts | 72 +++-- .../rule_data_plugin_service/index_options.ts | 4 +- .../resource_installer.ts | 256 +++++++++++------- .../rule_data_plugin_service.ts | 20 +- .../server/utils/create_lifecycle_executor.ts | 2 + .../utils/create_lifecycle_rule_type.test.ts | 2 + .../create_persistence_rule_type_factory.ts | 5 +- .../rule_registry_log_client.ts | 4 - .../rule_types/__mocks__/rule_type.ts | 15 +- .../security_solution/server/plugin.ts | 4 - .../server/lib/alerts/test_utils/index.ts | 18 +- x-pack/plugins/uptime/server/plugin.ts | 4 - .../tests/alerts/rule_registry.ts | 3 +- 25 files changed, 254 insertions(+), 237 deletions(-) delete mode 100644 x-pack/plugins/rule_registry/common/field_map/es_field_type_map.ts diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts index 26667dff90c6e..c78e93815bf85 100644 --- a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -8,7 +8,8 @@ import { Logger } from 'kibana/server'; import { of } from 'rxjs'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; -import type { IRuleDataClient } from '../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../rule_registry/server'; +import { ruleRegistryMocks } from '../../../../../rule_registry/server/mocks'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../alerting/server'; import { APMConfig, APM_SERVER_FEATURE_ID } from '../../..'; @@ -51,20 +52,9 @@ export const createRuleTypeMocks = () => { alerting, config$: mockedConfig$, logger: loggerMock, - ruleDataClient: ({ - getReader: () => { - return { - search: jest.fn(), - }; - }, - getWriter: () => { - return { - bulk: jest.fn(), - }; - }, - isWriteEnabled: jest.fn(() => true), - indexName: '.alerts-observability.apm.alerts', - } as unknown) as IRuleDataClient, + ruleDataClient: ruleRegistryMocks.createRuleDataClient( + '.alerts-observability.apm.alerts' + ) as IRuleDataClient, }, services, scheduleActions, diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 1e0e61bc2bf3a..fadeae338cbdb 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -122,7 +122,6 @@ export class APMPlugin componentTemplates: [ { name: 'mappings', - version: 0, mappings: mappingFromFieldMap( { [SERVICE_NAME]: { @@ -142,9 +141,6 @@ export class APMPlugin ), }, ], - indexTemplate: { - version: 0, - }, }); const resourcePlugins = mapValues(plugins, (value, key) => { diff --git a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts index 9b3f7edb97007..f6be29deeb322 100644 --- a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts +++ b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts @@ -31,12 +31,8 @@ export const createRuleDataClient = ({ componentTemplates: [ { name: 'mappings', - version: 0, mappings: {}, }, ], - indexTemplate: { - version: 0, - }, }); }; diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts index 2c1c8b55a631e..bc8a281afd7b5 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts @@ -30,7 +30,7 @@ export const technicalRuleFieldMap = { [Fields.ALERT_EVALUATION_THRESHOLD]: { type: 'scaled_float', scaling_factor: 100 }, [Fields.ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100 }, [Fields.VERSION]: { - type: 'keyword', + type: 'version', array: false, required: false, }, diff --git a/x-pack/plugins/rule_registry/common/field_map/es_field_type_map.ts b/x-pack/plugins/rule_registry/common/field_map/es_field_type_map.ts deleted file mode 100644 index df41a020d274b..0000000000000 --- a/x-pack/plugins/rule_registry/common/field_map/es_field_type_map.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import * as t from 'io-ts'; - -export const esFieldTypeMap = { - keyword: t.string, - text: t.string, - date: t.string, - boolean: t.boolean, - byte: t.number, - long: t.number, - integer: t.number, - short: t.number, - double: t.number, - float: t.number, - scaled_float: t.number, - unsigned_long: t.number, - flattened: t.record(t.string, t.array(t.string)), -}; diff --git a/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts index fe3504c84115b..11dff18721522 100644 --- a/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts +++ b/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts @@ -45,6 +45,7 @@ const BooleanFromString = new t.Type( const esFieldTypeMap = { keyword: t.string, + version: t.string, text: t.string, date: t.string, boolean: t.union([t.number, BooleanFromString]), diff --git a/x-pack/plugins/rule_registry/server/config.ts b/x-pack/plugins/rule_registry/server/config.ts index b50927fff4e93..830762c9b3741 100644 --- a/x-pack/plugins/rule_registry/server/config.ts +++ b/x-pack/plugins/rule_registry/server/config.ts @@ -24,3 +24,4 @@ export const config = { export type RuleRegistryPluginConfig = TypeOf; export const INDEX_PREFIX = '.alerts' as const; +export const INDEX_PREFIX_FOR_BACKING_INDICES = '.internal.alerts' as const; diff --git a/x-pack/plugins/rule_registry/server/mocks.ts b/x-pack/plugins/rule_registry/server/mocks.ts index 269eff182633f..e9ec25ddcdaba 100644 --- a/x-pack/plugins/rule_registry/server/mocks.ts +++ b/x-pack/plugins/rule_registry/server/mocks.ts @@ -6,11 +6,13 @@ */ import { alertsClientMock } from './alert_data_client/alerts_client.mock'; +import { createRuleDataClientMock } from './rule_data_client/rule_data_client.mock'; import { ruleDataPluginServiceMock } from './rule_data_plugin_service/rule_data_plugin_service.mock'; import { createLifecycleAlertServicesMock } from './utils/lifecycle_alert_services_mock'; export const ruleRegistryMocks = { createLifecycleAlertServices: createLifecycleAlertServicesMock, createRuleDataPluginService: ruleDataPluginServiceMock.create, + createRuleDataClient: createRuleDataClientMock, createAlertsClientMock: alertsClientMock, }; diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index cb1810420c2cd..a4122e3a1ffc1 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -19,7 +19,7 @@ import { import { PluginStartContract as AlertingStart } from '../../alerting/server'; import { SecurityPluginSetup } from '../../security/server'; -import { INDEX_PREFIX, RuleRegistryPluginConfig } from './config'; +import { RuleRegistryPluginConfig } from './config'; import { RuleDataPluginService } from './rule_data_plugin_service'; import { AlertsClientFactory } from './alert_data_client/alerts_client_factory'; import { AlertsClient } from './alert_data_client/alerts_client'; @@ -54,6 +54,7 @@ export class RuleRegistryPlugin private readonly config: RuleRegistryPluginConfig; private readonly legacyConfig: SharedGlobalConfig; private readonly logger: Logger; + private readonly kibanaVersion: string; private readonly alertsClientFactory: AlertsClientFactory; private ruleDataService: RuleDataPluginService | null; private security: SecurityPluginSetup | undefined; @@ -63,6 +64,7 @@ export class RuleRegistryPlugin // TODO: Can be removed in 8.0.0. Exists to work around multi-tenancy users. this.legacyConfig = initContext.config.legacy.get(); this.logger = initContext.logger.get(); + this.kibanaVersion = initContext.env.packageInfo.version; this.ruleDataService = null; this.alertsClientFactory = new AlertsClientFactory(); } @@ -71,7 +73,7 @@ export class RuleRegistryPlugin core: CoreSetup, plugins: RuleRegistryPluginSetupDependencies ): RuleRegistryPluginSetupContract { - const { logger } = this; + const { logger, kibanaVersion } = this; const startDependencies = core.getStartServices().then(([coreStart, pluginStart]) => { return { @@ -99,8 +101,8 @@ export class RuleRegistryPlugin this.ruleDataService = new RuleDataPluginService({ logger, + kibanaVersion, isWriteEnabled: isWriteEnabled(this.config, this.legacyConfig), - index: INDEX_PREFIX, getClusterClient: async () => { const deps = await startDependencies; return deps.core.elasticsearch.client.asInternalUser; diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts index 8aa5f8a6edf19..323adcc756674 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts @@ -18,23 +18,25 @@ type RuleDataClientMock = jest.Mocked) => MockInstances; }; -export function createRuleDataClientMock(): RuleDataClientMock { +export const createRuleDataClientMock = ( + indexName: string = '.alerts-security.alerts' +): RuleDataClientMock => { const bulk = jest.fn(); const search = jest.fn(); const getDynamicIndexPattern = jest.fn(); return { - indexName: '.alerts-security.alerts', - + indexName, + kibanaVersion: '7.16.0', isWriteEnabled: jest.fn(() => true), getReader: jest.fn((_options?: { namespace?: string }) => ({ - getDynamicIndexPattern, search, + getDynamicIndexPattern, })), getWriter: jest.fn(() => ({ bulk, })), }; -} +}; diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts index bbfa6abdd1a71..89ae479132de5 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts @@ -33,6 +33,10 @@ export class RuleDataClient implements IRuleDataClient { return this.options.indexInfo.baseName; } + public get kibanaVersion(): string { + return this.options.indexInfo.kibanaVersion; + } + public isWriteEnabled(): boolean { return this.options.isWriteEnabled; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 979aa7e264848..3e8b6b3413c6f 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -14,6 +14,7 @@ import { TechnicalRuleDataFieldName } from '../../common/technical_rule_data_fie export interface IRuleDataClient { indexName: string; + kibanaVersion: string; isWriteEnabled(): boolean; getReader(options?: { namespace?: string }): IRuleDataReader; getWriter(options?: { namespace?: string }): IRuleDataWriter; diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts index 4e64ea025a27a..52fef63a732f0 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts @@ -5,22 +5,13 @@ * 2.0. */ +import { INDEX_PREFIX, INDEX_PREFIX_FOR_BACKING_INDICES } from '../config'; import { IndexOptions } from './index_options'; import { joinWithDash } from './utils'; interface ConstructorOptions { - /** - * Prepends a relative resource name (defined in the code) with - * a full resource prefix, which starts with '.alerts' and can - * optionally include a user-defined part in it. - * @example 'security.alerts' => '.alerts-security.alerts' - */ - getResourceName(relativeName: string): string; - - /** - * Options provided by the plugin/solution defining the index. - */ indexOptions: IndexOptions; + kibanaVersion: string; } /** @@ -31,12 +22,17 @@ interface ConstructorOptions { */ export class IndexInfo { constructor(options: ConstructorOptions) { - const { getResourceName, indexOptions } = options; + const { indexOptions, kibanaVersion } = options; const { registrationContext, dataset } = indexOptions; this.indexOptions = indexOptions; - this.baseName = getResourceName(`${registrationContext}.${dataset}`); + this.kibanaVersion = kibanaVersion; + this.baseName = joinWithDash(INDEX_PREFIX, `${registrationContext}.${dataset}`); this.basePattern = joinWithDash(this.baseName, '*'); + this.baseNameForBackingIndices = joinWithDash( + INDEX_PREFIX_FOR_BACKING_INDICES, + `${registrationContext}.${dataset}` + ); } /** @@ -45,7 +41,13 @@ export class IndexInfo { public readonly indexOptions: IndexOptions; /** - * Base index name, prefixed with the full resource prefix. + * Current version of Kibana. We version our index resources and documents based on it. + * @example '7.16.0' + */ + public readonly kibanaVersion: string; + + /** + * Base index name, prefixed with the resource prefix. * @example '.alerts-security.alerts' */ public readonly baseName: string; @@ -56,6 +58,12 @@ export class IndexInfo { */ public readonly basePattern: string; + /** + * Base name for internal backing indices, prefixed with a special prefix. + * @example '.internal.alerts-security.alerts' + */ + private readonly baseNameForBackingIndices: string; + /** * Primary index alias. Includes a namespace. * Used as a write target when writing documents to the index. @@ -65,14 +73,6 @@ export class IndexInfo { return joinWithDash(this.baseName, namespace); } - /** - * Index pattern based on the primary alias. - * @example '.alerts-security.alerts-default-*' - */ - public getPrimaryAliasPattern(namespace: string): string { - return joinWithDash(this.baseName, namespace, '*'); - } - /** * Optional secondary alias that can be applied to concrete indices in * addition to the primary one. @@ -83,6 +83,26 @@ export class IndexInfo { return secondaryAlias ? joinWithDash(secondaryAlias, namespace) : null; } + /** + * Name of the initial concrete index, with the namespace and the ILM suffix. + * @example '.internal.alerts-security.alerts-default-000001' + */ + public getConcreteIndexInitialName(namespace: string): string { + return joinWithDash(this.baseNameForBackingIndices, namespace, '000001'); + } + + /** + * Index pattern for internal backing indices. Used in the index bootstrapping logic. + * Can include or exclude the namespace. + * + * WARNING: Must not be used for reading documents! If you use it, you should know what you're doing. + * + * @example '.internal.alerts-security.alerts-default-*', '.internal.alerts-security.alerts-*' + */ + public getPatternForBackingIndices(namespace?: string): string { + return joinWithDash(this.baseNameForBackingIndices, namespace, '*'); + } + /** * Index pattern that should be used when reading documents from the index. * Can include or exclude the namespace. @@ -100,14 +120,6 @@ export class IndexInfo { return `${joinWithDash(this.baseName, namespace)}*`; } - /** - * Name of the initial concrete index, with the namespace and the ILM suffix. - * @example '.alerts-security.alerts-default-000001' - */ - public getConcreteIndexInitialName(namespace: string): string { - return joinWithDash(this.baseName, namespace, '000001'); - } - /** * Name of the custom ILM policy (if it's provided by the plugin/solution). * Specific to the index. Shared between all namespaces of the index. diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts index 0f9486a068c24..e85331fb02a63 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts @@ -75,7 +75,7 @@ export interface IndexOptions { /** * Additional properties for the namespaced index template. */ - indexTemplate: IndexTemplateOptions; + indexTemplate?: IndexTemplateOptions; /** * Optional custom ILM policy for the index. @@ -120,7 +120,6 @@ export type Meta = estypes.Metadata; */ export interface ComponentTemplateOptions { name: string; - version: Version; // TODO: encapsulate versioning (base on Kibana version) mappings?: Mappings; settings?: Settings; _meta?: Meta; @@ -140,7 +139,6 @@ export interface ComponentTemplateOptions { * https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html */ export interface IndexTemplateOptions { - version: Version; // TODO: encapsulate versioning (base on Kibana version) _meta?: Meta; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts index e53e1db5391e7..73651ec298c36 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -132,7 +132,6 @@ export class ResourceInstaller { settings: ct.settings ?? {}, mappings: ct.mappings, }, - version: ct.version, _meta: ct._meta, }, }); @@ -146,29 +145,22 @@ export class ResourceInstaller { } private async updateIndexMappings(indexInfo: IndexInfo) { - const { logger, getClusterClient } = this.options; - const clusterClient = await getClusterClient(); + const { logger } = this.options; - logger.debug(`Updating mappings of existing concrete indices for ${indexInfo.baseName}`); + const aliases = indexInfo.basePattern; + const backingIndices = indexInfo.getPatternForBackingIndices(); - const { body: aliasesResponse } = await clusterClient.indices.getAlias({ - index: indexInfo.basePattern, - }); + logger.debug(`Updating mappings of existing concrete indices for ${indexInfo.baseName}`); - const writeIndicesAndAliases = Object.entries(aliasesResponse).flatMap(([index, { aliases }]) => - Object.entries(aliases) - .filter(([, aliasProperties]) => aliasProperties.is_write_index) - .map(([aliasName]) => ({ index, alias: aliasName })) - ); + // Find all concrete indices for all namespaces of the index. + const concreteIndices = await this.fetchConcreteIndices(aliases, backingIndices); + const concreteWriteIndices = concreteIndices.filter((item) => item.isWriteIndex); - await Promise.all( - writeIndicesAndAliases.map((indexAndAlias) => - this.updateAliasWriteIndexMapping(indexAndAlias) - ) - ); + // Update mappings of the found write indices. + await Promise.all(concreteWriteIndices.map((item) => this.updateAliasWriteIndexMapping(item))); } - private async updateAliasWriteIndexMapping({ index, alias }: { index: string; alias: string }) { + private async updateAliasWriteIndexMapping({ index, alias }: ConcreteIndexInfo) { const { logger, getClusterClient } = this.options; const clusterClient = await getClusterClient(); @@ -228,86 +220,59 @@ export class ResourceInstaller { indexInfo: IndexInfo, namespace: string ): Promise { - await this.createWriteTargetIfNeeded(indexInfo, namespace); + const { logger } = this.options; + + const alias = indexInfo.getPrimaryAlias(namespace); + + logger.info(`Installing namespace-level resources and creating concrete index for ${alias}`); + + // If we find a concrete backing index which is the write index for the alias here, we shouldn't + // be making a new concrete index. We return early because we don't need a new write target. + const indexExists = await this.checkIfConcreteWriteIndexExists(indexInfo, namespace); + if (indexExists) { + return; + } + + await this.installNamespacedIndexTemplate(indexInfo, namespace); + await this.createConcreteWriteIndex(indexInfo, namespace); } - private async createWriteTargetIfNeeded(indexInfo: IndexInfo, namespace: string) { - const { logger, getClusterClient } = this.options; - const clusterClient = await getClusterClient(); + private async checkIfConcreteWriteIndexExists( + indexInfo: IndexInfo, + namespace: string + ): Promise { + const { logger } = this.options; const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace); - const primaryNamespacedPattern = indexInfo.getPrimaryAliasPattern(namespace); - const initialIndexName = indexInfo.getConcreteIndexInitialName(namespace); + const indexPatternForBackingIndices = indexInfo.getPatternForBackingIndices(namespace); - logger.debug(`Creating write target for ${primaryNamespacedAlias}`); + logger.debug(`Checking if concrete write index exists for ${primaryNamespacedAlias}`); - try { - // When a new namespace is created we expect getAlias to return a 404 error, - // we'll catch it below and continue on. A non-404 error is a real problem so we throw. + const concreteIndices = await this.fetchConcreteIndices( + primaryNamespacedAlias, + indexPatternForBackingIndices + ); + const concreteIndicesExist = concreteIndices.some( + (item) => item.alias === primaryNamespacedAlias + ); + const concreteWriteIndicesExist = concreteIndices.some( + (item) => item.alias === primaryNamespacedAlias && item.isWriteIndex + ); - // It's critical that we specify *both* the index pattern and alias in this request. The alias prevents the - // request from finding other namespaces that could match the -* part of the index pattern - // (see https://github.com/elastic/kibana/issues/107704). The index pattern prevents the request from - // finding legacy .siem-signals indices that we add the alias to for backwards compatibility reasons. Together, - // the index pattern and alias should ensure that we retrieve only the "new" backing indices for this - // particular alias. - const { body: aliases } = await clusterClient.indices.getAlias({ - index: primaryNamespacedPattern, - name: primaryNamespacedAlias, - }); + // If we find backing indices for the alias here, we shouldn't be making a new concrete index - + // either one of the indices is the write index so we return early because we don't need a new write target, + // or none of them are the write index so we'll throw an error because one of the existing indices should have + // been the write target - // If we find backing indices for the alias here, we shouldn't be making a new concrete index - - // either one of the indices is the write index so we return early because we don't need a new write target, - // or none of them are the write index so we'll throw an error because one of the existing indices should have - // been the write target - if ( - Object.values(aliases).some( - (aliasesObject) => aliasesObject.aliases[primaryNamespacedAlias].is_write_index - ) - ) { - return; - } else { - throw new Error( - `Indices matching pattern ${primaryNamespacedPattern} exist but none are set as the write index for alias ${primaryNamespacedAlias}` - ); - } - } catch (err) { - // 404 is expected if the alerts-as-data index hasn't been created yet - if (err.statusCode !== 404) { - throw err; - } + // If there are some concrete indices but none of them are the write index, we'll throw an error + // because one of the existing indices should have been the write target. + if (concreteIndicesExist && !concreteWriteIndicesExist) { + throw new Error( + `Indices matching pattern ${indexPatternForBackingIndices} exist but none are set as the write index for alias ${primaryNamespacedAlias}` + ); } - await this.installNamespacedIndexTemplate(indexInfo, namespace); - - try { - await clusterClient.indices.create({ - index: initialIndexName, - body: { - aliases: { - [primaryNamespacedAlias]: { - is_write_index: true, - }, - }, - }, - }); - } catch (err) { - // If the index already exists and it's the write index for the alias, - // something else created it so suppress the error. If it's not the write - // index, that's bad, throw an error. - if (err?.meta?.body?.error?.type === 'resource_already_exists_exception') { - const { body: existingIndices } = await clusterClient.indices.get({ - index: initialIndexName, - }); - if (!existingIndices[initialIndexName]?.aliases?.[primaryNamespacedAlias]?.is_write_index) { - throw Error( - `Attempted to create index: ${initialIndexName} as the write index for alias: ${primaryNamespacedAlias}, but the index already exists and is not the write index for the alias` - ); - } - } else { - throw err; - } - } + return concreteWriteIndicesExist; } private async installNamespacedIndexTemplate(indexInfo: IndexInfo, namespace: string) { @@ -315,13 +280,13 @@ export class ResourceInstaller { const { componentTemplateRefs, componentTemplates, - indexTemplate, + indexTemplate = {}, ilmPolicy, } = indexInfo.indexOptions; const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace); - const primaryNamespacedPattern = indexInfo.getPrimaryAliasPattern(namespace); const secondaryNamespacedAlias = indexInfo.getSecondaryAlias(namespace); + const indexPatternForBackingIndices = indexInfo.getPatternForBackingIndices(namespace); logger.debug(`Installing index template for ${primaryNamespacedAlias}`); @@ -334,6 +299,15 @@ export class ResourceInstaller { ? indexInfo.getIlmPolicyName() : getResourceName(DEFAULT_ILM_POLICY_ID); + const indexMetadata: estypes.Metadata = { + ...indexTemplate._meta, + kibana: { + ...indexTemplate._meta?.kibana, + version: indexInfo.kibanaVersion, + }, + namespace, + }; + // TODO: need a way to update this template if/when we decide to make changes to the // built in index template. Probably do it as part of updateIndexMappingsForAsset? // (Before upgrading any indices, find and upgrade all namespaced index templates - component templates @@ -347,7 +321,7 @@ export class ResourceInstaller { await this.createOrUpdateIndexTemplate({ name: indexInfo.getIndexTemplateName(namespace), body: { - index_patterns: [primaryNamespacedPattern], + index_patterns: [indexPatternForBackingIndices], // Order matters: // - first go external component templates referenced by this index (e.g. the common full ECS template) @@ -369,6 +343,9 @@ export class ResourceInstaller { rollover_alias: primaryNamespacedAlias, }, }, + mappings: { + _meta: indexMetadata, + }, aliases: secondaryNamespacedAlias != null ? { @@ -379,12 +356,7 @@ export class ResourceInstaller { : undefined, }, - _meta: { - ...indexTemplate._meta, - namespace, - }, - - version: indexTemplate.version, + _meta: indexMetadata, // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace // then newly created indices will use the matching template with the *longest* namespace @@ -393,6 +365,45 @@ export class ResourceInstaller { }); } + private async createConcreteWriteIndex(indexInfo: IndexInfo, namespace: string) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace); + const initialIndexName = indexInfo.getConcreteIndexInitialName(namespace); + + logger.debug(`Creating concrete write index for ${primaryNamespacedAlias}`); + + try { + await clusterClient.indices.create({ + index: initialIndexName, + body: { + aliases: { + [primaryNamespacedAlias]: { + is_write_index: true, + }, + }, + }, + }); + } catch (err) { + // If the index already exists and it's the write index for the alias, + // something else created it so suppress the error. If it's not the write + // index, that's bad, throw an error. + if (err?.meta?.body?.error?.type === 'resource_already_exists_exception') { + const { body: existingIndices } = await clusterClient.indices.get({ + index: initialIndexName, + }); + if (!existingIndices[initialIndexName]?.aliases?.[primaryNamespacedAlias]?.is_write_index) { + throw Error( + `Attempted to create index: ${initialIndexName} as the write index for alias: ${primaryNamespacedAlias}, but the index already exists and is not the write index for the alias` + ); + } + } else { + throw err; + } + } + } + // ----------------------------------------------------------------------------------------------- // Helpers @@ -431,4 +442,55 @@ export class ResourceInstaller { return clusterClient.indices.putIndexTemplate(template); } + + private async fetchConcreteIndices( + aliasOrPatternForAliases: string, + indexPatternForBackingIndices: string + ): Promise { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + logger.debug(`Fetching concrete indices for ${indexPatternForBackingIndices}`); + + try { + // It's critical that we specify *both* the index pattern for backing indices and their alias(es) in this request. + // The alias prevents the request from finding other namespaces that could match the -* part of the index pattern + // (see https://github.com/elastic/kibana/issues/107704). The backing index pattern prevents the request from + // finding legacy .siem-signals indices that we add the alias to for backwards compatibility reasons. Together, + // the index pattern and alias should ensure that we retrieve only the "new" backing indices for this + // particular alias. + const { body: response } = await clusterClient.indices.getAlias({ + index: indexPatternForBackingIndices, + name: aliasOrPatternForAliases, + }); + + return createConcreteIndexInfo(response); + } catch (err) { + // 404 is expected if the alerts-as-data indices haven't been created yet + if (err.statusCode === 404) { + return createConcreteIndexInfo({}); + } + + // A non-404 error is a real problem so we re-throw. + throw err; + } + } } + +interface ConcreteIndexInfo { + index: string; + alias: string; + isWriteIndex: boolean; +} + +const createConcreteIndexInfo = ( + response: estypes.IndicesGetAliasResponse +): ConcreteIndexInfo[] => { + return Object.entries(response).flatMap(([index, { aliases }]) => + Object.entries(aliases).map(([aliasName, aliasProperties]) => ({ + index, + alias: aliasName, + isWriteIndex: aliasProperties.is_write_index ?? false, + })) + ); +}; diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts index a417ba289d83a..ed3d5340756e8 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts @@ -10,6 +10,7 @@ import { ValidFeatureId } from '@kbn/rule-data-utils'; import { ElasticsearchClient, Logger } from 'kibana/server'; +import { INDEX_PREFIX } from '../config'; import { IRuleDataClient, RuleDataClient, WaitResult } from '../rule_data_client'; import { IndexInfo } from './index_info'; import { Dataset, IndexOptions } from './index_options'; @@ -19,8 +20,8 @@ import { joinWithDash } from './utils'; interface ConstructorOptions { getClusterClient: () => Promise; logger: Logger; + kibanaVersion: string; isWriteEnabled: boolean; - index: string; } /** @@ -49,18 +50,16 @@ export class RuleDataPluginService { } /** - * Returns a full resource prefix. - * - it's '.alerts' by default - * - it can be adjusted by the user via Kibana config + * Returns a prefix used in the naming scheme of index aliases, templates + * and other Elasticsearch resources that this service creates + * for alerts-as-data indices. */ public getResourcePrefix(): string { - // TODO: https://github.com/elastic/kibana/issues/106432 - return this.options.index; + return INDEX_PREFIX; } /** - * Prepends a relative resource name with a full resource prefix, which - * starts with '.alerts' and can optionally include a user-defined part in it. + * Prepends a relative resource name with the resource prefix. * @returns Full name of the resource. * @example 'security.alerts' => '.alerts-security.alerts' */ @@ -106,10 +105,7 @@ export class RuleDataPluginService { ); } - const indexInfo = new IndexInfo({ - getResourceName: (name) => this.getResourceName(name), - indexOptions, - }); + const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: this.options.kibanaVersion }); const indicesAssociatedWithFeature = this.indicesByFeatureId.get(indexOptions.feature) ?? []; this.indicesByFeatureId.set(indexOptions.feature, [...indicesAssociatedWithFeature, indexInfo]); diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index abdffd2aa03cc..a3e830d6e0b2f 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -35,6 +35,7 @@ import { EVENT_KIND, SPACE_IDS, TIMESTAMP, + VERSION, } from '../../common/technical_rule_data_field_names'; import { IRuleDataClient } from '../rule_data_client'; import { AlertExecutorOptionsWithExtraServices } from '../types'; @@ -250,6 +251,7 @@ export const createLifecycleExecutor = ( [EVENT_KIND]: 'signal', [ALERT_RULE_CONSUMER]: rule.consumer, [ALERT_ID]: alertId, + [VERSION]: ruleDataClient.kibanaVersion, } as ParsedTechnicalFields; const isNew = !state.trackedAlerts[alertId]; diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index 7d798effcb9e5..71a0dee5deac7 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -203,6 +203,7 @@ describe('createLifecycleRuleTypeFactory', () => { "kibana.space_ids": Array [ "spaceId", ], + "kibana.version": "7.16.0", "service.name": "opbeans-java", "tags": Array [ "tags", @@ -226,6 +227,7 @@ describe('createLifecycleRuleTypeFactory', () => { "kibana.space_ids": Array [ "spaceId", ], + "kibana.version": "7.16.0", "service.name": "opbeans-node", "tags": Array [ "tags", diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts index caf14e8ba3000..30e17f1afca54 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ALERT_ID } from '@kbn/rule-data-utils'; +import { ALERT_ID, VERSION } from '@kbn/rule-data-utils'; import { CreatePersistenceRuleTypeFactory } from './persistence_types'; export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory = ({ @@ -28,8 +28,9 @@ export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory body: alerts.flatMap((event) => [ { index: {} }, { - [ALERT_ID]: event.id, ...event.fields, + [ALERT_ID]: event.id, + [VERSION]: ruleDataClient.kibanaVersion, }, ]), refresh, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts index 3f2f34c17679f..dc556fc988afa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts @@ -82,13 +82,9 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient { componentTemplates: [ { name: 'mappings', - version: 0, mappings: mappingFromFieldMap(ruleExecutionFieldMap, 'strict'), }, ], - indexTemplate: { - version: 0, - }, }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts index 3c28551a71deb..1a8389d450ab3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts @@ -12,12 +12,12 @@ import { Logger, SavedObject } from 'kibana/server'; import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; import type { IRuleDataClient } from '../../../../../../rule_registry/server'; +import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../../alerting/server'; import { ConfigType } from '../../../../config'; import { AlertAttributes } from '../../signals/types'; import { createRuleMock } from './rule'; import { listMock } from '../../../../../../lists/server/mocks'; -import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks'; import { RuleParams } from '../../schemas/rule_schemas'; export const createRuleTypeMocks = ( @@ -81,16 +81,9 @@ export const createRuleTypeMocks = ( config$: mockedConfig$, lists: listMock.createSetup(), logger: loggerMock, - ruleDataClient: ({ - getReader: jest.fn(() => ({ - search: jest.fn(), - })), - getWriter: jest.fn(() => ({ - bulk: jest.fn(), - })), - isWriteEnabled: jest.fn(() => true), - indexName: '.alerts-security.alerts', - } as unknown) as IRuleDataClient, + ruleDataClient: ruleRegistryMocks.createRuleDataClient( + '.alerts-security.alerts' + ) as IRuleDataClient, ruleDataService: ruleRegistryMocks.createRuleDataPluginService(), }, services, diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index e5c837293e2a8..0b803b9990701 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -233,16 +233,12 @@ export class Plugin implements IPlugin { - return { - search: jest.fn(), - }; - }, - getWriter: () => { - return { - bulk: jest.fn(), - }; - }, - isWriteEnabled: jest.fn(() => true), - indexName: '.alerts-observability.uptime.alerts', - } as unknown) as IRuleDataClient, + ruleDataClient: ruleRegistryMocks.createRuleDataClient( + '.alerts-observability.uptime.alerts' + ) as IRuleDataClient, }, services, scheduleActions, diff --git a/x-pack/plugins/uptime/server/plugin.ts b/x-pack/plugins/uptime/server/plugin.ts index a201eddc45345..736cbed51084c 100644 --- a/x-pack/plugins/uptime/server/plugin.ts +++ b/x-pack/plugins/uptime/server/plugin.ts @@ -43,13 +43,9 @@ export class Plugin implements PluginType { componentTemplates: [ { name: 'mappings', - version: 0, mappings: mappingFromFieldMap(uptimeRuleFieldMap, 'strict'), }, ], - indexTemplate: { - version: 0, - }, }); initServerWithKibana( diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts index 5400c3af64b55..678aebf502869 100644 --- a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -14,6 +14,7 @@ import { ALERT_STATUS, ALERT_UUID, EVENT_KIND, + VERSION, } from '@kbn/rule-data-utils'; import { merge, omit } from 'lodash'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -348,7 +349,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { any >; - const exclude = ['@timestamp', ALERT_START, ALERT_UUID, ALERT_RULE_UUID]; + const exclude = ['@timestamp', ALERT_START, ALERT_UUID, ALERT_RULE_UUID, VERSION]; const toCompare = omit(alertEvent, exclude);