diff --git a/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts new file mode 100644 index 0000000000000..959e38b3f7ee4 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * This function is similar to getIndexExists, but is limited to searching indices that match + * the index pattern used as concrete backing indices (e.g. .siem-signals-default-000001). + * This allows us to separate the indices that are actually .siem-signals indices from + * alerts as data indices that only share the .siem-signals alias. + * + * @param esClient Elasticsearch client to use to make the request + * @param index Index alias name to check for existence + */ +export const getBootstrapIndexExists = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + try { + const { body } = await esClient.indices.getAlias({ + index: `${index}-*`, + name: index, + }); + return Object.keys(body).length > 0; + } catch (err) { + if (err.body != null && err.body.status === 404) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/packages/kbn-securitysolution-es-utils/src/index.ts b/packages/kbn-securitysolution-es-utils/src/index.ts index f4b081ac1b0a0..a32cf8968967c 100644 --- a/packages/kbn-securitysolution-es-utils/src/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/index.ts @@ -13,6 +13,7 @@ export * from './delete_all_index'; export * from './delete_policy'; export * from './delete_template'; export * from './encode_hit_version'; +export * from './get_bootstrap_index_exists'; export * from './get_index_aliases'; export * from './get_index_count'; export * from './get_index_exists'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts index 974d18292a078..0040b5f30afb4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts @@ -43,10 +43,12 @@ export const templateNeedsUpdate = async ({ export const fieldAliasesOutdated = async (esClient: ElasticsearchClient, index: string) => { const { body: indexMappings } = await esClient.indices.get({ index }); - for (const [_, mapping] of Object.entries(indexMappings)) { - const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0; - if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) { - return true; + for (const [indexName, mapping] of Object.entries(indexMappings)) { + if (indexName.startsWith(`${index}-`)) { + const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0; + if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) { + return true; + } } } return false; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index b011fd3fcd247..09c76344ba8e7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { get } from 'lodash'; +import { chunk, get } from 'lodash'; import { estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient } from 'src/core/server'; import { transformError, - getIndexExists, + getBootstrapIndexExists, getPolicyExists, setPolicy, createBootstrapIndex, @@ -25,6 +25,8 @@ import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION, createBackwardsCompatibilityMapping, + ALIAS_VERSION_FIELD, + SIGNALS_FIELD_ALIASES_VERSION, } from './get_signals_template'; import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup'; import signalsPolicy from './signals_policy.json'; @@ -71,7 +73,10 @@ export const createDetectionIndex = async ( const spaceId = context.getSpaceId(); const index = siemClient.getSignalsIndex(); - const indexExists = await getIndexExists(esClient, index); + const indexExists = await getBootstrapIndexExists( + context.core.elasticsearch.client.asInternalUser, + index + ); const { ruleRegistryEnabled } = config.experimentalFeatures; // If using the rule registry implementation, we don't want to create new .siem-signals indices - @@ -124,6 +129,11 @@ export const createDetectionIndex = async ( } }; +// This function can be expensive if there are lots of existing .siem-signals indices +// because any new backwards compatibility mappings need to be applied to all of them +// while also preserving the original 'version' of the mapping. To do it somewhat efficiently, +// we first group the indices by version and exclude any that already have up-to-date +// aliases. Then we start updating the mappings sequentially in chunks. const addFieldAliasesToIndices = async ({ esClient, index, @@ -132,14 +142,34 @@ const addFieldAliasesToIndices = async ({ index: string; }) => { const { body: indexMappings } = await esClient.indices.get({ index }); + const indicesByVersion: Record = {}; + const versions: Set = new Set(); for (const [indexName, mapping] of Object.entries(indexMappings)) { - const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version'); - const body = createBackwardsCompatibilityMapping(currentVersion ?? 0); - await esClient.indices.putMapping({ - index: indexName, - body, - allow_no_indices: true, - } as estypes.IndicesPutMappingRequest); + const version: number = get(mapping.mappings?._meta, 'version') ?? 0; + const aliasesVersion: number = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0; + // Only attempt to add backwards compatibility mappings to indices whose names start with the alias + // This limits us to legacy .siem-signals indices, since alerts as data indices use a different naming + // scheme (but have the same alias, so will also be returned by the "get" request) + if ( + indexName.startsWith(`${index}-`) && + isOutdated({ current: aliasesVersion, target: SIGNALS_FIELD_ALIASES_VERSION }) + ) { + indicesByVersion[version] = indicesByVersion[version] + ? [...indicesByVersion[version], indexName] + : [indexName]; + versions.add(version); + } + } + for (const version of versions) { + const body = createBackwardsCompatibilityMapping(version); + const indexNameChunks = chunk(indicesByVersion[version], 20); + for (const indexNameChunk of indexNameChunks) { + await esClient.indices.putMapping({ + index: indexNameChunk, + body, + allow_no_indices: true, + } as estypes.IndicesPutMappingRequest); + } } }; @@ -152,7 +182,7 @@ const addFieldAliasesToIndices = async ({ // index: string; // aadIndexAliasName: string; // }) => { -// const { body: indices } = await esClient.indices.getAlias({ name: index }); +// const { body: indices } = await esClient.indices.getAlias({ index: `${index}-*`, name: index }); // const aliasActions = { // actions: Object.keys(indices).map((concreteIndexName) => { // return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts index 4cfedd5dcaa01..164743b8b8bb2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils'; +import { transformError, getBootstrapIndexExists } from '@kbn/securitysolution-es-utils'; import { parseExperimentalConfigValue } from '../../../../../common/experimental_features'; import { ConfigType } from '../../../../config'; import type { SecuritySolutionPluginRouter } from '../../../../types'; @@ -41,7 +41,10 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: Con const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental); const index = siemClient.getSignalsIndex(); - const indexExists = await getIndexExists(esClient, index); + const indexExists = await getBootstrapIndexExists( + context.core.elasticsearch.client.asInternalUser, + index + ); if (indexExists) { let mappingOutdated: boolean | null = null; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts index 4748e39cd3a46..56802a48adabe 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts @@ -20,6 +20,8 @@ import { createUserAndRole, deleteUserAndRole } from '../../../common/services/s export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const supertestWithoutAuth = getService('supertestWithoutAuth'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); describe('create_index', () => { afterEach(async () => { @@ -27,6 +29,65 @@ export default ({ getService }: FtrProviderContext) => { }); describe('elastic admin', () => { + describe('with another index that shares index alias', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/signals/index_alias_clash'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/signals/index_alias_clash'); + }); + + it('should report that signals index does not exist', async () => { + const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404); + expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 }); + }); + + it('should return 200 for create_index', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_INDEX_URL) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + expect(body).to.eql({ acknowledged: true }); + }); + }); + + describe('with an outdated signals index', () => { + beforeEach(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/endpoint/resolver/signals'); + }); + + afterEach(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/endpoint/resolver/signals'); + }); + + it('should report that signals index is outdated', async () => { + const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200); + expect(body).to.eql({ + index_mapping_outdated: true, + name: `${DEFAULT_SIGNALS_INDEX}-default`, + }); + }); + + it('should return 200 for create_index and add field aliases', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_INDEX_URL) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + expect(body).to.eql({ acknowledged: true }); + + const { body: mappings } = await es.indices.get({ + index: '.siem-signals-default-000001', + }); + // Make sure that aliases_version has been updated on the existing index + expect(mappings['.siem-signals-default-000001'].mappings?._meta?.aliases_version).to.eql( + 1 + ); + }); + }); + it('should return a 404 when the signal index has never been created', async () => { const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404); expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 00147a2ec2ef7..9993fda170fdf 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -19,6 +19,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./update_actions')); loadTestFile(require.resolve('./add_prepackaged_rules')); loadTestFile(require.resolve('./check_privileges')); + loadTestFile(require.resolve('./create_index')); loadTestFile(require.resolve('./create_rules')); loadTestFile(require.resolve('./create_rules_bulk')); loadTestFile(require.resolve('./create_index')); diff --git a/x-pack/test/functional/es_archives/signals/README.md b/x-pack/test/functional/es_archives/signals/README.md index 97c8c504a4039..01f8405d6ed58 100644 --- a/x-pack/test/functional/es_archives/signals/README.md +++ b/x-pack/test/functional/es_archives/signals/README.md @@ -24,3 +24,7 @@ A legacy signals index. It has no migration metadata fields and a very old mappi #### `signals/outdated_signals_index` A signals index that had previously been updated but is now out of date. It has migration metadata fields and a recent mapping version. + +#### `signals/index_alias_clash` + +An index that has the .siem-signals alias, but is NOT a signals index. Used for simulating an alerts-as-data index, which will have the .siem-signals alias but different mappings. This way we can test that functionality that needs to target only signals indices (e.g. mapping updates to apply field aliases) work correctly in the presence of alerts-as-data indices. diff --git a/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json b/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json new file mode 100644 index 0000000000000..8bba21bbfbae8 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json @@ -0,0 +1,11 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_name_clash", + "source": { + "@timestamp": "2020-10-28T05:08:53.000Z" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json b/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json new file mode 100644 index 0000000000000..3a77af645b118 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json @@ -0,0 +1,24 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-default": { + "is_write_index": false + } + }, + "index": "index_alias_clash", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}