diff --git a/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts new file mode 100644 index 0000000000000..959e38b3f7ee4 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * This function is similar to getIndexExists, but is limited to searching indices that match + * the index pattern used as concrete backing indices (e.g. .siem-signals-default-000001). + * This allows us to separate the indices that are actually .siem-signals indices from + * alerts as data indices that only share the .siem-signals alias. + * + * @param esClient Elasticsearch client to use to make the request + * @param index Index alias name to check for existence + */ +export const getBootstrapIndexExists = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + try { + const { body } = await esClient.indices.getAlias({ + index: `${index}-*`, + name: index, + }); + return Object.keys(body).length > 0; + } catch (err) { + if (err.body != null && err.body.status === 404) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/packages/kbn-securitysolution-es-utils/src/index.ts b/packages/kbn-securitysolution-es-utils/src/index.ts index f4b081ac1b0a0..a32cf8968967c 100644 --- a/packages/kbn-securitysolution-es-utils/src/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/index.ts @@ -13,6 +13,7 @@ export * from './delete_all_index'; export * from './delete_policy'; export * from './delete_template'; export * from './encode_hit_version'; +export * from './get_bootstrap_index_exists'; export * from './get_index_aliases'; export * from './get_index_count'; export * from './get_index_exists'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts index 974d18292a078..0040b5f30afb4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts @@ -43,10 +43,12 @@ export const templateNeedsUpdate = async ({ export const fieldAliasesOutdated = async (esClient: ElasticsearchClient, index: string) => { const { body: indexMappings } = await esClient.indices.get({ index }); - for (const [_, mapping] of Object.entries(indexMappings)) { - const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0; - if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) { - return true; + for (const [indexName, mapping] of Object.entries(indexMappings)) { + if (indexName.startsWith(`${index}-`)) { + const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0; + if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) { + return true; + } } } return false; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index 2e377e50530d1..cf40988c1d742 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { get } from 'lodash'; +import { chunk, get } from 'lodash'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ElasticsearchClient } from 'src/core/server'; import { transformError, - getIndexExists, + getBootstrapIndexExists, getPolicyExists, setPolicy, createBootstrapIndex, @@ -25,6 +25,8 @@ import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION, createBackwardsCompatibilityMapping, + ALIAS_VERSION_FIELD, + SIGNALS_FIELD_ALIASES_VERSION, } from './get_signals_template'; import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup'; import signalsPolicy from './signals_policy.json'; @@ -71,7 +73,10 @@ export const createDetectionIndex = async ( const spaceId = context.getSpaceId(); const index = siemClient.getSignalsIndex(); - const indexExists = await getIndexExists(esClient, index); + const indexExists = await getBootstrapIndexExists( + context.core.elasticsearch.client.asInternalUser, + index + ); const { ruleRegistryEnabled } = config.experimentalFeatures; // If using the rule registry implementation, we don't want to create new .siem-signals indices - @@ -124,6 +129,11 @@ export const createDetectionIndex = async ( } }; +// This function can be expensive if there are lots of existing .siem-signals indices +// because any new backwards compatibility mappings need to be applied to all of them +// while also preserving the original 'version' of the mapping. To do it somewhat efficiently, +// we first group the indices by version and exclude any that already have up-to-date +// aliases. Then we start updating the mappings sequentially in chunks. const addFieldAliasesToIndices = async ({ esClient, index, @@ -132,14 +142,34 @@ const addFieldAliasesToIndices = async ({ index: string; }) => { const { body: indexMappings } = await esClient.indices.get({ index }); + const indicesByVersion: Record = {}; + const versions: Set = new Set(); for (const [indexName, mapping] of Object.entries(indexMappings)) { - const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version'); - const body = createBackwardsCompatibilityMapping(currentVersion ?? 0); - await esClient.indices.putMapping({ - index: indexName, - body, - allow_no_indices: true, - } as estypes.IndicesPutMappingRequest); + const version: number = get(mapping.mappings?._meta, 'version') ?? 0; + const aliasesVersion: number = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0; + // Only attempt to add backwards compatibility mappings to indices whose names start with the alias + // This limits us to legacy .siem-signals indices, since alerts as data indices use a different naming + // scheme (but have the same alias, so will also be returned by the "get" request) + if ( + indexName.startsWith(`${index}-`) && + isOutdated({ current: aliasesVersion, target: SIGNALS_FIELD_ALIASES_VERSION }) + ) { + indicesByVersion[version] = indicesByVersion[version] + ? [...indicesByVersion[version], indexName] + : [indexName]; + versions.add(version); + } + } + for (const version of versions) { + const body = createBackwardsCompatibilityMapping(version); + const indexNameChunks = chunk(indicesByVersion[version], 20); + for (const indexNameChunk of indexNameChunks) { + await esClient.indices.putMapping({ + index: indexNameChunk, + body, + allow_no_indices: true, + } as estypes.IndicesPutMappingRequest); + } } }; @@ -152,7 +182,7 @@ const addIndexAliases = async ({ index: string; aadIndexAliasName: string; }) => { - const { body: indices } = await esClient.indices.getAlias({ name: index }); + const { body: indices } = await esClient.indices.getAlias({ index: `${index}-*`, name: index }); const aliasActions = { actions: Object.keys(indices).map((concreteIndexName) => { return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts index 242c78ceed28b..3b1b27176ef88 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts @@ -5,12 +5,16 @@ * 2.0. */ -import { transformError } from '@kbn/securitysolution-es-utils'; +import { transformError, getBootstrapIndexExists } from '@kbn/securitysolution-es-utils'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; import { buildSiemResponse } from '../utils'; import { RuleDataPluginService } from '../../../../../../rule_registry/server'; +import { fieldAliasesOutdated } from './check_template_version'; +import { getIndexVersion } from './get_index_version'; +import { isOutdated } from '../../migrations/helpers'; +import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; export const readIndexRoute = ( router: SecuritySolutionPluginRouter, @@ -29,6 +33,7 @@ export const readIndexRoute = ( try { const siemClient = context.securitySolution?.getAppClient(); + const esClient = context.core.elasticsearch.client.asCurrentUser; if (!siemClient) { return siemResponse.error({ statusCode: 404 }); @@ -37,12 +42,47 @@ export const readIndexRoute = ( const spaceId = context.securitySolution.getSpaceId(); const indexName = ruleDataService.getResourceName(`security.alerts-${spaceId}`); - return response.ok({ - body: { - name: indexName, - index_mapping_outdated: false, - }, - }); + const index = siemClient.getSignalsIndex(); + const indexExists = await getBootstrapIndexExists( + context.core.elasticsearch.client.asInternalUser, + index + ); + + if (indexExists) { + let mappingOutdated: boolean | null = null; + let aliasesOutdated: boolean | null = null; + try { + const indexVersion = await getIndexVersion(esClient, index); + mappingOutdated = isOutdated({ + current: indexVersion, + target: SIGNALS_TEMPLATE_VERSION, + }); + aliasesOutdated = await fieldAliasesOutdated(esClient, index); + } catch (err) { + const error = transformError(err); + // Some users may not have the view_index_metadata permission necessary to check the index mapping version + // so just continue and return null for index_mapping_outdated if the error is a 403 + if (error.statusCode !== 403) { + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + return response.ok({ + body: { + name: indexName, + index_mapping_outdated: mappingOutdated || aliasesOutdated, + }, + }); + } else { + return response.ok({ + body: { + name: indexName, + index_mapping_outdated: false, + }, + }); + } } catch (err) { const error = transformError(err); return siemResponse.error({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts new file mode 100644 index 0000000000000..856a9f2b70658 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + DEFAULT_ALERTS_INDEX, + DETECTION_ENGINE_INDEX_URL, +} from '../../../../plugins/security_solution/common/constants'; + +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { deleteSignalsIndex } from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('create_index', () => { + afterEach(async () => { + await deleteSignalsIndex(supertest); + }); + + describe('elastic admin', () => { + describe('with another index that shares index alias', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/signals/index_alias_clash'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/signals/index_alias_clash'); + }); + + it.skip('should report that signals index does not exist', async () => { + const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404); + expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 }); + }); + + it('should return 200 for create_index', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_INDEX_URL) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + expect(body).to.eql({ acknowledged: true }); + }); + }); + + describe('with an outdated signals index', () => { + beforeEach(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/endpoint/resolver/signals'); + }); + + afterEach(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/endpoint/resolver/signals'); + }); + + it('should report that signals index is outdated', async () => { + const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200); + expect(body).to.eql({ + index_mapping_outdated: true, + name: `${DEFAULT_ALERTS_INDEX}-default`, + }); + }); + + it('should return 200 for create_index and add field aliases', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_INDEX_URL) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + expect(body).to.eql({ acknowledged: true }); + + const mappings = await es.indices.get({ + index: '.siem-signals-default-000001', + }); + // Make sure that aliases_version has been updated on the existing index + expect(mappings['.siem-signals-default-000001'].mappings?._meta?.aliases_version).to.eql( + 1 + ); + }); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 31ecf6edb9bb2..db616baa0678a 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -19,6 +19,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./update_actions')); loadTestFile(require.resolve('./add_prepackaged_rules')); loadTestFile(require.resolve('./check_privileges')); + loadTestFile(require.resolve('./create_index')); loadTestFile(require.resolve('./create_rules')); loadTestFile(require.resolve('./create_rules_bulk')); loadTestFile(require.resolve('./create_ml')); diff --git a/x-pack/test/functional/es_archives/signals/README.md b/x-pack/test/functional/es_archives/signals/README.md index 97c8c504a4039..01f8405d6ed58 100644 --- a/x-pack/test/functional/es_archives/signals/README.md +++ b/x-pack/test/functional/es_archives/signals/README.md @@ -24,3 +24,7 @@ A legacy signals index. It has no migration metadata fields and a very old mappi #### `signals/outdated_signals_index` A signals index that had previously been updated but is now out of date. It has migration metadata fields and a recent mapping version. + +#### `signals/index_alias_clash` + +An index that has the .siem-signals alias, but is NOT a signals index. Used for simulating an alerts-as-data index, which will have the .siem-signals alias but different mappings. This way we can test that functionality that needs to target only signals indices (e.g. mapping updates to apply field aliases) work correctly in the presence of alerts-as-data indices. diff --git a/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json b/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json new file mode 100644 index 0000000000000..8bba21bbfbae8 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json @@ -0,0 +1,11 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_name_clash", + "source": { + "@timestamp": "2020-10-28T05:08:53.000Z" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json b/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json new file mode 100644 index 0000000000000..3a77af645b118 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json @@ -0,0 +1,24 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-default": { + "is_write_index": false + } + }, + "index": "index_alias_clash", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}