[7.16] [Security Solution] Only apply field aliases to legacy .siem-s…

…ignals indices (#115290) (#116841) * [Security Solution] Only apply field aliases to legacy .siem-signals indices (#115290) * Only apply field aliases to legacy .siem-signals indices * Fix unit test mocks * Add new function for special index existence check * Actually add new function for special index existence check * Undo getIndexVersion change * Add basic integration tests for field alias logic * Add back create_index to test list * Add missing markdown to readme * Revert change to delete_index_route Co-authored-by: Kibana Machine <[email protected]> # Conflicts: # x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts # x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts # x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts * Remove extra esClient definition * Adjust for old ES client Co-authored-by: Kibana Machine <[email protected]>
elastic · Oct 30, 2021 · 1815d7c · 1815d7c
1 parent 22e2973
commit 1815d7c
Show file tree

Hide file tree

Showing 10 changed files with 191 additions and 17 deletions.
diff --git a/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * This function is similar to getIndexExists, but is limited to searching indices that match
+ * the index pattern used as concrete backing indices (e.g. .siem-signals-default-000001).
+ * This allows us to separate the indices that are actually .siem-signals indices from
+ * alerts as data indices that only share the .siem-signals alias.
+ *
+ * @param esClient Elasticsearch client to use to make the request
+ * @param index Index alias name to check for existence
+ */
+export const getBootstrapIndexExists = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<boolean> => {
+  try {
+    const { body } = await esClient.indices.getAlias({
+      index: `${index}-*`,
+      name: index,
+    });
+    return Object.keys(body).length > 0;
+  } catch (err) {
+    if (err.body != null && err.body.status === 404) {
+      return false;
+    } else {
+      throw err.body ? err.body : err;
+    }
+  }
+};
diff --git a/packages/kbn-securitysolution-es-utils/src/index.ts b/packages/kbn-securitysolution-es-utils/src/index.ts
@@ -13,6 +13,7 @@ export * from './delete_all_index';
 export * from './delete_policy';
 export * from './delete_template';
 export * from './encode_hit_version';
+export * from './get_bootstrap_index_exists';
 export * from './get_index_aliases';
 export * from './get_index_count';
 export * from './get_index_exists';

diff --git a/...gins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts b/...gins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts
@@ -43,10 +43,12 @@ export const templateNeedsUpdate = async ({
 
 export const fieldAliasesOutdated = async (esClient: ElasticsearchClient, index: string) => {
   const { body: indexMappings } = await esClient.indices.get({ index });
-  for (const [_, mapping] of Object.entries(indexMappings)) {
-    const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
-    if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
-      return true;
+  for (const [indexName, mapping] of Object.entries(indexMappings)) {
+    if (indexName.startsWith(`${index}-`)) {
+      const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
+      if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
+        return true;
+      }
     }
   }
   return false;

diff --git a/.../plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/.../plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts
@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { get } from 'lodash';
+import { chunk, get } from 'lodash';
 import { estypes } from '@elastic/elasticsearch';
 import { ElasticsearchClient } from 'src/core/server';
 import {
   transformError,
-  getIndexExists,
+  getBootstrapIndexExists,
   getPolicyExists,
   setPolicy,
   createBootstrapIndex,
@@ -25,6 +25,8 @@ import {
   getSignalsTemplate,
   SIGNALS_TEMPLATE_VERSION,
   createBackwardsCompatibilityMapping,
+  ALIAS_VERSION_FIELD,
+  SIGNALS_FIELD_ALIASES_VERSION,
 } from './get_signals_template';
 import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup';
 import signalsPolicy from './signals_policy.json';
@@ -71,7 +73,10 @@ export const createDetectionIndex = async (
   const spaceId = context.getSpaceId();
   const index = siemClient.getSignalsIndex();
 
-  const indexExists = await getIndexExists(esClient, index);
+  const indexExists = await getBootstrapIndexExists(
+    context.core.elasticsearch.client.asInternalUser,
+    index
+  );
   const { ruleRegistryEnabled } = config.experimentalFeatures;
 
   // If using the rule registry implementation, we don't want to create new .siem-signals indices -
@@ -124,6 +129,11 @@ export const createDetectionIndex = async (
   }
 };
 
+// This function can be expensive if there are lots of existing .siem-signals indices
+// because any new backwards compatibility mappings need to be applied to all of them
+// while also preserving the original 'version' of the mapping. To do it somewhat efficiently,
+// we first group the indices by version and exclude any that already have up-to-date
+// aliases. Then we start updating the mappings sequentially in chunks.
 const addFieldAliasesToIndices = async ({
   esClient,
   index,
@@ -132,14 +142,34 @@ const addFieldAliasesToIndices = async ({
   index: string;
 }) => {
   const { body: indexMappings } = await esClient.indices.get({ index });
+  const indicesByVersion: Record<number, string[]> = {};
+  const versions: Set<number> = new Set();
   for (const [indexName, mapping] of Object.entries(indexMappings)) {
-    const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version');
-    const body = createBackwardsCompatibilityMapping(currentVersion ?? 0);
-    await esClient.indices.putMapping({
-      index: indexName,
-      body,
-      allow_no_indices: true,
-    } as estypes.IndicesPutMappingRequest);
+    const version: number = get(mapping.mappings?._meta, 'version') ?? 0;
+    const aliasesVersion: number = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
+    // Only attempt to add backwards compatibility mappings to indices whose names start with the alias
+    // This limits us to legacy .siem-signals indices, since alerts as data indices use a different naming
+    // scheme (but have the same alias, so will also be returned by the "get" request)
+    if (
+      indexName.startsWith(`${index}-`) &&
+      isOutdated({ current: aliasesVersion, target: SIGNALS_FIELD_ALIASES_VERSION })
+    ) {
+      indicesByVersion[version] = indicesByVersion[version]
+        ? [...indicesByVersion[version], indexName]
+        : [indexName];
+      versions.add(version);
+    }
+  }
+  for (const version of versions) {
+    const body = createBackwardsCompatibilityMapping(version);
+    const indexNameChunks = chunk(indicesByVersion[version], 20);
+    for (const indexNameChunk of indexNameChunks) {
+      await esClient.indices.putMapping({
+        index: indexNameChunk,
+        body,
+        allow_no_indices: true,
+      } as estypes.IndicesPutMappingRequest);
+    }
   }
 };
 
@@ -152,7 +182,7 @@ const addFieldAliasesToIndices = async ({
 //   index: string;
 //   aadIndexAliasName: string;
 // }) => {
-//   const { body: indices } = await esClient.indices.getAlias({ name: index });
+//   const { body: indices } = await esClient.indices.getAlias({ index: `${index}-*`, name: index });
 //   const aliasActions = {
 //     actions: Object.keys(indices).map((concreteIndexName) => {
 //       return {

diff --git a/...ck/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/...ck/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils';
+import { transformError, getBootstrapIndexExists } from '@kbn/securitysolution-es-utils';
 import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
 import { ConfigType } from '../../../../config';
 import type { SecuritySolutionPluginRouter } from '../../../../types';
@@ -41,7 +41,10 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: Con
         const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental);
 
         const index = siemClient.getSignalsIndex();
-        const indexExists = await getIndexExists(esClient, index);
+        const indexExists = await getBootstrapIndexExists(
+          context.core.elasticsearch.client.asInternalUser,
+          index
+        );
 
         if (indexExists) {
           let mappingOutdated: boolean | null = null;

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts
@@ -20,13 +20,74 @@ import { createUserAndRole, deleteUserAndRole } from '../../../common/services/s
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const supertestWithoutAuth = getService('supertestWithoutAuth');
+  const esArchiver = getService('esArchiver');
+  const es = getService('es');
 
   describe('create_index', () => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
     });
 
     describe('elastic admin', () => {
+      describe('with another index that shares index alias', () => {
+        before(async () => {
+          await esArchiver.load('x-pack/test/functional/es_archives/signals/index_alias_clash');
+        });
+
+        after(async () => {
+          await esArchiver.unload('x-pack/test/functional/es_archives/signals/index_alias_clash');
+        });
+
+        it('should report that signals index does not exist', async () => {
+          const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404);
+          expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 });
+        });
+
+        it('should return 200 for create_index', async () => {
+          const { body } = await supertest
+            .post(DETECTION_ENGINE_INDEX_URL)
+            .set('kbn-xsrf', 'true')
+            .send()
+            .expect(200);
+          expect(body).to.eql({ acknowledged: true });
+        });
+      });
+
+      describe('with an outdated signals index', () => {
+        beforeEach(async () => {
+          await esArchiver.load('x-pack/test/functional/es_archives/endpoint/resolver/signals');
+        });
+
+        afterEach(async () => {
+          await esArchiver.unload('x-pack/test/functional/es_archives/endpoint/resolver/signals');
+        });
+
+        it('should report that signals index is outdated', async () => {
+          const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200);
+          expect(body).to.eql({
+            index_mapping_outdated: true,
+            name: `${DEFAULT_SIGNALS_INDEX}-default`,
+          });
+        });
+
+        it('should return 200 for create_index and add field aliases', async () => {
+          const { body } = await supertest
+            .post(DETECTION_ENGINE_INDEX_URL)
+            .set('kbn-xsrf', 'true')
+            .send()
+            .expect(200);
+          expect(body).to.eql({ acknowledged: true });
+
+          const { body: mappings } = await es.indices.get({
+            index: '.siem-signals-default-000001',
+          });
+          // Make sure that aliases_version has been updated on the existing index
+          expect(mappings['.siem-signals-default-000001'].mappings?._meta?.aliases_version).to.eql(
+            1
+          );
+        });
+      });
+
       it('should return a 404 when the signal index has never been created', async () => {
         const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404);
         expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 });

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts
@@ -19,6 +19,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
       loadTestFile(require.resolve('./update_actions'));
       loadTestFile(require.resolve('./add_prepackaged_rules'));
       loadTestFile(require.resolve('./check_privileges'));
+      loadTestFile(require.resolve('./create_index'));
       loadTestFile(require.resolve('./create_rules'));
       loadTestFile(require.resolve('./create_rules_bulk'));
       loadTestFile(require.resolve('./create_index'));

diff --git a/x-pack/test/functional/es_archives/signals/README.md b/x-pack/test/functional/es_archives/signals/README.md
@@ -24,3 +24,7 @@ A legacy signals index. It has no migration metadata fields and a very old mappi
 #### `signals/outdated_signals_index`
 
 A signals index that had previously been updated but is now out of date. It has migration metadata fields and a recent mapping version.
+
+#### `signals/index_alias_clash`
+
+An index that has the .siem-signals alias, but is NOT a signals index. Used for simulating an alerts-as-data index, which will have the .siem-signals alias but different mappings. This way we can test that functionality that needs to target only signals indices (e.g. mapping updates to apply field aliases) work correctly in the presence of alerts-as-data indices.
diff --git a/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json b/x-pack/test/functional/es_archives/signals/index_alias_clash/data.json
@@ -0,0 +1,11 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "signal_name_clash",
+    "source": {
+      "@timestamp": "2020-10-28T05:08:53.000Z"
+    },
+    "type": "_doc"
+  }
+}
diff --git a/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json b/x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json
@@ -0,0 +1,24 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+      ".siem-signals-default": {
+        "is_write_index": false
+      }
+    },
+    "index": "index_alias_clash",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}