[Security Solution] Only apply field aliases to legacy .siem-signals indices

packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * This function is similar to getIndexExists, but is limited to searching indices that match
+ * the index pattern used as concrete backing indices (e.g. .siem-signals-default-000001).
+ * This allows us to separate the indices that are actually .siem-signals indices from
+ * alerts as data indices that only share the .siem-signals alias.
+ *
+ * @param esClient Elasticsearch client to use to make the request
+ * @param index Index alias name to check for existence
+ */
+export const getBootstrapIndexExists = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<boolean> => {
+  try {
+    const { body } = await esClient.indices.getAlias({
+      index: `${index}-*`,
+      name: index,
+    });
+    return Object.keys(body).length > 0;
+  } catch (err) {
+    if (err.body != null && err.body.status === 404) {
+      return false;
+    } else {
+      throw err.body ? err.body : err;
+    }
+  }
+};

packages/kbn-securitysolution-es-utils/src/index.ts

@@ -13,6 +13,7 @@ export * from './delete_all_index';
 export * from './delete_policy';
 export * from './delete_template';
 export * from './encode_hit_version';
+export * from './get_bootstrap_index_exists';
 export * from './get_index_aliases';
 export * from './get_index_count';
 export * from './get_index_exists';

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts

@@ -43,10 +43,12 @@ export const templateNeedsUpdate = async ({
 
 export const fieldAliasesOutdated = async (esClient: ElasticsearchClient, index: string) => {
   const { body: indexMappings } = await esClient.indices.get({ index });
-  for (const [_, mapping] of Object.entries(indexMappings)) {
-    const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
-    if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
-      return true;
+  for (const [indexName, mapping] of Object.entries(indexMappings)) {
+    if (indexName.startsWith(`${index}-`)) {
+      const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
+      if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
+        return true;
+      }
     }
   }
   return false;

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts

@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { get } from 'lodash';
+import { chunk, get } from 'lodash';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ElasticsearchClient } from 'src/core/server';
 import {
   transformError,
-  getIndexExists,
+  getBootstrapIndexExists,
   getPolicyExists,
   setPolicy,
   createBootstrapIndex,
@@ -25,6 +25,8 @@ import {
   getSignalsTemplate,
   SIGNALS_TEMPLATE_VERSION,
   createBackwardsCompatibilityMapping,
+  ALIAS_VERSION_FIELD,
+  SIGNALS_FIELD_ALIASES_VERSION,
 } from './get_signals_template';
 import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup';
 import signalsPolicy from './signals_policy.json';
@@ -71,7 +73,10 @@ export const createDetectionIndex = async (
   const spaceId = context.getSpaceId();
   const index = siemClient.getSignalsIndex();
 
-  const indexExists = await getIndexExists(esClient, index);
+  const indexExists = await getBootstrapIndexExists(
+    context.core.elasticsearch.client.asInternalUser,
+    index
+  );
   const { ruleRegistryEnabled } = config.experimentalFeatures;
 
   // If using the rule registry implementation, we don't want to create new .siem-signals indices -
@@ -124,6 +129,11 @@ export const createDetectionIndex = async (
   }
 };
 
+// This function can be expensive if there are lots of existing .siem-signals indices
+// because any new backwards compatibility mappings need to be applied to all of them
+// while also preserving the original 'version' of the mapping. To do it somewhat efficiently,
+// we first group the indices by version and exclude any that already have up-to-date
+// aliases. Then we start updating the mappings sequentially in chunks.
 const addFieldAliasesToIndices = async ({
   esClient,
   index,
@@ -132,14 +142,34 @@ const addFieldAliasesToIndices = async ({
   index: string;
 }) => {
   const { body: indexMappings } = await esClient.indices.get({ index });
+  const indicesByVersion: Record<number, string[]> = {};
+  const versions: Set<number> = new Set();
   for (const [indexName, mapping] of Object.entries(indexMappings)) {
-    const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version');
-    const body = createBackwardsCompatibilityMapping(currentVersion ?? 0);
-    await esClient.indices.putMapping({
-      index: indexName,
-      body,
-      allow_no_indices: true,
-    } as estypes.IndicesPutMappingRequest);
+    const version: number = get(mapping.mappings?._meta, 'version') ?? 0;
+    const aliasesVersion: number = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
+    // Only attempt to add backwards compatibility mappings to indices whose names start with the alias
+    // This limits us to legacy .siem-signals indices, since alerts as data indices use a different naming
+    // scheme (but have the same alias, so will also be returned by the "get" request)
+    if (
+      indexName.startsWith(`${index}-`) &&
+      isOutdated({ current: aliasesVersion, target: SIGNALS_FIELD_ALIASES_VERSION })
+    ) {
+      indicesByVersion[version] = indicesByVersion[version]
+        ? [...indicesByVersion[version], indexName]
+        : [indexName];
+      versions.add(version);
+    }
+  }
+  for (const version of versions) {
+    const body = createBackwardsCompatibilityMapping(version);
+    const indexNameChunks = chunk(indicesByVersion[version], 20);
+    for (const indexNameChunk of indexNameChunks) {
+      await esClient.indices.putMapping({
+        index: indexNameChunk,
+        body,
+        allow_no_indices: true,
+      } as estypes.IndicesPutMappingRequest);
+    }
   }
 };
 
@@ -152,7 +182,7 @@ const addIndexAliases = async ({
   index: string;
   aadIndexAliasName: string;
 }) => {
-  const { body: indices } = await esClient.indices.getAlias({ name: index });
+  const { body: indices } = await esClient.indices.getAlias({ index: `${index}-*`, name: index });
   const aliasActions = {
     actions: Object.keys(indices).map((concreteIndexName) => {
       return {

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts

@@ -5,12 +5,16 @@
  * 2.0.
  */
 
-import { transformError } from '@kbn/securitysolution-es-utils';
+import { transformError, getBootstrapIndexExists } from '@kbn/securitysolution-es-utils';
 import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants';
 
 import { buildSiemResponse } from '../utils';
 import { RuleDataPluginService } from '../../../../../../rule_registry/server';
+import { fieldAliasesOutdated } from './check_template_version';
+import { getIndexVersion } from './get_index_version';
+import { isOutdated } from '../../migrations/helpers';
+import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template';
 
 export const readIndexRoute = (
   router: SecuritySolutionPluginRouter,
@@ -29,6 +33,7 @@ export const readIndexRoute = (
 
       try {
         const siemClient = context.securitySolution?.getAppClient();
+        const esClient = context.core.elasticsearch.client.asCurrentUser;
 
         if (!siemClient) {
           return siemResponse.error({ statusCode: 404 });
@@ -37,12 +42,47 @@ export const readIndexRoute = (
         const spaceId = context.securitySolution.getSpaceId();
         const indexName = ruleDataService.getResourceName(`security.alerts-${spaceId}`);
 
-        return response.ok({
-          body: {
-            name: indexName,
-            index_mapping_outdated: false,
-          },
-        });
+        const index = siemClient.getSignalsIndex();
+        const indexExists = await getBootstrapIndexExists(
+          context.core.elasticsearch.client.asInternalUser,
+          index
+        );
+
+        if (indexExists) {
+          let mappingOutdated: boolean | null = null;
+          let aliasesOutdated: boolean | null = null;
+          try {
+            const indexVersion = await getIndexVersion(esClient, index);
+            mappingOutdated = isOutdated({
+              current: indexVersion,
+              target: SIGNALS_TEMPLATE_VERSION,
+            });
+            aliasesOutdated = await fieldAliasesOutdated(esClient, index);
+          } catch (err) {
+            const error = transformError(err);
+            // Some users may not have the view_index_metadata permission necessary to check the index mapping version
+            // so just continue and return null for index_mapping_outdated if the error is a 403
+            if (error.statusCode !== 403) {
+              return siemResponse.error({
+                body: error.message,
+                statusCode: error.statusCode,
+              });
+            }
+          }
+          return response.ok({
+            body: {
+              name: indexName,
+              index_mapping_outdated: mappingOutdated || aliasesOutdated,
+            },
+          });
+        } else {
+          return response.ok({
+            body: {
+              name: indexName,
+              index_mapping_outdated: false,
+            },
+          });
+        }
       } catch (err) {
         const error = transformError(err);
         return siemResponse.error({

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import {
+  DEFAULT_ALERTS_INDEX,
+  DETECTION_ENGINE_INDEX_URL,
+} from '../../../../plugins/security_solution/common/constants';
+
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { deleteSignalsIndex } from '../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext) => {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+  const es = getService('es');
+
+  describe('create_index', () => {
+    afterEach(async () => {
+      await deleteSignalsIndex(supertest);
+    });
+
+    describe('elastic admin', () => {
+      describe('with another index that shares index alias', () => {
+        before(async () => {
+          await esArchiver.load('x-pack/test/functional/es_archives/signals/index_alias_clash');
+        });
+
+        after(async () => {
+          await esArchiver.unload('x-pack/test/functional/es_archives/signals/index_alias_clash');
+        });
+
+        it.skip('should report that signals index does not exist', async () => {
+          const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404);
+          expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 });
+        });
+
+        it('should return 200 for create_index', async () => {
+          const { body } = await supertest
+            .post(DETECTION_ENGINE_INDEX_URL)
+            .set('kbn-xsrf', 'true')
+            .send()
+            .expect(200);
+          expect(body).to.eql({ acknowledged: true });
+        });
+      });
+
+      describe('with an outdated signals index', () => {
+        beforeEach(async () => {
+          await esArchiver.load('x-pack/test/functional/es_archives/endpoint/resolver/signals');
+        });
+
+        afterEach(async () => {
+          await esArchiver.unload('x-pack/test/functional/es_archives/endpoint/resolver/signals');
+        });
+
+        it('should report that signals index is outdated', async () => {
+          const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200);
+          expect(body).to.eql({
+            index_mapping_outdated: true,
+            name: `${DEFAULT_ALERTS_INDEX}-default`,
+          });
+        });
+
+        it('should return 200 for create_index and add field aliases', async () => {
+          const { body } = await supertest
+            .post(DETECTION_ENGINE_INDEX_URL)
+            .set('kbn-xsrf', 'true')
+            .send()
+            .expect(200);
+          expect(body).to.eql({ acknowledged: true });
+
+          const mappings = await es.indices.get({
+            index: '.siem-signals-default-000001',
+          });
+          // Make sure that aliases_version has been updated on the existing index
+          expect(mappings['.siem-signals-default-000001'].mappings?._meta?.aliases_version).to.eql(
+            1
+          );
+        });
+      });
+    });
+  });
+};

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts

@@ -19,6 +19,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
       loadTestFile(require.resolve('./update_actions'));
       loadTestFile(require.resolve('./add_prepackaged_rules'));
       loadTestFile(require.resolve('./check_privileges'));
+      loadTestFile(require.resolve('./create_index'));
       loadTestFile(require.resolve('./create_rules'));
       loadTestFile(require.resolve('./create_rules_bulk'));
       loadTestFile(require.resolve('./create_ml'));

x-pack/test/functional/es_archives/signals/README.md

@@ -24,3 +24,7 @@ A legacy signals index. It has no migration metadata fields and a very old mappi
 #### `signals/outdated_signals_index`
 
 A signals index that had previously been updated but is now out of date. It has migration metadata fields and a recent mapping version.
+
+#### `signals/index_alias_clash`
+
+An index that has the .siem-signals alias, but is NOT a signals index. Used for simulating an alerts-as-data index, which will have the .siem-signals alias but different mappings. This way we can test that functionality that needs to target only signals indices (e.g. mapping updates to apply field aliases) work correctly in the presence of alerts-as-data indices.

x-pack/test/functional/es_archives/signals/index_alias_clash/data.json

@@ -0,0 +1,11 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "signal_name_clash",
+    "source": {
+      "@timestamp": "2020-10-28T05:08:53.000Z"
+    },
+    "type": "_doc"
+  }
+}

x-pack/test/functional/es_archives/signals/index_alias_clash/mappings.json

@@ -0,0 +1,24 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+      ".siem-signals-default": {
+        "is_write_index": false
+      }
+    },
+    "index": "index_alias_clash",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}