[Security Solution] Only apply field aliases to legacy .siem-signals indices

packages/kbn-securitysolution-es-utils/src/get_bootstrap_index_exists/index.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * This function is similar to getIndexExists, but is limited to searching indices that match
+ * the index pattern used as concrete backing indices (e.g. .siem-signals-default-000001).
+ * This allows us to separate the indices that are actually .siem-signals indices from
+ * alerts as data indices that only share the .siem-signals alias.
+ *
+ * @param esClient Elasticsearch client to use to make the request
+ * @param index Index alias name to check for existence
+ */
+export const getBootstrapIndexExists = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<boolean> => {
+  try {
+    const { body } = await esClient.indices.getAlias({
+      index: `${index}-*`,
+      name: index,
+    });
+    return Object.keys(body).length > 0;
+  } catch (err) {
+    if (err.body != null && err.body.status === 404) {
+      return false;
+    } else {
+      throw err.body ? err.body : err;
+    }
+  }
+};

packages/kbn-securitysolution-es-utils/src/index.ts

@@ -13,6 +13,7 @@ export * from './delete_all_index';
 export * from './delete_policy';
 export * from './delete_template';
 export * from './encode_hit_version';
+export * from './get_bootstrap_index_exists';
 export * from './get_index_aliases';
 export * from './get_index_count';
 export * from './get_index_exists';

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts

@@ -43,10 +43,12 @@ export const templateNeedsUpdate = async ({
 
 export const fieldAliasesOutdated = async (esClient: ElasticsearchClient, index: string) => {
   const { body: indexMappings } = await esClient.indices.get({ index });
-  for (const [_, mapping] of Object.entries(indexMappings)) {
-    const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
-    if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
-      return true;
+  for (const [indexName, mapping] of Object.entries(indexMappings)) {
+    if (indexName.startsWith(`${index}-`)) {
+      const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
+      if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
+        return true;
+      }
     }
   }
   return false;

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts

@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { get } from 'lodash';
+import { chunk, get } from 'lodash';
 import { estypes } from '@elastic/elasticsearch';
 import { ElasticsearchClient } from 'src/core/server';
 import {
   transformError,
-  getIndexExists,
+  getBootstrapIndexExists,
   getPolicyExists,
   setPolicy,
   createBootstrapIndex,
@@ -26,6 +26,8 @@ import {
   getSignalsTemplate,
   SIGNALS_TEMPLATE_VERSION,
   createBackwardsCompatibilityMapping,
+  ALIAS_VERSION_FIELD,
+  SIGNALS_FIELD_ALIASES_VERSION,
 } from './get_signals_template';
 import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup';
 import signalsPolicy from './signals_policy.json';
@@ -94,7 +96,10 @@ export const createDetectionIndex = async (
 
   const index = siemClient.getSignalsIndex();
 
-  const indexExists = await getIndexExists(esClient, index);
+  const indexExists = await getBootstrapIndexExists(
+    context.core.elasticsearch.client.asInternalUser,
+    index
+  );
   // If using the rule registry implementation, we don't want to create new .siem-signals indices -
   // only create/update resources if there are existing indices
   if (ruleRegistryEnabled && !indexExists) {
@@ -142,6 +147,11 @@ export const createDetectionIndex = async (
   }
 };
 
+// This function can be expensive if there are lots of existing .siem-signals indices
+// because any new backwards compatibility mappings need to be applied to all of them
+// while also preserving the original 'version' of the mapping. To do it somewhat efficiently,
+// we first group the indices by version and exclude any that already have up-to-date
+// aliases. Then we start updating the mappings sequentially in chunks.
 const addFieldAliasesToIndices = async ({
   esClient,
   index,
@@ -150,14 +160,34 @@ const addFieldAliasesToIndices = async ({
   index: string;
 }) => {
   const { body: indexMappings } = await esClient.indices.get({ index });
+  const indicesByVersion: Record<number, string[]> = {};
+  const versions: Set<number> = new Set();
   for (const [indexName, mapping] of Object.entries(indexMappings)) {
-    const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version');
-    const body = createBackwardsCompatibilityMapping(currentVersion ?? 0);
-    await esClient.indices.putMapping({
-      index: indexName,
-      body,
-      allow_no_indices: true,
-    } as estypes.IndicesPutMappingRequest);
+    const version: number = get(mapping.mappings?._meta, 'version') ?? 0;
+    const aliasesVersion: number = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
+    // Only attempt to add backwards compatibility mappings to indices whose names start with the alias
+    // This limits us to legacy .siem-signals indices, since alerts as data indices use a different naming
+    // scheme (but have the same alias, so will also be returned by the "get" request)
+    if (
+      indexName.startsWith(`${index}-`) &&
+      isOutdated({ current: aliasesVersion, target: SIGNALS_FIELD_ALIASES_VERSION })
+    ) {
+      indicesByVersion[version] = indicesByVersion[version]
+        ? [...indicesByVersion[version], indexName]
+        : [indexName];
+      versions.add(version);
+    }
+  }
+  for (const version of versions) {
+    const body = createBackwardsCompatibilityMapping(version);
+    const indexNameChunks = chunk(indicesByVersion[version], 20);
+    for (const indexNameChunk of indexNameChunks) {
+      await esClient.indices.putMapping({
+        index: indexNameChunk,
+        body,
+        allow_no_indices: true,
+      } as estypes.IndicesPutMappingRequest);
+    }
   }
 };
 
@@ -170,7 +200,7 @@ const addFieldAliasesToIndices = async ({
 //   index: string;
 //   aadIndexAliasName: string;
 // }) => {
-//   const { body: indices } = await esClient.indices.getAlias({ name: index });
+//   const { body: indices } = await esClient.indices.getAlias({ index: `${index}-*`, name: index });
 //   const aliasActions = {
 //     actions: Object.keys(indices).map((concreteIndexName) => {
 //       return {

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/delete_index_route.ts

@@ -7,7 +7,7 @@
 
 import {
   transformError,
-  getIndexExists,
+  getBootstrapIndexExists,
   getPolicyExists,
   deletePolicy,
   deleteAllIndex,
@@ -49,7 +49,10 @@ export const deleteIndexRoute = (router: SecuritySolutionPluginRouter) => {
         }
 
         const index = siemClient.getSignalsIndex();
-        const indexExists = await getIndexExists(esClient, index);
+        const indexExists = await getBootstrapIndexExists(
+          context.core.elasticsearch.client.asInternalUser,
+          index
+        );
 
         if (!indexExists) {
           return siemResponse.error({

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils';
+import { transformError, getBootstrapIndexExists } from '@kbn/securitysolution-es-utils';
 import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
 import { ConfigType } from '../../../../config';
 import type { SecuritySolutionPluginRouter } from '../../../../types';
@@ -41,7 +41,10 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: Con
         const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental);
 
         const index = siemClient.getSignalsIndex();
-        const indexExists = await getIndexExists(esClient, index);
+        const indexExists = await getBootstrapIndexExists(
+          context.core.elasticsearch.client.asInternalUser,
+          index
+        );
 
         if (indexExists) {
           let mappingOutdated: boolean | null = null;