[7.17] Migrations: Don't auto-create temp index (#158182) (#188646)

# Backport This will backport the following commits from `main` to `7.17`: - [Migrations: Don't auto-create temp index (#158182)](#158182)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\nCo-authored-by: Rudolf Meijering <[email protected]>"}},{"branch":"8.16","label":"v8.16.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT-->
elastic · Jul 22, 2024 · 0869061 · 0869061
1 parent 584b29a
commit 0869061
Show file tree

Hide file tree

Showing 11 changed files with 67 additions and 11 deletions.
diff --git a/...ver/saved_objects/migrationsv2/__snapshots__/migrations_state_action_machine.test.ts.snap b/...ver/saved_objects/migrationsv2/__snapshots__/migrations_state_action_machine.test.ts.snap
diff --git a/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts b/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts
@@ -50,6 +50,12 @@ export interface BulkOverwriteTransformedDocumentsParams {
   index: string;
   transformedDocs: SavedObjectsRawDoc[];
   refresh?: estypes.Refresh;
+  /**
+   * If true, we prevent Elasticsearch from auto-creating the index if it
+   * doesn't exist. We use the ES paramater require_alias: true so `index`
+   * must be an alias, otherwise the bulk index will fail.
+   */
+  useAliasToPreventAutoCreate?: boolean;
 }
 
 /**
@@ -62,6 +68,7 @@ export const bulkOverwriteTransformedDocuments =
     index,
     transformedDocs,
     refresh = false,
+    useAliasToPreventAutoCreate = false,
   }: BulkOverwriteTransformedDocumentsParams): TaskEither.TaskEither<
     | RetryableEsClientError
     | TargetIndexHadWriteBlock
@@ -83,7 +90,7 @@ export const bulkOverwriteTransformedDocuments =
         // mappings. Such tampering could lead to many other problems and is
         // probably unlikely so for now we'll accept this risk and wait till
         // system indices puts in place a hard control.
-        require_alias: false,
+        require_alias: useAliasToPreventAutoCreate,
         wait_for_active_shards: WAIT_FOR_ALL_SHARDS_TO_BE_ACTIVE,
         refresh,
         filter_path: ['items.*.error'],

diff --git a/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts
@@ -70,6 +70,7 @@ describe('migration actions', () => {
     await createIndex({
       client,
       indexName: 'existing_index_with_docs',
+      aliases: ['existing_index_with_docs_alias'],
       mappings: {
         dynamic: true,
         properties: {},
@@ -133,7 +134,9 @@ describe('migration actions', () => {
       expect(res.right).toEqual(
         expect.objectContaining({
           existing_index_with_docs: {
-            aliases: {},
+            aliases: {
+              existing_index_with_docs_alias: {},
+            },
             mappings: expect.anything(),
             settings: expect.anything(),
           },
@@ -1519,6 +1522,30 @@ describe('migration actions', () => {
                 }
               `);
     });
+    it('resolves left index_not_found_exception if the index does not exist and useAliasToPreventAutoCreate=true', async () => {
+      const newDocs = [
+        { _source: { title: 'doc 5' } },
+        { _source: { title: 'doc 6' } },
+        { _source: { title: 'doc 7' } },
+      ] as unknown as SavedObjectsRawDoc[];
+      await expect(
+        bulkOverwriteTransformedDocuments({
+          client,
+          index: 'existing_index_with_docs_alias_that_does_not_exist',
+          useAliasToPreventAutoCreate: true,
+          transformedDocs: newDocs,
+          refresh: 'wait_for',
+        })()
+      ).resolves.toMatchInlineSnapshot(`
+          Object {
+            "_tag": "Left",
+            "left": Object {
+              "index": "existing_index_with_docs_alias_that_does_not_exist",
+              "type": "index_not_found_exception",
+            },
+          }
+      `);
+    });
     it('resolves left target_index_had_write_block if there are write_block errors', async () => {
       const newDocs = [
         { _source: { title: 'doc 5' } },

diff --git a/src/core/server/saved_objects/migrationsv2/initial_state.test.ts b/src/core/server/saved_objects/migrationsv2/initial_state.test.ts
@@ -72,6 +72,7 @@ describe('createInitialState', () => {
         },
       },
       tempIndex: '.kibana_task_manager_8.1.0_reindex_temp',
+      tempIndexAlias: '.kibana_task_manager_8.1.0_reindex_temp_alias',
       tempIndexMappings: {
         dynamic: false,
         properties: {

diff --git a/src/core/server/saved_objects/migrationsv2/initial_state.ts b/src/core/server/saved_objects/migrationsv2/initial_state.ts
@@ -73,6 +73,7 @@ export const createInitialState = ({
     versionAlias: `${indexPrefix}_${kibanaVersion}`,
     versionIndex: `${indexPrefix}_${kibanaVersion}_001`,
     tempIndex: `${indexPrefix}_${kibanaVersion}_reindex_temp`,
+    tempIndexAlias: `${indexPrefix}_${kibanaVersion}_reindex_temp_alias`,
     kibanaVersion,
     preMigrationScript: Option.fromNullable(preMigrationScript),
     targetIndexMappings: targetMappings,

diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts
@@ -43,7 +43,7 @@ describe('migration v2', function () {
         es: {
           license: 'basic',
           dataArchive: Path.join(__dirname, 'archives', '7.14.0_xpack_sample_saved_objects.zip'),
-          esArgs: ['http.max_content_length=1715277b'],
+          esArgs: ['http.max_content_length=1715329b'],
         },
       },
     }));
@@ -61,7 +61,7 @@ describe('migration v2', function () {
   });
 
   it('completes the migration even when a full batch would exceed ES http.max_content_length', async () => {
-    root = createRoot({ maxBatchSizeBytes: 1715277 });
+    root = createRoot({ maxBatchSizeBytes: 1715329 });
     esServer = await startES();
     await root.preboot();
     await root.setup();
@@ -90,7 +90,7 @@ describe('migration v2', function () {
     await root.preboot();
     await root.setup();
     await expect(root.start()).rejects.toMatchInlineSnapshot(
-      `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715277 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
+      `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715283 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
     );
 
     await retryAsync(
@@ -103,7 +103,7 @@ describe('migration v2', function () {
         expect(
           records.find((rec) =>
             rec.message.startsWith(
-              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715277 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
+              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715283 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
             )
           )
         ).toBeDefined();

diff --git a/...objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts b/...objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts
@@ -54,7 +54,7 @@ describe('migration v2', () => {
   });
 
   it('fails with a descriptive message when maxBatchSizeBytes exceeds ES http.max_content_length', async () => {
-    root = createRoot({ maxBatchSizeBytes: 1715277 });
+    root = createRoot({ maxBatchSizeBytes: 1715329 });
     esServer = await startES();
     await root.preboot();
     await root.setup();

diff --git a/src/core/server/saved_objects/migrationsv2/model/model.test.ts b/src/core/server/saved_objects/migrationsv2/model/model.test.ts
@@ -81,6 +81,7 @@ describe('migrations v2 model', () => {
     versionAlias: '.kibana_7.11.0',
     versionIndex: '.kibana_7.11.0_001',
     tempIndex: '.kibana_7.11.0_reindex_temp',
+    tempIndexAlias: '.kibana_7.11.0_reindex_temp_alias',
     unusedTypesQuery: {
       bool: {
         must_not: [

diff --git a/src/core/server/saved_objects/migrationsv2/model/model.ts b/src/core/server/saved_objects/migrationsv2/model/model.ts
@@ -504,7 +504,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
       if (stateP.corruptDocumentIds.length === 0 && stateP.transformErrors.length === 0) {
         const batches = createBatches(
           res.right.processedDocs,
-          stateP.tempIndex,
+          stateP.tempIndexAlias,
           stateP.maxBatchSizeBytes
         );
         if (Either.isRight(batches)) {

diff --git a/src/core/server/saved_objects/migrationsv2/next.ts b/src/core/server/saved_objects/migrationsv2/next.ts
@@ -87,6 +87,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
       Actions.createIndex({
         client,
         indexName: state.tempIndex,
+        aliases: [state.tempIndexAlias],
         mappings: state.tempIndexMappings,
       }),
     REINDEX_SOURCE_TO_TEMP_OPEN_PIT: (state: ReindexSourceToTempOpenPit) =>
@@ -110,8 +111,14 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
     REINDEX_SOURCE_TO_TEMP_INDEX_BULK: (state: ReindexSourceToTempIndexBulk) =>
       Actions.bulkOverwriteTransformedDocuments({
         client,
-        index: state.tempIndex,
+        index: state.tempIndexAlias,
         transformedDocs: state.transformedDocBatches[state.currentBatch],
+        /*
+         * Since other nodes can delete the temp index while we're busy writing
+         * to it, we use the alias to prevent the auto-creation of the index if
+         * it doesn't exist.
+         */
+        useAliasToPreventAutoCreate: true,
         /**
          * Since we don't run a search against the target index, we disable "refresh" to speed up
          * the migration process.

diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts
@@ -117,10 +117,16 @@ export interface BaseState extends ControlState {
    */
   readonly versionIndex: string;
   /**
-   * An alias on the target index used as part of an "reindex block" that
-   * prevents lost deletes e.g. `.kibana_7.11.0_reindex`.
+   * A temporary index used as part of an "reindex block" that
+   * prevents lost deletes e.g. `.kibana_7.11.0_reindex_temp`.
    */
   readonly tempIndex: string;
+  /**
+   * An alias to the tempIndex used to prevent ES from auto-creating the temp
+   * index if one node deletes it while another writes to it
+   * e.g. `.kibana_7.11.0_reindex_temp_alias`.
+   */
+  readonly tempIndexAlias: string;
   /**
    * When reindexing we use a source query to exclude saved objects types which
    * are no longer used. These saved objects will still be kept in the outdated