ensure migrations dont block the event loop

gmmorris · Jun 18, 2020 · 5655085 · 5655085
1 parent aac18a6
commit 5655085
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 57 deletions.
diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.ts b/src/core/server/saved_objects/migrations/core/index_migrator.ts
@@ -195,7 +195,7 @@ async function migrateSourceToDest(context: Context) {
     await Index.write(
       callCluster,
       dest.indexName,
-      migrateRawDocs(serializer, documentMigrator.migrate, docs, log)
+      await migrateRawDocs(serializer, documentMigrator.migrate, docs, log)
     );
   }
 }
diff --git a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts
@@ -26,7 +26,7 @@ import { createSavedObjectsMigrationLoggerMock } from '../../migrations/mocks';
 describe('migrateRawDocs', () => {
   test('converts raw docs to saved objects', async () => {
     const transform = jest.fn<any, any>((doc: any) => _.set(doc, 'attributes.name', 'HOI!'));
-    const result = migrateRawDocs(
+    const result = await migrateRawDocs(
       new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
       transform,
       [
@@ -55,7 +55,7 @@ describe('migrateRawDocs', () => {
     const transform = jest.fn<any, any>((doc: any) =>
       _.set(_.cloneDeep(doc), 'attributes.name', 'TADA')
     );
-    const result = migrateRawDocs(
+    const result = await migrateRawDocs(
       new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
       transform,
       [

diff --git a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts
@@ -33,26 +33,42 @@ import { SavedObjectsMigrationLogger } from '.';
  * @param {SavedObjectsRawDoc[]} rawDocs
  * @returns {SavedObjectsRawDoc[]}
  */
-export function migrateRawDocs(
+export async function migrateRawDocs(
   serializer: SavedObjectsSerializer,
   migrateDoc: TransformFn,
   rawDocs: SavedObjectsRawDoc[],
   log: SavedObjectsMigrationLogger
 ): SavedObjectsRawDoc[] {
-  return rawDocs.map((raw) => {
+  const migrateDocWithoutBlocking = transformNonBlocking(migrateDoc);
+  const processesDocs = [];
+  for (const raw of rawDocs) {
     if (serializer.isRawSavedObject(raw)) {
       const savedObject = serializer.rawToSavedObject(raw);
       savedObject.migrationVersion = savedObject.migrationVersion || {};
-      return serializer.savedObjectToRaw({
-        references: [],
-        ...migrateDoc(savedObject),
-      });
+      processesDocs.push(
+        serializer.savedObjectToRaw({
+          references: [],
+          ...(await migrateDocWithoutBlocking(savedObject)),
+        })
+      );
+    } else {
+      log.error(
+        `Error: Unable to migrate the corrupt Saved Object document ${raw._id}. To prevent Kibana from performing a migration on every restart, please delete or fix this document by ensuring that the namespace and type in the document's id matches the values in the namespace and type fields.`,
+        { rawDocument: raw }
+      );
+      processesDocs.push(raw);
     }
+  }
+  return processesDocs;
+}
 
-    log.error(
-      `Error: Unable to migrate the corrupt Saved Object document ${raw._id}. To prevent Kibana from performing a migration on every restart, please delete or fix this document by ensuring that the namespace and type in the document's id matches the values in the namespace and type fields.`,
-      { rawDocument: raw }
-    );
-    return raw;
-  });
+function transformNonBlocking(transform: TransformFn): Promise<ReturnType<TransformFn>> {
+  // promises aren't enough to unblock the event loop
+  return (...args) =>
+    new Promise((resolve) => {
+      // set immediate is though
+      setImmediate(() => {
+        resolve(transform(...args));
+      });
+    });
 }
diff --git a/...pted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects/data.json b/...pted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects/data.json
@@ -202,48 +202,6 @@
   }
 }
 
-{
-  "type": "doc",
-  "value": {
-    "id": "alert:74f3e6d7-b7bb-477d-ac28-92ee22728e6e",
-    "index": ".kibana_1",
-    "source": {
-      "alert": {
-        "actions": [
-        ],
-        "alertTypeId": "example.always-firing",
-        "apiKey": "erUMhI0y92leBCopRrSA9Ngm5Rd8RCglWRDcNLD0iw+/rcBLKfL/5CuaUM87OYb3ew/o3jPAIr1fD2El8dW4ZLseRecOGyDC39BKztmXRUtqp+mrNnozJx+kXlDJz/rbDRVAODsX6xI+xKQNLNna7SYn4ybDyZqVL+s6L128ujpEb6HD1HDHBwJguaXwHshGWM2GMpZDee/Swg==",
-        "apiKeyOwner": "elastic",
-        "consumer": "alerts",
-        "createdAt": "2020-06-17T15:35:38.497Z",
-        "createdBy": "elastic",
-        "enabled": true,
-        "muteAll": false,
-        "mutedInstanceIds": [
-        ],
-        "name": "always-firing-alert",
-        "params": {
-        },
-        "schedule": {
-          "interval": "1m"
-        },
-        "scheduledTaskId": "329798f0-b0b0-11ea-9510-fdf248d5f2a4",
-        "tags": [
-        ],
-        "throttle": null,
-        "updatedBy": "elastic"
-      },
-      "migrationVersion": {
-        "alert": "7.9.0"
-      },
-      "references": [
-      ],
-      "type": "alert",
-      "updated_at": "2020-06-17T15:35:39.839Z"
-    }
-  }
-}
-
 {
   "type": "doc",
   "value": {