Adapt es-archiver to call _migrate endpoint instead of creating a migrator

packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts

@@ -57,9 +57,28 @@ interface UpdateOptions<Attributes> extends IndexOptions<Attributes> {
   id: string;
 }
 
+interface MigrateResponse {
+  success: boolean;
+  result: Array<{ status: string }>;
+}
+
 export class KbnClientSavedObjects {
   constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {}
 
+  /**
+   * Run the saved objects migration
+   */
+  public async migrate() {
+    this.log.debug('Migrating saved objects');
+
+    return await this.requester.request<MigrateResponse>({
+      description: 'migrate saved objects',
+      path: uriencode`/api/saved_objects/_migrate`,
+      method: 'POST',
+      body: {},
+    });
+  }
+
   /**
    * Get an object
    */

packages/kbn-pm/dist/index.js

@@ -43781,6 +43781,18 @@ class KbnClientSavedObjects {
         this.log = log;
         this.requester = requester;
     }
+    /**
+     * Run the saved objects migration
+     */
+    async migrate() {
+        this.log.debug('Migrating saved objects');
+        return await this.requester.request({
+            description: 'migrate saved objects',
+            path: kbn_client_requester_1.uriencode `/api/saved_objects/_migrate`,
+            method: 'POST',
+            body: {},
+        });
+    }
     /**
      * Get an object
      */

src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts

@@ -95,8 +95,10 @@ export class KibanaMigrator {
    *    The promise resolves with an array of migration statuses, one for each
    *    elasticsearch index which was migrated.
    */
-  public runMigrations(): Promise<Array<{ status: string }>> {
-    if (this.migrationResult === undefined) {
+  public runMigrations({ rerun = false }: { rerun?: boolean } = {}): Promise<
+    Array<{ status: string }>
+  > {
+    if (this.migrationResult === undefined || rerun) {
       this.migrationResult = this.runMigrationsInternal();
     }
 

src/core/server/saved_objects/routes/index.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { InternalHttpServiceSetup } from '../../http';
+import { IKibanaMigrator } from '../migrations';
+import { registerMigrateRoute } from './migrate';
+
+interface RegisterRouteOptions {
+  http: InternalHttpServiceSetup;
+  migratorPromise: Promise<IKibanaMigrator>;
+}
+
+export function registerRoutes({ http, migratorPromise }: RegisterRouteOptions) {
+  const router = http.createRouter('');
+
+  registerMigrateRoute(router, migratorPromise);
+}

src/core/server/saved_objects/routes/migrate.ts

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { IRouter } from '../../http';
+import { IKibanaMigrator } from '../migrations';
+
+export const registerMigrateRoute = (
+  router: IRouter,
+  migratorPromise: Promise<IKibanaMigrator>
+) => {
+  router.post(
+    {
+      path: '/api/saved_objects/_migrate',
+      validate: false,
+    },
+    async (context, req, res) => {
+      const migrator = await migratorPromise;
+      const result = await migrator.runMigrations({ rerun: true });
+      return res.ok({
+        body: {
+          success: true,
+          result,
+        },
+      });
+    }
+  );
+};

src/core/server/saved_objects/saved_objects_service.test.ts

@@ -31,6 +31,7 @@ import { Env } from '../config';
 import { configServiceMock } from '../mocks';
 import { elasticsearchServiceMock } from '../elasticsearch/elasticsearch_service.mock';
 import { legacyServiceMock } from '../legacy/legacy_service.mock';
+import { httpServiceMock } from '../http/http_service.mock';
 import { SavedObjectsClientFactoryProvider } from './service/lib';
 import { BehaviorSubject } from 'rxjs';
 import { NodesVersionCompatibility } from '../elasticsearch/version_check/ensure_es_version';
@@ -39,6 +40,7 @@ describe('SavedObjectsService', () => {
   const createSetupDeps = () => {
     const elasticsearchMock = elasticsearchServiceMock.createInternalSetup();
     return {
+      http: httpServiceMock.createSetupContract(),
       elasticsearch: elasticsearchMock,
       legacyPlugins: legacyServiceMock.createDiscoverPlugins(),
     };

src/core/server/saved_objects/saved_objects_service.ts

@@ -17,8 +17,9 @@
  * under the License.
  */
 
-import { CoreService } from 'src/core/types';
+import { Subject } from 'rxjs';
 import { first, filter, take } from 'rxjs/operators';
+import { CoreService } from '../../types';
 import {
   SavedObjectsClient,
   SavedObjectsClientProvider,
@@ -32,7 +33,7 @@ import { InternalElasticsearchServiceSetup, APICaller } from '../elasticsearch';
 import { KibanaConfigType } from '../kibana_config';
 import { migrationsRetryCallCluster } from '../elasticsearch/retry_call_cluster';
 import { SavedObjectsConfigType } from './saved_objects_config';
-import { KibanaRequest } from '../http';
+import { InternalHttpServiceSetup, KibanaRequest } from '../http';
 import { SavedObjectsClientContract, SavedObjectsType } from './types';
 import { ISavedObjectsRepository, SavedObjectsRepository } from './service/lib/repository';
 import {
@@ -44,6 +45,7 @@ import { convertLegacyTypes } from './utils';
 import { SavedObjectTypeRegistry, ISavedObjectTypeRegistry } from './saved_objects_type_registry';
 import { PropertyValidators } from './validation';
 import { SavedObjectsSerializer } from './serialization';
+import { registerRoutes } from './routes';
 
 /**
  * Saved Objects is Kibana's data persistence mechanism allowing plugins to
@@ -196,6 +198,7 @@ export interface SavedObjectsRepositoryFactory {
 
 /** @internal */
 export interface SavedObjectsSetupDeps {
+  http: InternalHttpServiceSetup;
   legacyPlugins: LegacyServiceDiscoverPlugins;
   elasticsearch: InternalElasticsearchServiceSetup;
 }
@@ -218,6 +221,7 @@ export class SavedObjectsService
   private clientFactoryProvider?: SavedObjectsClientFactoryProvider;
   private clientFactoryWrappers: WrappedClientFactoryWrapper[] = [];
 
+  private migrator$ = new Subject<KibanaMigrator>();
   private typeRegistry = new SavedObjectTypeRegistry();
   private validations: PropertyValidators = {};
 
@@ -237,6 +241,11 @@ export class SavedObjectsService
     legacyTypes.forEach(type => this.typeRegistry.registerType(type));
     this.validations = setupDeps.legacyPlugins.uiExports.savedObjectValidations || {};
 
+    registerRoutes({
+      http: setupDeps.http,
+      migratorPromise: this.migrator$.pipe(first()).toPromise(),
+    });
+
     return {
       setClientFactoryProvider: provider => {
         if (this.clientFactoryProvider) {
@@ -278,6 +287,8 @@ export class SavedObjectsService
     const adminClient = this.setupDeps!.elasticsearch.adminClient;
     const migrator = this.createMigrator(kibanaConfig, savedObjectsConfig, migrationsRetryDelay);
 
+    this.migrator$.next(migrator);
+
     /**
      * Note: We want to ensure that migrations have completed before
      * continuing with further Core start steps that might use SavedObjects

src/core/server/server.ts

@@ -132,6 +132,7 @@ export class Server {
     });
 
     const savedObjectsSetup = await this.savedObjects.setup({
+      http: httpSetup,
       elasticsearch: elasticsearchServiceSetup,
       legacyPlugins,
     });

src/es_archiver/actions/empty_kibana_index.ts

@@ -32,9 +32,8 @@ export async function emptyKibanaIndexAction({
   kbnClient: KbnClient;
 }) {
   const stats = createStats('emptyKibanaIndex', log);
-  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
   await deleteKibanaIndices({ client, stats, log });
-  await migrateKibanaIndex({ client, log, kibanaPluginIds });
+  await migrateKibanaIndex({ client, kbnClient });
   return stats;
 }

src/es_archiver/actions/load.ts

@@ -106,7 +106,7 @@ export async function loadAction({
 
   // If we affected the Kibana index, we need to ensure it's migrated...
   if (Object.keys(result).some(k => k.startsWith('.kibana'))) {
-    await migrateKibanaIndex({ client, log, kibanaPluginIds });
+    await migrateKibanaIndex({ client, kbnClient });
 
     if (kibanaPluginIds.includes('spaces')) {
       await createDefaultSpace({ client, index: '.kibana' });

src/es_archiver/lib/indices/kibana_index.ts

@@ -17,42 +17,10 @@
  * under the License.
  */
 
-import { get } from 'lodash';
-import fs from 'fs';
-import Path from 'path';
-import { promisify } from 'util';
-import { toArray } from 'rxjs/operators';
 import { Client, CreateDocumentParams } from 'elasticsearch';
-import { ToolingLog } from '@kbn/dev-utils';
-
+import { ToolingLog, KbnClient } from '@kbn/dev-utils';
 import { Stats } from '../stats';
 import { deleteIndex } from './delete_index';
-import { KibanaMigrator } from '../../../core/server/saved_objects/migrations';
-import { LegacyConfig } from '../../../core/server';
-import { convertLegacyTypes } from '../../../core/server/saved_objects/utils';
-import { SavedObjectTypeRegistry } from '../../../core/server/saved_objects';
-// @ts-ignore
-import { collectUiExports } from '../../../legacy/ui/ui_exports';
-// @ts-ignore
-import { findPluginSpecs } from '../../../legacy/plugin_discovery';
-
-/**
- * Load the uiExports for a Kibana instance, only load uiExports from xpack if
- * it is enabled in the Kibana server.
- */
-const getUiExports = async (kibanaPluginIds: string[]) => {
-  const xpackEnabled = kibanaPluginIds.includes('xpack_main');
-
-  const { spec$ } = await findPluginSpecs({
-    plugins: {
-      scanDirs: [Path.resolve(__dirname, '../../../legacy/core_plugins')],
-      paths: xpackEnabled ? [Path.resolve(__dirname, '../../../../x-pack')] : [],
-    },
-  });
-
-  const specs = await spec$.pipe(toArray()).toPromise();
-  return collectUiExports(specs);
-};
 
 /**
  * Deletes all indices that start with `.kibana`
@@ -93,61 +61,21 @@ export async function deleteKibanaIndices({
  */
 export async function migrateKibanaIndex({
   client,
-  log,
-  kibanaPluginIds,
+  kbnClient,
 }: {
   client: Client;
-  log: ToolingLog;
-  kibanaPluginIds: string[];
+  kbnClient: KbnClient;
 }) {
-  const uiExports = await getUiExports(kibanaPluginIds);
-  const kibanaVersion = await loadKibanaVersion();
-
-  const configKeys: Record<string, string> = {
-    'xpack.task_manager.index': '.kibana_task_manager',
-  };
-  const config = { get: (path: string) => configKeys[path] };
-
-  const savedObjectTypes = convertLegacyTypes(uiExports, config as LegacyConfig);
-  const typeRegistry = new SavedObjectTypeRegistry();
-  savedObjectTypes.forEach(type => typeRegistry.registerType(type));
-
-  const logger = {
-    trace: log.verbose.bind(log),
-    debug: log.debug.bind(log),
-    info: log.info.bind(log),
-    warn: log.warning.bind(log),
-    error: log.error.bind(log),
-    fatal: log.error.bind(log),
-    log: (entry: any) => log.info(entry.message),
-    get: () => logger,
-  };
-
-  const migratorOptions = {
-    savedObjectsConfig: {
-      scrollDuration: '5m',
-      batchSize: 100,
-      pollInterval: 100,
-      skip: false,
+  // we allow dynamic mappings on the index, as some interceptors are accessing documents before
+  // the migration is actually performed. The migrator will put the value back to `strict` after migration.
+  await client.indices.putMapping({
+    index: '.kibana',
+    body: {
+      dynamic: true,
     },
-    kibanaConfig: {
-      index: '.kibana',
-    } as any,
-    logger,
-    kibanaVersion,
-    typeRegistry,
-    savedObjectValidations: uiExports.savedObjectValidations,
-    callCluster: (path: string, ...args: any[]) =>
-      (get(client, path) as Function).call(client, ...args),
-  };
-
-  return await new KibanaMigrator(migratorOptions).runMigrations();
-}
+  } as any);
 
-async function loadKibanaVersion() {
-  const readFile = promisify(fs.readFile);
-  const packageJson = await readFile(Path.join(__dirname, '../../../../package.json'));
-  return JSON.parse(packageJson.toString('utf-8')).version;
+  return await kbnClient.savedObjects.migrate();
 }
 
 /**