Adapt es-archiver to call _migrate endpoint instead of creating a migrator

packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts

@@ -57,9 +57,28 @@ interface UpdateOptions<Attributes> extends IndexOptions<Attributes> {
   id: string;
 }
 
+interface MigrateResponse {
+  success: boolean;
+  result: Array<{ status: string }>;
+}
+
 export class KbnClientSavedObjects {
   constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {}
 
+  /**
+   * Run the saved objects migration
+   */
+  public async migrate() {
+    this.log.debug('Migrating saved objects');
+
+    return await this.requester.request<MigrateResponse>({
+      description: 'migrate saved objects',
+      path: uriencode`/api/saved_objects/_migrate`,
+      method: 'POST',
+      body: {},
+    });
+  }
+
   /**
    * Get an object
    */

packages/kbn-pm/dist/index.js

@@ -43777,6 +43777,18 @@ class KbnClientSavedObjects {
         this.log = log;
         this.requester = requester;
     }
+    /**
+     * Run the saved objects migration
+     */
+    async migrate() {
+        this.log.debug('Migrating saved objects');
+        return await this.requester.request({
+            description: 'migrate saved objects',
+            path: kbn_client_requester_1.uriencode `/api/saved_objects/_migrate`,
+            method: 'POST',
+            body: {},
+        });
+    }
     /**
      * Get an object
      */

src/core/server/saved_objects/migrations/core/build_active_mappings.test.ts

@@ -21,6 +21,11 @@ import { IndexMapping } from './../../mappings';
 import { buildActiveMappings, diffMappings } from './build_active_mappings';
 
 describe('buildActiveMappings', () => {
+  test('creates a strict mapping', () => {
+    const mappings = buildActiveMappings({});
+    expect(mappings.dynamic).toEqual('strict');
+  });
+
   test('combines all mappings and includes core mappings', () => {
     const properties = {
       aaa: { type: 'text' },

src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts

@@ -88,15 +88,27 @@ export class KibanaMigrator {
   }
 
   /**
-   * Migrates the mappings and documents in the Kibana index. This will run only
+   * Migrates the mappings and documents in the Kibana index. By default, this will run only
    * once and subsequent calls will return the result of the original call.
    *
+   * @param rerun - If true, method will run a new migration when called again instead of
+   * returning the result of the initial migration. This should only be used when factors external
+   * to Kibana itself alter the kibana index causing the saved objects mappings or data to change
+   * after the Kibana server performed the initial migration.
+   *
+   * @remarks When the `rerun` parameter is set to true, no checks are performed to ensure that no migration
+   * is currently running. Chained or concurrent calls to `runMigrations({ rerun: true })` can lead to
+   * multiple migrations running at the same time. When calling with this parameter, it's expected that the calling
+   * code should ensure that the initial call resolves before calling the function again.
+   *
    * @returns - A promise which resolves once all migrations have been applied.
    *    The promise resolves with an array of migration statuses, one for each
    *    elasticsearch index which was migrated.
    */
-  public runMigrations(): Promise<Array<{ status: string }>> {
-    if (this.migrationResult === undefined) {
+  public runMigrations({ rerun = false }: { rerun?: boolean } = {}): Promise<
+    Array<{ status: string }>
+  > {
+    if (this.migrationResult === undefined || rerun) {
       this.migrationResult = this.runMigrationsInternal();
     }
 

src/core/server/saved_objects/routes/index.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { InternalHttpServiceSetup } from '../../http';
+import { IKibanaMigrator } from '../migrations';
+import { registerMigrateRoute } from './migrate';
+
+interface RegisterRouteOptions {
+  http: InternalHttpServiceSetup;
+  migratorPromise: Promise<IKibanaMigrator>;
+}
+
+export function registerRoutes({ http, migratorPromise }: RegisterRouteOptions) {
+  const router = http.createRouter('/api/saved_objects');
+
+  registerMigrateRoute(router, migratorPromise);
+}

src/core/server/saved_objects/routes/integration_tests/migrate.test.mocks.ts

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock';
+
+export const migratorInstanceMock = mockKibanaMigrator.create();
+export const KibanaMigratorMock = jest.fn().mockImplementation(() => migratorInstanceMock);
+jest.doMock('../../migrations/kibana/kibana_migrator', () => ({
+  KibanaMigrator: KibanaMigratorMock,
+}));

src/core/server/saved_objects/routes/integration_tests/migrate.test.ts

@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { migratorInstanceMock } from './migrate.test.mocks';
+import * as kbnTestServer from '../../../../../test_utils/kbn_server';
+
+describe('SavedObjects /_migrate endpoint', () => {
+  let root: ReturnType<typeof kbnTestServer.createRoot>;
+
+  beforeEach(async () => {
+    root = kbnTestServer.createRoot({ migrations: { skip: true } });
+    await root.setup();
+    await root.start();
+    migratorInstanceMock.runMigrations.mockClear();
+  }, 30000);
+
+  afterEach(async () => {
+    await root.shutdown();
+  });
+
+  it('calls runMigrations on the migrator with rerun=true when accessed', async () => {
+    await kbnTestServer.request
+      .post(root, '/api/saved_objects/_migrate')
+      .send({})
+      .expect(200);
+
+    expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1);
+    expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith({ rerun: true });
+  });
+
+  it('calls runMigrations multiple time when multiple access', async () => {
+    await kbnTestServer.request
+      .post(root, '/api/saved_objects/_migrate')
+      .send({})
+      .expect(200);
+
+    expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1);
+
+    await kbnTestServer.request
+      .post(root, '/api/saved_objects/_migrate')
+      .send({})
+      .expect(200);
+
+    expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(2);
+  });
+});

src/core/server/saved_objects/routes/migrate.ts

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { IRouter } from '../../http';
+import { IKibanaMigrator } from '../migrations';
+
+export const registerMigrateRoute = (
+  router: IRouter,
+  migratorPromise: Promise<IKibanaMigrator>
+) => {
+  router.post(
+    {
+      path: '/_migrate',
+      validate: false,
+    },
+    router.handleLegacyErrors(async (context, req, res) => {
+      const migrator = await migratorPromise;
+      const result = await migrator.runMigrations({ rerun: true });
+      return res.ok({
+        body: {
+          success: true,
+          result,
+        },
+      });
+    })
+  );
+};

src/core/server/saved_objects/saved_objects_service.test.ts

@@ -31,6 +31,7 @@ import { Env } from '../config';
 import { configServiceMock } from '../mocks';
 import { elasticsearchServiceMock } from '../elasticsearch/elasticsearch_service.mock';
 import { legacyServiceMock } from '../legacy/legacy_service.mock';
+import { httpServiceMock } from '../http/http_service.mock';
 import { SavedObjectsClientFactoryProvider } from './service/lib';
 import { BehaviorSubject } from 'rxjs';
 import { NodesVersionCompatibility } from '../elasticsearch/version_check/ensure_es_version';
@@ -39,6 +40,7 @@ describe('SavedObjectsService', () => {
   const createSetupDeps = () => {
     const elasticsearchMock = elasticsearchServiceMock.createInternalSetup();
     return {
+      http: httpServiceMock.createSetupContract(),
       elasticsearch: elasticsearchMock,
       legacyPlugins: legacyServiceMock.createDiscoverPlugins(),
     };

src/core/server/saved_objects/saved_objects_service.ts

@@ -17,8 +17,9 @@
  * under the License.
  */
 
-import { CoreService } from 'src/core/types';
+import { Subject } from 'rxjs';
 import { first, filter, take } from 'rxjs/operators';
+import { CoreService } from '../../types';
 import {
   SavedObjectsClient,
   SavedObjectsClientProvider,
@@ -32,7 +33,7 @@ import { InternalElasticsearchServiceSetup, APICaller } from '../elasticsearch';
 import { KibanaConfigType } from '../kibana_config';
 import { migrationsRetryCallCluster } from '../elasticsearch/retry_call_cluster';
 import { SavedObjectsConfigType } from './saved_objects_config';
-import { KibanaRequest } from '../http';
+import { InternalHttpServiceSetup, KibanaRequest } from '../http';
 import { SavedObjectsClientContract, SavedObjectsType } from './types';
 import { ISavedObjectsRepository, SavedObjectsRepository } from './service/lib/repository';
 import {
@@ -44,6 +45,7 @@ import { convertLegacyTypes } from './utils';
 import { SavedObjectTypeRegistry, ISavedObjectTypeRegistry } from './saved_objects_type_registry';
 import { PropertyValidators } from './validation';
 import { SavedObjectsSerializer } from './serialization';
+import { registerRoutes } from './routes';
 
 /**
  * Saved Objects is Kibana's data persistence mechanism allowing plugins to
@@ -196,6 +198,7 @@ export interface SavedObjectsRepositoryFactory {
 
 /** @internal */
 export interface SavedObjectsSetupDeps {
+  http: InternalHttpServiceSetup;
   legacyPlugins: LegacyServiceDiscoverPlugins;
   elasticsearch: InternalElasticsearchServiceSetup;
 }
@@ -218,6 +221,7 @@ export class SavedObjectsService
   private clientFactoryProvider?: SavedObjectsClientFactoryProvider;
   private clientFactoryWrappers: WrappedClientFactoryWrapper[] = [];
 
+  private migrator$ = new Subject<KibanaMigrator>();
   private typeRegistry = new SavedObjectTypeRegistry();
   private validations: PropertyValidators = {};
 
@@ -237,6 +241,11 @@ export class SavedObjectsService
     legacyTypes.forEach(type => this.typeRegistry.registerType(type));
     this.validations = setupDeps.legacyPlugins.uiExports.savedObjectValidations || {};
 
+    registerRoutes({
+      http: setupDeps.http,
+      migratorPromise: this.migrator$.pipe(first()).toPromise(),
+    });
+
     return {
       setClientFactoryProvider: provider => {
         if (this.clientFactoryProvider) {
@@ -278,6 +287,8 @@ export class SavedObjectsService
     const adminClient = this.setupDeps!.elasticsearch.adminClient;
     const migrator = this.createMigrator(kibanaConfig, savedObjectsConfig, migrationsRetryDelay);
 
+    this.migrator$.next(migrator);
+
     /**
      * Note: We want to ensure that migrations have completed before
      * continuing with further Core start steps that might use SavedObjects

src/core/server/server.ts

@@ -132,6 +132,7 @@ export class Server {
     });
 
     const savedObjectsSetup = await this.savedObjects.setup({
+      http: httpSetup,
       elasticsearch: elasticsearchServiceSetup,
       legacyPlugins,
     });

src/es_archiver/actions/empty_kibana_index.ts

@@ -32,9 +32,8 @@ export async function emptyKibanaIndexAction({
   kbnClient: KbnClient;
 }) {
   const stats = createStats('emptyKibanaIndex', log);
-  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
   await deleteKibanaIndices({ client, stats, log });
-  await migrateKibanaIndex({ client, log, kibanaPluginIds });
+  await migrateKibanaIndex({ client, kbnClient });
   return stats;
 }

src/es_archiver/actions/load.ts

@@ -106,7 +106,7 @@ export async function loadAction({
 
   // If we affected the Kibana index, we need to ensure it's migrated...
   if (Object.keys(result).some(k => k.startsWith('.kibana'))) {
-    await migrateKibanaIndex({ client, log, kibanaPluginIds });
+    await migrateKibanaIndex({ client, kbnClient });
 
     if (kibanaPluginIds.includes('spaces')) {
       await createDefaultSpace({ client, index: '.kibana' });