Add reindex feature to Upgrade Assistant (elastic#27457)

* Fixup saved objects

* WIP reindex state machine

* WIP UI

* Separate status from current step

* Add error messages

* Copy types to CallWithInternalUser

* Use backend worker

* Add progress bar for large indices

* Cleanup worker implementation

* Add tests for ReindexService

* Fix types

* Fix CI

* Add support for mapping coerced boolean fields

* Add basic functional integration test

* Cleanup reindexing backend, add more checks

* Cleanup frontend code and add tests

* Support removal of _default_ mapping + add tests

* Add reindex warnings to reindex service

* Add confirmation modal for reindex warnings

* Cleanup flyout appearance

* Generate new index name intelligently

* Design tweaks

* Show reindex button in both grouping modes

* Add data archive integration tests

* Change flyout design to two step process

* Reorganize flyout files

* Use custom steps component

* Allow writes to indices if anything fails

* Misc cleanup

* Design edits

* Only show warnings panel when reindex has not begun

* Fix types

* Handle moving existing aliases

* Move integration tests to separate config

* Fix ReindexButton bugs when changing pages

* Fix polling service tests

* Refactor ReindexService and ML handling (backend only)

* Add credential caching and paused state to backend

* Add paused state to frontend

* Fix ts errors

* Update copy

* Add check for node version before ML step

* Update data archive format

* Update snapshots for React upgrade

* Handle _default_ mappings correctly in warning checks

* Use hashed state of reindex operation for credential storage

* Address tyler's comments

* Add watcher stop/starting

* Use json-stable-stringify and sha256 for CredentialStore

* Add types for json-stable-stringify

src/server/kbn_server.d.ts

@@ -18,9 +18,10 @@
  */
 
 import { Server } from 'hapi';
+
 import { CallClusterWithRequest, ElasticsearchPlugin } from '../legacy/core_plugins/elasticsearch';
 import { IndexPatternsServiceFactory } from './index_patterns';
-import { SavedObjectsService } from './saved_objects';
+import { SavedObjectsClient, SavedObjectsService } from './saved_objects';
 
 export interface KibanaConfig {
   get<T>(key: string): T;
@@ -31,6 +32,7 @@ declare module 'hapi' {
   interface PluginProperties {
     elasticsearch: ElasticsearchPlugin;
     kibana: any;
+    spaces: any;
     // add new plugin types here
   }
 
@@ -41,11 +43,9 @@ declare module 'hapi' {
   }
 
   interface Request {
-    getBasePath: () => string;
-  }
-
-  interface Request {
-    getUiSettingsService: () => any;
+    getSavedObjectsClient(): SavedObjectsClient;
+    getBasePath(): string;
+    getUiSettingsService(): any;
   }
 }
 

src/server/saved_objects/service/create_saved_objects_service.d.ts

@@ -17,19 +17,16 @@
  * under the License.
  */
 
-import { SavedObjectsRepository, ScopedSavedObjectsClientProvider } from './lib';
+import { ScopedSavedObjectsClientProvider } from './lib';
 import { SavedObjectsClient } from './saved_objects_client';
 
 export interface SavedObjectsService<Request = any> {
   // ATTENTION: these types are incomplete
-
   addScopedSavedObjectsClientWrapperFactory: ScopedSavedObjectsClientProvider<
     Request
   >['addClientWrapperFactory'];
-  getSavedObjectsRepository: (
-    callCluster: (endpoint: string, clientParams: any, options: any) => Promise<any>
-  ) => SavedObjectsRepository;
   getScopedSavedObjectsClient: ScopedSavedObjectsClientProvider<Request>['getClient'];
   SavedObjectsClient: typeof SavedObjectsClient;
   types: string[];
+  getSavedObjectsRepository(...rest: any[]): any;
 }

src/server/saved_objects/service/lib/repository.js

@@ -313,7 +313,7 @@ export class SavedObjectsRepository {
     }
 
     if (fields && !Array.isArray(fields)) {
-      throw new TypeError('options.searchFields must be an array');
+      throw new TypeError('options.fields must be an array');
     }
 
     const esOptions = {

src/server/saved_objects/service/saved_objects_client.d.ts

@@ -28,30 +28,32 @@ export interface CreateOptions extends BaseOptions {
   override?: boolean;
 }
 
-export interface BulkCreateObject {
+export interface BulkCreateObject<T extends SavedObjectAttributes = any> {
   id?: string;
   type: string;
-  attributes: SavedObjectAttributes;
+  attributes: T;
   extraDocumentProperties?: string[];
 }
 
-export interface BulkCreateResponse {
-  savedObjects: SavedObject[];
+export interface BulkCreateResponse<T extends SavedObjectAttributes = any> {
+  savedObjects: Array<SavedObject<T>>;
 }
 
 export interface FindOptions extends BaseOptions {
+  type?: string | string[];
   page?: number;
   perPage?: number;
   sortField?: string;
   sortOrder?: string;
   fields?: string[];
-  type?: string | string[];
+  search?: string;
+  searchFields?: string[];
 }
 
-export interface FindResponse {
-  saved_objects: SavedObject[];
+export interface FindResponse<T extends SavedObjectAttributes = any> {
+  saved_objects: Array<SavedObject<T>>;
   total: number;
-  perPage: number;
+  per_page: number;
   page: number;
 }
 
@@ -65,46 +67,57 @@ export interface BulkGetObject {
 }
 export type BulkGetObjects = BulkGetObject[];
 
-export interface BulkGetResponse {
-  savedObjects: SavedObject[];
+export interface BulkGetResponse<T extends SavedObjectAttributes = any> {
+  savedObjects: Array<SavedObject<T>>;
 }
 
 export interface SavedObjectAttributes {
   [key: string]: SavedObjectAttributes | string | number | boolean | null;
 }
 
-export interface SavedObject {
+export interface SavedObject<T extends SavedObjectAttributes = any> {
   id: string;
   type: string;
   version?: number;
   updated_at?: string;
   error?: {
     message: string;
   };
-  attributes: SavedObjectAttributes;
+  attributes: T;
 }
 
 export declare class SavedObjectsClient {
   public static errors: typeof errors;
   public errors: typeof errors;
-  public create: (
+
+  constructor(repository: SavedObjectsRepository);
+
+  public create<T extends SavedObjectAttributes = any>(
     type: string,
-    attributes: SavedObjectAttributes,
+    attributes: T,
     options?: CreateOptions
-  ) => Promise<SavedObject>;
-  public bulkCreate: (
-    objects: BulkCreateObject[],
+  ): Promise<SavedObject<T>>;
+  public bulkCreate<T extends SavedObjectAttributes = any>(
+    objects: Array<BulkCreateObject<T>>,
     options?: CreateOptions
-  ) => Promise<BulkCreateResponse>;
-  public delete: (type: string, id: string, options?: BaseOptions) => Promise<{}>;
-  public find: (options: FindOptions) => Promise<FindResponse>;
-  public bulkGet: (objects: BulkGetObjects, options?: BaseOptions) => Promise<BulkGetResponse>;
-  public get: (type: string, id: string, options?: BaseOptions) => Promise<SavedObject>;
-  public update: (
+  ): Promise<BulkCreateResponse<T>>;
+  public delete(type: string, id: string, options?: BaseOptions): Promise<{}>;
+  public find<T extends SavedObjectAttributes = any>(
+    options: FindOptions
+  ): Promise<FindResponse<T>>;
+  public bulkGet<T extends SavedObjectAttributes = any>(
+    objects: BulkGetObjects,
+    options?: BaseOptions
+  ): Promise<BulkGetResponse<T>>;
+  public get<T extends SavedObjectAttributes = any>(
+    type: string,
+    id: string,
+    options?: BaseOptions
+  ): Promise<SavedObject<T>>;
+  public update<T extends SavedObjectAttributes = any>(
     type: string,
     id: string,
-    attributes: SavedObjectAttributes,
+    attributes: Partial<T>,
     options?: UpdateOptions
-  ) => Promise<SavedObject>;
-  constructor(repository: SavedObjectsRepository);
+  ): Promise<SavedObject<T>>;
 }

x-pack/package.json

@@ -132,6 +132,7 @@
     "@scant/router": "^0.1.0",
     "@slack/client": "^4.8.0",
     "@turf/boolean-contains": "6.0.1",
+    "@types/json-stable-stringify": "^1.0.32",
     "angular-resource": "1.4.9",
     "angular-sanitize": "1.6.5",
     "angular-ui-ace": "0.2.3",
@@ -187,6 +188,7 @@
     "io-ts": "^1.4.2",
     "joi": "^13.5.2",
     "jquery": "^3.3.1",
+    "json-stable-stringify": "^1.0.1",
     "jsonwebtoken": "^8.3.0",
     "lodash": "npm:@elastic/[email protected]",
     "lodash.keyby": "^4.6.0",

x-pack/plugins/__mocks__/ui/chrome.js

@@ -36,8 +36,13 @@ function getInjected(key) {
   }
 }
 
+function getXsrfToken() {
+  return 'kbn';
+}
+
 export default {
   getInjected,
   addBasePath,
-  getUiSettingsClient
+  getUiSettingsClient,
+  getXsrfToken
 };

x-pack/plugins/spaces/server/lib/saved_objects_client/spaces_saved_objects_client.ts

@@ -84,7 +84,11 @@ export class SpacesSavedObjectsClient implements SavedObjectsClient {
    * @property {string} [options.namespace]
    * @returns {promise} - { id, type, version, attributes }
    */
-  public async create(type: string, attributes = {}, options: CreateOptions = {}) {
+  public async create<T extends SavedObjectAttributes>(
+    type: string,
+    attributes: T = {} as T,
+    options: CreateOptions = {}
+  ) {
     throwErrorIfTypeIsSpace(type);
     throwErrorIfNamespaceSpecified(options);
 
@@ -215,10 +219,10 @@ export class SpacesSavedObjectsClient implements SavedObjectsClient {
    * @property {string} [options.namespace]
    * @returns {promise}
    */
-  public async update(
+  public async update<T extends SavedObjectAttributes>(
     type: string,
     id: string,
-    attributes: SavedObjectAttributes,
+    attributes: Partial<T>,
     options: UpdateOptions = {}
   ) {
     throwErrorIfTypeIsSpace(type);

x-pack/plugins/spaces/server/routes/api/__fixtures__/create_test_handler.ts

@@ -5,11 +5,12 @@
  */
 
 // @ts-ignore
-import { PluginProperties, Server } from 'hapi';
+import { Server } from 'hapi';
+import { Legacy } from 'kibana';
 import { SpacesClient } from '../../../lib/spaces_client';
 import { createSpaces } from './create_spaces';
 
-interface KibanaServer extends Server {
+interface KibanaServer extends Legacy.Server {
   savedObjects: any;
 }
 
@@ -46,13 +47,6 @@ const baseConfig: TestConfig = {
   'xpack.spaces.maxSpaces': 1000,
 };
 
-// Merge / extend default interfaces for hapi. This is all faked out below.
-declare module 'hapi' {
-  interface PluginProperties {
-    spaces: any;
-  }
-}
-
 export function createTestHandler(initApiFn: (server: any, preCheckLicenseImpl: any) => void) {
   const teardowns: TeardownFn[] = [];
 

x-pack/plugins/upgrade_assistant/common/types.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  SavedObject,
+  SavedObjectAttributes,
+} from 'src/server/saved_objects/service/saved_objects_client';
+
+export enum ReindexStep {
+  // Enum values are spaced out by 10 to give us room to insert steps in between.
+  created = 0,
+  indexConsumersStopped = 10,
+  readonly = 20,
+  newIndexCreated = 30,
+  reindexStarted = 40,
+  reindexCompleted = 50,
+  aliasCreated = 60,
+  indexConsumersStarted = 70,
+}
+
+export enum ReindexStatus {
+  inProgress,
+  completed,
+  failed,
+  paused,
+}
+
+export const REINDEX_OP_TYPE = 'upgrade-assistant-reindex-operation';
+export interface ReindexOperation extends SavedObjectAttributes {
+  indexName: string;
+  newIndexName: string;
+  status: ReindexStatus;
+  lastCompletedStep: ReindexStep;
+  locked: string | null;
+  reindexTaskId: string | null;
+  reindexTaskPercComplete: number | null;
+  errorMessage: string | null;
+  mlReindexCount: number | null;
+}
+
+export type ReindexSavedObject = SavedObject<ReindexOperation>;
+
+export enum ReindexWarning {
+  allField,
+  booleanFields,
+}

x-pack/plugins/upgrade_assistant/common/version.ts

@@ -12,3 +12,4 @@ const matches = currentVersionNum.match(/^([1-9]+)\.([0-9]+)\.([0-9]+)$/)!;
 
 export const CURRENT_MAJOR_VERSION = matches[1];
 export const NEXT_MAJOR_VERSION = (parseInt(CURRENT_MAJOR_VERSION, 10) + 1).toString();
+export const PREV_MAJOR_VERSION = (parseInt(CURRENT_MAJOR_VERSION, 10) - 1).toString();

x-pack/plugins/upgrade_assistant/index.ts

@@ -15,6 +15,12 @@ export function upgradeAssistant(kibana: any) {
     uiExports: {
       managementSections: ['plugins/upgrade_assistant'],
       styleSheetPaths: resolve(__dirname, 'public/index.scss'),
+      mappings: require('./mappings.json'),
+      savedObjectSchemas: {
+        'upgrade-assistant-reindex-operation': {
+          isNamespaceAgnostic: true,
+        },
+      },
     },
     publicDir: resolve(__dirname, 'public'),
 

x-pack/plugins/upgrade_assistant/mappings.json

@@ -0,0 +1,13 @@
+{
+  "upgrade-assistant-reindex-operation": {
+    "dynamic": true,
+    "properties": {
+      "indexName": {
+        "type": "keyword"
+      },
+      "status": {
+        "type": "integer"
+      }
+    }
+  }
+}

x-pack/plugins/upgrade_assistant/public/components/tabs.test.tsx

@@ -9,6 +9,7 @@ import { mountWithIntl } from 'test_utils/enzyme_helpers';
 
 jest.mock('axios', () => ({
   get: jest.fn(),
+  create: jest.fn(),
 }));
 
 import { UpgradeAssistantTabs } from './tabs';

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/_index.scss

@@ -1,4 +1,5 @@
 @import './cell';
+@import './reindex/index';
 
 .upgDeprecations {
   // Pull the container through the padding of EuiPageContent