Implement version rollback for server restore (#1680)

Signed-off-by: Mykola Morhun <[email protected]>

README.md

@@ -669,16 +669,27 @@ OPTIONS
   -r, --repository-url=repository-url                    Full address of backup repository. Format is identical to
                                                          restic.
 
-  -s, --snapshot-id=snapshot-id                          ID of a snapshot to restore from
+  -s, --snapshot-id=snapshot-id                          snapshot identificator to restore from. Value "latest" means
+                                                         restoring from the most recent snapshot.
+
+  -v, --version=version                                  Che Operator version to restore to (e.g. 7.35.1). If the flag
+                                                         is not set, restore to the current version.
 
   --aws-access-key-id=aws-access-key-id                  AWS access key ID
 
   --aws-secret-access-key=aws-secret-access-key          AWS secret access key
 
+  --backup-cr-name=backup-cr-name                        Name of a backup custom resource to restore from
+
   --backup-server-config-name=backup-server-config-name  Name of custom resource with backup server config
 
+  --batch                                                Batch mode. Running a command without end user interaction.
+
   --password=password                                    Authentication password for backup REST server
 
+  --rollback                                             Rolling back to previous version of Eclipse Che only if backup
+                                                         exists
+
   --ssh-key=ssh-key                                      Private SSH key for authentication on SFTP server
 
   --ssh-key-file=ssh-key-file                            Path to file with private SSH key for authentication on SFTP
@@ -687,16 +698,20 @@ OPTIONS
   --username=username                                    Username for authentication in backup REST server
 
 EXAMPLES
-  # Reuse existing backup configuration:
-  chectl server:restore
-  # Restore from specific backup snapshot using previos backup configuration:
-  chectl server:restore -s 585421f3
-  # Create and use configuration for REST backup server:
-  chectl server:resotre -r rest:http://my-sert-server.net:4000/che-backup -p repopassword
-  # Create and use configuration for AWS S3 (and API compatible) backup server (bucket should be precreated):
-  chectl server:backup -r s3:s3.amazonaws.com/bucketche -p repopassword
-  # Create and use configuration for SFTP backup server:
-  chectl server:backup -r=sftp:[email protected]:/srv/sftp/che-data -p repopassword
+  # Restore from the latest snapshot from a provided REST backup server:
+  chectl server:restore -r rest:http://my-sert-server.net:4000/che-backup -p repopassword --snapshot-id=latest
+  # Restore from the latest snapshot from a provided AWS S3 (or API compatible) backup server (bucket must be 
+  precreated):
+  chectl server:restore -r s3:s3.amazonaws.com/bucketche -p repopassword --snapshot-id=latest
+  # Restore from the latest snapshot from a provided SFTP backup server:
+  chectl server:restore -r sftp:[email protected]:/srv/sftp/che-data -p repopassword --snapshot-id=latest
+  # Restore from a specific snapshot to a given Eclipse Che version from a provided REST backup server:
+  chectl server:restore -r rest:http://my-sert-server.net:4000/che-backup -p repopassword --version=7.35.2 
+  --snapshot-id=9ea02f58
+  # Rollback to a previous version only if backup exists:
+  chectl server:restore --rollback
+  # Restore from a specific backup object:
+  chectl server:restore --backup-cr-name=backup-object-name
 ```
 
 _See code: [src/commands/server/restore.ts](https://github.com/che-incubator/chectl/blob/v0.0.2/src/commands/server/restore.ts)_

src/api/backup-restore.ts

@@ -52,9 +52,6 @@ export interface SftpBackupServerCredentials {
   sshKey: string
 }
 
-export const BACKUP_CR_NAME = 'eclipse-che-backup'
-export const RESTORE_CR_NAME = 'eclipse-che-restore'
-
 export const BACKUP_SERVER_CONFIG_NAME = 'eclipse-che-backup-server-config'
 
 export const BACKUP_REPOSITORY_PASSWORD_SECRET_NAME = 'chectl-backup-repository-password'
@@ -80,26 +77,28 @@ export function getBackupServerType(url: string): BackupServerType {
 /**
  * Submits backup of Che installation task.
  * @param namespace namespace in which Che is installed
+ * @param name name of the backup CR to create
  * @param backupServerConfig backup server configuration data or name of the config CR
  */
-export async function requestBackup(namespace: string, backupServerConfig?: BackupServerConfig | string): Promise<V1CheClusterBackup> {
+export async function requestBackup(namespace: string, name: string, backupServerConfig?: BackupServerConfig | string): Promise<V1CheClusterBackup> {
   const kube = new KubeHelper()
   const backupServerConfigName = await getBackupServerConfigurationName(namespace, backupServerConfig)
-  return kube.recreateBackupCr(namespace, BACKUP_CR_NAME, backupServerConfigName)
+  return kube.recreateBackupCr(namespace, name, backupServerConfigName)
 }
 
 /**
  * Submits Che restore task.
  * @param namespace namespace in which Che should be restored
+ * @param name name of the restore CR to create
  * @param backupServerConfig backup server configuration data or name of the config CR
  */
-export async function requestRestore(namespace: string, backupServerConfig?: BackupServerConfig | string, snapshotId?: string): Promise<V1CheClusterBackup> {
+export async function requestRestore(namespace: string, name: string, backupServerConfig?: BackupServerConfig | string, snapshotId?: string): Promise<V1CheClusterBackup> {
   const kube = new KubeHelper()
   const backupServerConfigName = await getBackupServerConfigurationName(namespace, backupServerConfig)
   if (!backupServerConfigName) {
     throw new Error(`No backup server configuration found in ${namespace} namespace`)
   }
-  return kube.recreateRestoreCr(namespace, RESTORE_CR_NAME, backupServerConfigName, snapshotId)
+  return kube.recreateRestoreCr(namespace, name, backupServerConfigName, snapshotId)
 }
 
 /**
@@ -111,14 +110,14 @@ export async function requestRestore(namespace: string, backupServerConfig?: Bac
  * @param backupServerConfig backup server configuration data or name of the backup server config CR
  * @returns name of existing backup server configuration in the given namespace or empty string if none suitable
  */
-async function getBackupServerConfigurationName(namespace: string, backupServerConfig?: BackupServerConfig | string): Promise<string> {
+export async function getBackupServerConfigurationName(namespace: string, backupServerConfig?: BackupServerConfig | string): Promise<string> {
   const kube = new KubeHelper()
 
   if (backupServerConfig) {
     if (typeof backupServerConfig === 'string') {
       // Name of CR with backup server configuration provided
       // Check if it exists
-      const backupServerConfigCr = await kube.getCustomResource(namespace, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_BACKUP_SERVER_CONFIG_KIND_PLURAL)
+      const backupServerConfigCr = await kube.getCustomResource(namespace, backupServerConfig, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_BACKUP_SERVER_CONFIG_KIND_PLURAL)
       if (!backupServerConfigCr) {
         throw new Error(`Backup server configuration with '${backupServerConfig}' name not found in '${namespace}' namespace.`)
       }

src/api/context.ts

@@ -16,7 +16,7 @@ import * as os from 'os'
 import * as path from 'path'
 
 import { CHE_OPERATOR_CR_PATCH_YAML_KEY, CHE_OPERATOR_CR_YAML_KEY, LOG_DIRECTORY_KEY } from '../common-flags'
-import { CHECTL_PROJECT_NAME, DEFAULT_CHE_NAMESPACE, DEFAULT_OPENSHIFT_OPERATORS_NS_NAME, STABLE_ALL_NAMESPACES_CHANNEL_NAME } from '../constants'
+import { CHECTL_PROJECT_NAME, DEFAULT_CHE_NAMESPACE, DEFAULT_OPENSHIFT_OPERATORS_NS_NAME, OLM_STABLE_ALL_NAMESPACES_CHANNEL_NAME } from '../constants'
 import { getProjectName, getProjectVersion, readCRFile } from '../util'
 
 import { CHECTL_DEVELOPMENT_VERSION } from './version'
@@ -45,12 +45,12 @@ export namespace ChectlContext {
   export async function init(flags: any, command: Command): Promise<void> {
     ctx.isChectl = getProjectName() === CHECTL_PROJECT_NAME
     ctx.isDevVersion = getProjectVersion().includes('next') || getProjectVersion() === CHECTL_DEVELOPMENT_VERSION
-    ctx.operatorNamespace = flags.chenamespace || DEFAULT_CHE_NAMESPACE
     if (flags['listr-renderer'] as any) {
       ctx.listrOptions = { renderer: (flags['listr-renderer'] as any), collapse: false } as Listr.ListrOptions
     }
 
-    if (flags['olm-channel'] === STABLE_ALL_NAMESPACES_CHANNEL_NAME) {
+    ctx.operatorNamespace = flags.chenamespace || DEFAULT_CHE_NAMESPACE
+    if (flags['olm-channel'] === OLM_STABLE_ALL_NAMESPACES_CHANNEL_NAME) {
       ctx.operatorNamespace = DEFAULT_OPENSHIFT_OPERATORS_NS_NAME
     }
 

src/api/kube.ts

@@ -1199,6 +1199,18 @@ export class KubeHelper {
     }
   }
 
+  async deleteDeployment(namespace: string, name: string): Promise<void> {
+    const k8sAppsApi = this.kubeConfig.makeApiClient(AppsV1Api)
+    try {
+      k8sAppsApi.deleteNamespacedDeployment(name, namespace)
+    } catch (error) {
+      if (error.response && error.response.statusCode === 404) {
+        return
+      }
+      throw this.wrapK8sClientError(error)
+    }
+  }
+
   async deleteAllDeployments(namespace: string): Promise<void> {
     const k8sAppsApi = this.kubeConfig.makeApiClient(AppsV1Api)
     try {
@@ -1627,13 +1639,36 @@ export class KubeHelper {
    * Returns `checlusters.org.eclipse.che' in the given namespace.
    */
   async getCheCluster(cheNamespace: string): Promise<any | undefined> {
-    return this.getCustomResource(cheNamespace, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_KIND_PLURAL)
+    return this.findCustomResource(cheNamespace, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_KIND_PLURAL)
+  }
+
+  /**
+   * Deletes `checlusters.org.eclipse.che' resources in the given namespace.
+   */
+  async getAllCheClusters(): Promise<any[]> {
+    return this.getAllCustomResources(CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_KIND_PLURAL)
+  }
+
+  /**
+   * Returns custom resource object by its name in the given namespace.
+   */
+  async getCustomResource(namespace: string, name: string, resourceAPIGroup: string, resourceAPIVersion: string, resourcePlural: string): Promise<any | undefined> {
+    const customObjectsApi = this.kubeConfig.makeApiClient(CustomObjectsApi)
+    try {
+      const res = await customObjectsApi.getNamespacedCustomObject(resourceAPIGroup, resourceAPIVersion, namespace, resourcePlural, name)
+      return res.body
+    } catch (e) {
+      if (e.response && e.response.statusCode !== 404) {
+        throw this.wrapK8sClientError(e)
+      }
+    }
   }
 
   /**
-   * Returns custom resource in the given namespace.
+   * Returns the only custom resource in the given namespace.
+   * Throws error if there is more than one object of given kind.
    */
-  async getCustomResource(namespace: string, resourceAPIGroup: string, resourceAPIVersion: string, resourcePlural: string): Promise<any | undefined> {
+  async findCustomResource(namespace: string, resourceAPIGroup: string, resourceAPIVersion: string, resourcePlural: string): Promise<any | undefined> {
     const customObjectsApi = this.kubeConfig.makeApiClient(CustomObjectsApi)
     try {
       const { body } = await customObjectsApi.listNamespacedCustomObject(resourceAPIGroup, resourceAPIVersion, namespace, resourcePlural)
@@ -1656,13 +1691,6 @@ export class KubeHelper {
     }
   }
 
-  /**
-   * Deletes `checlusters.org.eclipse.che' resources in the given namespace.
-   */
-  async getAllCheClusters(): Promise<any[]> {
-    return this.getAllCustomResources(CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_KIND_PLURAL)
-  }
-
   /**
    * Returns all custom resources
    */

src/api/typings/backup-restore-crds.d.ts

@@ -29,6 +29,7 @@ export interface V1CheClusterBackupStatus {
   message?: string
   state?: string
   stage?: string
+  cheVersion?: string
   snapshotId?: string
 }
 

src/commands/server/backup.ts

@@ -92,6 +92,8 @@ export const backupServerConfigName = string({
   exclusive: [BACKUP_REPOSITORY_URL_KEY, BACKUP_REPOSITORY_PASSWORD_KEY],
 })
 
+const BACKUP_CR_NAME = 'eclipse-che-backup'
+
 export default class Backup extends Command {
   static description = 'Backup Eclipse Che installation'
 
@@ -150,7 +152,7 @@ export default class Backup extends Command {
         title: 'Scheduling backup...',
         task: async (_ctx: any, task: any) => {
           const backupServerConfig = getBackupServerConfiguration(flags)
-          await requestBackup(flags.chenamespace, backupServerConfig)
+          await requestBackup(flags.chenamespace, BACKUP_CR_NAME, backupServerConfig)
           task.title = `${task.title}OK`
         },
       },
@@ -161,7 +163,7 @@ export default class Backup extends Command {
           let backupStatus: V1CheClusterBackupStatus = {}
           do {
             await cli.wait(1000)
-            const backupCr: V1CheClusterBackup = await kube.getCustomResource(flags.chenamespace, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL)
+            const backupCr: V1CheClusterBackup = await kube.getCustomResource(flags.chenamespace, BACKUP_CR_NAME, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL)
             if (!backupCr.status) {
               continue
             }
@@ -170,7 +172,7 @@ export default class Backup extends Command {
             if (backupStatus.stage) {
               task.title = `Waiting until backup process finishes: ${backupStatus.stage}`
             }
-          } while (backupStatus.state === 'InProgress')
+          } while (!backupStatus.state || backupStatus.state === 'InProgress')
 
           if (backupStatus.state === 'Failed') {
             throw new Error(`Failed to create backup: ${backupStatus.message}`)

src/commands/server/deploy.ts

@@ -18,7 +18,7 @@ import * as semver from 'semver'
 import { ChectlContext } from '../../api/context'
 import { KubeHelper } from '../../api/kube'
 import { batch, cheDeployment, cheDeployVersion, cheNamespace, cheOperatorCRPatchYaml, cheOperatorCRYaml, CHE_OPERATOR_CR_PATCH_YAML_KEY, CHE_OPERATOR_CR_YAML_KEY, CHE_TELEMETRY, DEPLOY_VERSION_KEY, devWorkspaceControllerNamespace, k8sPodDownloadImageTimeout, K8SPODDOWNLOADIMAGETIMEOUT_KEY, k8sPodErrorRecheckTimeout, K8SPODERRORRECHECKTIMEOUT_KEY, k8sPodReadyTimeout, K8SPODREADYTIMEOUT_KEY, k8sPodWaitTimeout, K8SPODWAITTIMEOUT_KEY, listrRenderer, logsDirectory, LOG_DIRECTORY_KEY, skipKubeHealthzCheck as skipK8sHealthCheck } from '../../common-flags'
-import { DEFAULT_ANALYTIC_HOOK_NAME, DEFAULT_CHE_NAMESPACE, DEFAULT_OLM_SUGGESTED_NAMESPACE, DOCS_LINK_INSTALL_RUNNING_CHE_LOCALLY, MIN_CHE_OPERATOR_INSTALLER_VERSION, MIN_HELM_INSTALLER_VERSION, MIN_OLM_INSTALLER_VERSION, STABLE_ALL_NAMESPACES_CHANNEL_NAME } from '../../constants'
+import { DEFAULT_ANALYTIC_HOOK_NAME, DEFAULT_CHE_NAMESPACE, DEFAULT_OLM_SUGGESTED_NAMESPACE, DOCS_LINK_INSTALL_RUNNING_CHE_LOCALLY, MIN_CHE_OPERATOR_INSTALLER_VERSION, MIN_HELM_INSTALLER_VERSION, MIN_OLM_INSTALLER_VERSION, OLM_STABLE_ALL_NAMESPACES_CHANNEL_NAME } from '../../constants'
 import { CheTasks } from '../../tasks/che'
 import { DevWorkspaceTasks } from '../../tasks/component-installers/devfile-workspace-operator-installer'
 import { checkChectlAndCheVersionCompatibility, downloadTemplates, getPrintHighlightedMessagesTask, retrieveCheCaCertificateTask } from '../../tasks/installers/common-tasks'
@@ -210,7 +210,7 @@ export default class Deploy extends Command {
     }
 
     if (!flags.installer) {
-      await this.setDefaultInstaller(flags, ctx)
+      await setDefaultInstaller(flags)
       cli.info(`› Installer type is set to: '${flags.installer}'`)
     }
 
@@ -295,7 +295,7 @@ export default class Deploy extends Command {
         this.error(`🛑 The specified installer ${flags.installer} does not support Minishift`)
       }
 
-      if (flags['olm-channel'] === STABLE_ALL_NAMESPACES_CHANNEL_NAME && isKubernetesPlatformFamily(flags.platform)) {
+      if (flags['olm-channel'] === OLM_STABLE_ALL_NAMESPACES_CHANNEL_NAME && isKubernetesPlatformFamily(flags.platform)) {
         this.error('"stable-all-namespaces" channel is supported only in "openshift" platform')
       }
 
@@ -306,9 +306,6 @@ export default class Deploy extends Command {
         if (flags['starting-csv']) {
           this.error('"starting-csv" and "version" flags are mutually exclusive. Please specify only one of them.')
         }
-        if (flags['olm-channel']) {
-          this.error('"starting-csv" and "version" flags are mutually exclusive. Use "starting-csv" with "olm-channel" flag.')
-        }
         if (flags['auto-update']) {
           this.error('enabled "auto-update" flag cannot be used with version flag. Deploy latest version instead.')
         }
@@ -418,7 +415,7 @@ export default class Deploy extends Command {
       enabled: () => (this.isDevWorkspaceEnabled(ctx) || flags['workspace-engine'] === 'dev-workspace') && !ctx.isOpenShift,
       task: () => new Listr(devWorkspaceTasks.getInstallTasks(flags)),
     })
-    const installTasks = new Listr(installerTasks.installTasks(flags, this), ctx.listrOptions)
+    const installTasks = new Listr(await installerTasks.installTasks(flags, this), ctx.listrOptions)
 
     // Post Install Checks
     const postInstallTasks = new Listr([
@@ -460,24 +457,24 @@ export default class Deploy extends Command {
     notifyCommandCompletedSuccessfully()
     this.exit(0)
   }
+}
 
-  /**
-   * Sets default installer which is `olm` for OpenShift 4 with stable version of chectl
-   * and `operator` for other cases.
-   */
-  async setDefaultInstaller(flags: any, _ctx: any): Promise<void> {
-    const kubeHelper = new KubeHelper(flags)
+/**
+ * Sets default installer which is `olm` for OpenShift 4 with stable version of chectl
+ * and `operator` for other cases.
+ */
+export async function setDefaultInstaller(flags: any): Promise<void> {
+  const kubeHelper = new KubeHelper(flags)
 
-    const isOlmPreinstalled = await kubeHelper.isPreInstalledOLM()
-    if ((flags['catalog-source-name'] || flags['catalog-source-yaml']) && isOlmPreinstalled) {
-      flags.installer = 'olm'
-      return
-    }
+  const isOlmPreinstalled = await kubeHelper.isPreInstalledOLM()
+  if ((flags['catalog-source-name'] || flags['catalog-source-yaml']) && isOlmPreinstalled) {
+    flags.installer = 'olm'
+    return
+  }
 
-    if (flags.platform === 'openshift' && await kubeHelper.isOpenShift4() && isOlmPreinstalled) {
-      flags.installer = 'olm'
-    } else {
-      flags.installer = 'operator'
-    }
+  if (flags.platform === 'openshift' && await kubeHelper.isOpenShift4() && isOlmPreinstalled) {
+    flags.installer = 'olm'
+  } else {
+    flags.installer = 'operator'
   }
 }