e2e rollback test (#1734)

Che rollback after update integration test

Signed-off-by: Mykola Morhun <[email protected]>

.github/workflows/e2e-minikube-rollback-olm.yml

@@ -0,0 +1,40 @@
+#
+#  Copyright (c) 2012-2021 Red Hat, Inc.
+#    This program and the accompanying materials are made
+#    available under the terms of the Eclipse Public License 2.0
+#    which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+#  SPDX-License-Identifier: EPL-2.0
+#
+#  Contributors:
+#    Red Hat, Inc. - initial API and implementation
+name: Minikube E2E
+on: pull_request
+jobs:
+  minikube-e2e-olm-rollback:
+    name: OLM installer rollback update
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: Start minikube cluster
+      id: run-minikube
+      uses: che-incubator/setup-minikube-action@next
+      with:
+        minikube-version: v1.21.0
+    - name: Install chectl dependencies
+      run: yarn
+    - name: Install OLM
+      run: |
+        export OLM_VERSION=v0.17.0
+        curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/install.sh | bash -s ${OLM_VERSION}
+        sleep 60
+    - name: Run OLM e2e rollback tests
+      run: |
+        export PLATFORM=minikube
+        export INSTALLER=olm
+        yarn test --coverage=false --forceExit --testRegex=test/e2e/e2e-rollback.test.ts
+    - uses: actions/upload-artifact@v2
+      if: ${{ always() }}
+      with:
+        name: test-artifacts
+        path: /tmp/logs/*

README.md

@@ -691,6 +691,9 @@ OPTIONS
   --ssh-key-file=ssh-key-file                            Path to file with private SSH key for authentication on SFTP
                                                          server
 
+  --telemetry=on|off                                     Enable or disable telemetry. This flag skips a prompt and
+                                                         enable/disable telemetry
+
   --username=username                                    Username for authentication in backup REST server
 
 EXAMPLES

src/commands/server/restore.ts

@@ -15,7 +15,7 @@ import { boolean, string } from '@oclif/parser/lib/flags'
 import * as Listr from 'listr'
 
 import { CHE_BACKUP_SERVER_CONFIG_KIND_PLURAL, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL, CHE_CLUSTER_RESTORE_KIND_PLURAL, DEFAULT_ANALYTIC_HOOK_NAME, DEFAULT_OPENSHIFT_OPERATORS_NS_NAME, OPERATOR_DEPLOYMENT_NAME } from '../../constants'
-import { batch, listrRenderer } from '../../common-flags'
+import { batch, CHE_TELEMETRY, listrRenderer } from '../../common-flags'
 import { ChectlContext } from '../../api/context'
 import { KubeHelper } from '../../api/kube'
 import { CheHelper } from '../../api/che'
@@ -55,6 +55,7 @@ export default class Restore extends Command {
     help: flags.help({ char: 'h' }),
     chenamespace: cheNamespace,
     'listr-renderer': listrRenderer,
+    telemetry: CHE_TELEMETRY,
     batch,
     [BACKUP_REPOSITORY_URL_KEY]: backupRepositoryUrl,
     [BACKUP_REPOSITORY_PASSWORD_KEY]: backupRepositoryPassword,
@@ -125,7 +126,9 @@ export default class Restore extends Command {
     }
 
     cli.info(getCommandSuccessMessage())
-    notifyCommandCompletedSuccessfully()
+    if (!flags.batch) {
+      notifyCommandCompletedSuccessfully()
+    }
   }
 
   private getRestoreTasks(flags: any): Listr.ListrTask[] {
@@ -383,7 +386,7 @@ export default class Restore extends Command {
             flags.platform = 'openshift'
             flags['cluster-monitoring'] = true
           } else {
-            flags.platform = 'kubernetes'
+            flags.platform = 'k8s'
           }
 
           if (flags.installer === 'operator') {

src/util.ts

@@ -366,6 +366,7 @@ export function confirmYN(): Promise<boolean> {
     const removeKeyPressHandler = () => {
       process.stdin.removeListener('keypress', keyPressHandler)
       process.stdin.setRawMode(false)
+      process.stdin.destroy()
     }
     const keyPressHandler = (_string: any, key: any) => {
       // Handle brake

test/e2e/e2e-rollback.test.ts

@@ -0,0 +1,74 @@
+/*********************************************************************
+ * Copyright (c) 2021 Red Hat, Inc.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ **********************************************************************/
+
+// tslint:disable: no-console
+import { E2eHelper, NAMESPACE } from './util'
+
+const helper = new E2eHelper()
+jest.setTimeout(1000000)
+
+const binChectl = E2eHelper.getChectlBinaries()
+
+const PLATFORM = process.env.PLATFORM || 'minikube'
+
+const INSTALLER = 'olm'
+const OLM_CHANNEL = 'stable'
+
+const CHE_VERSION_TIMEOUT_MS = 12 * 60 * 1000
+const CHE_BACKUP_TIMEOUT_MS = 2 * 60 * 1000
+
+describe('Test rollback Che update', () => {
+  let previousCheVersion: string
+  let latestCheVersion: string
+
+  describe('Prepare pre-latest stable Che', () => {
+    it(`Deploy Che using ${INSTALLER} installer from ${OLM_CHANNEL} channel`, async () => {
+      // Retrieve pre-latest and latest stable Che version
+      [previousCheVersion, latestCheVersion] = await helper.getTwoLatestReleasedVersions()
+
+      const deployCommand = `${binChectl} server:deploy --batch --platform=${PLATFORM} --installer=${INSTALLER} --olm-channel=${OLM_CHANNEL} --version=${previousCheVersion} --chenamespace=${NAMESPACE} --telemetry=off --che-operator-cr-patch-yaml=test/e2e/resources/cr-patch.yaml`
+      await helper.runCliCommand(deployCommand)
+
+      await helper.waitForVersionInCheCR(previousCheVersion, CHE_VERSION_TIMEOUT_MS)
+    })
+  })
+
+  describe('Update Che to the latest stable version', () => {
+    it('Update Eclipse Che Version to the latest', async () => {
+      console.log(`Updating from ${previousCheVersion} to ${latestCheVersion}`)
+
+      await helper.runCliCommand(binChectl, ['server:update', '-y', `-n ${NAMESPACE}`, '--telemetry=off'])
+    })
+
+    it('Wait backup done', async () => {
+      const backupCrName = 'backup-before-update-to-' + latestCheVersion.replace(/\./g, '-')
+      await helper.waitForSuccessfulBackup(backupCrName, CHE_BACKUP_TIMEOUT_MS)
+    })
+
+    it('Wait updated Che version', async () => {
+      await helper.waitForVersionInCheCR(latestCheVersion, CHE_VERSION_TIMEOUT_MS)
+      // Wait some time to reconcile old resources
+      await helper.sleep(60 * 1000)
+    })
+  })
+
+  describe('Rollback Che update', () => {
+    it('Rollback Che to the previous version', async () => {
+      console.log(`Rolling back from ${latestCheVersion} to ${previousCheVersion}`)
+
+      await helper.runCliCommandVerbose(binChectl, ['server:restore', '--batch', '--rollback', '-n', NAMESPACE, '--telemetry=off'])
+    })
+
+    it('Wait previous Che', async () => {
+      // It is possible to reduce awaiting timeout, because rollback itself waits for the restore to complete.
+      await helper.waitForVersionInCheCR(previousCheVersion, 2 * 60 * 1000)
+    })
+  })
+})

test/e2e/util.ts

@@ -9,13 +9,14 @@
  **********************************************************************/
 import { Octokit } from '@octokit/rest'
 import * as execa from 'execa'
+import { spawn } from 'child_process'
 import * as fs from 'fs-extra'
 
 import { CheHelper } from '../../src/api/che'
 import { CheGithubClient, TagInfo } from '../../src/api/github-client'
 import { KubeHelper } from '../../src/api/kube'
 import { OpenShiftHelper } from '../../src/api/openshift'
-import { DEFAULT_OLM_SUGGESTED_NAMESPACE } from '../../src/constants'
+import { CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL, DEFAULT_OLM_SUGGESTED_NAMESPACE } from '../../src/constants'
 
 // Fields which chectl returns for workspace:list commands
 interface WorkspaceInfo {
@@ -65,6 +66,10 @@ export class E2eHelper {
     return `${process.cwd()}/bin/run`
   }
 
+  /**
+   * Runs given command and returns its output (including error stream if any).
+   * See also runCliCommandVerbose for debug purposes.
+   */
   async runCliCommand(command: string, args?: string[], printOutput = true): Promise<string> {
     if (printOutput) {
       // tslint:disable-next-line: no-console
@@ -87,6 +92,37 @@ export class E2eHelper {
     return stdout
   }
 
+  /**
+   * Runs given cli command.
+   * Unlike runCliCommand, it listens to the command output and prints it immediatly.
+   * This function is useful for debigging, especially if the command hungs at some point.
+   */
+  async runCliCommandVerbose(command: string, args?: string[]): Promise<void> {
+    const argsString = args ? args.join(' ') : ''
+    // tslint:disable-next-line: no-console
+    console.log(`Running command ${command} ${argsString}`)
+
+    return new Promise(resolve => {
+      const process = spawn(command, args)
+
+      process.stdout.on('data', data => {
+        // tslint:disable-next-line: no-console
+        console.log(`[output] ${data}`)
+      })
+      process.stderr.on('data', data => {
+        // tslint:disable-next-line: no-console
+        console.log(`[error] ${data}`)
+      })
+
+      process.on('exit', code => {
+        // tslint:disable-next-line: no-console
+        console.log(`Command ${command} ${argsString} exited with code ${code}`)
+
+        resolve()
+      })
+    })
+  }
+
   // Return an array with all user workspaces
   // async getAllWorkspaces(isOpenshiftPlatformFamily: string): Promise<chetypes.workspace.Workspace[]> {
   private async getAllWorkspaces(): Promise<WorkspaceInfo[]> {
@@ -215,6 +251,28 @@ export class E2eHelper {
     throw new Error(`Che server image tag ${tag} has not appeared in ${timeoutMs / 1000}s `)
   }
 
+  async waitForSuccessfulBackup(backupCrName: string, timeoutMs: number): Promise<void> {
+    const delayMs = 5 * 1000
+
+    let totalTimeMs = 0
+    while (totalTimeMs < timeoutMs) {
+      const backupCr = await this.kubeHelper.getCustomResource(NAMESPACE, backupCrName, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL)
+      if (backupCr && backupCr.status && backupCr.status.state) {
+        if (backupCr.status.state === 'Succeeded') {
+          // Backup successfully created
+          return
+        } else if (backupCr.status.state === 'Failed') {
+          throw new Error(`Backup '${backupCrName}' failed: ${backupCr.status.message}`)
+        }
+        // Wait more
+      }
+
+      await this.sleep(delayMs)
+      totalTimeMs += delayMs
+    }
+    throw new Error(`Backup CR '${backupCrName}' has not appeared in ${timeoutMs / 1000}s`)
+  }
+
   /**
    * Gets last 50 tags from the given repository.
    * @param repo repository name to list tag in
@@ -234,6 +292,15 @@ export class E2eHelper {
     return latestTag.name
   }
 
+  /**
+   * Gets pre-latest and latest released version from chectl repository
+   */
+   async getTwoLatestReleasedVersions(): Promise<[string, string]> {
+    const githubClient = new CheGithubClient()
+    const latestTags = (githubClient as any).sortSemanticTags(await this.listLatestTags(CHECTL_REPONAME))
+    return [latestTags[1].name, latestTags[0].name]
+  }
+
   /**
    * Check if VERSION file exists and return content. In case if Version file doesn't exists
    * We are not in release branch and return `next`