From fb3f966b3c4023f7b90c1a34af040b042d7e4876 Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Thu, 7 Oct 2021 17:23:20 +0300 Subject: [PATCH] e2e rollback test (#1734) Che rollback after update integration test Signed-off-by: Mykola Morhun --- .../workflows/e2e-minikube-rollback-olm.yml | 40 ++++++++++ README.md | 3 + src/commands/server/restore.ts | 9 ++- src/util.ts | 1 + test/e2e/e2e-rollback.test.ts | 74 +++++++++++++++++++ test/e2e/util.ts | 69 ++++++++++++++++- 6 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/e2e-minikube-rollback-olm.yml create mode 100644 test/e2e/e2e-rollback.test.ts diff --git a/.github/workflows/e2e-minikube-rollback-olm.yml b/.github/workflows/e2e-minikube-rollback-olm.yml new file mode 100644 index 000000000..44fbeb90f --- /dev/null +++ b/.github/workflows/e2e-minikube-rollback-olm.yml @@ -0,0 +1,40 @@ +# +# Copyright (c) 2012-2021 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation +name: Minikube E2E +on: pull_request +jobs: + minikube-e2e-olm-rollback: + name: OLM installer rollback update + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v1 + - name: Start minikube cluster + id: run-minikube + uses: che-incubator/setup-minikube-action@next + with: + minikube-version: v1.21.0 + - name: Install chectl dependencies + run: yarn + - name: Install OLM + run: | + export OLM_VERSION=v0.17.0 + curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/install.sh | bash -s ${OLM_VERSION} + sleep 60 + - name: Run OLM e2e rollback tests + run: | + export PLATFORM=minikube + export INSTALLER=olm + yarn test --coverage=false --forceExit --testRegex=test/e2e/e2e-rollback.test.ts + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: test-artifacts + path: /tmp/logs/* diff --git a/README.md b/README.md index aa997937c..95ca553b9 100644 --- a/README.md +++ b/README.md @@ -691,6 +691,9 @@ OPTIONS --ssh-key-file=ssh-key-file Path to file with private SSH key for authentication on SFTP server + --telemetry=on|off Enable or disable telemetry. This flag skips a prompt and + enable/disable telemetry + --username=username Username for authentication in backup REST server EXAMPLES diff --git a/src/commands/server/restore.ts b/src/commands/server/restore.ts index ec0c186be..637359847 100644 --- a/src/commands/server/restore.ts +++ b/src/commands/server/restore.ts @@ -15,7 +15,7 @@ import { boolean, string } from '@oclif/parser/lib/flags' import * as Listr from 'listr' import { CHE_BACKUP_SERVER_CONFIG_KIND_PLURAL, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL, CHE_CLUSTER_RESTORE_KIND_PLURAL, DEFAULT_ANALYTIC_HOOK_NAME, DEFAULT_OPENSHIFT_OPERATORS_NS_NAME, OPERATOR_DEPLOYMENT_NAME } from '../../constants' -import { batch, listrRenderer } from '../../common-flags' +import { batch, CHE_TELEMETRY, listrRenderer } from '../../common-flags' import { ChectlContext } from '../../api/context' import { KubeHelper } from '../../api/kube' import { CheHelper } from '../../api/che' @@ -55,6 +55,7 @@ export default class Restore extends Command { help: flags.help({ char: 'h' }), chenamespace: cheNamespace, 'listr-renderer': listrRenderer, + telemetry: CHE_TELEMETRY, batch, [BACKUP_REPOSITORY_URL_KEY]: backupRepositoryUrl, [BACKUP_REPOSITORY_PASSWORD_KEY]: backupRepositoryPassword, @@ -125,7 +126,9 @@ export default class Restore extends Command { } cli.info(getCommandSuccessMessage()) - notifyCommandCompletedSuccessfully() + if (!flags.batch) { + notifyCommandCompletedSuccessfully() + } } private getRestoreTasks(flags: any): Listr.ListrTask[] { @@ -383,7 +386,7 @@ export default class Restore extends Command { flags.platform = 'openshift' flags['cluster-monitoring'] = true } else { - flags.platform = 'kubernetes' + flags.platform = 'k8s' } if (flags.installer === 'operator') { diff --git a/src/util.ts b/src/util.ts index 90e850c2f..0925b17f7 100644 --- a/src/util.ts +++ b/src/util.ts @@ -366,6 +366,7 @@ export function confirmYN(): Promise { const removeKeyPressHandler = () => { process.stdin.removeListener('keypress', keyPressHandler) process.stdin.setRawMode(false) + process.stdin.destroy() } const keyPressHandler = (_string: any, key: any) => { // Handle brake diff --git a/test/e2e/e2e-rollback.test.ts b/test/e2e/e2e-rollback.test.ts new file mode 100644 index 000000000..60c0dd098 --- /dev/null +++ b/test/e2e/e2e-rollback.test.ts @@ -0,0 +1,74 @@ +/********************************************************************* + * Copyright (c) 2021 Red Hat, Inc. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + **********************************************************************/ + +// tslint:disable: no-console +import { E2eHelper, NAMESPACE } from './util' + +const helper = new E2eHelper() +jest.setTimeout(1000000) + +const binChectl = E2eHelper.getChectlBinaries() + +const PLATFORM = process.env.PLATFORM || 'minikube' + +const INSTALLER = 'olm' +const OLM_CHANNEL = 'stable' + +const CHE_VERSION_TIMEOUT_MS = 12 * 60 * 1000 +const CHE_BACKUP_TIMEOUT_MS = 2 * 60 * 1000 + +describe('Test rollback Che update', () => { + let previousCheVersion: string + let latestCheVersion: string + + describe('Prepare pre-latest stable Che', () => { + it(`Deploy Che using ${INSTALLER} installer from ${OLM_CHANNEL} channel`, async () => { + // Retrieve pre-latest and latest stable Che version + [previousCheVersion, latestCheVersion] = await helper.getTwoLatestReleasedVersions() + + const deployCommand = `${binChectl} server:deploy --batch --platform=${PLATFORM} --installer=${INSTALLER} --olm-channel=${OLM_CHANNEL} --version=${previousCheVersion} --chenamespace=${NAMESPACE} --telemetry=off --che-operator-cr-patch-yaml=test/e2e/resources/cr-patch.yaml` + await helper.runCliCommand(deployCommand) + + await helper.waitForVersionInCheCR(previousCheVersion, CHE_VERSION_TIMEOUT_MS) + }) + }) + + describe('Update Che to the latest stable version', () => { + it('Update Eclipse Che Version to the latest', async () => { + console.log(`Updating from ${previousCheVersion} to ${latestCheVersion}`) + + await helper.runCliCommand(binChectl, ['server:update', '-y', `-n ${NAMESPACE}`, '--telemetry=off']) + }) + + it('Wait backup done', async () => { + const backupCrName = 'backup-before-update-to-' + latestCheVersion.replace(/\./g, '-') + await helper.waitForSuccessfulBackup(backupCrName, CHE_BACKUP_TIMEOUT_MS) + }) + + it('Wait updated Che version', async () => { + await helper.waitForVersionInCheCR(latestCheVersion, CHE_VERSION_TIMEOUT_MS) + // Wait some time to reconcile old resources + await helper.sleep(60 * 1000) + }) + }) + + describe('Rollback Che update', () => { + it('Rollback Che to the previous version', async () => { + console.log(`Rolling back from ${latestCheVersion} to ${previousCheVersion}`) + + await helper.runCliCommandVerbose(binChectl, ['server:restore', '--batch', '--rollback', '-n', NAMESPACE, '--telemetry=off']) + }) + + it('Wait previous Che', async () => { + // It is possible to reduce awaiting timeout, because rollback itself waits for the restore to complete. + await helper.waitForVersionInCheCR(previousCheVersion, 2 * 60 * 1000) + }) + }) +}) diff --git a/test/e2e/util.ts b/test/e2e/util.ts index add7f0b51..a75fdc817 100644 --- a/test/e2e/util.ts +++ b/test/e2e/util.ts @@ -9,13 +9,14 @@ **********************************************************************/ import { Octokit } from '@octokit/rest' import * as execa from 'execa' +import { spawn } from 'child_process' import * as fs from 'fs-extra' import { CheHelper } from '../../src/api/che' import { CheGithubClient, TagInfo } from '../../src/api/github-client' import { KubeHelper } from '../../src/api/kube' import { OpenShiftHelper } from '../../src/api/openshift' -import { DEFAULT_OLM_SUGGESTED_NAMESPACE } from '../../src/constants' +import { CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL, DEFAULT_OLM_SUGGESTED_NAMESPACE } from '../../src/constants' // Fields which chectl returns for workspace:list commands interface WorkspaceInfo { @@ -65,6 +66,10 @@ export class E2eHelper { return `${process.cwd()}/bin/run` } + /** + * Runs given command and returns its output (including error stream if any). + * See also runCliCommandVerbose for debug purposes. + */ async runCliCommand(command: string, args?: string[], printOutput = true): Promise { if (printOutput) { // tslint:disable-next-line: no-console @@ -87,6 +92,37 @@ export class E2eHelper { return stdout } + /** + * Runs given cli command. + * Unlike runCliCommand, it listens to the command output and prints it immediatly. + * This function is useful for debigging, especially if the command hungs at some point. + */ + async runCliCommandVerbose(command: string, args?: string[]): Promise { + const argsString = args ? args.join(' ') : '' + // tslint:disable-next-line: no-console + console.log(`Running command ${command} ${argsString}`) + + return new Promise(resolve => { + const process = spawn(command, args) + + process.stdout.on('data', data => { + // tslint:disable-next-line: no-console + console.log(`[output] ${data}`) + }) + process.stderr.on('data', data => { + // tslint:disable-next-line: no-console + console.log(`[error] ${data}`) + }) + + process.on('exit', code => { + // tslint:disable-next-line: no-console + console.log(`Command ${command} ${argsString} exited with code ${code}`) + + resolve() + }) + }) + } + // Return an array with all user workspaces // async getAllWorkspaces(isOpenshiftPlatformFamily: string): Promise { private async getAllWorkspaces(): Promise { @@ -215,6 +251,28 @@ export class E2eHelper { throw new Error(`Che server image tag ${tag} has not appeared in ${timeoutMs / 1000}s `) } + async waitForSuccessfulBackup(backupCrName: string, timeoutMs: number): Promise { + const delayMs = 5 * 1000 + + let totalTimeMs = 0 + while (totalTimeMs < timeoutMs) { + const backupCr = await this.kubeHelper.getCustomResource(NAMESPACE, backupCrName, CHE_CLUSTER_API_GROUP, CHE_CLUSTER_API_VERSION, CHE_CLUSTER_BACKUP_KIND_PLURAL) + if (backupCr && backupCr.status && backupCr.status.state) { + if (backupCr.status.state === 'Succeeded') { + // Backup successfully created + return + } else if (backupCr.status.state === 'Failed') { + throw new Error(`Backup '${backupCrName}' failed: ${backupCr.status.message}`) + } + // Wait more + } + + await this.sleep(delayMs) + totalTimeMs += delayMs + } + throw new Error(`Backup CR '${backupCrName}' has not appeared in ${timeoutMs / 1000}s`) + } + /** * Gets last 50 tags from the given repository. * @param repo repository name to list tag in @@ -234,6 +292,15 @@ export class E2eHelper { return latestTag.name } + /** + * Gets pre-latest and latest released version from chectl repository + */ + async getTwoLatestReleasedVersions(): Promise<[string, string]> { + const githubClient = new CheGithubClient() + const latestTags = (githubClient as any).sortSemanticTags(await this.listLatestTags(CHECTL_REPONAME)) + return [latestTags[1].name, latestTags[0].name] + } + /** * Check if VERSION file exists and return content. In case if Version file doesn't exists * We are not in release branch and return `next`