Improve Eclipse Che deploy / update flow (#1082)

Improve Eclipse Che deploying with chectl
Signed-off-by: Mykola Morhun <[email protected]>

README.md

@@ -415,6 +415,9 @@ OPTIONS
                                                                            answer to all prompts and run
                                                                            non-interactively
 
+  --batch                                                                  Batch mode. Running a command without end
+                                                                           user interaction.
+
   --delete-namespace                                                       Indicates that a Eclipse Che namespace will
                                                                            be deleted as well
 
@@ -442,7 +445,7 @@ USAGE
   $ chectl server:deploy
 
 OPTIONS
-  -a, --installer=helm|operator|olm|minishift-addon
+  -a, --installer=helm|operator|olm
       Installer type. If not set, default is "olm" for OpenShift 4.x platform otherwise "operator".
 
   -b, --domain=domain
@@ -479,12 +482,18 @@ OPTIONS
   -t, --templates=templates
       Path to the templates folder
 
+  -v, --version=version
+      Version to deploy (e.g. 7.15.2). Defaults to the same as chectl.
+
   --[no-]auto-update
       Auto update approval strategy for installation Eclipse Che.
                            With this strategy will be provided auto-update Eclipse Che without any human interaction.
                            By default this flag is enabled.
                            This parameter is used only when the installer is 'olm'.
 
+  --batch
+      Batch mode. Running a command without end user interaction.
+
   --catalog-source-name=catalog-source-name
       OLM catalog source to install Eclipse Che operator.
                            This parameter is used only when the installer is the 'olm'.
@@ -508,8 +517,7 @@ OPTIONS
       is the 'operator' or the 'olm'.
 
   --che-operator-image=che-operator-image
-      [default: quay.io/eclipse/che-operator:nightly] Container image of the operator. This parameter is used only when 
-      the installer is the operator
+      Container image of the operator. This parameter is used only when the installer is the operator
 
   --debug
       Enables the debug mode for Eclipse Che server. To debug Eclipse Che server from localhost use 'server:debug' 
@@ -714,19 +722,20 @@ USAGE
 OPTIONS
   -h, --help                                               show CLI help
   -n, --chenamespace=chenamespace                          Eclipse Che Kubernetes namespace. Default to 'eclipse-che'
-  -t, --templates=templates                                [default: templates] Path to the templates folder
+  -t, --templates=templates                                Path to the templates folder
+
+  -v, --version=version                                    Version to deploy (e.g. 7.15.2). Defaults to the same as
+                                                           chectl.
 
   -y, --yes                                                Automatic yes to prompts; assume "yes" as answer to all
                                                            prompts and run non-interactively
 
+  --batch                                                  Batch mode. Running a command without end user interaction.
+
   --che-operator-cr-patch-yaml=che-operator-cr-patch-yaml  Path to a yaml file that overrides the default values in
                                                            CheCluster CR used by the operator. This parameter is used
                                                            only when the installer is the 'operator' or the 'olm'.
 
-  --che-operator-image=che-operator-image                  [default: quay.io/eclipse/che-operator:nightly] Container
-                                                           image of the operator. This parameter is used only when the
-                                                           installer is the operator
-
   --deployment-name=deployment-name                        [default: che] Eclipse Che deployment name
 
   --skip-kubernetes-health-check                           Skip Kubernetes health check

package.json

@@ -15,13 +15,7 @@
     "@oclif/plugin-autocomplete": "^0.2.0",
     "@oclif/plugin-help": "^3",
     "@oclif/plugin-update": "^1.3.10",
-    "@types/command-exists": "^1.2.0",
-    "@types/fs-extra": "^9.0.1",
-    "@types/inquirer": "^7.3.1",
-    "@types/node-notifier": "^8.0.0",
-    "@types/request": "^2.48.5",
-    "@types/websocket": "^1.0.1",
-    "@types/ws": "^7.2.6",
+    "@octokit/rest": "^18.0.12",
     "analytics-node": "^3.4.0-beta.3",
     "ansi-colors": "4.1.1",
     "axios": "^0.19.2",
@@ -47,20 +41,33 @@
     "node-forge": "^0.10.0",
     "node-notifier": "^8.0.0",
     "querystring": "^0.2.0",
+    "rimraf": "^3.0.2",
+    "semver": "^7.3.4",
     "stream-buffers": "^3.0.2",
-    "tslib": "^1"
+    "tslib": "^1",
+    "unzipper": "0.10.11"
   },
   "devDependencies": {
     "@eclipse-che/api": "latest",
     "@oclif/dev-cli": "^1",
     "@oclif/test": "^1",
     "@oclif/tslint": "^3",
     "@types/chai": "^4",
+    "@types/command-exists": "^1.2.0",
+    "@types/fs-extra": "^9.0.1",
+    "@types/inquirer": "^7.3.1",
     "@types/jest": "26.0.14",
     "@types/js-yaml": "^3.12.5",
     "@types/listr": "^0.14.2",
     "@types/node": "^12",
     "@types/node-forge": "^0.9.5",
+    "@types/node-notifier": "^8.0.0",
+    "@types/request": "^2.48.5",
+    "@types/rimraf": "^3.0.0",
+    "@types/semver": "^7.3.4",
+    "@types/unzipper": "^0.10.3",
+    "@types/websocket": "^1.0.1",
+    "@types/ws": "^7.2.6",
     "chai": "^4.2.0",
     "cpx": "^1.5.0",
     "globby": "^11",
@@ -90,13 +97,14 @@
   "main": "lib/index.js",
   "oclif": {
     "commands": "./lib/commands",
+    "hooks": {
+      "prerun": "./lib/hooks/prerun/new-version-warning",
+      "analytics": "./lib/hooks/analytics/analytics"
+    },
     "bin": "chectl",
     "macos": {
       "identifier": "che-incubator.chectl"
     },
-    "hooks": {
-      "analytics": "./lib/hooks/analytics/analytics"
-    },
     "plugins": [
       "@oclif/plugin-autocomplete",
       "@oclif/plugin-help",
@@ -133,12 +141,11 @@
   },
   "repository": "che-incubator/chectl",
   "scripts": {
-    "postinstall": "npm run -s postinstall-repositories && npm run -s postinstall-helm && npm run -s postinstall-cert-manager && npm run -s postinstall-operator && npm run -s postinstall-minishift-addon && npm run -s postinstall-devfile-api && npm run -s postinstall-dev-workspace && npm run -s postinstall-cleanup",
+    "postinstall": "npm run -s postinstall-repositories && npm run -s postinstall-helm && npm run -s postinstall-cert-manager && npm run -s postinstall-operator && npm run -s postinstall-devfile-api && npm run -s postinstall-dev-workspace && npm run -s postinstall-cleanup",
     "postinstall-helm": "rimraf templates/kubernetes && cpx 'node_modules/eclipse-che/deploy/kubernetes/**' 'templates/kubernetes'",
     "postinstall-cert-manager": "rimraf templates/cert-manager && cpx 'node_modules/eclipse-che/deploy/cert-manager/**' 'templates/cert-manager'",
     "postinstall-devfile-api": "rimraf templates/devfile-api && cpx 'node_modules/eclipse-che-devfile-api/deploy/**' 'templates/devfile-api'",
     "postinstall-dev-workspace": "rimraf templates/devworkspace && cpx 'node_modules/eclipse-che-devfile-workspace-operator/deploy/**' 'templates/devworkspace'",
-    "postinstall-minishift-addon": "rimraf templates/minishift-addon && cpx 'node_modules/eclipse-che-minishift/addons/che/**' 'templates/minishift-addon/che'",
     "postinstall-operator": "rimraf templates/che-operator && cpx 'node_modules/eclipse-che-operator/deploy/**' 'templates/che-operator'",
     "postinstall-repositories": "yarn upgrade eclipse-che eclipse-che-operator eclipse-che-minishift eclipse-che-devfile-api eclipse-che-devfile-workspace-operator",
     "postinstall-cleanup": "rimraf node_modules/eclipse-che && rimraf node_modules/eclipse-che-operator && rimraf node_modules/eclipse-che-minishift",

src/api/che.ts

@@ -20,10 +20,12 @@ import * as yaml from 'js-yaml'
 import * as nodeforge from 'node-forge'
 import * as os from 'os'
 import * as path from 'path'
+import * as rimraf from 'rimraf'
+import * as unzipper from 'unzipper'
 
 import { OpenShiftHelper } from '../api/openshift'
 import { CHE_ROOT_CA_SECRET_NAME, DEFAULT_CA_CERT_FILE_NAME } from '../constants'
-import { base64Decode } from '../util'
+import { base64Decode, downloadFile } from '../util'
 
 import { CheApiClient } from './che-api-client'
 import { ChectlContext } from './context'
@@ -484,4 +486,69 @@ export class CheHelper {
     return fileName
   }
 
+  /**
+   * Gets install templates for given installer.
+   * @param installer Che installer
+   * @param url link to zip archive with sources of Che operator
+   * @param destDir destination directory into which the templates should be unpacked
+   */
+  async downloadAndUnpackTemplates(installer: string, url: string, destDir: string): Promise<void> {
+    // Add che-operator folder for operator templates
+    if (installer === 'operator') {
+      destDir = path.join(destDir, 'che-operator')
+    }
+    // No need to add kubernetes folder for Helm installer as it already present in the archive
+
+    const tempDir = path.join(os.tmpdir(), Date.now().toString())
+    await fs.mkdirp(tempDir)
+    const zipFile = path.join(tempDir, `che-templates-${installer}.zip`)
+    await downloadFile(url, zipFile)
+    await this.unzipTemplates(zipFile, destDir)
+    // Clean up zip. Do not wait when finishes.
+    rimraf(tempDir, () => {})
+  }
+
+  /**
+   * Unpacks repository deploy templates into specified folder
+   * @param zipFile path to zip archive with source code
+   * @param destDir target directory into which templates should be unpacked
+   */
+  private async unzipTemplates(zipFile: string, destDir: string) {
+    // Gets path from: repo-name/deploy/path
+    const deployDirRegex = new RegExp('(?:^[\\\w-]*\\\/deploy\\\/)(.*)')
+
+    const zip = fs.createReadStream(zipFile).pipe(unzipper.Parse({ forceStream: true }))
+    for await (const entry of zip) {
+      const entryPathInZip: string = entry.path
+      const templatesPathMatch = entryPathInZip.match(deployDirRegex)
+      if (templatesPathMatch && templatesPathMatch.length > 1 && templatesPathMatch[1]) {
+        // Remove prefix from in-zip path
+        const entryPathWhenExtracted = templatesPathMatch[1]
+        // Path to the item in target location
+        const dest = path.join(destDir, entryPathWhenExtracted)
+
+        // Extract item
+        if (entry.type === 'File') {
+          const parentDirName = path.dirname(dest)
+          if (!fs.existsSync(parentDirName)) {
+            await fs.mkdirp(parentDirName)
+          }
+          entry.pipe(fs.createWriteStream(dest))
+        } else if (entry.type === 'Directory') {
+          if (!fs.existsSync(dest)) {
+            await fs.mkdirp(dest)
+          }
+          // The folder is created above
+          entry.autodrain()
+        } else {
+          // Ignore the item as we do not need to handle links and etc.
+          entry.autodrain()
+        }
+      } else {
+        // No need to extract this item
+        entry.autodrain()
+      }
+    }
+  }
+
 }

src/api/context.ts

@@ -14,9 +14,10 @@ import * as os from 'os'
 import * as path from 'path'
 
 import { CHE_OPERATOR_CR_PATCH_YAML_KEY, CHE_OPERATOR_CR_YAML_KEY, LOG_DIRECTORY_KEY } from '../common-flags'
-import { readCRFile } from '../util'
+import { getProjectName, getProjectVersion, readCRFile } from '../util'
 
 import { KubeHelper } from './kube'
+import { CHECTL_DEVELOPMENT_VERSION } from './version'
 
 /**
  * chectl command context.
@@ -28,13 +29,14 @@ export namespace ChectlContext {
   export const START_TIME = 'startTime'
   export const END_TIME = 'endTime'
   export const CONFIG_DIR = 'configDir'
+  export const CACHE_DIR = 'cacheDir'
   export const ERROR_LOG = 'errorLog'
   export const COMMAND_ID = 'commandId'
 
   // command specific attributes
   export const CUSTOM_CR = 'customCR'
   export const CR_PATCH = 'crPatch'
-  export const LOGS_DIRECTORY = 'directory'
+  export const LOGS_DIR = 'directory'
 
   const ctx: any = {}
 
@@ -43,6 +45,9 @@ export namespace ChectlContext {
     ctx[IS_OPENSHIFT] = await kube.isOpenShift()
     ctx[IS_OPENSHIFT4] = await kube.isOpenShift4()
 
+    ctx.isChectl = getProjectName() === 'chectl'
+    ctx.isNightly = getProjectVersion().includes('next') || getProjectVersion() === CHECTL_DEVELOPMENT_VERSION
+
     if (flags['listr-renderer'] as any) {
       ctx.listrOptions = { renderer: (flags['listr-renderer'] as any), collapse: false } as Listr.ListrOptions
     }
@@ -51,9 +56,10 @@ export namespace ChectlContext {
     ctx[START_TIME] = Date.now()
 
     ctx[CONFIG_DIR] = command.config.configDir
+    ctx[CACHE_DIR] = command.config.cacheDir
     ctx[ERROR_LOG] = command.config.errlog
     ctx[COMMAND_ID] = command.id
-    ctx[LOGS_DIRECTORY] = path.resolve(flags[LOG_DIRECTORY_KEY] ? flags[LOG_DIRECTORY_KEY] : path.resolve(os.tmpdir(), 'chectl-logs', Date.now().toString()))
+    ctx[LOGS_DIR] = path.resolve(flags[LOG_DIRECTORY_KEY] ? flags[LOG_DIRECTORY_KEY] : path.resolve(os.tmpdir(), 'chectl-logs', Date.now().toString()))
 
     ctx[CUSTOM_CR] = readCRFile(flags, CHE_OPERATOR_CR_YAML_KEY)
     ctx[CR_PATCH] = readCRFile(flags, CHE_OPERATOR_CR_PATCH_YAML_KEY)