[ws-deployment] Prelim checkin to create a workspace cluster using cli

components/blobserve/go.mod

@@ -52,10 +52,10 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 )
 
-replace github.com/gitpod-io/gitpod/common-go => ../common-go // leeway
-
 replace github.com/gitpod-io/gitpod/gitpod-protocol => ../gitpod-protocol/go // leeway
 
+replace github.com/gitpod-io/gitpod/common-go => ../common-go // leeway
+
 replace github.com/gitpod-io/gitpod/registry-facade => ../registry-facade // leeway
 
 replace github.com/gitpod-io/gitpod/registry-facade/api => ../registry-facade-api/go // leeway

dev/gpctl/go.mod

@@ -139,3 +139,5 @@ replace k8s.io/controller-manager => k8s.io/controller-manager v0.22.2 // leeway
 replace k8s.io/kubectl => k8s.io/kubectl v0.22.2 // leeway indirect from components/common-go:lib
 
 replace k8s.io/mount-utils => k8s.io/mount-utils v0.22.2 // leeway indirect from components/common-go:lib
+
+replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.2 // leeway indirect from components/common-go:lib

dev/loadgen/go.mod

@@ -86,3 +86,5 @@ replace k8s.io/controller-manager => k8s.io/controller-manager v0.22.2 // leeway
 replace k8s.io/kubectl => k8s.io/kubectl v0.22.2 // leeway indirect from components/common-go:lib
 
 replace k8s.io/mount-utils => k8s.io/mount-utils v0.22.2 // leeway indirect from components/common-go:lib
+
+replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.2 // leeway indirect from components/common-go:lib

dev/poolkeeper/go.mod

@@ -104,3 +104,5 @@ replace k8s.io/controller-manager => k8s.io/controller-manager v0.22.2 // leeway
 replace k8s.io/kubectl => k8s.io/kubectl v0.22.2 // leeway indirect from components/common-go:lib
 
 replace k8s.io/mount-utils => k8s.io/mount-utils v0.22.2 // leeway indirect from components/common-go:lib
+
+replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.2 // leeway indirect from components/common-go:lib

dev/telepresence-hack/go.mod

@@ -100,3 +100,5 @@ replace k8s.io/controller-manager => k8s.io/controller-manager v0.22.2 // leeway
 replace k8s.io/kubectl => k8s.io/kubectl v0.22.2 // leeway indirect from components/common-go:lib
 
 replace k8s.io/mount-utils => k8s.io/mount-utils v0.22.2 // leeway indirect from components/common-go:lib
+
+replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.2 // leeway indirect from components/common-go:lib

installer/go.mod

@@ -217,12 +217,6 @@ require (
 
 replace github.com/gitpod-io/gitpod/image-builder => ../components/image-builder-mk3 // leeway
 
-replace github.com/gitpod-io/gitpod/image-builder/api => ../components/image-builder-api/go // leeway
-
-replace github.com/gitpod-io/gitpod/ws-scheduler => ../components/ee/ws-scheduler // leeway
-
-replace github.com/gitpod-io/gitpod/ws-proxy => ../components/ws-proxy // leeway
-
 replace github.com/gitpod-io/gitpod/agent-smith => ../components/ee/agent-smith // leeway
 
 replace github.com/gitpod-io/gitpod/blobserve => ../components/blobserve // leeway
@@ -235,6 +229,8 @@ replace github.com/gitpod-io/gitpod/content-service/api => ../components/content
 
 replace github.com/gitpod-io/gitpod/gitpod-protocol => ../components/gitpod-protocol/go // leeway
 
+replace github.com/gitpod-io/gitpod/image-builder/api => ../components/image-builder-api/go // leeway
+
 replace github.com/gitpod-io/gitpod/registry-facade => ../components/registry-facade // leeway
 
 replace github.com/gitpod-io/gitpod/registry-facade/api => ../components/registry-facade-api/go // leeway
@@ -245,6 +241,10 @@ replace github.com/gitpod-io/gitpod/ws-daemon/api => ../components/ws-daemon-api
 
 replace github.com/gitpod-io/gitpod/ws-manager/api => ../components/ws-manager-api/go // leeway
 
+replace github.com/gitpod-io/gitpod/ws-proxy => ../components/ws-proxy // leeway
+
+replace github.com/gitpod-io/gitpod/ws-scheduler => ../components/ee/ws-scheduler // leeway
+
 replace k8s.io/api => k8s.io/api v0.22.2 // leeway indirect from components/common-go:lib
 
 replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.2 // leeway indirect from components/common-go:lib

operations/workspace/deployment/.gitignore

@@ -0,0 +1 @@
+ws-deployment

operations/workspace/deployment/BUILD.yaml

@@ -0,0 +1,14 @@
+packages:
+  - name: app
+    type: go
+    srcs:
+      - "**/*.go"
+      - "go.mod"
+      - "go.sum"
+    deps:
+      - components/common-go:lib
+    env:
+      - CGO_ENABLED=0
+    config:
+      packaging: app
+

operations/workspace/deployment/OWNERS

@@ -0,0 +1,9 @@
+
+options:
+  no_parent_owners: true
+
+approvers:
+  - engineering-workspace
+
+labels:
+  - "team: workspace"

operations/workspace/deployment/README.md

@@ -0,0 +1,87 @@
+# Overview
+ws-deployment is a cli used to deploy gitpod workspace to a new cluster. It uses a combination of existing tools, scripts and methodologies to achieve this.
+
+Please refer to [this design doc](https://www.notion.so/gitpod/Deployment-Process-Workspace-5f082cc8387447f5940ffb8389bb4fc7) where we had first set of proposals and discussions. For the first iteration we decided to do things with the following in mind:
+`Create immutable workspace clusters and avoid as much manual intervention as possible.`
+
+What this means is we can create same terraform modules and helm values yaml files when a set of same parameters is passed (Note: We are however creating a random id in code for a new cluster). This will help us avoid checking in terraform modules to ops repository. We can generate the same set of modules by running with a bunch of parameters. This is particulary required when we want to destroy a cluster through terraform.
+
+# What
+Broadly these are the steps/things that it does in the given order:
+
+1. Use [ops repository](https://github.com/gitpod-io/ops) to create Terraform Modules based on the given config
+1. Use terraform to apply these terraform module. This creates workspace cluster/(s) and associated infrastructure
+1. Installs gitpod on the cluster/(s) created in previous step using helm
+1. Run a smoke test on these clusters
+1. Register the clusters with right cordoned and govern flag to appropriate meta clusters as defined in the config
+1. Shift Traffic to the new clusters
+
+
+# How
+## Quick Start
+
+Use the below command to get started:
+```sh
+go run main.go deploy --config /path/to/config.yaml
+```
+
+## Overall Flow Diagram
+![Gitpod Workspace Deployment](https://user-images.githubusercontent.com/32481722/137714660-7083d730-d3d6-481d-8bd3-82ec2bb83dad.png "Gitpod Worksapce Deployment")
+
+## Flow Steps
+This is the flow that we are building to achive automated gitpod deployment:
+
+1. A config (default `$HOME/ws-deployment.yaml`) is read by the cli `ws-deployment` and validated based on the version. Currently the config looks similar to below but is constantly being updated as we make progress.
+    ```yaml
+    version: v1
+    project:
+      id: gitpod-dev-staging
+      environment: dev-staging
+      gcpSACredFile: /mnt/secrets/gcp-sa/service-account.json
+      network: gitpod-dev-staging
+      dnsZone: gitpod-dev-staging-com
+      bucket: gitpod-dev-staging-bucket
+    metaClusters:
+    - name: dev-stag-meta-eu01
+      region: europe-west1
+    - name: dev-stag-meta-us01
+      region: us-west1
+    workspaceClusters:
+    - region: europe-west1
+      prefix: eu
+      governedBy: dev-stag-meta-eu01
+      type: gke
+    - region: us-west1
+      prefix: us
+      governedBy: dev-stag-meta-us01
+      type: gke
+    ```
+    TODO(prs) Explanation of each field
+1. A random cluster id is generated which is used to generate cluster names. e.g. a cluster id 17 with prefix value in above config set to `eu` will result in a cluster being created with name `eu17`. This is generated by terraform scripts in the [ops repository](https://github.com/gitpod-io/ops)
+1. We follow a set of steps as mentioned below. Each step may or may not have a pre run check. For example creating a workspace cluster will likely have a check to see if a cluster with the generated name already exists
+    1. **CreateClusterStep** - Use [build-cluster.sh](https://github.com/gitpod-io/ops/blob/main/dev/build-ws-cluster/build-ws-cluster.sh) script to generate terraform modules.
+    1. **InstallGitpodStep** - Use `helm install` to install gitpod
+    1. **InstallJaegerStep** - TODO
+    1. **InstallMonitoringStep** - TODO
+    1. **SmokeTestStep** - TODO
+    1. **RegisterClusterStep** - TODO
+    1. **TrafficShiftStep** - TODO
+
+
+# Where does this Run
+I propose we use a werft job (e.g. [branch](https://github.com/gitpod-io/ops/tree/prs/wspace-cluster-auto) where I am trying to do this. This is outdated though) inside the [ops repository](https://github.com/gitpod-io/ops) and then build this cli in the submodule `gitpod-core`.
+
+I propose we give the ops repository's commit as an input to this werft job (just like how we do it now). The version of gitpod in the values version yaml file will be used to deploy gitpod.
+
+Alternatively, once we have installer production ready, we can use gitpod commit ids to deploy a specific version of gitpod.
+
+# Current Status
+We are far away from having anything executable. I have been only able to create the framework for writing prerun checks and steps. We need to do these (in parallel):
+
+* We have support to create a workspace cluster
+
+# Next Phase
+In the next phase we would want to achive the following:
+1. Metrics based traffic shift
+1. Rollback on failure metrics
+1. Cluster destruction post complete traffic migration

operations/workspace/deployment/cmd/deploy.go

@@ -0,0 +1,49 @@
+/*
+Copyright © 2021 NAME HERE <EMAIL ADDRESS>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"fmt"
+	"math/rand"
+	"os/exec"
+
+	"github.com/gitpod-io/gitpod/common-go/log"
+	"github.com/gitpod-io/gitpod/ws-deployment/pkg/orchestrate"
+	"github.com/spf13/cobra"
+)
+
+// deployCmd represents the deploy command
+var deployCmd = &cobra.Command{
+	Use:   "deploy",
+	Short: "Creates a new workspace cluster and installs gitpod on it",
+	Run: func(cmd *cobra.Command, args []string) {
+		cfg := getConfig()
+		randomId := fmt.Sprintf("%d", rand.Intn(200)+100)
+		cfg.InitializeWorkspaceClusterNames(randomId) // TODO(prs):revisit and update this
+		log.Log.Infof("%+v", cfg)
+		// Activate google service account
+		// mapping command: `gcloud auth activate-service-account --key-file ${GCLOUD_SERVICE_ACCOUNT_CRED_PATH}`
+		out, err := exec.Command("gcloud", "auth", "activate-service-account", "--key-file", cfg.Project.GCPSACredFile).CombinedOutput()
+		if err != nil {
+			log.Log.Infof("error while trying to activate service account: %s. Assuming SA is already activated and configured %s", out)
+		}
+		orchestrate.Deploy(&cfg.Project, cfg.WorkspaceClusters)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(deployCmd)
+}

operations/workspace/deployment/cmd/root.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2020 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"github.com/gitpod-io/gitpod/common-go/log"
+	v1 "github.com/gitpod-io/gitpod/ws-deployment/pkg/config/v1"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	// ServiceName is the name we use for tracing/logging
+	ServiceName = "ws-deployment"
+	// Version of this service - set during build
+	Version = ""
+)
+
+var cfgFile string
+var jsonLog bool
+var verbose bool
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "ws-deployment",
+	Short: "ws-deployment manages the workspace clusters",
+	Long:  "ws-deployment manages the the creation of workspace clusters, installation of gitpod in those clusters and traffic shifting",
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		log.Init(ServiceName, Version, jsonLog, verbose)
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "c", "config file (default is $HOME/ws-deployment.yaml)")
+	rootCmd.PersistentFlags().BoolVarP(&jsonLog, "json-log", "j", true, "produce JSON log output on verbose level")
+}
+
+func getConfig() *v1.Config {
+	ctnt, err := os.ReadFile(cfgFile)
+	if err != nil {
+		log.WithError(err).Fatal("cannot read configuration. Maybe missing --config?")
+	}
+
+	var cfg v1.Config
+	dec := yaml.NewDecoder(bytes.NewReader(ctnt))
+	err = dec.Decode(&cfg)
+	if err != nil {
+		log.WithError(err).Fatal("cannot decode configuration. Maybe missing --config?")
+	}
+
+	return &cfg
+}

operations/workspace/deployment/example-config.yaml

@@ -0,0 +1,22 @@
+version: v1
+project:
+  id: gitpod-dev-staging
+  environment: dev-staging
+  gcpSACredFile: /mnt/secrets/gcp-sa/service-account.json
+  network: gitpod-dev-staging
+  dnsZone: gitpod-dev-staging-com
+  bucket: gitpod-dev-staging-bucket
+metaClusters:
+- name: dev-stag-meta-eu01
+  region: europe-west1
+- name: dev-stag-meta-us01
+  region: us-west1
+workspaceClusters:
+- region: europe-west1
+  prefix: eu
+  governedBy: dev-stag-meta-eu01
+  type: gke
+- region: us-west1
+  prefix: us
+  governedBy: dev-stag-meta-us01
+  type: gke