[werft] Support installation of gitpod workspace component in a separ…

…ate k3s cluster (#4664)

* Support workspace deployment in a separate k3s cluster using flag k3s-ws

.werft/util/certs.ts

@@ -9,8 +9,7 @@ export class IssueCertificateParams {
     dnsZoneDomain: string
     domain: string
     ip: string
-    additionalWsSubdomains: string[]
-    includeDefaults: boolean
+    additionalSubdomains: string[]
     pathToKubeConfig: string
     bucketPrefixTail: string
     certNamespace: string
@@ -24,14 +23,11 @@ export class InstallCertificateParams {
     destinationNamespace: string
 }
 
-function getDefaultSubDomains(): string[] {
-    return ["", "*.", "*.ws-dev."];
-}
-
 export async function issueCertficate(werft, params: IssueCertificateParams) {
-    const subdomains = params.includeDefaults ? getDefaultSubDomains() : [];
-    for (const sd of params.additionalWsSubdomains) {
-        subdomains.push(`*.ws-${sd}.`);
+    var subdomains = [];
+    werft.log("certificate", `Subdomains: ${params.additionalSubdomains}`)
+    for (const sd of params.additionalSubdomains) {
+        subdomains.push(sd);
     }
 
     // sanity: check if there is a "SAN short enough to fit into CN (63 characters max)"
@@ -63,10 +59,10 @@ export async function issueCertficate(werft, params: IssueCertificateParams) {
 
     werft.log('certificate', `waiting until certificate ${params.certNamespace}/${params.namespace} is ready...`)
     let notReadyYet = true;
-    while (notReadyYet) {
-        werft.log('certificate', `polling state of certs/${params.namespace}...`)
+    for (let i = 0; i < 90 && notReadyYet; i++) {
+        werft.log('certificate', `polling state of ${params.certNamespace}/${params.namespace}...`)
         const result = exec(`export KUBECONFIG=${params.pathToKubeConfig} && kubectl -n ${params.certNamespace} get certificate ${params.namespace} -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}"`, { silent: true, dontCheckRc: true });
-        if (result.code === 0 && result.stdout === "True") {
+        if (result != undefined && result.code === 0 && result.stdout === "True") {
             notReadyYet = false;
             break;
         }
@@ -76,13 +72,26 @@ export async function issueCertficate(werft, params: IssueCertificateParams) {
 }
 
 export async function installCertficate(werft, params: InstallCertificateParams) {
+    let notReadyYet = true;
     werft.log('certificate', `copying certificate from "${params.certNamespace}/${params.certName}" to "${params.destinationNamespace}/${params.certSecretName}"`);
-    // certmanager is configured to create a secret in the namespace "certs" with the name "${namespace}".
-    exec(`export KUBECONFIG=${params.pathToKubeConfig} && kubectl get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \
-        | yq d - 'metadata.namespace' \
-        | yq d - 'metadata.uid' \
-        | yq d - 'metadata.resourceVersion' \
-        | yq d - 'metadata.creationTimestamp' \
-        | sed 's/${params.certName}/${params.certSecretName}/g' \
-        | kubectl apply --namespace=${params.destinationNamespace} -f -`);
+    const cmd = `export KUBECONFIG=${params.pathToKubeConfig} && kubectl get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \
+    | yq d - 'metadata.namespace' \
+    | yq d - 'metadata.uid' \
+    | yq d - 'metadata.resourceVersion' \
+    | yq d - 'metadata.creationTimestamp' \
+    | sed 's/${params.certName}/${params.certSecretName}/g' \
+    | kubectl apply --namespace=${params.destinationNamespace} -f -`
+
+    for (let i = 0; i < 60 && notReadyYet; i++) {
+        const result = exec(cmd, { silent: true, dontCheckRc: true });
+        if (result != undefined && result.code === 0) {
+            notReadyYet = false;
+            break;
+        }
+        werft.log('certificate', `Could not copy "${params.certNamespace}/${params.certName}", will retry`);
+        await sleep(5000);
+    }
+    if (!notReadyYet) {
+        werft.log('certificate', `copied certificate from "${params.certNamespace}/${params.certName}" to "${params.destinationNamespace}/${params.certSecretName}"`);
+    }
 }

.werft/util/gpctl.ts

@@ -0,0 +1,43 @@
+import * as shell from 'shelljs';
+
+function buildRequiredFlags(pathToKubeConfig: string): string {
+    if (pathToKubeConfig != "") {
+        return ` --kubeconfig=${pathToKubeConfig}`
+    }
+    return ""
+}
+
+export function buildGpctlBinary() {
+    shell.exec(`cd /workspace/dev/gpctl && go build && cd -`)
+}
+
+export function printClustersList(pathToKubeConfig: string): string {
+    let cmd = `/workspace/dev/gpctl/gpctl clusters list` + buildRequiredFlags(pathToKubeConfig)
+    const result = shell.exec(cmd).trim()
+    return result
+}
+
+export function uncordonCluster(pathToKubeConfig: string, name: string): string {
+    let cmd = `/workspace/dev/gpctl/gpctl clusters uncordon --name=${name}` + buildRequiredFlags(pathToKubeConfig)
+    const result = shell.exec(cmd).trim()
+    return result
+}
+
+export function registerCluster(pathToKubeConfig: string, name: string, url: string): string {
+    let cmd = `/workspace/dev/gpctl/gpctl clusters register \
+	--name ${name} \
+	--hint-cordoned \
+	--hint-govern \
+	--tls-path ./wsman-tls \
+	--url ${url}` + buildRequiredFlags(pathToKubeConfig)
+
+    const result = shell.exec(cmd).trim()
+    return result
+}
+
+export function getClusterTLS(pathToKubeConfig: string): string {
+    let cmd = `/workspace/dev/gpctl/gpctl clusters get-tls-config` + buildRequiredFlags(pathToKubeConfig)
+    const result = shell.exec(cmd).trim()
+    return result
+}
+

.werft/util/kubectl.ts

@@ -4,10 +4,10 @@ import { exec, ExecOptions, werft } from './shell';
 
 export const IS_PREVIEW_APP_LABEL: string = "isPreviewApp";
 
-export function setKubectlContextNamespace(namespace, shellOpts) {
+export function setKubectlContextNamespace(pathToKubeConfig, namespace, shellOpts) {
     [
-        "kubectl config current-context",
-        `kubectl config set-context --current --namespace=${namespace}`
+        `export KUBECONFIG=${pathToKubeConfig} && kubectl config current-context`,
+        `export KUBECONFIG=${pathToKubeConfig} && kubectl config set-context --current --namespace=${namespace}`
     ].forEach(cmd => exec(cmd, shellOpts));
 }
 

.werft/values.wsCluster.yaml → .werft/values.disableMeta.yaml

@@ -3,9 +3,6 @@ components:
   proxy:
     disabled: true
 
-  wsProxy:
-    hostHeader: Host
-
   wsManagerBridge:
     disabled: true
 
@@ -35,5 +32,3 @@ minio:
 mysql:
   enabled: false
 
-registry-facade:
-  enabled: false

.werft/values.k3sWsCluster.yaml

@@ -0,0 +1,175 @@
+installation:
+  stage: devstaging
+  tenant: gitpod-core
+  region: europe-west1
+  cluster: "00"
+  shortname: "k3s"
+hostname: staging.gitpod-dev.com
+imagePrefix: eu.gcr.io/gitpod-core-dev/build/
+certificatesSecret:
+  secretName: proxy-config-certificates
+version: not-set
+imagePullPolicy: Always
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: gitpod.io/workload_services
+          operator: In
+          values:
+          - "true"
+authProviders: []
+tracing:
+  endoint: http://jaeger-collector:14268/api/traces
+  samplerType: const
+  samplerParam: "1"
+
+components:
+  wsManagerBridge:
+    disabled: true
+
+  agentSmith:
+    name: "agent-smith"
+    disabled: false
+    # in preview envs, we never want DaemonSets not to be scheduled (because they don't trigger scaleup)
+    resources:
+      cpu: 1m
+      memory: 32Mi
+
+  registryFacade:
+    daemonSet: true
+    # in preview envs, we never want DaemonSets not to be scheduled (because they don't trigger scaleup)
+    resources:
+      cpu: 1m
+      memory: 32Mi
+
+  contentService:
+    remoteStorage:
+      blobQuota: 1073741824 # 1 GiB
+
+  workspace:
+    # configure GCP registry
+    pullSecret:
+      secretName: gcp-sa-registry-auth
+    affinity:
+      default: "gitpod.io/workload_workspace"
+    templates:
+      default:
+        spec:
+          dnsConfig:
+            nameservers:
+            - 1.1.1.1
+            - 8.8.8.8
+          dnsPolicy: None   # do NOT query against K8s DNS (https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/)
+          env:
+          - name: THEIA_PREVENT_METADATA_ACCESS
+            value: true
+      regular:
+        spec:
+          containers:
+          - name: "workspace"
+            env:
+            - name: THEIA_RATELIMIT_LOG
+              value: "50"
+            - name: SUPERVISOR_DEBUG_ENABLE
+              value: "true"
+      prebuild:
+        spec:
+          containers:
+          - name: workspace
+            # Intended to reduce the density for prebuilds
+            resources:
+              limits:
+                cpu: "5"
+                memory: 12Gi
+              requests:
+                cpu: 1m
+                ephemeral-storage: 5Gi
+                memory: 4608Mi  # = 2 * 2304Mi
+
+  # Enable events trace
+  wsManager:
+    eventTraceLogLocation: "/tmp/evts.json"
+
+  wsDaemon:
+    hostWorkspaceArea: /mnt/disks/ssd0/workspaces
+    volumes:
+    - name: gcloud-tmp
+      hostPath:
+        path: /mnt/disks/ssd0/sync-tmp
+        type: DirectoryOrCreate
+    volumeMounts:
+    - mountPath: /mnt/sync-tmp
+      name: gcloud-tmp
+    userNamespaces:
+      fsShift: fuse
+      shiftfsModuleLoader:
+        enabled: false
+      seccompProfileInstaller:
+        enabled: true
+    # in preview envs, we never want DaemonSets not to be scheduled (because they don't trigger scaleup)
+    resources:
+      cpu: 1m
+      memory: 32Mi
+
+  wsScheduler:
+    scaler:
+      enabled: true
+      controller:
+        kind: "constant"
+        constant:
+          setpoint: 1
+
+  # Enable ws-proxy in dev
+  wsProxy:
+    useHTTPS: true
+    hostHeader: Host
+    name: "ws-proxy"
+    disabled: false
+    replicas: 1
+    serviceType: "LoadBalancer"
+    wsManagerProxy:
+      enabled: true
+    ports:
+      wsManagerProxy:
+        expose: true
+        containerPort: 8081
+        servicePort: 8081
+      httpsProxy:
+        expose: true
+        containerPort: 9090
+        servicePort: 443
+      httpProxy:
+        expose: true
+        containerPort: 8080
+        servicePort: 80
+
+  imageBuilder:
+    disabled: true
+    hostDindData: "/mnt/disks/ssd0/builder"
+    # configure GCP registry
+    registry:
+      name: eu.gcr.io/gitpod-core-dev/registry
+      secretName: gcp-sa-registry-auth
+      path: gcp-sa-registry-auth
+    registryCerts: []
+
+
+# configure GCP registry
+docker-registry:
+  enabled: false
+
+rabbitmq:
+  # ensure shovels are configured on boot
+  disabled: true
+  enabled: false
+
+cert-manager:
+  enabled: true
+
+dbMigrations:
+  disabled: true
+
+db:
+  autoMigrate: false