gitpod-installer-job.yaml

# Copyright (c) 2022 Gitpod GmbH. All rights reserved.
# Licensed under the MIT License. See License-MIT.txt in the project root for license information.

# The installer job is where the magic happens. It generates
# the config, installs Gitpod and then deletes itself when
# it's finished
apiVersion: batch/v1
kind: Job
metadata:
  # Appending cursor allows us to replace with new versions
  name: installer-{{repl Cursor }}
  labels:
    app: gitpod
    component: gitpod-installer
    cursor: "{{repl Cursor }}"
spec:
  backoffLimit: 1
  ttlSecondsAfterFinished: 0
  template:
    metadata:
      labels:
        app: gitpod
        component: gitpod-installer
        cursor: "{{repl Cursor }}"
    spec:
      affinity:
        nodeAffinity:
          # Same affinity as ws-daemon as detecting the containerd location
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                - key: gitpod.io/workload_workspace_regular
                  operator: Exists
              - matchExpressions:
                - key: gitpod.io/workload_workspace_headless
                  operator: Exists
      serviceAccountName: installer
      restartPolicy: OnFailure
      containers:
        - name: installer
          # This will normally be the release tag
          image: "eu.gcr.io/gitpod-core-dev/build/installer:release-2022.08.0.0"
          volumeMounts:
            - mountPath: /config-patch
              name: config-patch
              readOnly: true
            - mountPath: /mnt/node0
              name: node-fs0
              readOnly: true
          env:
            - name: CONFIG_FILE
              value: /tmp/gitpod-config.yaml
            - name: CONFIG_PATCH_FILE
              value: /config-patch/gitpod-config-patch.yaml
            - name: CONTAINERD_DIR_K3S
              value: /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io
            - name: CONTAINERD_SOCKET_K3S
              value: /run/k3s/containerd/containerd.sock
            - name: CONTAINERD_DIR_AL
              value: /run/containerd/io.containerd.runtime.v2.task/k8s.io
            - name: CONTAINERD_SOCKET_AL
              value: /run/containerd/containerd.sock
            - name: GITPOD_OBJECTS
              value: /tmp/gitpod
          command:
            - /bin/sh
            - -c
          args:
            - |
              set -e

              echo "Gitpod: Killing any in-progress installations"

              kubectl delete jobs.batch -n {{repl Namespace }} -l component=gitpod-installer,cursor!={{repl Cursor }} --force --grace-period 0 || true
              kubectl delete pod -n {{repl Namespace }} -l component=gitpod-installer,cursor!={{repl Cursor }} --force --grace-period 0 || true

              if [ "$(helm status -n {{repl Namespace }} gitpod -o json | jq '.info.status == "deployed"')" = "false" ];
              then
                echo "Gitpod: Deployment in-progress - clearing"

                VERSION="$(helm status -n {{repl Namespace }} gitpod -o json | jq '.version')"
                if [ "${VERSION}" -le 1 ];
                then
                  echo "Gitpod: Uninstall application"
                  helm uninstall -n {{repl Namespace }} gitpod --wait || true
                else
                  echo "Gitpod: Rolling back application"
                  helm rollback -n {{repl Namespace }} gitpod --wait || true
                fi
              fi

              echo "Gitpod: Create a Helm template directory"
              rm -Rf "${GITPOD_OBJECTS}"
              mkdir -p "${GITPOD_OBJECTS}/templates"
              cat <<EOF >> "${GITPOD_OBJECTS}/Chart.yaml"
              apiVersion: v2
              name: gitpod-kots
              description: Always ready-to-code
              version: "1.0.0"
              appVersion: "$(/app/installer version | yq e '.version' -)"
              EOF

              echo "Gitpod: Generate the base Installer config"
              /app/installer init > "${CONFIG_FILE}"

              echo "Gitpod: auto-detecting ShiftFS support on host machine"
              kubectl wait job -n {{repl Namespace }} --for=condition=complete -l component=shiftfs-module-loader --timeout=30s || true
              ENABLE_SHIFTFS=$(kubectl get jobs.batch -n {{repl Namespace }} -l component=shiftfs-module-loader -o jsonpath='{.items[0].status.succeeded}')

              if [ "${ENABLE_SHIFTFS}" = "1" ]; then
                echo "Gitpod: enabling ShiftFS support"

                yq e -i '.workspace.runtime.fsShiftMethod = "shiftfs"' "${CONFIG_FILE}"
              fi

              echo "Gitpod: auto-detecting containerd location on host machine"
              if [ -d "/mnt/node0${CONTAINERD_DIR_K3S}" ]; then
                echo "Gitpod: containerd dir detected as k3s"

                yq e -i ".workspace.runtime.containerdRuntimeDir = \"${CONTAINERD_DIR_K3S}\"" "${CONFIG_FILE}"
              elif [ -d "/mnt/node0${CONTAINERD_DIR_AL}" ]; then
                echo "Gitpod: containerd dir detected as ${CONTAINERD_DIR_AL}"

                yq e -i ".workspace.runtime.containerdRuntimeDir = \"${CONTAINERD_DIR_AL}\"" "${CONFIG_FILE}"
              fi

              if [ -S "/mnt/node0${CONTAINERD_SOCKET_K3S}" ]; then
                echo "Gitpod: containerd socket detected as k3s"

                yq e -i ".workspace.runtime.containerdSocket = \"${CONTAINERD_SOCKET_K3S}\"" "${CONFIG_FILE}"
              elif [ -S "/mnt/node0${CONTAINERD_SOCKET_AL}" ]; then
                echo "Gitpod: containerd socket detected as ${CONTAINERD_SOCKET_AL}"

                yq e -i ".workspace.runtime.containerdSocket = \"${CONTAINERD_SOCKET_AL}\"" "${CONFIG_FILE}"
              fi

              echo "Gitpod: Inject the Replicated variables into the config"
              yq e -i '.domain = "{{repl ConfigOption "domain" }}"' "${CONFIG_FILE}"
              yq e -i '.license.kind = "secret"' "${CONFIG_FILE}"
              yq e -i '.license.name = "gitpod-license"' "${CONFIG_FILE}"

              if [ '{{repl ConfigOptionNotEquals "openVsxUrl" "" }}' = "true" ];
              then
                echo "Gitpod: Setting Open VSX Registry URL"
                yq e -i ".openVSX.url = \"{{repl ConfigOption "openVsxUrl" }}\"" "${CONFIG_FILE}"
              fi

              if [ '{{repl and (ConfigOptionEquals "db_incluster" "0") (ConfigOptionEquals "db_cloudsql_enabled" "1") }}' = "true" ];
              then
                echo "Gitpod: configuring CloudSQLProxy"

                yq e -i ".database.inCluster = false" "${CONFIG_FILE}"
                yq e -i ".database.cloudSQL.instance = \"{{repl ConfigOption "db_cloudsql_instance" }}\"" "${CONFIG_FILE}"
                yq e -i ".database.cloudSQL.serviceAccount.kind = \"secret\"" "${CONFIG_FILE}"
                yq e -i ".database.cloudSQL.serviceAccount.name = \"cloudsql\"" "${CONFIG_FILE}"
              fi

              if [ '{{repl and (ConfigOptionEquals "db_incluster" "0") (ConfigOptionEquals "db_cloudsql_enabled" "0") }}' = "true" ];
              then
                echo "Gitpod: configuring external database"

                yq e -i ".database.inCluster = false" "${CONFIG_FILE}"
                yq e -i ".database.external.certificate.kind = \"secret\"" "${CONFIG_FILE}"
                yq e -i ".database.external.certificate.name = \"database\"" "${CONFIG_FILE}"
              fi

              if [ '{{repl HasLocalRegistry }}' = "true" ];
              then
                echo "Gitpod: configuring mirrored container registry for airgapped installation"

                yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}"
                yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}"
                yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}"
                yq e -i '.dropImageRepo = true' "${CONFIG_FILE}"

                # Add the registry to the server allowlist - keep docker.io in case it's just using the mirrored registry functionality without being airgapped
                yq e -i ".containerRegistry.privateBaseImageAllowList += \"{{repl LocalRegistryHost }}\"" "${CONFIG_FILE}"
                yq e -i ".containerRegistry.privateBaseImageAllowList += \"docker.io\"" "${CONFIG_FILE}"
              fi

              if [ '{{repl ConfigOptionNotEquals "reg_docker_config" "" }}' = "true" ];
              then
                DOCKER_CONFIG='{{repl ConfigOptionData "reg_docker_config" | Base64Encode }}'
                echo "${DOCKER_CONFIG}" | base64 -d  > /tmp/userconfig.json
                # Add the registries to the server allowlist
                yq e -i ".containerRegistry.privateBaseImageAllowList += $(cat /tmp/userconfig.json | jq '.auths' | jq -rc 'keys')" "${CONFIG_FILE}"
                yq e -i ".containerRegistry.privateBaseImageAllowList += \"docker.io\"" "${CONFIG_FILE}"
              fi

              # Output the local registry secret - this is proxy.replicated.com if user hasn't set their own
              echo "{{repl LocalRegistryImagePullSecret }}" | base64 -d > /tmp/kotsregistry.json

              if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ];
              then
                echo "Gitpod: configuring external container registry"

                # Create a container-registry secret merging the external registry and KOTS registry keys
                echo '{{repl printf "{\"auths\": {\"%s\": {\"username\": \"%s\", \"password\": %s, \"auth\": \"%s\"}}}" (ConfigOption "reg_server" | default (ConfigOption "reg_url")) (ConfigOption "reg_username") (ConfigOption "reg_password" | toJson) (printf "%s:%s" (ConfigOption "reg_username") (ConfigOption "reg_password") | Base64Encode) }}' \
                  | yq -o=json '.' - \
                  > /tmp/gitpodregistry.json

                cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - > /tmp/container-registry-secret

                echo "Gitpod: create the container-registry secret"
                kubectl create secret docker-registry container-registry \
                  --namespace "{{repl Namespace }}" \
                  --from-file=.dockerconfigjson=/tmp/container-registry-secret \
                  -o yaml --dry-run=client >  "${GITPOD_OBJECTS}/templates/gitpod.yaml"

                yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}"
                yq e -i ".containerRegistry.external.url = \"{{repl ConfigOption "reg_url" }}\"" "${CONFIG_FILE}"
                yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}"
                yq e -i ".containerRegistry.external.certificate.name = \"container-registry\"" "${CONFIG_FILE}"
              else
                if [ '{{repl ConfigOptionEquals "reg_incluster_storage" "s3" }}' = "true" ];
                then
                  echo "Gitpod: configuring container registry S3 backend"

                  yq e -i ".containerRegistry.s3storage.region = \"{{repl ConfigOption "reg_incluster_storage_s3_region" }}\"" "${CONFIG_FILE}"
                  yq e -i ".containerRegistry.s3storage.endpoint = \"{{repl ConfigOption "reg_incluster_storage_s3_endpoint" }}\"" "${CONFIG_FILE}"
                  yq e -i ".containerRegistry.s3storage.bucket = \"{{repl ConfigOption "reg_incluster_storage_s3_bucketname" }}\"" "${CONFIG_FILE}"
                  yq e -i ".containerRegistry.s3storage.certificate.kind = \"secret\"" "${CONFIG_FILE}"
                  yq e -i ".containerRegistry.s3storage.certificate.name = \"container-registry-s3-backend\"" "${CONFIG_FILE}"
                fi
              fi

              if [ '{{repl ConfigOptionNotEquals "store_provider" "incluster" }}' = "true" ];
              then
                echo "Gitpod: configuring the storage"

                yq e -i ".metadata.region = \"{{repl ConfigOption "store_region" }}\"" "${CONFIG_FILE}"
                yq e -i ".objectStorage.inCluster = false" "${CONFIG_FILE}"

                if [ '{{repl ConfigOptionEquals "store_provider" "azure" }}' = "true" ];
                then
                  echo "Gitpod: configuring storage for Azure"

                  yq e -i ".objectStorage.azure.credentials.kind = \"secret\"" "${CONFIG_FILE}"
                  yq e -i ".objectStorage.azure.credentials.name = \"storage-azure\"" "${CONFIG_FILE}"
                fi

                if [ '{{repl ConfigOptionEquals "store_provider" "gcp" }}' = "true" ];
                then
                  echo "Gitpod: configuring storage for GCP"

                  yq e -i ".objectStorage.cloudStorage.project = \"{{repl ConfigOption "store_gcp_project" }}\"" "${CONFIG_FILE}"
                  yq e -i ".objectStorage.cloudStorage.serviceAccount.kind = \"secret\"" "${CONFIG_FILE}"
                  yq e -i ".objectStorage.cloudStorage.serviceAccount.name = \"storage-gcp\"" "${CONFIG_FILE}"
                fi

                if [ '{{repl ConfigOptionEquals "store_provider" "s3" }}' = "true" ];
                then
                  echo "Gitpod: configuring storage for S3"

                  yq e -i ".objectStorage.s3.endpoint = \"{{repl ConfigOption "store_s3_endpoint" }}\"" "${CONFIG_FILE}"
                  yq e -i ".objectStorage.s3.bucket = \"{{repl ConfigOption "store_s3_bucket" }}\"" "${CONFIG_FILE}"
                  yq e -i ".objectStorage.s3.credentials.kind = \"secret\"" "${CONFIG_FILE}"
                  yq e -i ".objectStorage.s3.credentials.name = \"storage-s3\"" "${CONFIG_FILE}"
                fi
              fi

              if [ '{{repl ConfigOptionEquals "ssh_gateway" "1" }}' = "true" ];
              then
                echo "Gitpod: Generate SSH host key"
                ssh-keygen -t rsa -q -N "" -f host.key
                kubectl create secret generic ssh-gateway-host-key --from-file=host.key -n {{repl Namespace }} || echo "SSH Gateway Host Key secret has not been created. Does it exist already?"
                yq e -i '.sshGatewayHostKey.kind = "secret"' "${CONFIG_FILE}"
                yq e -i '.sshGatewayHostKey.name = "ssh-gateway-host-key"' "${CONFIG_FILE}"
              fi

              if [ '{{repl ConfigOptionEquals "tls_self_signed_enabled" "1" }}' = "true" ];
              then
                echo "Gitpod: Generating a self-signed certificate with the internal CA"
                yq e -i '.customCACert.kind = "secret"' "${CONFIG_FILE}"
                yq e -i '.customCACert.name = "ca-issuer-ca"' "${CONFIG_FILE}"
              elif [ '{{repl and (ConfigOptionEquals "tls_self_signed_enabled" "0") (ConfigOptionEquals "cert_manager_enabled" "0") (ConfigOptionNotEquals "tls_ca_crt" "") }}' = "true" ];
              then
                echo "Gitpod: Setting CA to be used for certificate"
                yq e -i '.customCACert.kind = "secret"' "${CONFIG_FILE}"
                yq e -i '.customCACert.name = "ca-certificate"' "${CONFIG_FILE}"
              fi

              if [ '{{repl ConfigOptionEquals "user_management_block_enabled" "1" }}' = "true" ];
              then
                echo "Gitpod: Adding blockNewUsers to config"
                yq e -i '.blockNewUsers.enabled = true' "${CONFIG_FILE}"

                for domain in {{repl ConfigOption "user_management_block_passlist" }}
                do
                  echo "Gitpod: Adding domain \"${domain}\" to blockNewUsers config"
                  yq e -i ".blockNewUsers.passlist += \"${domain}\"" "${CONFIG_FILE}"
                done
              fi

              if [ '{{repl ConfigOptionEquals "advanced_mode_enabled" "1" }}' = "true" ];
              then
                echo "Gitpod: Applying advanced configuration"

                if [ '{{repl ConfigOptionNotEquals "component_proxy_service_serviceType" "" }}' = "true" ];
                then
                  # Empty string defaults to LoadBalancer. This maintains backwards compatibility with the deprecated experimental value
                  echo "Gitpod: Applying Proxy service type"
                  yq e -i ".components.proxy.service.serviceType = \"{{repl ConfigOption "component_proxy_service_serviceType" }}\"" "${CONFIG_FILE}"
                fi

                if [ '{{repl ConfigOptionNotEquals "customization_patch" "" }}' = "true" ];
                then
                  CUSTOMIZATION='{{repl ConfigOptionData "customization_patch" | Base64Encode }}'
                  echo "Gitpod: Applying customization patch ${CUSTOMIZATION}"

                  # Apply the customization property - if something else is set, this will be ignored
                  yq e -i ".customization = $(echo "${CUSTOMIZATION}" | base64 -d | yq e -o json '.customization' - | jq -rc) // []" "${CONFIG_FILE}"

                  # Delete any old resources
                  echo "Gitpod: Deleting old Gitpod resources"
                  kubectl delete all -n {{repl Namespace }} -l app=gitpod,component!=gitpod-installer,component!=gitpod-installer-status --wait || true
                fi
              else
                echo "Gitpod: No advanced configuration applied"
              fi

              echo "Gitpod: Update platform telemetry value"
              yq eval-all --inplace '.experimental.telemetry.data.platform = "{{repl Distribution }}"' "${CONFIG_FILE}"

              echo "Gitpod: Patch Gitpod config"
              base64 -d "${CONFIG_PATCH_FILE}" > /tmp/patch.yaml
              config_patch=$(cat /tmp/patch.yaml)
              echo "Gitpod: ${CONFIG_PATCH_FILE}=${config_patch}"
              yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' "${CONFIG_FILE}" /tmp/patch.yaml

              echo "Gitpod: Generate the Kubernetes objects"
              config=$(cat "${CONFIG_FILE}")
              echo "Gitpod: ${CONFIG_FILE}=${config}"

              echo "Gitpod: render Kubernetes manifests"
              /app/installer render -c "${CONFIG_FILE}" --namespace {{repl Namespace }} --use-experimental-config >> "${GITPOD_OBJECTS}/templates/gitpod.yaml"

              if [ '{{repl ConfigOptionEquals "reg_incluster" "1" }}' = "true" ];
              then
                echo "Gitpod: Add the local registry secret to the in-cluster registry secret"

                # Get the in-cluster registry secret
                yq eval-all '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson")' \
                  "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
                  | base64 -d \
                  > /tmp/gitpodregistry.json

                export REGISTRY_SECRET=$(cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - | base64 -w 0)

                echo "Gitpod: update the in-cluster registry secret"
                yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
                  "${GITPOD_OBJECTS}/templates/gitpod.yaml"
              fi

              # figure out the secret to use
              if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ];
              then
                export REGISTRY_SECRET_NAME="container-registry"
              else
                export REGISTRY_SECRET_NAME="builtin-registry-auth"
              fi

              if [ '{{repl ConfigOptionNotEquals "reg_docker_config" "" }}' = "true" ];
              then
                echo "Gitpod: Add given extra docker config json file to ${REGISTRY_SECRET_NAME}"

                yq eval-all '(select(.kind == "Secret" and .metadata.name == env(REGISTRY_SECRET_NAME)) | .data.".dockerconfigjson")' \
                  "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
                  | base64 -d \
                  > /tmp/currentconfig.json

                export REGISTRY_SECRET=$(jq -s '.[0] * .[1]' /tmp/userconfig.json /tmp/currentconfig.json | base64 -w 0)

                echo "Gitpod: update the in-cluster registry secret"
                yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == env(REGISTRY_SECRET_NAME)) | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
                  "${GITPOD_OBJECTS}/templates/gitpod.yaml"
              fi

              echo "Gitpod: Escape any Golang template values"
              sed -i -r 's/(.*\{\{.*)/{{`\1`}}/' "${GITPOD_OBJECTS}/templates/gitpod.yaml"

              # This is to overcome upgrade problems with PodSecurityPolicy removal
              echo "Gitpod: Delete Minio deployment"
              kubectl delete deployments.apps --wait -n {{repl Namespace }} minio || true

              echo "Gitpod: Delete RabbitMQ statefulset"
              kubectl delete statefulsets.apps --wait -n {{repl Namespace }} messagebus || true

              echo "Gitpod: Delete MySQL statefulset"
              kubectl delete statefulsets.apps --wait -n {{repl Namespace }} mysql || true

              echo "Gitpod: Delete container-registry secret"
              kubectl delete secret --wait -n {{repl Namespace }} container-registry || true
              # End of PSP removal fixes

              # If certificate secret already exists, set the timeout to 5m
              CERT_SECRET=$(kubectl get secrets -n {{repl Namespace }} https-certificates -o jsonpath='{.metadata.name}' || echo '')
              HELM_TIMEOUT="10m"
              if [ "${CERT_SECRET}" = "" ]; then
                HELM_TIMEOUT="1h"
              fi

              # The long timeout is to ensure the TLS cert is created (if required)
              echo "Gitpod: Apply the Kubernetes objects with timeout of ${HELM_TIMEOUT}"
              helm upgrade \
                --atomic \
                --cleanup-on-fail \
                --create-namespace \
                --install \
                --namespace="{{repl Namespace }}" \
                --reset-values \
                --timeout "${HELM_TIMEOUT}" \
                --wait \
                gitpod \
                "${GITPOD_OBJECTS}"

              echo "Gitpod: Restarting installation status job"
              kubectl delete pod -n {{repl Namespace }} -l component=gitpod-installer-status || true

              echo "Gitpod: Installer job finished - goodbye"
      volumes:
        - name: config-patch
          configMap:
            name: gitpod-config-patch
        - name: node-fs0
          hostPath:
            path: /
            type: Directory