From ce04c184518762368f3f5f3862e1d404a54b99e0 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Mon, 8 Nov 2021 17:46:39 +0000 Subject: [PATCH] [installer]: allow use of external container registry --- .../pkg/components/blobserve/configmap.go | 1 - .../pkg/components/blobserve/deployment.go | 87 +++++++++---------- .../components/image-builder-mk3/configmap.go | 35 ++++---- .../image-builder-mk3/deployment.go | 49 ++++++----- installer/pkg/components/proxy/configmap.go | 77 ++++++++-------- .../components/registry-facade/configmap.go | 1 - .../components/registry-facade/daemonset.go | 49 ++++++----- .../components/ws-manager/configmap_test.go | 18 +++- 8 files changed, 166 insertions(+), 151 deletions(-) diff --git a/installer/pkg/components/blobserve/configmap.go b/installer/pkg/components/blobserve/configmap.go index 62799dde19329d..810f066b48c39b 100644 --- a/installer/pkg/components/blobserve/configmap.go +++ b/installer/pkg/components/blobserve/configmap.go @@ -78,7 +78,6 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { MaxSize: MaxSizeBytes, }, }, - // todo(sje): make conditional on the workspace having a pull secret AuthCfg: "/mnt/pull-secret.json", PProfAddr: ":6060", PrometheusAddr: "127.0.0.1:9500", diff --git a/installer/pkg/components/blobserve/deployment.go b/installer/pkg/components/blobserve/deployment.go index 48f1e618c4b079..e5d4a11cdacf70 100644 --- a/installer/pkg/components/blobserve/deployment.go +++ b/installer/pkg/components/blobserve/deployment.go @@ -5,6 +5,7 @@ package blobserve import ( + "fmt" "github.com/gitpod-io/gitpod/installer/pkg/common" dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry" appsv1 "k8s.io/api/apps/v1" @@ -18,6 +19,16 @@ import ( func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { labels := common.DefaultLabels(Component) + volumeName := "pull-secret" + var secretName string + if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { + secretName = dockerregistry.BuiltInRegistryAuth + } else if ctx.Config.ContainerRegistry.External != nil { + secretName = ctx.Config.ContainerRegistry.External.Certificate.Name + } else { + return nil, fmt.Errorf("%s: invalid container registry config", Component) + } + var hashObj []runtime.Object if objs, err := configmap(ctx); err != nil { return nil, err @@ -25,48 +36,10 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { hashObj = append(hashObj, objs...) } - volumes := []corev1.Volume{{ - Name: "cache", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }, { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: Component}, - }, - }, - }} - - volumeMounts := []corev1.VolumeMount{ - { - Name: "config", - MountPath: "/mnt/config", - ReadOnly: true, - }, { - Name: "cache", - MountPath: "/mnt/cache", - }, - } - - if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { - volumeName := "pull-secret" - volumes = append(volumes, corev1.Volume{ - Name: volumeName, - VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ - SecretName: dockerregistry.BuiltInRegistryAuth, - }}, - }) - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: volumeName, - MountPath: "/mnt/pull-secret.json", - SubPath: ".dockerconfigjson", - }) - - if objs, err := common.DockerRegistryHash(ctx); err != nil { - return nil, err - } else { - hashObj = append(hashObj, objs...) - } + if objs, err := common.DockerRegistryHash(ctx); err != nil { + return nil, err + } else { + hashObj = append(hashObj, objs...) } configHash, err := common.ObjectHash(hashObj, nil) @@ -99,7 +72,22 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Affinity: &corev1.Affinity{}, ServiceAccountName: Component, EnableServiceLinks: pointer.Bool(false), - Volumes: volumes, + Volumes: []corev1.Volume{{ + Name: "cache", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: Component}, + }, + }, + }, { + Name: volumeName, + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + }}, + }}, Containers: []corev1.Container{{ Name: Component, Args: []string{"run", "-v", "/mnt/config/config.json"}, @@ -123,7 +111,18 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { common.DefaultEnv(&ctx.Config), common.TracingEnv(&ctx.Config), ), - VolumeMounts: volumeMounts, + VolumeMounts: []corev1.VolumeMount{{ + Name: "config", + MountPath: "/mnt/config", + ReadOnly: true, + }, { + Name: "cache", + MountPath: "/mnt/cache", + }, { + Name: volumeName, + MountPath: "/mnt/pull-secret.json", + SubPath: ".dockerconfigjson", + }}, }, *common.KubeRBACProxyContainer()}, }, }, diff --git a/installer/pkg/components/image-builder-mk3/configmap.go b/installer/pkg/components/image-builder-mk3/configmap.go index cfdcdcb8920ada..42e1bf9b0fa0b6 100644 --- a/installer/pkg/components/image-builder-mk3/configmap.go +++ b/installer/pkg/components/image-builder-mk3/configmap.go @@ -9,6 +9,7 @@ import ( "fmt" dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry" "k8s.io/utils/pointer" + "strings" "time" "github.com/gitpod-io/gitpod/common-go/util" @@ -23,6 +24,15 @@ import ( ) func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { + var registryName string + if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { + registryName = fmt.Sprintf("%s.%s", dockerregistry.RegistryName, ctx.Config.Domain) + } else if ctx.Config.ContainerRegistry.External != nil { + registryName = strings.TrimSuffix(ctx.Config.ContainerRegistry.External.URL, "/") + } else { + return nil, fmt.Errorf("%s: invalid container registry config", Component) + } + orchestrator := config.Configuration{ WorkspaceManager: config.WorkspaceManagerConfig{ Address: fmt.Sprintf("%s:%d", wsmanager.Component, wsmanager.RPCPort), @@ -32,28 +42,13 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { PrivateKey: "/wsman-certs/tls.key", }, }, - BuilderImage: common.ImageName(ctx.Config.Repository, BuilderImage, ctx.VersionManifest.Components.ImageBuilderMk3.BuilderImage.Version), - BuilderAuthKeyFile: "/config/authkey", + AuthFile: PullSecretFile, + BaseImageRepository: fmt.Sprintf("%s/base-images", registryName), + BuilderImage: common.ImageName(ctx.Config.Repository, BuilderImage, ctx.VersionManifest.Components.ImageBuilderMk3.BuilderImage.Version), + BuilderAuthKeyFile: "/config/authkey", + WorkspaceImageRepository: fmt.Sprintf("%s/workspace-images", registryName), } - var baseImageRepo string - var workspaceImgRepo string - if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { - // todo(sje): handle external registry - registryName := fmt.Sprintf("%s.%s", dockerregistry.RegistryName, ctx.Config.Domain) - - baseImageRepo = fmt.Sprintf("%s/base-images", registryName) - workspaceImgRepo = fmt.Sprintf("%s/workspace-images", registryName) - - orchestrator.AuthFile = PullSecretFile - } else { - // todo(sje): handle outside cluster values for image builder mk3 - return nil, fmt.Errorf("in cluster container currently only supported option") - } - - orchestrator.BaseImageRepository = baseImageRepo - orchestrator.WorkspaceImageRepository = workspaceImgRepo - imgcfg := config.ServiceConfig{ Orchestrator: orchestrator, RefCache: config.RefCacheConfig{ diff --git a/installer/pkg/components/image-builder-mk3/deployment.go b/installer/pkg/components/image-builder-mk3/deployment.go index 739a7ec7929ce7..cb57460baf61df 100644 --- a/installer/pkg/components/image-builder-mk3/deployment.go +++ b/installer/pkg/components/image-builder-mk3/deployment.go @@ -34,27 +34,21 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { hashObj = append(hashObj, objs...) } - var volumes []corev1.Volume - var volumeMounts []corev1.VolumeMount - + var secretName string if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "pull-secret", - MountPath: PullSecretFile, - SubPath: ".dockerconfigjson", - }) - volumes = append(volumes, corev1.Volume{ - Name: "pull-secret", - VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ - SecretName: dockerregistry.BuiltInRegistryAuth, - }}, - }) - if objs, err := common.DockerRegistryHash(ctx); err != nil { - return nil, err - } else { - hashObj = append(hashObj, objs...) - } + secretName = dockerregistry.BuiltInRegistryAuth + } else if ctx.Config.ContainerRegistry.External != nil { + secretName = ctx.Config.ContainerRegistry.External.Certificate.Name + } else { + return nil, fmt.Errorf("%s: invalid container registry config", Component) + } + + if objs, err := common.DockerRegistryHash(ctx); err != nil { + return nil, err + } else { + hashObj = append(hashObj, objs...) } + configHash, err := common.ObjectHash(hashObj, nil) if err != nil { return nil, err @@ -87,7 +81,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { DNSPolicy: "ClusterFirst", RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(30), - Volumes: append([]corev1.Volume{{ + Volumes: []corev1.Volume{{ Name: "configuration", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ @@ -108,7 +102,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { SecretName: wsmanager.TLSSecretNameClient, }, }, - }}, volumes...), + }, { + Name: "pull-secret", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + }}, + }}, Containers: []corev1.Container{{ Name: Component, Image: common.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.ImageBuilderMk3.Version), @@ -136,7 +135,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Privileged: pointer.Bool(false), RunAsUser: pointer.Int64(33333), }, - VolumeMounts: append([]corev1.VolumeMount{{ + VolumeMounts: []corev1.VolumeMount{{ Name: "configuration", MountPath: "/config/image-builder.json", SubPath: "image-builder.json", @@ -148,7 +147,11 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Name: "wsman-tls-certs", MountPath: "/wsman-certs", ReadOnly: true, - }}, volumeMounts...), + }, { + Name: "pull-secret", + MountPath: PullSecretFile, + SubPath: ".dockerconfigjson", + }}, }, *common.KubeRBACProxyContainer()}, }, }, diff --git a/installer/pkg/components/proxy/configmap.go b/installer/pkg/components/proxy/configmap.go index 375cb9432ae13b..2f76ac3ad54b76 100644 --- a/installer/pkg/components/proxy/configmap.go +++ b/installer/pkg/components/proxy/configmap.go @@ -10,6 +10,7 @@ import ( "encoding/base64" "fmt" minioComponent "github.com/gitpod-io/gitpod/installer/pkg/components/minio" + openvsxproxy "github.com/gitpod-io/gitpod/installer/pkg/components/openvsx-proxy" "text/template" "github.com/gitpod-io/gitpod/installer/pkg/common" @@ -18,6 +19,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" ) //go:embed templates/configmap/vhost.docker-registry.tpl @@ -88,36 +90,9 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { return nil, err } - // todo(sje) make conditional - // todo(sje): allow value to be set via config - username := ctx.Values.InternalRegistryUsername - if username == "" { - return nil, fmt.Errorf("unknown value: internal registry username") - } - - password := ctx.Values.InternalRegistryPassword - if password == "" { - return nil, fmt.Errorf("unknown value: internal registry password") - } - - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - return nil, err - } - - dockerRegistry, err := renderTemplate(vhostDockerRegistry, dockerRegistryTpl{ - Domain: ctx.Config.Domain, - ReverseProxy: fmt.Sprintf("https://%s.%s.%s", common.DockerRegistryName, ctx.Namespace, kubeDomain), - Username: username, - Password: base64.StdEncoding.EncodeToString(hashedPassword), - }) - if err != nil { - return nil, err - } - openVSX, err := renderTemplate(vhostOpenVSXTmpl, openVSXTpl{ Domain: ctx.Config.Domain, - RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, 8080), // todo(sje): get port from (future) config + RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, openvsxproxy.ServicePort), }) if err != nil { return nil, err @@ -140,6 +115,43 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { return nil, err } + data := map[string]string{ + "vhost.empty": *empty, + "vhost.minio": *minio, + "vhost.open-vsx": *openVSX, + "vhost.payment-endpoint": *paymentEndpoint, + "vhost.kedge": *kedge, + } + + if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { + username := ctx.Values.InternalRegistryUsername + if username == "" { + return nil, fmt.Errorf("unknown value: internal registry username") + } + + password := ctx.Values.InternalRegistryPassword + if password == "" { + return nil, fmt.Errorf("unknown value: internal registry password") + } + + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return nil, err + } + + dockerRegistry, err := renderTemplate(vhostDockerRegistry, dockerRegistryTpl{ + Domain: ctx.Config.Domain, + ReverseProxy: fmt.Sprintf("https://%s.%s.%s", common.DockerRegistryName, ctx.Namespace, kubeDomain), + Username: username, + Password: base64.StdEncoding.EncodeToString(hashedPassword), + }) + if err != nil { + return nil, err + } + + data["vhost.docker-registry"] = *dockerRegistry + } + return []runtime.Object{ &corev1.ConfigMap{ TypeMeta: common.TypeMetaConfigmap, @@ -148,14 +160,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { Namespace: ctx.Namespace, Labels: common.DefaultLabels(Component), }, - Data: map[string]string{ - "vhost.empty": *empty, - "vhost.minio": *minio, - "vhost.docker-registry": *dockerRegistry, - "vhost.open-vsx": *openVSX, - "vhost.payment-endpoint": *paymentEndpoint, - "vhost.kedge": *kedge, - }, + Data: data, }, }, nil } diff --git a/installer/pkg/components/registry-facade/configmap.go b/installer/pkg/components/registry-facade/configmap.go index 0e8afba5299b62..ece927cebdfec0 100644 --- a/installer/pkg/components/registry-facade/configmap.go +++ b/installer/pkg/components/registry-facade/configmap.go @@ -48,7 +48,6 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { Type: "image", }}, }, - // todo(sje): only enabled if the pullSecret is not nil in daemonset AuthCfg: "/mnt/pull-secret.json", PProfAddr: ":6060", PrometheusAddr: "127.0.0.1:9500", diff --git a/installer/pkg/components/registry-facade/daemonset.go b/installer/pkg/components/registry-facade/daemonset.go index 61cc9525a70442..20a52dd77e2e21 100644 --- a/installer/pkg/components/registry-facade/daemonset.go +++ b/installer/pkg/components/registry-facade/daemonset.go @@ -5,6 +5,7 @@ package registryfacade import ( + "fmt" "github.com/gitpod-io/gitpod/installer/pkg/cluster" "github.com/gitpod-io/gitpod/installer/pkg/common" dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry" @@ -48,34 +49,27 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) { }) } - if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { - name := "pull-secret" - volumes = append(volumes, corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: dockerregistry.BuiltInRegistryAuth, - }, - }, - }) - - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: name, - MountPath: "/mnt/pull-secret.json", - SubPath: ".dockerconfigjson", - }) - - if objs, err := common.DockerRegistryHash(ctx); err != nil { - return nil, err - } else { - hashObj = append(hashObj, objs...) - } + if objs, err := common.DockerRegistryHash(ctx); err != nil { + return nil, err + } else { + hashObj = append(hashObj, objs...) } + configHash, err := common.ObjectHash(hashObj, nil) if err != nil { return nil, err } + name := "pull-secret" + var secretName string + if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { + secretName = dockerregistry.BuiltInRegistryAuth + } else if ctx.Config.ContainerRegistry.External != nil { + secretName = ctx.Config.ContainerRegistry.External.Certificate.Name + } else { + return nil, fmt.Errorf("%s: invalid container registry config", Component) + } + return []runtime.Object{&appsv1.DaemonSet{ TypeMeta: common.TypeMetaDaemonset, ObjectMeta: metav1.ObjectMeta{ @@ -141,6 +135,10 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) { Name: "ws-manager-client-tls-certs", MountPath: "/ws-manager-client-tls-certs", ReadOnly: true, + }, { + Name: name, + MountPath: "/mnt/pull-secret.json", + SubPath: ".dockerconfigjson", }}, volumeMounts...), }, *common.KubeRBACProxyContainer()}, Volumes: append([]corev1.Volume{{ @@ -158,6 +156,13 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) { SecretName: wsmanager.TLSSecretNameClient, }, }, + }, { + Name: name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + }, + }, }}, volumes...), }, }, diff --git a/installer/pkg/components/ws-manager/configmap_test.go b/installer/pkg/components/ws-manager/configmap_test.go index cdb4a76a97960d..f169acd42a2709 100644 --- a/installer/pkg/components/ws-manager/configmap_test.go +++ b/installer/pkg/components/ws-manager/configmap_test.go @@ -14,6 +14,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" ) func TestBuildWorkspaceTemplates(t *testing.T) { @@ -22,9 +23,10 @@ func TestBuildWorkspaceTemplates(t *testing.T) { Data map[string]bool } tests := []struct { - Name string - Config *configv1.WorkspaceTemplates - Expectation Expectation + Name string + Config *configv1.WorkspaceTemplates + ContainerRegistry *configv1.ContainerRegistry + Expectation Expectation }{ { Name: "no templates", @@ -124,7 +126,15 @@ func TestBuildWorkspaceTemplates(t *testing.T) { objs []runtime.Object err error ) - act.TplConfig, objs, err = buildWorkspaceTemplates(&common.RenderContext{Config: configv1.Config{Workspace: configv1.Workspace{Templates: test.Config}}}) + + if test.ContainerRegistry == nil { + test.ContainerRegistry = &configv1.ContainerRegistry{InCluster: pointer.Bool(true)} + } + + act.TplConfig, objs, err = buildWorkspaceTemplates(&common.RenderContext{Config: configv1.Config{ + ContainerRegistry: *test.ContainerRegistry, + Workspace: configv1.Workspace{Templates: test.Config}, + }}) if err != nil { t.Error(err) }