Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[installer]: allow use of external container registry #6621

Merged
merged 1 commit into from
Nov 11, 2021
Merged

[installer]: allow use of external container registry #6621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion installer/pkg/components/blobserve/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
MaxSize: MaxSizeBytes,
},
},
// todo(sje): make conditional on the workspace having a pull secret
AuthCfg: "/mnt/pull-secret.json",
PProfAddr: ":6060",
PrometheusAddr: "127.0.0.1:9500",
Expand Down
87 changes: 43 additions & 44 deletions installer/pkg/components/blobserve/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package blobserve

import (
"fmt"
"github.com/gitpod-io/gitpod/installer/pkg/common"
dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
appsv1 "k8s.io/api/apps/v1"
Expand All @@ -18,55 +19,27 @@ import (
func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
labels := common.DefaultLabels(Component)

volumeName := "pull-secret"
var secretName string
if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
secretName = dockerregistry.BuiltInRegistryAuth
} else if ctx.Config.ContainerRegistry.External != nil {
secretName = ctx.Config.ContainerRegistry.External.Certificate.Name
} else {
return nil, fmt.Errorf("%s: invalid container registry config", Component)
}

var hashObj []runtime.Object
if objs, err := configmap(ctx); err != nil {
return nil, err
} else {
hashObj = append(hashObj, objs...)
}

volumes := []corev1.Volume{{
Name: "cache",
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
}, {
Name: "config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{Name: Component},
},
},
}}

volumeMounts := []corev1.VolumeMount{
{
Name: "config",
MountPath: "/mnt/config",
ReadOnly: true,
}, {
Name: "cache",
MountPath: "/mnt/cache",
},
}

if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
volumeName := "pull-secret"
volumes = append(volumes, corev1.Volume{
Name: volumeName,
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
SecretName: dockerregistry.BuiltInRegistryAuth,
}},
})
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: volumeName,
MountPath: "/mnt/pull-secret.json",
SubPath: ".dockerconfigjson",
})

if objs, err := common.DockerRegistryHash(ctx); err != nil {
return nil, err
} else {
hashObj = append(hashObj, objs...)
}
if objs, err := common.DockerRegistryHash(ctx); err != nil {
return nil, err
} else {
hashObj = append(hashObj, objs...)
}

configHash, err := common.ObjectHash(hashObj, nil)
Expand Down Expand Up @@ -99,7 +72,22 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Affinity: &corev1.Affinity{},
ServiceAccountName: Component,
EnableServiceLinks: pointer.Bool(false),
Volumes: volumes,
Volumes: []corev1.Volume{{
Name: "cache",
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
}, {
Name: "config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{Name: Component},
},
},
}, {
Name: volumeName,
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
SecretName: secretName,
}},
}},
Containers: []corev1.Container{{
Name: Component,
Args: []string{"run", "-v", "/mnt/config/config.json"},
Expand All @@ -123,7 +111,18 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
common.DefaultEnv(&ctx.Config),
common.TracingEnv(&ctx.Config),
),
VolumeMounts: volumeMounts,
VolumeMounts: []corev1.VolumeMount{{
Name: "config",
MountPath: "/mnt/config",
ReadOnly: true,
}, {
Name: "cache",
MountPath: "/mnt/cache",
}, {
Name: volumeName,
MountPath: "/mnt/pull-secret.json",
SubPath: ".dockerconfigjson",
}},
}, *common.KubeRBACProxyContainer()},
},
},
Expand Down
35 changes: 15 additions & 20 deletions installer/pkg/components/image-builder-mk3/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"fmt"
dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
"k8s.io/utils/pointer"
"strings"
"time"

"github.com/gitpod-io/gitpod/common-go/util"
Expand All @@ -23,6 +24,15 @@ import (
)

func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
var registryName string
if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
registryName = fmt.Sprintf("%s.%s", dockerregistry.RegistryName, ctx.Config.Domain)
} else if ctx.Config.ContainerRegistry.External != nil {
registryName = strings.TrimSuffix(ctx.Config.ContainerRegistry.External.URL, "/")
} else {
mrsimonemms marked this conversation as resolved.
Show resolved Hide resolved
return nil, fmt.Errorf("%s: invalid container registry config", Component)
}

orchestrator := config.Configuration{
WorkspaceManager: config.WorkspaceManagerConfig{
Address: fmt.Sprintf("%s:%d", wsmanager.Component, wsmanager.RPCPort),
Expand All @@ -32,28 +42,13 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
PrivateKey: "/wsman-certs/tls.key",
},
},
BuilderImage: common.ImageName(ctx.Config.Repository, BuilderImage, ctx.VersionManifest.Components.ImageBuilderMk3.BuilderImage.Version),
BuilderAuthKeyFile: "/config/authkey",
AuthFile: PullSecretFile,
BaseImageRepository: fmt.Sprintf("%s/base-images", registryName),
BuilderImage: common.ImageName(ctx.Config.Repository, BuilderImage, ctx.VersionManifest.Components.ImageBuilderMk3.BuilderImage.Version),
BuilderAuthKeyFile: "/config/authkey",
WorkspaceImageRepository: fmt.Sprintf("%s/workspace-images", registryName),
}

var baseImageRepo string
var workspaceImgRepo string
if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
// todo(sje): handle external registry
registryName := fmt.Sprintf("%s.%s", dockerregistry.RegistryName, ctx.Config.Domain)

baseImageRepo = fmt.Sprintf("%s/base-images", registryName)
workspaceImgRepo = fmt.Sprintf("%s/workspace-images", registryName)

orchestrator.AuthFile = PullSecretFile
} else {
// todo(sje): handle outside cluster values for image builder mk3
return nil, fmt.Errorf("in cluster container currently only supported option")
}

orchestrator.BaseImageRepository = baseImageRepo
orchestrator.WorkspaceImageRepository = workspaceImgRepo

imgcfg := config.ServiceConfig{
Orchestrator: orchestrator,
RefCache: config.RefCacheConfig{
Expand Down
49 changes: 26 additions & 23 deletions installer/pkg/components/image-builder-mk3/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,27 +34,21 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
hashObj = append(hashObj, objs...)
}

var volumes []corev1.Volume
var volumeMounts []corev1.VolumeMount

var secretName string
if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "pull-secret",
MountPath: PullSecretFile,
SubPath: ".dockerconfigjson",
})
volumes = append(volumes, corev1.Volume{
Name: "pull-secret",
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
SecretName: dockerregistry.BuiltInRegistryAuth,
}},
})
if objs, err := common.DockerRegistryHash(ctx); err != nil {
return nil, err
} else {
hashObj = append(hashObj, objs...)
}
secretName = dockerregistry.BuiltInRegistryAuth
} else if ctx.Config.ContainerRegistry.External != nil {
secretName = ctx.Config.ContainerRegistry.External.Certificate.Name
} else {
return nil, fmt.Errorf("%s: invalid container registry config", Component)
}

if objs, err := common.DockerRegistryHash(ctx); err != nil {
return nil, err
} else {
hashObj = append(hashObj, objs...)
}

configHash, err := common.ObjectHash(hashObj, nil)
if err != nil {
return nil, err
Expand Down Expand Up @@ -87,7 +81,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
DNSPolicy: "ClusterFirst",
RestartPolicy: "Always",
TerminationGracePeriodSeconds: pointer.Int64(30),
Volumes: append([]corev1.Volume{{
Volumes: []corev1.Volume{{
Name: "configuration",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
Expand All @@ -108,7 +102,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
SecretName: wsmanager.TLSSecretNameClient,
},
},
}}, volumes...),
}, {
Name: "pull-secret",
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
SecretName: secretName,
}},
}},
Containers: []corev1.Container{{
Name: Component,
Image: common.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.ImageBuilderMk3.Version),
Expand Down Expand Up @@ -136,7 +135,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Privileged: pointer.Bool(false),
RunAsUser: pointer.Int64(33333),
},
VolumeMounts: append([]corev1.VolumeMount{{
VolumeMounts: []corev1.VolumeMount{{
Name: "configuration",
MountPath: "/config/image-builder.json",
SubPath: "image-builder.json",
Expand All @@ -148,7 +147,11 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
Name: "wsman-tls-certs",
MountPath: "/wsman-certs",
ReadOnly: true,
}}, volumeMounts...),
}, {
Name: "pull-secret",
MountPath: PullSecretFile,
SubPath: ".dockerconfigjson",
}},
}, *common.KubeRBACProxyContainer()},
},
},
Expand Down
77 changes: 41 additions & 36 deletions installer/pkg/components/proxy/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"encoding/base64"
"fmt"
minioComponent "github.com/gitpod-io/gitpod/installer/pkg/components/minio"
openvsxproxy "github.com/gitpod-io/gitpod/installer/pkg/components/openvsx-proxy"
"text/template"

"github.com/gitpod-io/gitpod/installer/pkg/common"
Expand All @@ -18,6 +19,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/pointer"
)

//go:embed templates/configmap/vhost.docker-registry.tpl
Expand Down Expand Up @@ -88,36 +90,9 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
return nil, err
}

// todo(sje) make conditional
// todo(sje): allow value to be set via config
username := ctx.Values.InternalRegistryUsername
if username == "" {
return nil, fmt.Errorf("unknown value: internal registry username")
}

password := ctx.Values.InternalRegistryPassword
if password == "" {
return nil, fmt.Errorf("unknown value: internal registry password")
}

hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return nil, err
}

dockerRegistry, err := renderTemplate(vhostDockerRegistry, dockerRegistryTpl{
Domain: ctx.Config.Domain,
ReverseProxy: fmt.Sprintf("https://%s.%s.%s", common.DockerRegistryName, ctx.Namespace, kubeDomain),
Username: username,
Password: base64.StdEncoding.EncodeToString(hashedPassword),
})
if err != nil {
return nil, err
}

openVSX, err := renderTemplate(vhostOpenVSXTmpl, openVSXTpl{
Domain: ctx.Config.Domain,
RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, 8080), // todo(sje): get port from (future) config
RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, openvsxproxy.ServicePort),
})
if err != nil {
return nil, err
Expand All @@ -140,6 +115,43 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
return nil, err
}

data := map[string]string{
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, this section has been rearranged to make the registry proxy conditional

"vhost.empty": *empty,
"vhost.minio": *minio,
"vhost.open-vsx": *openVSX,
"vhost.payment-endpoint": *paymentEndpoint,
"vhost.kedge": *kedge,
}

if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
username := ctx.Values.InternalRegistryUsername
if username == "" {
return nil, fmt.Errorf("unknown value: internal registry username")
}

password := ctx.Values.InternalRegistryPassword
if password == "" {
return nil, fmt.Errorf("unknown value: internal registry password")
}

hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return nil, err
}

dockerRegistry, err := renderTemplate(vhostDockerRegistry, dockerRegistryTpl{
Domain: ctx.Config.Domain,
ReverseProxy: fmt.Sprintf("https://%s.%s.%s", common.DockerRegistryName, ctx.Namespace, kubeDomain),
Username: username,
Password: base64.StdEncoding.EncodeToString(hashedPassword),
})
if err != nil {
return nil, err
}

data["vhost.docker-registry"] = *dockerRegistry
}

return []runtime.Object{
&corev1.ConfigMap{
TypeMeta: common.TypeMetaConfigmap,
Expand All @@ -148,14 +160,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
Namespace: ctx.Namespace,
Labels: common.DefaultLabels(Component),
},
Data: map[string]string{
"vhost.empty": *empty,
"vhost.minio": *minio,
"vhost.docker-registry": *dockerRegistry,
"vhost.open-vsx": *openVSX,
"vhost.payment-endpoint": *paymentEndpoint,
"vhost.kedge": *kedge,
},
Data: data,
},
}, nil
}
1 change: 0 additions & 1 deletion installer/pkg/components/registry-facade/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
Type: "image",
}},
},
// todo(sje): only enabled if the pullSecret is not nil in daemonset
AuthCfg: "/mnt/pull-secret.json",
PProfAddr: ":6060",
PrometheusAddr: "127.0.0.1:9500",
Expand Down
Loading