From dd26d45935f6a0295ccd021c50df0d9a208cf0b7 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Fri, 22 Oct 2021 10:13:58 +0100 Subject: [PATCH] [installer]: general fixes for the meta components This now installs the meta components to a cluster using an InCluster database and registry --- installer/pkg/common/common.go | 34 ++++++++++---- .../components/content-service/configmap.go | 9 ++-- .../components/docker-registry/constants.go | 1 + .../pkg/components/docker-registry/helm.go | 1 + .../components/image-builder-mk3/configmap.go | 46 +++++++++++++------ installer/pkg/components/mysql/secret.go | 29 ++++++++++-- installer/pkg/components/proxy/configmap.go | 2 +- .../templates/configmap/vhost.open-vsx.tpl | 43 +---------------- installer/pkg/components/server/configmap.go | 9 +++- installer/pkg/components/server/deployment.go | 3 +- .../ws-manager-bridge/deployment.go | 8 ++-- .../charts/docker-registry/values.yaml | 3 +- 12 files changed, 107 insertions(+), 81 deletions(-) diff --git a/installer/pkg/common/common.go b/installer/pkg/common/common.go index 51b03cb096d7cd..aa64861513c7ac 100644 --- a/installer/pkg/common/common.go +++ b/installer/pkg/common/common.go @@ -61,6 +61,16 @@ func DefaultEnv(cfg *config.Config) []corev1.EnvVar { return []corev1.EnvVar{ {Name: "GITPOD_DOMAIN", Value: cfg.Domain}, + {Name: "GITPOD_INSTALLATION_LONGNAME", Value: cfg.Domain}, // todo(sje): figure out these values + {Name: "GITPOD_INSTALLATION_SHORTNAME", Value: cfg.Domain}, // todo(sje): figure out these values + {Name: "GITPOD_REGION", Value: cfg.Metadata.Region}, + {Name: "HOST_URL", Value: "https://" + cfg.Domain}, + {Name: "KUBE_NAMESPACE", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }}, + {Name: "KUBE_DOMAIN", Value: "svc.cluster.local"}, {Name: "LOG_LEVEL", Value: strings.ToLower(logLevel)}, } } @@ -102,7 +112,7 @@ func AnalyticsEnv(cfg *config.Config) (res []corev1.EnvVar) { }} } -func MessageBusEnv(cfg *config.Config) (res []corev1.EnvVar) { +func MessageBusEnv(_ *config.Config) (res []corev1.EnvVar) { clusterObj := corev1.LocalObjectReference{Name: InClusterMessageQueueName} tlsObj := corev1.LocalObjectReference{Name: InClusterMessageQueueTLS} @@ -173,14 +183,22 @@ func DatabaseEnv(cfg *config.Config) (res []corev1.EnvVar) { LocalObjectReference: obj, Key: "password", }}, + }, { + Name: "DB_USERNAME", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: obj, + Key: "username", + }}, }, { // todo(sje): conditional Name: "DB_DELETED_ENTRIES_GC_ENABLED", Value: "false", }, { Name: "DB_ENCRYPTION_KEYS", - // todo(sje): either Value or ValueFrom - Value: "todo", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: obj, + Key: "encryptionKeys", + }}, //ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ // LocalObjectReference: corev1.LocalObjectReference{ // Name: "", @@ -193,7 +211,7 @@ func DatabaseEnv(cfg *config.Config) (res []corev1.EnvVar) { func DatabaseWaiterContainer(ctx *RenderContext) *corev1.Container { return &corev1.Container{ Name: "database-waiter", - Image: ImageName(ctx.Config.Repository, "service-waiter", "latest"), + Image: ImageName(ctx.Config.Repository, "service-waiter", ctx.VersionManifest.Components.ServiceWaiter.Version), Args: []string{ "-v", "database", @@ -211,7 +229,7 @@ func DatabaseWaiterContainer(ctx *RenderContext) *corev1.Container { func MessageBusWaiterContainer(ctx *RenderContext) *corev1.Container { return &corev1.Container{ Name: "msgbus-waiter", - Image: ImageName(ctx.Config.Repository, "service-waiter", "latest"), + Image: ImageName(ctx.Config.Repository, "service-waiter", ctx.VersionManifest.Components.ServiceWaiter.Version), Args: []string{ "-v", "messagebus", @@ -250,10 +268,10 @@ func KubeRBACProxyContainer() *corev1.Container { }, }, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ - corev1.ResourceName("cpu"): resource.MustParse("1m"), - corev1.ResourceName("memory"): resource.MustParse("30Mi"), + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("30Mi"), }}, - TerminationMessagePolicy: corev1.TerminationMessagePolicy("FallbackToLogsOnError"), + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, SecurityContext: &corev1.SecurityContext{ RunAsUser: pointer.Int64(65532), RunAsGroup: pointer.Int64(65532), diff --git a/installer/pkg/components/content-service/configmap.go b/installer/pkg/components/content-service/configmap.go index 383eb782a8a55b..fbacbf7609242c 100644 --- a/installer/pkg/components/content-service/configmap.go +++ b/installer/pkg/components/content-service/configmap.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/gitpod-io/gitpod/content-service/api/config" - apiconfig "github.com/gitpod-io/gitpod/content-service/api/config" "github.com/gitpod-io/gitpod/installer/pkg/common" corev1 "k8s.io/api/core/v1" @@ -18,6 +17,11 @@ import ( ) func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { + storageConfig, err := common.StorageConfiguration(ctx) + if err != nil { + return nil, err + } + cscfg := config.ServiceConfig{ Service: config.Service{ Addr: fmt.Sprintf(":%d", RPCPort), @@ -28,8 +32,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { PProf: config.PProf{ Addr: fmt.Sprintf(":%d", PProfPort), }, - // todo(sje): work out how to cater for different storages - Storage: apiconfig.StorageConfig{}, + Storage: *storageConfig, } fc, err := json.MarshalIndent(cscfg, "", " ") diff --git a/installer/pkg/components/docker-registry/constants.go b/installer/pkg/components/docker-registry/constants.go index 267b8cff2f784e..e50f60235ba0d6 100644 --- a/installer/pkg/components/docker-registry/constants.go +++ b/installer/pkg/components/docker-registry/constants.go @@ -8,4 +8,5 @@ const ( BuiltInRegistryAuth = "builtin-registry-auth" BuiltInRegistryCerts = "builtin-registry-certs" Component = "docker-registry" + RegistryName = "registry" ) diff --git a/installer/pkg/components/docker-registry/helm.go b/installer/pkg/components/docker-registry/helm.go index 729051d426e719..2d68d6698a158f 100644 --- a/installer/pkg/components/docker-registry/helm.go +++ b/installer/pkg/components/docker-registry/helm.go @@ -21,6 +21,7 @@ var Helm = common.CompositeHelmFunc( Enabled: pointer.BoolDeref(cfg.Config.ContainerRegistry.InCluster, false), Values: &values.Options{ Values: []string{ + helm.KeyValue("docker-registry.fullnameOverride", RegistryName), helm.KeyValue("docker-registry.service.port", strconv.Itoa(proxy.ContainerHTTPSPort)), helm.KeyValue("docker-registry.tlsSecretName", proxy.RegistryTLSCertSecret), }, diff --git a/installer/pkg/components/image-builder-mk3/configmap.go b/installer/pkg/components/image-builder-mk3/configmap.go index ca6d0edd19530e..bd620440d966f8 100644 --- a/installer/pkg/components/image-builder-mk3/configmap.go +++ b/installer/pkg/components/image-builder-mk3/configmap.go @@ -7,6 +7,8 @@ package image_builder_mk3 import ( "encoding/json" "fmt" + dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry" + "k8s.io/utils/pointer" "time" "github.com/gitpod-io/gitpod/common-go/util" @@ -21,22 +23,38 @@ import ( ) func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { - imgcfg := config.ServiceConfig{ - Orchestrator: config.Configuration{ - WorkspaceManager: config.WorkspaceManagerConfig{ - Address: fmt.Sprintf("%s:%d", wsmanager.Component, wsmanager.RPCPort), - TLS: config.TLS{ - Authority: "/wsman-certs/ca.crt", - Certificate: "/wsman-certs/tls.crt", - PrivateKey: "/wsman-certs/tls.key", - }, + orchestrator := config.Configuration{ + WorkspaceManager: config.WorkspaceManagerConfig{ + Address: fmt.Sprintf("%s:%d", wsmanager.Component, wsmanager.RPCPort), + TLS: config.TLS{ + Authority: "/wsman-certs/ca.crt", + Certificate: "/wsman-certs/tls.crt", + PrivateKey: "/wsman-certs/tls.key", }, - AuthFile: PullSecretFile, // todo(sje): make conditional - BaseImageRepository: "", // todo(sje): get conditional value - WorkspaceImageRepository: "", // todo(sje): get conditional value - BuilderImage: common.ImageName(ctx.Config.Repository, BuilderImage, BuilderImageVersion), - BuilderAuthKeyFile: "/config/authkey", }, + BuilderImage: common.ImageName(ctx.Config.Repository, BuilderImage, BuilderImageVersion), + BuilderAuthKeyFile: "/config/authkey", + } + + var baseImageRepo string + var workspaceImgRepo string + if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { + registryName := dockerregistry.RegistryName + + baseImageRepo = fmt.Sprintf("%s/base-images", registryName) + workspaceImgRepo = fmt.Sprintf("%s/workspace-images", registryName) + + orchestrator.AuthFile = PullSecretFile + } else { + // todo(sje): handle outside cluster values for image builder mk3 + return nil, fmt.Errorf("in cluster container currently only supported option") + } + + orchestrator.BaseImageRepository = baseImageRepo + orchestrator.WorkspaceImageRepository = workspaceImgRepo + + imgcfg := config.ServiceConfig{ + Orchestrator: orchestrator, RefCache: config.RefCacheConfig{ Interval: util.Duration(time.Hour * 6).String(), Refs: []string{ diff --git a/installer/pkg/components/mysql/secret.go b/installer/pkg/components/mysql/secret.go index a68bfdb148c94d..d02b0206de0c6c 100644 --- a/installer/pkg/components/mysql/secret.go +++ b/installer/pkg/components/mysql/secret.go @@ -5,6 +5,7 @@ package mysql import ( + "encoding/json" "fmt" "github.com/gitpod-io/gitpod/installer/pkg/common" corev1 "k8s.io/api/core/v1" @@ -12,6 +13,13 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +type EncryptionKey struct { + Name string `json:"name"` + Version int `json:"version"` + Primary bool `json:"primary"` + Material string `json:"material"` +} + func secrets(ctx *common.RenderContext) ([]runtime.Object, error) { if !enabled(ctx) { return nil, nil @@ -27,6 +35,16 @@ func secrets(ctx *common.RenderContext) ([]runtime.Object, error) { return nil, err } + encryptionKeys, err := json.MarshalIndent([]EncryptionKey{{ + Name: "general", + Version: 1, + Primary: true, + Material: "4uGh1q8y2DYryJwrVMHs0kWXJlqvHWWt/KJuNi04edI=", + }}, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal mysql encryptionKeys: %w", err) + } + return []runtime.Object{&corev1.Secret{ TypeMeta: common.TypeMetaSecret, ObjectMeta: metav1.ObjectMeta{ @@ -46,11 +64,12 @@ func secrets(ctx *common.RenderContext) ([]runtime.Object, error) { Labels: common.DefaultLabels(Component), }, Data: map[string][]byte{ - "database": []byte(Database), - "host": []byte(Component), - "port": []byte(fmt.Sprintf("%d", Port)), - "password": []byte(password), - "username": []byte(Username), + "database": []byte(Database), + "encryptionKeys": encryptionKeys, + "host": []byte(Component), + "port": []byte(fmt.Sprintf("%d", Port)), + "password": []byte(password), + "username": []byte(Username), }, }}, nil } diff --git a/installer/pkg/components/proxy/configmap.go b/installer/pkg/components/proxy/configmap.go index 11d450b038a7de..3d1b8ec6ebef26 100644 --- a/installer/pkg/components/proxy/configmap.go +++ b/installer/pkg/components/proxy/configmap.go @@ -118,7 +118,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { openVSX, err := renderTemplate(vhostOpenVSXTmpl, openVSXTpl{ Domain: ctx.Config.Domain, - RepoURL: "open-vsx.org", // todo(sje) allow this to be configurable + RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, 8080), // todo(sje): get port from (future) config }) if err != nil { return nil, err diff --git a/installer/pkg/components/proxy/templates/configmap/vhost.open-vsx.tpl b/installer/pkg/components/proxy/templates/configmap/vhost.open-vsx.tpl index 120fd1c90f9e80..a9f695d07bb086 100644 --- a/installer/pkg/components/proxy/templates/configmap/vhost.open-vsx.tpl +++ b/installer/pkg/components/proxy/templates/configmap/vhost.open-vsx.tpl @@ -1,48 +1,9 @@ -# We cache the requests to the VSX registry and in case of an upstream server error we serve the the cached results. https://open-vsx.{{.Domain}} { import enable_log_debug import remove_server_header import ssl_configuration - # The http_cache plugin does not allow to cache the HTTP OPTIONS method. - # That's why we simply serve a static respond instead of asking the upstream server. - @options method OPTIONS - header @options { - Access-Control-Allow-Credentials "true" - Access-Control-Allow-Headers "content-type,x-market-client-id,x-market-user-id,x-client-commit,x-client-name,x-client-version,x-machine-id" - Access-Control-Allow-Methods "OPTIONS,GET,POST,PATCH,PUT,DELETE" - Access-Control-Allow-Origin "*" - } - respond @options 204 { - close - } - - reverse_proxy { - to https://{{.RepoURL}} - - # health_uri /api/-/search - - header_up Host "{{.RepoURL}}" - header_up -Connection - - # Override/remove existing cache control headers from the upstream server. - header_down Cache-Control "max-age=30, public" # cache for 30 seconds - header_down -Vary - header_down -Pragma - header_down -Expires - } - - gitpod.body_intercept { - search "{{.RepoURL}}" - replace "open-vsx.{{.Domain}}" - } - - http_cache { - cache_type file - path /tmp/openvsx-cache - match_path / - match_methods GET HEAD POST - stale_max_age 72h # 3 days - cache_key "{http.request.method} {http.request.host}{http.request.uri.path}?{http.request.uri.query} {http.request.contentlength} {http.request.bodyhash}" + reverse_proxy { + to {{.RepoURL}} } } \ No newline at end of file diff --git a/installer/pkg/components/server/configmap.go b/installer/pkg/components/server/configmap.go index 51261eb81e81f3..c2069224b99906 100644 --- a/installer/pkg/components/server/configmap.go +++ b/installer/pkg/components/server/configmap.go @@ -7,9 +7,9 @@ package server import ( "encoding/json" "fmt" - "github.com/gitpod-io/gitpod/installer/pkg/common" "github.com/gitpod-io/gitpod/installer/pkg/components/workspace" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,6 +22,11 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { return nil, err } + authProviders := make([]config.AuthProviderConfigs, 0) + if len(ctx.Config.AuthProviders) > 0 { + authProviders = ctx.Config.AuthProviders + } + // todo(sje): all these values are configurable scfg := ConfigSerialized{ Version: ctx.VersionManifest.Version, @@ -56,7 +61,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { MinAgePrebuildDays: 7, }, EnableLocalApp: true, - AuthProviderConfigs: ctx.Config.AuthProviders, + AuthProviderConfigs: authProviders, DisableDynamicAuthProviderLogin: false, BrandingConfig: BrandingConfig{ Logo: "/images/gitpod-ddd.svg", diff --git a/installer/pkg/components/server/deployment.go b/installer/pkg/components/server/deployment.go index 22c18f0567331c..98f375b88779d3 100644 --- a/installer/pkg/components/server/deployment.go +++ b/installer/pkg/components/server/deployment.go @@ -61,7 +61,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Name: "config", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: Component}, + LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf("%s-config", Component)}, }, }, }, { @@ -104,6 +104,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { // todo(sje): do we need to cater for serverContainer.env from values.yaml? Env: common.MergeEnv( common.DefaultEnv(&ctx.Config), + common.DatabaseEnv(&ctx.Config), common.TracingEnv(&ctx.Config), common.AnalyticsEnv(&ctx.Config), common.MessageBusEnv(&ctx.Config), diff --git a/installer/pkg/components/ws-manager-bridge/deployment.go b/installer/pkg/components/ws-manager-bridge/deployment.go index 4e4711349a98c2..cf3dee9a0aa8a6 100644 --- a/installer/pkg/components/ws-manager-bridge/deployment.go +++ b/installer/pkg/components/ws-manager-bridge/deployment.go @@ -5,6 +5,7 @@ package wsmanagerbridge import ( + "fmt" "github.com/gitpod-io/gitpod/installer/pkg/common" wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager" @@ -50,7 +51,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Name: "config", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: Component}, + LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf("%s-config", Component)}, }, }, }, { @@ -64,8 +65,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { InitContainers: []corev1.Container{*common.DatabaseWaiterContainer(ctx), *common.MessageBusWaiterContainer(ctx)}, Containers: []corev1.Container{{ Name: Component, - Args: []string{"run", "-v", "/mnt/config/config.json"}, - Image: common.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.Blobserve.Version), + Image: common.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerBridge.Version), ImagePullPolicy: corev1.PullIfNotPresent, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -90,7 +90,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { ), VolumeMounts: []corev1.VolumeMount{{ Name: "config", - MountPath: "/mnt/config", + MountPath: "/config", ReadOnly: true, }, { Name: "ws-manager-client-tls-certs", diff --git a/installer/third_party/charts/docker-registry/values.yaml b/installer/third_party/charts/docker-registry/values.yaml index 9b5e6bffa13b0f..b3fc162a1f90d9 100644 --- a/installer/third_party/charts/docker-registry/values.yaml +++ b/installer/third_party/charts/docker-registry/values.yaml @@ -2,5 +2,4 @@ # Licensed under the GNU Affero General Public License (AGPL). # See License-AGPL.txt in the project root for license information. -docker-registry: - fullnameOverride: registry +docker-registry: { }