diff --git a/installer/pkg/common/constants.go b/installer/pkg/common/constants.go index ac98b383c400ea..207a28679ef452 100644 --- a/installer/pkg/common/constants.go +++ b/installer/pkg/common/constants.go @@ -15,8 +15,14 @@ const ( InClusterMessageQueueTLS = "messagebus-certificates-secret-core" MonitoringChart = "monitoring" ProxyComponent = "proxy" + RegistryFacadeComponent = "registry-facade" + RegistryFacadeServicePort = 3000 ServerComponent = "server" SystemNodeCritical = "system-node-critical" + WSManagerComponent = "ws-manager" + WSManagerBridgeComponent = "ws-manager-bridge" + WSProxyComponent = "ws-proxy" + WSSchedulerComponent = "ws-scheduler" ) const ( diff --git a/installer/pkg/common/objects.go b/installer/pkg/common/objects.go index 17eead47f766fa..0633de1ff53a45 100644 --- a/installer/pkg/common/objects.go +++ b/installer/pkg/common/objects.go @@ -5,6 +5,8 @@ package common import ( + "fmt" + storageconfig "github.com/gitpod-io/gitpod/content-service/api/config" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -68,3 +70,28 @@ func GenerateService(component string, ports map[string]ServicePort, assignClust }}, nil } } + +func StorageConfiguration(ctx *RenderContext) (*storageconfig.StorageConfig, error) { + accessKey, found := ctx.Values[ValueStorageAccessKey] + if !found { + return nil, fmt.Errorf("unknown value: %s", ValueStorageAccessKey) + } + secretKey, found := ctx.Values[ValueStorageSecretKey] + if !found { + return nil, fmt.Errorf("unknown value: %s", ValueStorageSecretKey) + } + + // todo(sje): support non-Minio storage configuration + // todo(sje): this has been set up with only the default values - receive configuration + return &storageconfig.StorageConfig{ + Kind: "minio", + BlobQuota: 0, + MinIOConfig: storageconfig.MinIOConfig{ + Endpoint: fmt.Sprintf("minio.%s", ctx.Config.Domain), + AccessKeyID: accessKey, + SecretAccessKey: secretKey, + Secure: false, + Region: "local", + }, + }, nil +} diff --git a/installer/pkg/components/blobserve/deployment.go b/installer/pkg/components/blobserve/deployment.go index 62827d1ace272e..f7d27bd2aa5785 100644 --- a/installer/pkg/components/blobserve/deployment.go +++ b/installer/pkg/components/blobserve/deployment.go @@ -6,7 +6,7 @@ package blobserve import ( "github.com/gitpod-io/gitpod/installer/pkg/common" - + dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -18,6 +18,42 @@ import ( func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { labels := common.DefaultLabels(Component) + volumes := []corev1.Volume{{ + Name: "cache", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, { + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: Component}, + }, + }, + }} + + volumeMounts := []corev1.VolumeMount{{ + Name: "config", + MountPath: "/mnt/config", + ReadOnly: true, + }, { + Name: "cache", + MountPath: "/mnt/cache", + }} + + if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) { + volumeName := "pull-secret" + volumes = append(volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: dockerregistry.BuiltInRegistryAuth, + }}, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: "/mnt/pull-secret.json", + SubPath: ".dockerconfigjson", + }) + } + return []runtime.Object{ &appsv1.Deployment{ TypeMeta: common.TypeMetaDeployment, @@ -41,22 +77,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Affinity: &corev1.Affinity{}, ServiceAccountName: Component, EnableServiceLinks: pointer.Bool(false), - Volumes: []corev1.Volume{{ - Name: "cache", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }, { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: Component}, - }, - }, - }, { - Name: "pull-secret", - VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ - SecretName: "", - }}, - }}, + Volumes: volumes, Containers: []corev1.Container{{ Name: Component, Args: []string{"run", "-v", "/mnt/config/config.json"}, @@ -80,14 +101,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { common.DefaultEnv(&ctx.Config), common.TracingEnv(&ctx.Config), ), - VolumeMounts: []corev1.VolumeMount{{ - Name: "config", - MountPath: "/mnt/config", - ReadOnly: true, - }, { - Name: "cache", - MountPath: "/mnt/cache", - }}, + VolumeMounts: volumeMounts, }, *common.KubeRBACProxyContainer()}, }, }, diff --git a/installer/pkg/components/registry-facade/configmap.go b/installer/pkg/components/registry-facade/configmap.go index 1df78aa1f02d30..0e8afba5299b62 100644 --- a/installer/pkg/components/registry-facade/configmap.go +++ b/installer/pkg/components/registry-facade/configmap.go @@ -40,12 +40,11 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { TLS: &tls, Store: "/mnt/cache/registry", RequireAuth: false, - // todo(sje): figure out these values StaticLayer: []regfac.StaticLayerCfg{{ - Ref: common.ImageName(ctx.Config.Repository, Component, "todo"), + Ref: common.ImageName(ctx.Config.Repository, SupervisorImage, ctx.VersionManifest.Components.Workspace.Supervisor.Version), Type: "image", }, { - Ref: common.ImageName(ctx.Config.Repository, Component, "todo"), + Ref: common.ImageName(ctx.Config.Repository, DockerUpImage, ctx.VersionManifest.Components.Workspace.DockerUp.Version), Type: "image", }}, }, diff --git a/installer/pkg/components/registry-facade/constants.go b/installer/pkg/components/registry-facade/constants.go index 16a9267565f868..3c144f22258b73 100644 --- a/installer/pkg/components/registry-facade/constants.go +++ b/installer/pkg/components/registry-facade/constants.go @@ -4,9 +4,16 @@ package registryfacade +import ( + "github.com/gitpod-io/gitpod/installer/pkg/common" + "github.com/gitpod-io/gitpod/installer/pkg/components/workspace" +) + const ( - Component = "registry-facade" + Component = common.RegistryFacadeComponent ContainerPortName = "registry" ContainerPort = 32223 - ServicePort = 3000 + ServicePort = common.RegistryFacadeServicePort + DockerUpImage = workspace.DockerUpImage + SupervisorImage = workspace.SupervisorImage ) diff --git a/installer/pkg/components/registry-facade/podsecuritypolicy.go b/installer/pkg/components/registry-facade/podsecuritypolicy.go index 62e486ec6d4243..82d72289de9f5f 100644 --- a/installer/pkg/components/registry-facade/podsecuritypolicy.go +++ b/installer/pkg/components/registry-facade/podsecuritypolicy.go @@ -5,6 +5,7 @@ package registryfacade import ( + "fmt" "github.com/gitpod-io/gitpod/installer/pkg/common" "k8s.io/api/policy/v1beta1" @@ -16,7 +17,7 @@ func podsecuritypolicy(ctx *common.RenderContext) ([]runtime.Object, error) { return []runtime.Object{&v1beta1.PodSecurityPolicy{ TypeMeta: common.TypeMetaPodSecurityPolicy, ObjectMeta: metav1.ObjectMeta{ - Name: Component, + Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component), Namespace: ctx.Namespace, Labels: common.DefaultLabels(Component), Annotations: map[string]string{ diff --git a/installer/pkg/components/workspace/constants.go b/installer/pkg/components/workspace/constants.go index a00afc1b64edf0..09c2d8b6c5c96f 100644 --- a/installer/pkg/components/workspace/constants.go +++ b/installer/pkg/components/workspace/constants.go @@ -5,7 +5,11 @@ package workspace const ( + ContainerPort = 23000 DefaultWorkspaceImage = "gitpod/workspace-full" DefaultWorkspaceImageVersion = "latest" IDEImageRepo = "ide/code" // todo(sje): does this need to be config driven? + DockerUpImage = "docker-up" + SupervisorImage = "supervisor" + SupervisorPort = 22999 ) diff --git a/installer/pkg/components/ws-daemon/clusterrole.go b/installer/pkg/components/ws-daemon/clusterrole.go index 0f52226b06370f..85c9322aa50c8d 100644 --- a/installer/pkg/components/ws-daemon/clusterrole.go +++ b/installer/pkg/components/ws-daemon/clusterrole.go @@ -21,7 +21,7 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) { &rbacv1.ClusterRole{ TypeMeta: common.TypeMetaClusterRole, ObjectMeta: metav1.ObjectMeta{ - Name: Component, + Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component), Namespace: ctx.Namespace, Labels: labels, }, diff --git a/installer/pkg/components/ws-daemon/configmap.go b/installer/pkg/components/ws-daemon/configmap.go index 0b2f81b0b1e426..cdb48612389206 100644 --- a/installer/pkg/components/ws-daemon/configmap.go +++ b/installer/pkg/components/ws-daemon/configmap.go @@ -27,11 +27,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -const ( - locContainerWorkingArea = "/mnt/workingarea" - locNodeWorkingArea = "/mnt/disks/ssd0/workspaces" -) - func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { var fsshift wsdapi.FSShiftMethod switch ctx.Config.Workspace.Runtime.FSShiftMethod { @@ -46,26 +41,38 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { wsdcfg := wsdconfig.Config{ Daemon: daemon.Config{ Runtime: daemon.RuntimeConfig{ + KubernetesNamespace: ctx.Namespace, Container: &container.Config{ Runtime: container.RuntimeContainerd, + Mapping: map[string]string{ + ctx.Config.Workspace.Runtime.ContainerDRuntimeDir: "/mnt/node0", + }, Mounts: container.NodeMountsLookupConfig{ - ProcLoc: "/mnt/rootfs/proc", + ProcLoc: "/mnt/mounts", }, Containerd: &container.ContainerdConfig{ - SocketPath: "/mnt/rootfs/run/containerd/containerd.sock", + SocketPath: "/mnt/containerd.sock", }, }, }, Content: content.Config{ - WorkingArea: locContainerWorkingArea, - WorkingAreaNode: locNodeWorkingArea, + WorkingArea: "/mnt/workingarea", + WorkingAreaNode: HostWorkspacePath, + TmpDir: "/tmp", UserNamespaces: content.UserNamespacesConfig{ FSShift: content.FSShiftMethod(fsshift), }, Storage: common.StorageConfig(&ctx.Config), + Backup: content.BackupConfig{ + Timeout: util.Duration(time.Minute * 5), + Attempts: 3, + }, + Initializer: content.InitializerConfig{ + Command: "/app/content-initializer", + }, }, Uidmapper: iws.UidmapperConfig{ - ProcLocation: "/mnt/rootfs/proc", + ProcLocation: "/proc", RootRange: iws.UIDRange{ Start: 33333, Size: 1, @@ -84,7 +91,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }, ControlPeriod: "15m", SamplingPeriod: "10s", - CGroupsBasePath: "/mnt/rootfs/sys/fs/cgroup", + CGroupsBasePath: "/mnt/node-cgroups", ProcessPriorities: map[resources.ProcessType]int{ resources.ProcessSupervisor: 0, resources.ProcessTheia: 5, @@ -94,7 +101,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }, Hosts: hosts.Config{ Enabled: true, - NodeHostsFile: "/mnt/rootfs/etc/hosts", + NodeHostsFile: "/mnt/hosts", FixedHosts: map[string][]hosts.Host{ "registryFacade": {{ Name: fmt.Sprintf("reg.%s", ctx.Config.Domain), @@ -110,11 +117,19 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { Enabled: true, Interval: util.Duration(5 * time.Minute), Locations: []diskguard.LocationConfig{{ - Path: locContainerWorkingArea, + Path: "/mnt/wsdaemon-workingarea", MinBytesAvail: 21474836480, }}, }, }, + Service: wsdconfig.AddrTLS{ + Addr: fmt.Sprintf(":%d", ServicePort), + TLS: &wsdconfig.TLS{ + Authority: "/certs/ca.crt", + Certificate: "/certs/tls.crt", + PrivateKey: "/certs/tls.key", + }, + }, Prometheus: wsdconfig.Addr{ Addr: "localhost:9500", }, diff --git a/installer/pkg/components/ws-daemon/constants.go b/installer/pkg/components/ws-daemon/constants.go index 26ba0c36217fd4..60a943ce818c27 100644 --- a/installer/pkg/components/ws-daemon/constants.go +++ b/installer/pkg/components/ws-daemon/constants.go @@ -5,8 +5,9 @@ package wsdaemon const ( - Component = "ws-daemon" - ServicePort = 8080 - TLSSecretName = "ws-daemon-tls" - VolumeTLSCerts = "ws-daemon-tls-certs" + Component = "ws-daemon" + ServicePort = 8080 + HostWorkspacePath = "/var/gitpod/workspaces" + TLSSecretName = "ws-daemon-tls" + VolumeTLSCerts = "ws-daemon-tls-certs" ) diff --git a/installer/pkg/components/ws-daemon/daemonset.go b/installer/pkg/components/ws-daemon/daemonset.go index 09b7aca61bab72..d9f584d00dacff 100644 --- a/installer/pkg/components/ws-daemon/daemonset.go +++ b/installer/pkg/components/ws-daemon/daemonset.go @@ -6,7 +6,7 @@ package wsdaemon import ( "github.com/gitpod-io/gitpod/installer/pkg/common" - config "github.com/gitpod-io/gitpod/installer/pkg/config/v1" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -38,12 +38,12 @@ set -euExo pipefail systemctl status kube-container-runtime-monitor.service || true if [ "$(systemctl is-active kube-container-runtime-monitor.service)" == "active" ] then -echo "kube-container-runtime-monitor.service is active" -systemctl stop kube-container-runtime-monitor.service -systemctl disable kube-container-runtime-monitor.service -systemctl status kube-container-runtime-monitor.service || true + echo "kube-container-runtime-monitor.service is active" + systemctl stop kube-container-runtime-monitor.service + systemctl disable kube-container-runtime-monitor.service + systemctl status kube-container-runtime-monitor.service || true else -echo "kube-container-runtime-monitor.service is not active, not doing anything" + echo "kube-container-runtime-monitor.service is not active, not doing anything" fi `}, SecurityContext: &corev1.SecurityContext{ @@ -72,13 +72,13 @@ fi "sh", "-c", `( -echo "running sysctls" && -sysctl -w net.core.somaxconn=4096 && -sysctl -w "net.ipv4.ip_local_port_range=5000 65000" && -sysctl -w "net.ipv4.tcp_tw_reuse=1" && -sysctl -w fs.inotify.max_user_watches=1000000 && -sysctl -w "kernel.dmesg_restrict=1" && -sysctl -w vm.unprivileged_userfaultfd=0 + echo "running sysctls" && + sysctl -w net.core.somaxconn=4096 && + sysctl -w "net.ipv4.ip_local_port_range=5000 65000" && + sysctl -w "net.ipv4.tcp_tw_reuse=1" && + sysctl -w fs.inotify.max_user_watches=1000000 && + sysctl -w "kernel.dmesg_restrict=1" && + sysctl -w vm.unprivileged_userfaultfd=0 ) && echo "done!" || echo "failed!" `, }, @@ -125,7 +125,7 @@ sysctl -w vm.unprivileged_userfaultfd=0 { Name: "working-area", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: "/mnt/disks/ssd0/workspaces", + Path: HostWorkspacePath, Type: func() *corev1.HostPathType { r := corev1.HostPathDirectoryOrCreate; return &r }(), }}, }, @@ -136,27 +136,20 @@ sysctl -w vm.unprivileged_userfaultfd=0 { Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: "ws-daemon-config"}, + LocalObjectReference: corev1.LocalObjectReference{Name: Component}, }}, }, { Name: "containerd-socket", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/containerd/containerd.sock", + Path: ctx.Config.Workspace.Runtime.ContainerDSocket, Type: func() *corev1.HostPathType { r := corev1.HostPathSocket; return &r }(), }}, }, { Name: "node-fs0", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib", - Type: func() *corev1.HostPathType { r := corev1.HostPathDirectory; return &r }(), - }}, - }, - { - Name: "node-fs1", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/containerd/io.containerd.runtime.v2.task/k8s.io", + Path: ctx.Config.Workspace.Runtime.ContainerDRuntimeDir, Type: func() *corev1.HostPathType { r := corev1.HostPathDirectory; return &r }(), }}, }, @@ -204,7 +197,7 @@ sysctl -w vm.unprivileged_userfaultfd=0 Containers: []corev1.Container{ { Name: Component, - Image: "eu.gcr.io/gitpod-core-dev/build/ws-daemon:not-set", + Image: common.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSDaemon.Version), Args: []string{ "run", "-v", @@ -219,10 +212,18 @@ sysctl -w vm.unprivileged_userfaultfd=0 Env: common.MergeEnv( common.DefaultEnv(&cfg), common.TracingEnv(&cfg), + []corev1.EnvVar{{ + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }}, ), Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ - corev1.ResourceName("cpu"): resource.MustParse("1m"), - corev1.ResourceName("memory"): resource.MustParse("1Mi"), + "cpu": resource.MustParse("1m"), + "memory": resource.MustParse("1Mi"), }}, VolumeMounts: []corev1.VolumeMount{ { @@ -242,10 +243,6 @@ sysctl -w vm.unprivileged_userfaultfd=0 Name: "node-fs0", MountPath: "/mnt/node0", }, - { - Name: "node-fs1", - MountPath: "/mnt/node1", - }, { Name: "node-mounts", ReadOnly: true, @@ -294,27 +291,27 @@ sysctl -w vm.unprivileged_userfaultfd=0 }, *common.KubeRBACProxyContainer(), }, - RestartPolicy: corev1.RestartPolicy("Always"), + RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(30), - DNSPolicy: corev1.DNSPolicy("ClusterFirst"), + DNSPolicy: "ClusterFirst", ServiceAccountName: Component, HostPID: true, Affinity: common.Affinity(common.AffinityLabelWorkspacesRegular, common.AffinityLabelWorkspacesHeadless), Tolerations: []corev1.Toleration{ { Key: "node.kubernetes.io/disk-pressure", - Operator: corev1.TolerationOperator("Exists"), - Effect: corev1.TaintEffect("NoExecute"), + Operator: "Exists", + Effect: "NoExecute", }, { Key: "node.kubernetes.io/memory-pressure", - Operator: corev1.TolerationOperator("Exists"), - Effect: corev1.TaintEffect("NoExecute"), + Operator: "Exists", + Effect: "NoExecute", }, { Key: "node.kubernetes.io/out-of-disk", - Operator: corev1.TolerationOperator("Exists"), - Effect: corev1.TaintEffect("NoExecute"), + Operator: "Exists", + Effect: "NoExecute", }, }, PriorityClassName: common.SystemNodeCritical, diff --git a/installer/pkg/components/ws-daemon/networkpolicy.go b/installer/pkg/components/ws-daemon/networkpolicy.go index a1fd9cc90a65cd..a79005bc9c35d9 100644 --- a/installer/pkg/components/ws-daemon/networkpolicy.go +++ b/installer/pkg/components/ws-daemon/networkpolicy.go @@ -6,7 +6,6 @@ package wsdaemon import ( "github.com/gitpod-io/gitpod/installer/pkg/common" - networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -37,7 +36,7 @@ func networkpolicy(ctx *common.RenderContext) ([]runtime.Object, error) { }, From: []networkingv1.NetworkPolicyPeer{ { - PodSelector: &metav1.LabelSelector{MatchLabels: labels}, + PodSelector: &metav1.LabelSelector{MatchLabels: common.DefaultLabels(common.WSManagerComponent)}, }, }, }, diff --git a/installer/pkg/components/ws-daemon/rolebinding.go b/installer/pkg/components/ws-daemon/rolebinding.go index b71e70eb59f189..f2a5a1272c88dd 100644 --- a/installer/pkg/components/ws-daemon/rolebinding.go +++ b/installer/pkg/components/ws-daemon/rolebinding.go @@ -40,7 +40,7 @@ func rolebinding(ctx *common.RenderContext) ([]runtime.Object, error) { &rbacv1.RoleBinding{ TypeMeta: common.TypeMetaRoleBinding, ObjectMeta: metav1.ObjectMeta{ - Name: Component, + Name: fmt.Sprintf("%s-rb", Component), Namespace: ctx.Namespace, Labels: labels, }, @@ -50,7 +50,7 @@ func rolebinding(ctx *common.RenderContext) ([]runtime.Object, error) { }}, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", - Name: fmt.Sprintf("%s-ns-ws-daemon", ctx.Namespace), + Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component), APIGroup: "rbac.authorization.k8s.io", }, }, diff --git a/installer/pkg/components/ws-daemon/tlssecret.go b/installer/pkg/components/ws-daemon/tlssecret.go index 09cc75e58d6411..9eaf381e5d6f4c 100644 --- a/installer/pkg/components/ws-daemon/tlssecret.go +++ b/installer/pkg/components/ws-daemon/tlssecret.go @@ -5,11 +5,41 @@ package wsdaemon import ( + "fmt" "github.com/gitpod-io/gitpod/installer/pkg/common" + certmanagerv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" "k8s.io/apimachinery/pkg/runtime" ) func tlssecret(ctx *common.RenderContext) ([]runtime.Object, error) { - return []runtime.Object{}, nil + oneYear := &metav1.Duration{Duration: time.Hour * 24 * 365} + + return []runtime.Object{ + &certmanagerv1.Certificate{ + TypeMeta: common.TypeMetaCertificate, + ObjectMeta: metav1.ObjectMeta{ + Name: TLSSecretName, + Namespace: ctx.Namespace, + Labels: common.DefaultLabels(Component), + }, + Spec: certmanagerv1.CertificateSpec{ + Duration: oneYear, + SecretName: TLSSecretName, + DNSNames: []string{ + fmt.Sprintf("gitpod.%s", ctx.Namespace), + fmt.Sprintf("%s.%s.svc", Component, ctx.Namespace), + Component, + }, + IssuerRef: cmmeta.ObjectReference{ + Name: common.CertManagerCAIssuer, + Kind: "Issuer", + Group: "cert-manager.io", + }, + }, + }, + }, nil } diff --git a/installer/pkg/components/ws-manager-bridge/constants.go b/installer/pkg/components/ws-manager-bridge/constants.go index 17ed44b2e10e82..b816fee2e2c5c8 100644 --- a/installer/pkg/components/ws-manager-bridge/constants.go +++ b/installer/pkg/components/ws-manager-bridge/constants.go @@ -4,6 +4,8 @@ package wsmanagerbridge +import "github.com/gitpod-io/gitpod/installer/pkg/common" + const ( - Component = "ws-manager-bridge" + Component = common.WSManagerBridgeComponent ) diff --git a/installer/pkg/components/ws-manager/configmap.go b/installer/pkg/components/ws-manager/configmap.go index 9d7e9379236cba..bd881742fc8d23 100644 --- a/installer/pkg/components/ws-manager/configmap.go +++ b/installer/pkg/components/ws-manager/configmap.go @@ -7,7 +7,10 @@ package wsmanager import ( "encoding/json" "fmt" + wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon" + "k8s.io/utils/pointer" "path/filepath" + "sigs.k8s.io/yaml" "time" "github.com/gitpod-io/gitpod/common-go/grpc" @@ -20,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/yaml" ) func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { @@ -37,12 +39,16 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { return (&q).String() } + storage, err := common.StorageConfiguration(ctx) + if err != nil { + return nil, err + } + wsmcfg := config.ServiceConfiguration{ - // todo(sje): put in config values Manager: config.Configuration{ Namespace: ctx.Namespace, SchedulerName: "workspace-scheduler", - SeccompProfile: "", + SeccompProfile: "/mnt/dst/workspace_default_not-set.json", DryRun: false, WorkspaceDaemon: config.WorkspaceDaemonConfiguration{ Port: 8080, @@ -73,13 +79,13 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }, HeartbeatInterval: util.Duration(30 * time.Second), GitpodHostURL: "https://" + ctx.Config.Domain, - WorkspaceClusterHost: "", + WorkspaceClusterHost: fmt.Sprintf("ws.%s", ctx.Config.Domain), InitProbe: config.InitProbeConfiguration{ Timeout: (1 * time.Second).String(), }, - WorkspaceURLTemplate: "", - WorkspacePortURLTemplate: "", - WorkspaceHostPath: "", + WorkspaceURLTemplate: fmt.Sprintf("https://{{ .Prefix }}.ws.%s", ctx.Config.Domain), + WorkspacePortURLTemplate: fmt.Sprintf("https://{{ .WorkspacePort }}-{{ .Prefix }}.ws.%s", ctx.Config.Domain), + WorkspaceHostPath: wsdaemon.HostWorkspacePath, WorkspacePodTemplate: templatesCfg, Timeouts: config.WorkspaceTimeoutConfiguration{ AfterClose: util.Duration(2 * time.Minute), @@ -91,14 +97,14 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { Stopping: util.Duration(1 * time.Hour), Interrupted: util.Duration(5 * time.Minute), }, - EventTraceLog: "", // todo(sje): make conditional based on config + //EventTraceLog: "", // todo(sje): make conditional based on config ReconnectionInterval: util.Duration(30 * time.Second), - RegistryFacadeHost: "", + RegistryFacadeHost: fmt.Sprintf("reg.%s:%v", common.RegistryFacadeComponent, common.RegistryFacadeServicePort), EnforceWorkspaceNodeAffinity: true, }, Content: struct { Storage storageconfig.StorageConfig `json:"storage"` - }{Storage: storageconfig.StorageConfig{}}, + }{Storage: *storage}, RPCServer: struct { Addr string `json:"addr"` TLS struct { @@ -108,7 +114,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { } `json:"tls"` RateLimits map[string]grpc.RateLimit `json:"ratelimits"` }{ - Addr: "8080", + Addr: fmt.Sprintf(":%d", RPCPort), TLS: struct { CA string `json:"ca"` Certificate string `json:"crt"` @@ -160,6 +166,19 @@ func buildWorkspaceTemplates(ctx *common.RenderContext) (config.WorkspacePodTemp cfgTpls = &configv1.WorkspaceTemplates{} } + cfgTpls.Default = &corev1.Pod{ + Spec: corev1.PodSpec{ + EnableServiceLinks: pointer.Bool(true), + DNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{ + "1.1.1.1", + "8.8.8.8", + }, + }, + DNSPolicy: corev1.DNSNone, + }, + } + ops := []struct { Name string Path *string @@ -170,17 +189,16 @@ func buildWorkspaceTemplates(ctx *common.RenderContext) (config.WorkspacePodTemp {Name: "imagebuild", Path: &cfg.ImagebuildPath, Tpl: cfgTpls.ImageBuild}, {Name: "prebuild", Path: &cfg.PrebuildPath, Tpl: cfgTpls.Prebuild}, {Name: "regular", Path: &cfg.RegularPath, Tpl: cfgTpls.Regular}, + {Name: "probe", Path: &cfg.ProbePath, Tpl: cfgTpls.Probe}, } for _, op := range ops { if op.Tpl == nil { continue } - fc, err := yaml.Marshal(op.Tpl) if err != nil { return cfg, nil, fmt.Errorf("unable to marshal %s workspace template: %w", op.Name, err) } - fn := op.Name + ".yaml" *op.Path = filepath.Join(WorkspaceTemplatePath, fn) tpls[fn] = string(fc) diff --git a/installer/pkg/components/ws-manager/constants.go b/installer/pkg/components/ws-manager/constants.go index 9ee5f3c3681080..48f876f53b5945 100644 --- a/installer/pkg/components/ws-manager/constants.go +++ b/installer/pkg/components/ws-manager/constants.go @@ -4,8 +4,10 @@ package wsmanager +import "github.com/gitpod-io/gitpod/installer/pkg/common" + const ( - Component = "ws-manager" + Component = common.WSManagerComponent RPCPort = 8080 RPCPortName = "rpc" TLSSecretNameSecret = "ws-manager-tls" diff --git a/installer/pkg/components/ws-manager/deployment.go b/installer/pkg/components/ws-manager/deployment.go index 28f06530f3a9cb..98da2f5ad20951 100644 --- a/installer/pkg/components/ws-manager/deployment.go +++ b/installer/pkg/components/ws-manager/deployment.go @@ -7,7 +7,6 @@ package wsmanager import ( "github.com/gitpod-io/gitpod/installer/pkg/common" wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -75,7 +74,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { ReadOnly: true, }, { Name: VolumeWorkspaceTemplate, - MountPath: "/workspace-template", + MountPath: WorkspaceTemplatePath, ReadOnly: true, }, { Name: wsdaemon.VolumeTLSCerts, @@ -87,39 +86,34 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { ReadOnly: true, }}, }}, - Volumes: []corev1.Volume{ - { - Name: VolumeConfig, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: Component}, - }, + Volumes: []corev1.Volume{{ + Name: VolumeConfig, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: Component}, }, }, - { - Name: wsdaemon.VolumeTLSCerts, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: wsdaemon.TLSSecretName, - }, - }, + }, { + Name: VolumeWorkspaceTemplate, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: WorkspaceTemplateConfigMap}}, }, - { - Name: VolumeTLSCerts, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: TLSSecretNameSecret, - }, + }, { + Name: wsdaemon.VolumeTLSCerts, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: wsdaemon.TLSSecretName, }, }, - { - Name: VolumeWorkspaceTemplate, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: WorkspaceTemplateConfigMap}}, + }, { + Name: VolumeTLSCerts, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: TLSSecretNameSecret, }, }, - }, + }}, }, }, }, diff --git a/installer/pkg/components/ws-manager/tlssecret.go b/installer/pkg/components/ws-manager/tlssecret.go index c8e22203b1888e..753406d4ff0b1a 100644 --- a/installer/pkg/components/ws-manager/tlssecret.go +++ b/installer/pkg/components/ws-manager/tlssecret.go @@ -19,17 +19,17 @@ import ( func tlssecret(ctx *common.RenderContext) ([]runtime.Object, error) { serverAltNames := []string{ fmt.Sprintf("gitpod.%s", ctx.Namespace), - fmt.Sprintf("%s.ws-manager.svc", ctx.Namespace), - "ws-manager", - "ws-manager-dev", + fmt.Sprintf("%s.%s.svc", Component, ctx.Namespace), + Component, + fmt.Sprintf("%s-dev", Component), } clientAltNames := []string{ - "registry-facade", - "server", - "ws-manager-bridge", - "ws-scheduler", - "ws-proxy", - "ws-manager", + common.RegistryFacadeComponent, + common.ServerComponent, + common.WSManagerBridgeComponent, + common.WSSchedulerComponent, + common.WSProxyComponent, + Component, } sixMonths := &metav1.Duration{Duration: time.Hour * 4380} @@ -39,7 +39,7 @@ func tlssecret(ctx *common.RenderContext) ([]runtime.Object, error) { &certmanagerv1.Certificate{ TypeMeta: common.TypeMetaCertificate, ObjectMeta: metav1.ObjectMeta{ - Name: Component, + Name: TLSSecretNameSecret, Namespace: ctx.Namespace, Labels: common.DefaultLabels(Component), }, @@ -49,7 +49,7 @@ func tlssecret(ctx *common.RenderContext) ([]runtime.Object, error) { DNSNames: serverAltNames, IssuerRef: cmmeta.ObjectReference{ Name: issuer, - Kind: "ClusterIssuer", + Kind: "Issuer", Group: "cert-manager.io", }, }, @@ -67,7 +67,7 @@ func tlssecret(ctx *common.RenderContext) ([]runtime.Object, error) { DNSNames: clientAltNames, IssuerRef: cmmeta.ObjectReference{ Name: issuer, - Kind: "ClusterIssuer", + Kind: "Issuer", Group: "cert-manager.io", }, }, diff --git a/installer/pkg/components/ws-proxy/configmap.go b/installer/pkg/components/ws-proxy/configmap.go index 9bdb0bc3566265..841b38d447b8f4 100644 --- a/installer/pkg/components/ws-proxy/configmap.go +++ b/installer/pkg/components/ws-proxy/configmap.go @@ -7,6 +7,7 @@ package wsproxy import ( "encoding/json" "fmt" + "github.com/gitpod-io/gitpod/installer/pkg/components/workspace" "time" "github.com/gitpod-io/gitpod/common-go/util" @@ -24,8 +25,8 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { // todo(sje): wsManagerProxy seems to be unused wspcfg := config.Config{ Ingress: proxy.HostBasedIngressConfig{ - HttpAddress: string(rune(HTTPProxyPort)), - HttpsAddress: string(rune(HTTPSProxyPort)), + HttpAddress: fmt.Sprintf(":%d", HTTPProxyPort), + HttpsAddress: fmt.Sprintf(":%d", HTTPSProxyPort), Header: HostHeader, }, Proxy: proxy.Config{ @@ -44,21 +45,27 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { }, BlobServer: &proxy.BlobServerConfig{ Scheme: "http", - // todo(sje): get blob service port from (future) blob service package - Host: fmt.Sprintf("blobserve.%s.svc.cluster.local:%d", ctx.Namespace, common.BlobServeServicePort), + Host: fmt.Sprintf("blobserve.%s.svc.cluster.local:%d", ctx.Namespace, common.BlobServeServicePort), }, - // todo(sje): import gitpod values from (future) gitpod package GitpodInstallation: &proxy.GitpodInstallation{ - Scheme: "http", + Scheme: "https", + HostName: ctx.Config.Domain, + WorkspaceHostSuffix: fmt.Sprintf(".ws.%s", ctx.Config.Domain), + WorkspaceHostSuffixRegex: fmt.Sprintf("\\.ws[^\\.]*\\.%s", ctx.Config.Domain), + }, + WorkspacePodConfig: &proxy.WorkspacePodConfig{ + ServiceTemplate: fmt.Sprintf("http://ws-{{ .workspaceID }}-theia.%s.svc.cluster.local:{{ .port }}", ctx.Namespace), + PortServiceTemplate: fmt.Sprintf("http://ws-{{ .workspaceID }}-ports.%s.svc.cluster.local:{{ .port }}", ctx.Namespace), + TheiaPort: workspace.ContainerPort, + SupervisorPort: workspace.SupervisorPort, + SupervisorImage: common.ImageName(ctx.Config.Repository, workspace.SupervisorImage, ctx.VersionManifest.Components.Workspace.Supervisor.Version), }, - // todo(sje): import wspod config from (future) workspace package - WorkspacePodConfig: &proxy.WorkspacePodConfig{}, BuiltinPages: proxy.BuiltinPagesConfig{ Location: "/app/public", }, }, WorkspaceInfoProviderConfig: proxy.WorkspaceInfoProviderConfig{ - WsManagerAddr: fmt.Sprintf("ws-manager:%d", wsmanager.RPCPort), + WsManagerAddr: fmt.Sprintf("%s:%d", wsmanager.Component, wsmanager.RPCPort), ReconnectInterval: util.Duration(time.Second * 3), TLS: struct { CA string `json:"ca"` diff --git a/installer/pkg/components/ws-proxy/constants.go b/installer/pkg/components/ws-proxy/constants.go index 25fbc7293de7e1..8a64344a463899 100644 --- a/installer/pkg/components/ws-proxy/constants.go +++ b/installer/pkg/components/ws-proxy/constants.go @@ -4,8 +4,10 @@ package wsproxy +import "github.com/gitpod-io/gitpod/installer/pkg/common" + const ( - Component = "ws-proxy" + Component = common.WSProxyComponent HostHeader = "x-wsproxy-host" HTTPProxyPort = 8080 HTTPProxyPortName = "http-proxy" diff --git a/installer/pkg/components/ws-scheduler/clusterrole.go b/installer/pkg/components/ws-scheduler/clusterrole.go index 249a3d6b872987..e4f23045276c38 100644 --- a/installer/pkg/components/ws-scheduler/clusterrole.go +++ b/installer/pkg/components/ws-scheduler/clusterrole.go @@ -21,7 +21,7 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) { &rbacv1.ClusterRole{ TypeMeta: common.TypeMetaClusterRole, ObjectMeta: metav1.ObjectMeta{ - Name: Component, + Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component), Namespace: ctx.Namespace, Labels: labels, }, diff --git a/installer/pkg/components/ws-scheduler/clusterrolebinding.go b/installer/pkg/components/ws-scheduler/clusterrolebinding.go index fadbd8ebcc0eec..2df574b5bda977 100644 --- a/installer/pkg/components/ws-scheduler/clusterrolebinding.go +++ b/installer/pkg/components/ws-scheduler/clusterrolebinding.go @@ -27,7 +27,7 @@ func clusterrolebinding(ctx *common.RenderContext) ([]runtime.Object, error) { }, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", - Name: fmt.Sprintf("%s-ns-ws-scheduler", ctx.Namespace), + Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component), APIGroup: "rbac.authorization.k8s.io", }, Subjects: []rbacv1.Subject{ diff --git a/installer/pkg/components/ws-scheduler/constants.go b/installer/pkg/components/ws-scheduler/constants.go index d87d949786bfbe..6613fb81df086d 100644 --- a/installer/pkg/components/ws-scheduler/constants.go +++ b/installer/pkg/components/ws-scheduler/constants.go @@ -4,6 +4,8 @@ package wsscheduler +import "github.com/gitpod-io/gitpod/installer/pkg/common" + const ( - Component = "ws-scheduler" + Component = common.WSSchedulerComponent ) diff --git a/installer/pkg/config/v1/config.go b/installer/pkg/config/v1/config.go index c21417ca4e790f..a744b3fee5d0f7 100644 --- a/installer/pkg/config/v1/config.go +++ b/installer/pkg/config/v1/config.go @@ -195,6 +195,7 @@ type WorkspaceTemplates struct { Ghost *corev1.Pod `json:"ghost"` ImageBuild *corev1.Pod `json:"image_build"` Regular *corev1.Pod `json:"regular"` + Probe *corev1.Pod `json:"probe"` } type Workspace struct {