diff --git a/components/workspacekit/cmd/fsprep.go b/components/workspacekit/cmd/fsprep.go new file mode 100644 index 00000000000000..d7859c701cc69e --- /dev/null +++ b/components/workspacekit/cmd/fsprep.go @@ -0,0 +1,121 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package cmd + +import ( + "bytes" + "os" + "os/exec" + "time" + + "github.com/spf13/cobra" + "golang.org/x/sys/unix" + "golang.org/x/xerrors" + + "github.com/gitpod-io/gitpod/common-go/log" +) + +const ( + workspaceDevice = "/dev/workspace" +) + +var fsPrepCmd = &cobra.Command{ + Use: "fsprep", + Short: "does fs prep and call supervisor", + RunE: func(_ *cobra.Command, args []string) (err error) { + defer func() { + if err != nil { + time.Sleep(5 * time.Minute) + } + }() + + isReady, err := isWorkspaceDeviceReady(workspaceDevice) + if err != nil { + return err + } + + if !isReady { + err = prepareWorkspaceDevice(workspaceDevice) + if err != nil { + return err + } + } + + err = os.RemoveAll("/workspace") + if err != nil { + log.WithError(err).Error("cannot remove /workspace") + } + + err = mountWorkspaceDevice(workspaceDevice, "/pvc/workspace") + if err != nil { + return err + } + + // write a test file into /pvc/workspace folder + // this is to make sure that the workspace folder is writable + // and the workspace disk is mounted + err = os.WriteFile("/pvc/workspace/test", []byte("test"), 0644) + if err != nil { + log.WithError(err).Error("cannot write test file to /pvc/workspace") + } + + log.Info("All done") + + return nil + }, +} + +func init() { + rootCmd.AddCommand(fsPrepCmd) +} + +func prepareWorkspaceDevice(device string) error { + var stderr bytes.Buffer + cmd := exec.Command("mkfs.ext4", "-m1", device) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.WithError(err).WithField("reason", stderr.String()).Error("cannot mount workspace disk") + return xerrors.Errorf("cannot format workspace disk using ext4: %w", err) + } + + return nil +} + +func mountWorkspaceDevice(device, target string) error { + if err := os.MkdirAll(target, 0755); err != nil { + return xerrors.Errorf("cannot create directory %v: %w", target, err) + } + + // chown it so that it is owned by gitpod:gitpod when workspace starts + if err := os.Chown(target, 133332, 133332); err != nil { + return xerrors.Errorf("cannot chown directory %v: %w", target, err) + } + + if err := unix.Mount(device, target, "ext4", uintptr(0), "user_xattr"); err != nil { + return xerrors.Errorf("cannot mount workspace disk in %v: %w", target, err) + } + + return nil +} + +func isWorkspaceDeviceReady(device string) (bool, error) { + var stderr bytes.Buffer + cmd := exec.Command("blkid", "-o", "value", "-s", "TYPE", device) + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if exitErr.ExitCode() == 2 { + // unformatted device + return false, nil + } + } + + log.WithError(err).WithField("stdout", string(out)).WithField("stderr", stderr.String()).Error("cannot obtain details from the workspace disk") + return false, xerrors.Errorf("cannot obtain details from the workspace disk: %w", err) + } + + return string(out) == "ext4", nil +} diff --git a/components/workspacekit/cmd/rings.go b/components/workspacekit/cmd/rings.go index d5249043b8e1b2..c43d267ec15c01 100644 --- a/components/workspacekit/cmd/rings.go +++ b/components/workspacekit/cmd/rings.go @@ -324,6 +324,21 @@ var ring1Cmd = &cobra.Command{ mnts = append(mnts, mnte{Target: "/workspace", Flags: unix.MS_BIND | unix.MS_REC}, ) + } else { + // remove /workspace folder + err = os.RemoveAll("/workspace") + if err != nil { + log.WithError(err).Error("cannot remove /workspace") + } + + err = os.Link("/pvc/workspace", "/workspace") + if err != nil { + log.WithError(err).Error("cannot link /pvc/workspace to /workspace") + } + + //mnts = append(mnts, + // mnte{Target: "/workspace", Source: "/pvc/workspace", Flags: unix.MS_BIND | unix.MS_REC}, + //) } for _, m := range mnts { diff --git a/components/ws-manager/pkg/manager/create.go b/components/ws-manager/pkg/manager/create.go index 57e9be8a091e6d..61948b4aed3ee8 100644 --- a/components/ws-manager/pkg/manager/create.go +++ b/components/ws-manager/pkg/manager/create.go @@ -258,6 +258,7 @@ func (m *Manager) createPVCForWorkspacePod(startContext *startWorkspaceContext) PVCConfig = startContext.Class.PVC } + volumeMode := corev1.PersistentVolumeBlock PVC := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", prefix, req.Id), @@ -266,6 +267,7 @@ func (m *Manager) createPVCForWorkspacePod(startContext *startWorkspaceContext) }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + VolumeMode: &volumeMode, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName(corev1.ResourceStorage): PVCConfig.Size, @@ -582,28 +584,50 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext }, } - // SubPath so that lost+found is not visible - pod.Spec.Containers[0].VolumeMounts[0].SubPath = "workspace" - // not needed, since it is using dedicated disk - pod.Spec.Containers[0].VolumeMounts[0].MountPropagation = nil + volumeDevice := corev1.VolumeDevice{ + Name: workspaceVolumeName, + DevicePath: "/dev/workspace", + } + pod.Spec.Containers[0].VolumeDevices = append(pod.Spec.Containers[0].VolumeDevices, volumeDevice) + + // get rid of first volume mount as PVC is mounted as block device and will be mounted by workspacekit + pod.Spec.Containers[0].VolumeMounts = pod.Spec.Containers[0].VolumeMounts[1:] - // pavel: 133332 is the Gitpod UID (33333) shifted by 99999. The shift happens inside the workspace container due to the user namespace use. - // We set this magical ID to make sure that gitpod user inside the workspace can write into /workspace folder mounted by PVC - gitpodGUID := int64(133332) - pod.Spec.SecurityContext.FSGroup = &gitpodGUID + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "test", + MountPath: "/pvc", + }) - // add init container to chown workspace subpath, so that it is owned by gitpod user (there is no k8s native way of doing this as of right now) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "test", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumDefault, + }, + }, + }) + + rootUID := int64(0) + // add init container to mount PVC block device pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ - Name: "chown-workspace", - Image: "busybox", + Name: "mount-workspace-volume", + Image: pod.Spec.Containers[0].Image, ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"chown", "133332:133332", "/workspace"}, + Command: []string{"/.supervisor/workspacekit", "fsprep"}, + VolumeDevices: []corev1.VolumeDevice{volumeDevice}, VolumeMounts: []corev1.VolumeMount{ { - Name: workspaceVolumeName, - SubPath: "workspace", - MountPath: "/workspace", + Name: "test", + MountPath: "/pvc", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_ADMIN"}, }, + Privileged: &boolTrue, + RunAsUser: &rootUID, + RunAsGroup: &rootUID, }, }) diff --git a/components/ws-manager/pkg/manager/testdata/cpwp_custom_storage.golden b/components/ws-manager/pkg/manager/testdata/cpwp_custom_storage.golden index 6748fe06fe22e4..e213bb0da057b8 100644 --- a/components/ws-manager/pkg/manager/testdata/cpwp_custom_storage.golden +++ b/components/ws-manager/pkg/manager/testdata/cpwp_custom_storage.golden @@ -27,7 +27,8 @@ "storage": "30Gi" } }, - "storageClassName": "test-storage" + "storageClassName": "test-storage", + "volumeMode": "Block" }, "status": {} } diff --git a/components/ws-manager/pkg/manager/testdata/cpwp_default_storage.golden b/components/ws-manager/pkg/manager/testdata/cpwp_default_storage.golden index d26feb8fbc7514..d0f5b34723d108 100644 --- a/components/ws-manager/pkg/manager/testdata/cpwp_default_storage.golden +++ b/components/ws-manager/pkg/manager/testdata/cpwp_default_storage.golden @@ -26,7 +26,8 @@ "requests": { "storage": "30Gi" } - } + }, + "volumeMode": "Block" }, "status": {} } diff --git a/components/ws-manager/pkg/manager/testdata/cpwp_from_volume_snapshot.golden b/components/ws-manager/pkg/manager/testdata/cpwp_from_volume_snapshot.golden index b575b5a4685701..d0313c09af2c1c 100644 --- a/components/ws-manager/pkg/manager/testdata/cpwp_from_volume_snapshot.golden +++ b/components/ws-manager/pkg/manager/testdata/cpwp_from_volume_snapshot.golden @@ -28,6 +28,7 @@ } }, "storageClassName": "test-storage", + "volumeMode": "Block", "dataSource": { "apiGroup": "snapshot.storage.k8s.io", "kind": "VolumeSnapshot", diff --git a/components/ws-manager/pkg/manager/testdata/cpwp_no_class.golden b/components/ws-manager/pkg/manager/testdata/cpwp_no_class.golden index d8835d894532da..2def82d5b8a64a 100644 --- a/components/ws-manager/pkg/manager/testdata/cpwp_no_class.golden +++ b/components/ws-manager/pkg/manager/testdata/cpwp_no_class.golden @@ -26,7 +26,8 @@ "requests": { "storage": "0" } - } + }, + "volumeMode": "Block" }, "status": {} } diff --git a/components/ws-manager/pkg/manager/testdata/cpwp_no_pvc.golden b/components/ws-manager/pkg/manager/testdata/cpwp_no_pvc.golden index c2be9eb9ce2fa7..a3107c12d8c52d 100644 --- a/components/ws-manager/pkg/manager/testdata/cpwp_no_pvc.golden +++ b/components/ws-manager/pkg/manager/testdata/cpwp_no_pvc.golden @@ -26,7 +26,8 @@ "requests": { "storage": "0" } - } + }, + "volumeMode": "Block" }, "status": {} }