mount PVC as block device

components/workspacekit/cmd/fsprep.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package cmd
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
+
+	"github.com/gitpod-io/gitpod/common-go/log"
+)
+
+const (
+	workspaceDevice = "/dev/workspace"
+)
+
+var fsPrepCmd = &cobra.Command{
+	Use:   "fsprep",
+	Short: "does fs prep and call supervisor",
+	RunE: func(_ *cobra.Command, args []string) (err error) {
+		defer func() {
+			if err != nil {
+				time.Sleep(5 * time.Minute)
+			}
+		}()
+
+		isReady, err := isWorkspaceDeviceReady(workspaceDevice)
+		if err != nil {
+			return err
+		}
+
+		if !isReady {
+			err = prepareWorkspaceDevice(workspaceDevice)
+			if err != nil {
+				return err
+			}
+		}
+
+		err = os.RemoveAll("/workspace")
+		if err != nil {
+			log.WithError(err).Error("cannot remove /workspace")
+		}
+
+		err = mountWorkspaceDevice(workspaceDevice, "/pvc/workspace")
+		if err != nil {
+			return err
+		}
+
+		// write a test file into /pvc/workspace folder
+		// this is to make sure that the workspace folder is writable
+		// and the workspace disk is mounted
+		err = os.WriteFile("/pvc/workspace/test", []byte("test"), 0644)
+		if err != nil {
+			log.WithError(err).Error("cannot write test file to /pvc/workspace")
+		}
+
+		log.Info("All done")
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(fsPrepCmd)
+}
+
+func prepareWorkspaceDevice(device string) error {
+	var stderr bytes.Buffer
+	cmd := exec.Command("mkfs.ext4", "-m1", device)
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		log.WithError(err).WithField("reason", stderr.String()).Error("cannot mount workspace disk")
+		return xerrors.Errorf("cannot format workspace disk using ext4: %w", err)
+	}
+
+	return nil
+}
+
+func mountWorkspaceDevice(device, target string) error {
+	if err := os.MkdirAll(target, 0755); err != nil {
+		return xerrors.Errorf("cannot create directory %v: %w", target, err)
+	}
+
+	// chown it so that it is owned by gitpod:gitpod when workspace starts
+	if err := os.Chown(target, 133332, 133332); err != nil {
+		return xerrors.Errorf("cannot chown directory %v: %w", target, err)
+	}
+
+	if err := unix.Mount(device, target, "ext4", uintptr(0), "user_xattr"); err != nil {
+		return xerrors.Errorf("cannot mount workspace disk in %v: %w", target, err)
+	}
+
+	return nil
+}
+
+func isWorkspaceDeviceReady(device string) (bool, error) {
+	var stderr bytes.Buffer
+	cmd := exec.Command("blkid", "-o", "value", "-s", "TYPE", device)
+	cmd.Stderr = &stderr
+	out, err := cmd.Output()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			if exitErr.ExitCode() == 2 {
+				// unformatted device
+				return false, nil
+			}
+		}
+
+		log.WithError(err).WithField("stdout", string(out)).WithField("stderr", stderr.String()).Error("cannot obtain details from the workspace disk")
+		return false, xerrors.Errorf("cannot obtain details from the workspace disk: %w", err)
+	}
+
+	return string(out) == "ext4", nil
+}

components/workspacekit/cmd/rings.go

@@ -324,6 +324,21 @@ var ring1Cmd = &cobra.Command{
 			mnts = append(mnts,
 				mnte{Target: "/workspace", Flags: unix.MS_BIND | unix.MS_REC},
 			)
+		} else {
+			// remove /workspace folder
+			err = os.RemoveAll("/workspace")
+			if err != nil {
+				log.WithError(err).Error("cannot remove /workspace")
+			}
+
+			err = os.Link("/pvc/workspace", "/workspace")
+			if err != nil {
+				log.WithError(err).Error("cannot link /pvc/workspace to /workspace")
+			}
+
+			//mnts = append(mnts,
+			//	mnte{Target: "/workspace", Source: "/pvc/workspace", Flags: unix.MS_BIND | unix.MS_REC},
+			//)
 		}
 
 		for _, m := range mnts {

components/ws-manager/pkg/manager/create.go

@@ -258,6 +258,7 @@ func (m *Manager) createPVCForWorkspacePod(startContext *startWorkspaceContext)
 		PVCConfig = startContext.Class.PVC
 	}
 
+	volumeMode := corev1.PersistentVolumeBlock
 	PVC := &corev1.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-%s", prefix, req.Id),
@@ -266,6 +267,7 @@ func (m *Manager) createPVCForWorkspacePod(startContext *startWorkspaceContext)
 		},
 		Spec: corev1.PersistentVolumeClaimSpec{
 			AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+			VolumeMode:  &volumeMode,
 			Resources: corev1.ResourceRequirements{
 				Requests: corev1.ResourceList{
 					corev1.ResourceName(corev1.ResourceStorage): PVCConfig.Size,
@@ -582,28 +584,50 @@ func (m *Manager) createDefiniteWorkspacePod(startContext *startWorkspaceContext
 				},
 			}
 
-			// SubPath so that lost+found is not visible
-			pod.Spec.Containers[0].VolumeMounts[0].SubPath = "workspace"
-			// not needed, since it is using dedicated disk
-			pod.Spec.Containers[0].VolumeMounts[0].MountPropagation = nil
+			volumeDevice := corev1.VolumeDevice{
+				Name:       workspaceVolumeName,
+				DevicePath: "/dev/workspace",
+			}
+			pod.Spec.Containers[0].VolumeDevices = append(pod.Spec.Containers[0].VolumeDevices, volumeDevice)
+
+			// get rid of first volume mount as PVC is mounted as block device and will be mounted by workspacekit
+			pod.Spec.Containers[0].VolumeMounts = pod.Spec.Containers[0].VolumeMounts[1:]
 
-			// pavel: 133332 is the Gitpod UID (33333) shifted by 99999. The shift happens inside the workspace container due to the user namespace use.
-			// We set this magical ID to make sure that gitpod user inside the workspace can write into /workspace folder mounted by PVC
-			gitpodGUID := int64(133332)
-			pod.Spec.SecurityContext.FSGroup = &gitpodGUID
+			pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
+				Name:      "test",
+				MountPath: "/pvc",
+			})
 
-			// add init container to chown workspace subpath, so that it is owned by gitpod user (there is no k8s native way of doing this as of right now)
+			pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+				Name: "test",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{
+						Medium: corev1.StorageMediumDefault,
+					},
+				},
+			})
+
+			rootUID := int64(0)
+			// add init container to mount PVC block device
 			pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
-				Name:            "chown-workspace",
-				Image:           "busybox",
+				Name:            "mount-workspace-volume",
+				Image:           pod.Spec.Containers[0].Image,
 				ImagePullPolicy: corev1.PullIfNotPresent,
-				Command:         []string{"chown", "133332:133332", "/workspace"},
+				Command:         []string{"/.supervisor/workspacekit", "fsprep"},
+				VolumeDevices:   []corev1.VolumeDevice{volumeDevice},
 				VolumeMounts: []corev1.VolumeMount{
 					{
-						Name:      workspaceVolumeName,
-						SubPath:   "workspace",
-						MountPath: "/workspace",
+						Name:      "test",
+						MountPath: "/pvc",
+					},
+				},
+				SecurityContext: &corev1.SecurityContext{
+					Capabilities: &corev1.Capabilities{
+						Add: []corev1.Capability{"SYS_ADMIN"},
 					},
+					Privileged: &boolTrue,
+					RunAsUser:  &rootUID,
+					RunAsGroup: &rootUID,
 				},
 			})
 

components/ws-manager/pkg/manager/testdata/cpwp_custom_storage.golden

@@ -27,7 +27,8 @@
                     "storage": "30Gi"
                 }
             },
-            "storageClassName": "test-storage"
+            "storageClassName": "test-storage",
+            "volumeMode": "Block"
         },
         "status": {}
     }

components/ws-manager/pkg/manager/testdata/cpwp_default_storage.golden

@@ -26,7 +26,8 @@
                 "requests": {
                     "storage": "30Gi"
                 }
-            }
+            },
+            "volumeMode": "Block"
         },
         "status": {}
     }

components/ws-manager/pkg/manager/testdata/cpwp_from_volume_snapshot.golden

@@ -28,6 +28,7 @@
                 }
             },
             "storageClassName": "test-storage",
+            "volumeMode": "Block",
             "dataSource": {
                 "apiGroup": "snapshot.storage.k8s.io",
                 "kind": "VolumeSnapshot",

components/ws-manager/pkg/manager/testdata/cpwp_no_class.golden

@@ -26,7 +26,8 @@
                 "requests": {
                     "storage": "0"
                 }
-            }
+            },
+            "volumeMode": "Block"
         },
         "status": {}
     }

components/ws-manager/pkg/manager/testdata/cpwp_no_pvc.golden

@@ -26,7 +26,8 @@
                 "requests": {
                     "storage": "0"
                 }
-            }
+            },
+            "volumeMode": "Block"
         },
         "status": {}
     }