Add persistent volume support for workspaces

fix

components/content-service/pkg/layer/provider.go

@@ -227,6 +227,116 @@ func (s *Provider) GetContentLayer(ctx context.Context, owner, workspaceID strin
 	return nil, nil, xerrors.Errorf("no backup or valid initializer present")
 }
 
+// GetContentLayerPVC provides the content layer for a workspace that uses PVC feature
+func (s *Provider) GetContentLayerPVC(ctx context.Context, owner, workspaceID string, initializer *csapi.WorkspaceInitializer) (l []Layer, manifest *csapi.WorkspaceContentManifest, err error) {
+	span, ctx := tracing.FromContext(ctx, "GetContentLayer")
+	defer tracing.FinishSpan(span, &err)
+	tracing.ApplyOWI(span, log.OWI(owner, workspaceID, ""))
+
+	defer func() {
+		// we never return a nil manifest, just maybe an empty one
+		if manifest == nil {
+			manifest = &csapi.WorkspaceContentManifest{
+				Type: csapi.TypeFullWorkspaceContentV1,
+			}
+		}
+	}()
+
+	// check if workspace has an FWB
+	var (
+		bucket = s.Storage.Bucket(owner)
+		mfobj  = fmt.Sprintf(fmtWorkspaceManifest, workspaceID)
+	)
+	span.LogKV("bucket", bucket, "mfobj", mfobj)
+	manifest, _, err = s.downloadContentManifest(ctx, bucket, mfobj)
+	if err != nil && err != storage.ErrNotFound {
+		return nil, nil, err
+	}
+	if manifest != nil {
+		span.LogKV("backup found", "full workspace backup")
+
+		l, err = s.layerFromContentManifestPVC(ctx, manifest, csapi.WorkspaceInitFromBackup, true)
+		return l, manifest, err
+	}
+
+	// check if legacy workspace backup is present
+	var layer *Layer
+	info, err := s.Storage.SignDownload(ctx, bucket, fmt.Sprintf(fmtLegacyBackupName, workspaceID), &storage.SignedURLOptions{})
+	if err != nil && !xerrors.Is(err, storage.ErrNotFound) {
+		return nil, nil, err
+	}
+	if err == nil {
+		span.LogKV("backup found", "legacy workspace backup")
+
+		cdesc, err := executor.PrepareFromBackup(info.URL)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		layer, err = contentDescriptorToLayerPVC(cdesc)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		l = []Layer{*layer}
+		return l, manifest, nil
+	}
+
+	// At this point we've found neither a full-workspace-backup, nor a legacy backup.
+	// It's time to use the initializer.
+	if gis := initializer.GetSnapshot(); gis != nil {
+		return s.getSnapshotContentLayer(ctx, gis)
+	}
+	if pis := initializer.GetPrebuild(); pis != nil {
+		l, manifest, err = s.getPrebuildContentLayer(ctx, pis)
+		if err != nil {
+			log.WithError(err).WithFields(log.OWI(owner, workspaceID, "")).Warn("cannot initialize from prebuild - falling back to Git")
+			span.LogKV("fallback-to-git", err.Error())
+
+			// we failed creating a prebuild initializer, so let's try falling back to the Git part.
+			var init []*csapi.WorkspaceInitializer
+			for _, gi := range pis.Git {
+				init = append(init, &csapi.WorkspaceInitializer{
+					Spec: &csapi.WorkspaceInitializer_Git{
+						Git: gi,
+					},
+				})
+			}
+			initializer = &csapi.WorkspaceInitializer{
+				Spec: &csapi.WorkspaceInitializer_Composite{
+					Composite: &csapi.CompositeInitializer{
+						Initializer: init,
+					},
+				},
+			}
+		} else {
+			// creating the initializer worked - we're done here
+			return
+		}
+	}
+	if gis := initializer.GetGit(); gis != nil {
+		span.LogKV("initializer", "Git")
+
+		cdesc, err := executor.Prepare(initializer, nil)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		layer, err = contentDescriptorToLayerPVC(cdesc)
+		if err != nil {
+			return nil, nil, err
+		}
+		return []Layer{*layer}, nil, nil
+	}
+	if initializer.GetBackup() != nil {
+		// We were asked to restore a backup and have tried above. We've failed to restore the backup,
+		// hance the backup initializer failed.
+		return nil, nil, xerrors.Errorf("no backup found")
+	}
+
+	return nil, nil, xerrors.Errorf("no backup or valid initializer present")
+}
+
 func (s *Provider) getSnapshotContentLayer(ctx context.Context, sp *csapi.SnapshotInitializer) (l []Layer, manifest *csapi.WorkspaceContentManifest, err error) {
 	span, ctx := tracing.FromContext(ctx, "getSnapshotContentLayer")
 	defer tracing.FinishSpan(span, &err)
@@ -360,6 +470,36 @@ func (s *Provider) layerFromContentManifest(ctx context.Context, mf *csapi.Works
 	return l, nil
 }
 
+func (s *Provider) layerFromContentManifestPVC(ctx context.Context, mf *csapi.WorkspaceContentManifest, initsrc csapi.WorkspaceInitSource, ready bool) (l []Layer, err error) {
+	// we have a valid full workspace backup
+	l = make([]Layer, len(mf.Layers))
+	for i, mfl := range mf.Layers {
+		info, err := s.Storage.SignDownload(ctx, mfl.Bucket, mfl.Object, &storage.SignedURLOptions{})
+		if err != nil {
+			return nil, err
+		}
+		if info.Meta.Digest != mfl.Digest.String() {
+			return nil, xerrors.Errorf("digest mismatch for %s/%s: expected %s, got %s", mfl.Bucket, mfl.Object, mfl.Digest, info.Meta.Digest)
+		}
+		l[i] = Layer{
+			DiffID:    mfl.DiffID.String(),
+			Digest:    mfl.Digest.String(),
+			MediaType: mfl.MediaType,
+			URL:       info.URL,
+			Size:      mfl.Size,
+		}
+	}
+
+	if ready {
+		rl, err := workspaceReadyLayerPVC(initsrc)
+		if err != nil {
+			return nil, err
+		}
+		l = append(l, *rl)
+	}
+	return l, nil
+}
+
 func contentDescriptorToLayer(cdesc []byte) (*Layer, error) {
 	return layerFromContent(
 		fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/workspace", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
@@ -368,6 +508,17 @@ func contentDescriptorToLayer(cdesc []byte) (*Layer, error) {
 	)
 }
 
+// version of this function for persistent volume claim feature
+// we cannot use /workspace folder as when mounting /workspace folder through PVC
+// it will mask anything that was in container layer, hence we are using /.workspace instead here
+func contentDescriptorToLayerPVC(cdesc []byte) (*Layer, error) {
+	return layerFromContent(
+		fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
+		fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace/.gitpod", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
+		fileInLayer{&tar.Header{Typeflag: tar.TypeReg, Name: "/.workspace/.gitpod/content.json", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755, Size: int64(len(cdesc))}, cdesc},
+	)
+}
+
 func workspaceReadyLayer(src csapi.WorkspaceInitSource) (*Layer, error) {
 	msg := csapi.WorkspaceReadyMessage{
 		Source: src,
@@ -384,6 +535,22 @@ func workspaceReadyLayer(src csapi.WorkspaceInitSource) (*Layer, error) {
 	)
 }
 
+func workspaceReadyLayerPVC(src csapi.WorkspaceInitSource) (*Layer, error) {
+	msg := csapi.WorkspaceReadyMessage{
+		Source: src,
+	}
+	ctnt, err := json.Marshal(msg)
+	if err != nil {
+		return nil, err
+	}
+
+	return layerFromContent(
+		fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
+		fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace/.gitpod", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
+		fileInLayer{&tar.Header{Typeflag: tar.TypeReg, Name: "/.workspace/.gitpod/ready", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755, Size: int64(len(ctnt))}, []byte(ctnt)},
+	)
+}
+
 type fileInLayer struct {
 	Header  *tar.Header
 	Content []byte

components/supervisor/pkg/supervisor/supervisor.go

@@ -1236,6 +1236,12 @@ func startContentInit(ctx context.Context, cfg *Config, wg *sync.WaitGroup, cst
 	}()
 
 	fn := "/workspace/.gitpod/content.json"
+	fnReady := "/workspace/.gitpod/ready"
+	if _, err := os.Stat("/.workspace/.gitpod/content.json"); !os.IsNotExist(err) {
+		fn = "/.workspace/.gitpod/content.json"
+		fnReady = "/.workspace/.gitpod/ready"
+		log.Info("Detected content.json in /.workspace folder, assuming PVC feature enabled")
+	}
 	f, err := os.Open(fn)
 	if os.IsNotExist(err) {
 		log.WithError(err).Info("no content init descriptor found - not trying to run it")
@@ -1245,7 +1251,7 @@ func startContentInit(ctx context.Context, cfg *Config, wg *sync.WaitGroup, cst
 		// TODO: rewrite using fsnotify
 		t := time.NewTicker(100 * time.Millisecond)
 		for range t.C {
-			b, err := os.ReadFile("/workspace/.gitpod/ready")
+			b, err := os.ReadFile(fnReady)
 			if err != nil {
 				if !os.IsNotExist(err) {
 					log.WithError(err).Error("cannot read content ready file")

components/workspacekit/cmd/rings.go

@@ -112,6 +112,7 @@ var ring0Cmd = &cobra.Command{
 		cmd.Env = append(os.Environ(),
 			"WORKSPACEKIT_FSSHIFT="+prep.FsShift.String(),
 			fmt.Sprintf("WORKSPACEKIT_FULL_WORKSPACE_BACKUP=%v", prep.FullWorkspaceBackup),
+			fmt.Sprintf("WORKSPACEKIT_PERSISTENT_VOLUME_CLAIM=%v", prep.PersistentVolumeClaim),
 		)
 
 		if err := cmd.Start(); err != nil {
@@ -303,7 +304,8 @@ var ring1Cmd = &cobra.Command{
 
 		// FWB workspaces do not require mounting /workspace
 		// if that is done, the backup will not contain any change in the directory
-		if os.Getenv("WORKSPACEKIT_FULL_WORKSPACE_BACKUP") != "true" {
+		// same applies to persistent volume claims, we cannot mount /workspace folder when PVC is used
+		if os.Getenv("WORKSPACEKIT_FULL_WORKSPACE_BACKUP") != "true" && os.Getenv("WORKSPACEKIT_PERSISTENT_VOLUME_CLAIM") != "true" {
 			mnts = append(mnts,
 				mnte{Target: "/workspace", Flags: unix.MS_BIND | unix.MS_REC},
 			)
@@ -416,6 +418,7 @@ var ring1Cmd = &cobra.Command{
 			log.WithError(err).Error("cannot mount proc")
 			return
 		}
+
 		_, err = client.EvacuateCGroup(ctx, &daemonapi.EvacuateCGroupRequest{})
 		if err != nil {
 			client.Close()

components/ws-daemon-api/daemon.proto

@@ -55,6 +55,11 @@ message InitWorkspaceRequest {
 
     // storage_quota_bytes enforces a storage quate for the workspace if set to a value != 0
     int64 storage_quota_bytes = 8;
+
+    // persistent_volume_claim means that we use PVC instead of HostPath to mount /workspace folder and content-init
+    // happens inside workspacekit instead of in ws-daemon. We also use k8s Snapshots to store\restore workspace content
+    // instead of GCS\tar.
+    bool persistent_volume_claim = 9;
 }
 
 // WorkspaceMetadata is data associated with a workspace that's required for other parts of the system to function

components/ws-daemon-api/go.mod

@@ -1,3 +1,5 @@
 module github.com/gitpod-io/generated_code_dependencies
 
 go 1.18
+
+require google.golang.org/protobuf v1.28.0 // indirect

components/ws-daemon-api/go.sum

@@ -0,0 +1,6 @@
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=