[ws-daemon] Refactor unmount

components/workspacekit/cmd/rings.go

@@ -61,6 +61,8 @@ var ring0Cmd = &cobra.Command{
 		exitCode := 1
 		defer handleExit(&exitCode)
 
+		defer log.Info("done")
+
 		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
 
@@ -76,16 +78,6 @@ var ring0Cmd = &cobra.Command{
 			log.WithError(err).Fatal("cannot prepare for user namespaces")
 			return
 		}
-		defer func() {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-
-			_, err = client.Teardown(ctx, &daemonapi.TeardownRequest{})
-			if err != nil {
-				log.WithError(err).Error("cannot trigger teardown")
-				return
-			}
-		}()
 
 		cmd := exec.Command("/proc/self/exe", "ring1")
 		cmd.SysProcAttr = &syscall.SysProcAttr{
@@ -167,11 +159,12 @@ var ring1Cmd = &cobra.Command{
 	Run: func(_cmd *cobra.Command, args []string) {
 		log.Init(ServiceName, Version, true, true)
 		log := log.WithField("ring", 1)
-		defer log.Info("done")
 
 		exitCode := 1
 		defer handleExit(&exitCode)
 
+		defer log.Info("done")
+
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
 
@@ -182,6 +175,16 @@ var ring1Cmd = &cobra.Command{
 		}
 		defer conn.Close()
 
+		defer func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			_, err = client.Teardown(ctx, &daemonapi.TeardownRequest{})
+			if err != nil {
+				log.WithError(err).Error("cannot trigger teardown")
+			}
+		}()
+
 		mapping := []*daemonapi.WriteIDMappingRequest_Mapping{
 			{ContainerId: 0, HostId: 33333, Size: 1},
 			{ContainerId: 1, HostId: 100000, Size: 65534},
@@ -344,6 +347,7 @@ var ring1Cmd = &cobra.Command{
 			log.WithError(err).Error("cannot mount proc")
 			return
 		}
+
 		_, err = client.MountProc(ctx, &daemonapi.MountProcRequest{
 			Target: procLoc,
 			Pid:    int64(cmd.Process.Pid),
@@ -584,18 +588,21 @@ var ring2Cmd = &cobra.Command{
 	Run: func(_cmd *cobra.Command, args []string) {
 		log.Init(ServiceName, Version, true, true)
 		log := log.WithField("ring", 2)
-		defer log.Info("done")
 
 		exitCode := 1
 		defer handleExit(&exitCode)
 
+		defer log.Info("done")
+
 		// we talk to ring1 using a Unix socket, so that we can send the seccomp fd across.
 		rconn, err := net.Dial("unix", args[0])
 		if err != nil {
 			log.WithError(err).Error("cannot connect to parent")
 			return
 		}
 		conn := rconn.(*net.UnixConn)
+		defer conn.Close()
+
 		log.Info("connected to parent socket")
 
 		// Before we do anything, we wait for the parent to make /proc available to us.
@@ -632,7 +639,6 @@ var ring2Cmd = &cobra.Command{
 
 		sktfd := int(connf.Fd())
 		err = unix.Sendmsg(sktfd, nil, unix.UnixRights(int(scmpFd)), nil, 0)
-		connf.Close()
 		if err != nil {
 			log.WithError(err).Error("cannot send seccomp fd")
 			return
@@ -777,6 +783,7 @@ func connectToInWorkspaceDaemonService(ctx context.Context) (daemonapi.InWorkspa
 	if err != nil {
 		return nil, nil, err
 	}
+
 	return daemonapi.NewInWorkspaceServiceClient(conn), conn, nil
 }
 

components/ws-daemon/pkg/content/service.go

@@ -12,14 +12,17 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 
 	"github.com/opencontainers/go-digest"
 	ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/procfs"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 	"golang.org/x/xerrors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -358,9 +361,11 @@ func (s *WorkspaceService) DisposeWorkspace(ctx context.Context, req *api.Dispos
 		return nil, status.Error(codes.Internal, "cannot delete workspace from store")
 	}
 
+	if err := unmountMark(sess.ServiceLocDaemon); err != nil {
+		log.WithError(err).WithField("workspaceId", req.Id).Error("cannot unmount mark mount")
+	}
 	// remove workspace daemon directory in the node
-	err = os.RemoveAll(sess.ServiceLocDaemon)
-	if err != nil {
+	if err := os.RemoveAll(sess.ServiceLocDaemon); err != nil {
 		log.WithError(err).WithField("workspaceId", req.Id).Error("cannot delete workspace daemon directory")
 	}
 
@@ -763,6 +768,29 @@ func workspaceLifecycleHooks(cfg Config, kubernetesNamespace string, workspaceEx
 	return map[session.WorkspaceState][]session.WorkspaceLivecycleHook{
 		session.WorkspaceInitializing: {setupWorkspace, startIWS},
 		session.WorkspaceReady:        {setupWorkspace, startIWS},
-		session.WorkspaceDisposing:    {iws.StopServingWorkspace},
+		session.WorkspaceDisposed:     {iws.StopServingWorkspace},
+	}
+}
+
+// if the mark mount still exists in /proc/mounts it means we failed to unmount it and
+// we cannot remove the content. As a side effect the pod will stay in Terminating state
+func unmountMark(path string) error {
+	mounts, err := procfs.GetMounts()
+	if err != nil {
+		return xerrors.Errorf("unexpected error reading /proc/mounts: %w", err)
 	}
+
+	for _, mount := range mounts {
+		if !strings.Contains(mount.MountPoint, path) {
+			continue
+		}
+
+		log.WithField("path", path).Debug("Unmounting pending mark")
+		err = unix.Unmount(mount.MountPoint, 0)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	return nil
 }

components/ws-manager/pkg/manager/monitor.go

@@ -355,12 +355,13 @@ func (m *Monitor) actOnPodEvent(ctx context.Context, status *api.WorkspaceStatus
 				break
 			}
 		}
-		if _, gone := wso.Pod.Annotations[wsk8s.ContainerIsGoneAnnotation]; !terminated && gone {
+		if _, gone := wso.Pod.Annotations[wsk8s.ContainerIsGoneAnnotation]; gone {
 			// workaround for https://github.com/containerd/containerd/pull/4214 which can prevent pod status
 			// propagation. ws-daemon observes the pods and propagates this state out-of-band via the annotation.
-			terminated = true
-		}
-		if terminated {
+			if !terminated {
+				go m.finalizeWorkspaceContent(ctx, wso)
+			}
+		} else {
 			go m.finalizeWorkspaceContent(ctx, wso)
 		}
 	}