Device: Don't remove an existing target directory when unmounting a disk device if the original dir hasn't been created by LXD

doc/api-extensions.md

@@ -2520,3 +2520,7 @@ contains the total available logical CPUs available when LXD started.
 ## `vm_limits_cpu_pin_strategy`
 
 Adds a new {config:option}`instance-resource-limits:limits.cpu.pin_strategy` configuration option for virtual machines. This option controls the CPU pinning strategy. When set to `none`, CPU auto pinning is disabled. When set to `auto`, CPU auto pinning is enabled.
+
+## `disk_state_created`
+
+This API extension provides the ability to check if a target directory was created within the instance file system at mount time. If a host directory is mounted to an existing container directory (e.g., `/opt`), the target directory won't be removed upon unmounting. However, if LXD creates a target directory during the mount, like `/new_dir`, it will be deleted when the device is unmounted.

doc/metadata.txt

@@ -2427,6 +2427,14 @@ Specify either a cron expression (`<minute> <hour> <dom> <month> <dow>`), a comm
 
 <!-- config group instance-snapshots end -->
 <!-- config group instance-volatile start -->
+```{config:option} volatile.<disk_dev_name>.last_state.created instance-volatile
+:shortdesc: "Path of the directory created from mounting a `disk` device"
+:type: "string"
+If mounting a host directory through a `disk` device creates the target directory, this option contains the path of this newly created target directory.
+For example, if the target directory is `/opt` and `/opt` already exists in the instance, this key is not set.
+However, if the target directory is `/opt/foo` and `/opt/foo` doesn't exist in the instance, this key is set to `/opt/foo`.
+```
+
 ```{config:option} volatile.<name>.apply_quota instance-volatile
 :shortdesc: "Disk quota"
 :type: "string"

doc/rest-api.yaml

@@ -1,4 +1,29 @@
 definitions:
+    AgentDeviceRemove:
+        properties:
+            config:
+                additionalProperties:
+                    type: string
+                description: Device configuration map
+                type: object
+                x-go-name: Config
+            name:
+                description: Device name
+                type: string
+                x-go-name: Name
+            type:
+                description: Type of device ('disk', 'nic', etc.)
+                type: string
+                x-go-name: Type
+            volatile:
+                additionalProperties:
+                    type: string
+                description: Optional device volatile configuration keys
+                type: object
+                x-go-name: Volatile
+        title: AgentDeviceRemove represents the fields of an device removal request that needs to occur inside the VM agent.
+        type: object
+        x-go-package: github.com/canonical/lxd/shared/api
     AuthGroup:
         properties:
             description:

lxd-agent/api_1.0.go

@@ -29,6 +29,7 @@ var api10 = []APIEndpoint{
 	api10Cmd,
 	execCmd,
 	eventsCmd,
+	devicesCmd,
 	metricsCmd,
 	operationsCmd,
 	operationCmd,

lxd-agent/devices.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/canonical/lxd/lxd/response"
+	"github.com/canonical/lxd/shared/api"
+)
+
+var devicesCmd = APIEndpoint{
+	Path: "devices",
+
+	Delete: APIEndpointAction{Handler: deviceDelete},
+}
+
+// deviceDelete handles the removal of a device from the VM agent.
+// e.g, if the device is a disk mount, this will cleanly unmount it and remove it if necessary.
+func deviceDelete(d *Daemon, r *http.Request) response.Response {
+	var device api.AgentDeviceRemove
+
+	err := json.NewDecoder(r.Body).Decode(&device)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	// We only support disk devices for now
+	if device.Type != "disk" {
+		return response.BadRequest(fmt.Errorf("Device type %q not supported for removal within VM agent", device.Type))
+	}
+
+	targetPath := device.Config["path"]
+
+	if !filepath.IsAbs(targetPath) {
+		return response.SmartError(fmt.Errorf("The device path must be absolute: %q", device.Config["path"]))
+	}
+
+	file, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return response.SmartError(fmt.Errorf("Error opening /proc/self/mountinfo: %v", err))
+	}
+
+	defer file.Close()
+
+	var mountPoints []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) >= 10 && strings.HasPrefix(fields[4], strings.TrimSuffix(targetPath, "/")) {
+			mountPoints = append(mountPoints, fields[4])
+		}
+	}
+
+	err = scanner.Err()
+	if err != nil {
+		return response.SmartError(fmt.Errorf("Error reading /proc/self/mountinfo: %v", err))
+	}
+
+	if len(mountPoints) == 0 {
+		return response.SmartError(fmt.Errorf("No mount points found for %s", targetPath))
+	}
+
+	// Reverse the slice to unmount in reverse order.
+	// This is needed to unmount potential over-mounts first.
+	for i, j := 0, len(mountPoints)-1; i < j; i, j = i+1, j-1 {
+		mountPoints[i], mountPoints[j] = mountPoints[j], mountPoints[i]
+	}
+
+	for _, mountPoint := range mountPoints {
+		err = unix.Unmount(mountPoint, unix.MNT_DETACH)
+		if err != nil {
+			return response.SmartError(fmt.Errorf("Error unmounting %s: %v", mountPoint, err))
+		}
+	}
+
+	// Now that the unmount has occurred,
+	// check if we need to remove the target path.
+	if device.Volatile != nil {
+		path, ok := device.Volatile["last_state.created"]
+		if ok {
+			if targetPath == path {
+				err = os.Remove(filepath.Clean(targetPath))
+				if err != nil {
+					return response.SmartError(fmt.Errorf("Failed to remove last state file: %v", err))
+				}
+			}
+		}
+	}
+
+	return response.EmptySyncResponse
+}

lxd/device/disk.go

@@ -2007,6 +2007,7 @@ func (d *disk) Stop() (*deviceConfig.RunConfig, error) {
 
 	// Request an unmount of the device inside the instance.
 	runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{
+		DevName:    d.Name(),
 		TargetPath: relativeDestPath,
 	})
 

lxd/instance/drivers/driver_lxc.go

@@ -1486,7 +1486,7 @@ func (d *lxc) deviceStart(dev device.Device, instanceRunning bool) (*deviceConfi
 func (d *lxc) deviceStaticShiftMounts(mounts []deviceConfig.MountEntryItem) error {
 	idmapSet, err := d.CurrentIdmap()
 	if err != nil {
-		return fmt.Errorf("Failed to get idmap for device: %s", err)
+		return fmt.Errorf("Failed to get idmap for device: %w", err)
 	}
 
 	// If there is an idmap being applied and LXD not running in a user namespace then shift the
@@ -1693,6 +1693,15 @@ func (d *lxc) deviceDetachNIC(configCopy map[string]string, netIF []deviceConfig
 func (d *lxc) deviceHandleMounts(mounts []deviceConfig.MountEntryItem) error {
 	reverter := revert.New()
 	defer reverter.Fail()
+
+	// Connect to files API.
+	files, err := d.FileSFTP()
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = files.Close() }()
+
 	for _, mount := range mounts {
 		if mount.DevPath != "" {
 			flags := 0
@@ -1716,35 +1725,60 @@ func (d *lxc) deviceHandleMounts(mounts []deviceConfig.MountEntryItem) error {
 				}
 			}
 
-			// Mount it into the container.
-			err := d.insertMount(mount.DevPath, mount.TargetPath, mount.FSType, flags, idmapType)
+			_, err = files.Lstat(mount.TargetPath)
 			if err != nil {
-				return fmt.Errorf("Failed to add mount for device inside container: %s", err)
+				absTargetPath := mount.TargetPath
+				if !strings.HasPrefix(mount.TargetPath, "/") {
+					absTargetPath = fmt.Sprintf("/%s", mount.TargetPath)
+				}
+
+				err = d.deviceVolatileSetFunc(mount.DevName)(map[string]string{"last_state.created": absTargetPath}) // We want to store the absolute path.
+				if err != nil {
+					return fmt.Errorf("Error updating volatile for the device: %w", err)
+				}
 			}
-		} else {
-			relativeTargetPath := strings.TrimPrefix(mount.TargetPath, "/")
 
-			// Connect to files API.
-			files, err := d.FileSFTP()
+			// Mount it into the container.
+			err = d.insertMount(mount.DevPath, mount.TargetPath, mount.FSType, flags, idmapType)
 			if err != nil {
-				return err
+				return fmt.Errorf("Failed to add mount for device inside container: %w", err)
 			}
+		} else {
+			_, err = files.Lstat(mount.TargetPath)
+			if err == nil {
+				removeTargetFiles := false
+
+				// Check if the target path hasn't been created by LXD
+				mountConf := d.deviceVolatileGetFunc(mount.DevName)()
+				targetPath := mountConf["last_state.created"]
+				absMountTargetPath := mount.TargetPath
+				if !strings.HasPrefix(mount.TargetPath, "/") {
+					absMountTargetPath = fmt.Sprintf("/%s", mount.TargetPath)
+				}
 
-			reverter.Add(func() { _ = files.Close() })
+				if targetPath == absMountTargetPath {
+					removeTargetFiles = true
+				}
 
-			_, err = files.Lstat(relativeTargetPath)
-			if err == nil {
-				err := d.removeMount(mount.TargetPath)
+				err = d.removeMount(mount.TargetPath)
 				if err != nil {
-					return fmt.Errorf("Error unmounting the device path inside container: %s", err)
+					return fmt.Errorf("Error unmounting the device path inside container: %w", err)
 				}
 
-				err = files.Remove(relativeTargetPath)
-				if err != nil {
-					// Only warn here and don't fail as removing a directory
-					// mount may fail if there was already files inside
-					// directory before it was mouted over preventing delete.
-					d.logger.Warn("Could not remove the device path inside container", logger.Ctx{"err": err})
+				if removeTargetFiles {
+					err = files.Remove(mount.TargetPath)
+					if err != nil {
+						// Only warn here and don't fail as removing a directory
+						// mount may fail if there was already files inside
+						// directory before it was mouted over preventing delete.
+						d.logger.Warn("Could not remove the device path inside container", logger.Ctx{"err": err})
+					}
+				} else {
+					// Remove option from the device.
+					err = d.deviceVolatileSetFunc(mount.DevName)(map[string]string{"last_state.created": ""})
+					if err != nil {
+						return fmt.Errorf("Error updating volatile for the device: %w", err)
+					}
 				}
 			}
 
@@ -4149,7 +4183,7 @@ func (d *lxc) Update(args db.InstanceArgs, userRequested bool) error {
 	if args.Architecture != 0 {
 		_, err = osarch.ArchitectureName(args.Architecture)
 		if err != nil {
-			return fmt.Errorf("Invalid architecture id: %s", err)
+			return fmt.Errorf("Invalid architecture id: %w", err)
 		}
 	}
 
@@ -7607,12 +7641,12 @@ func (d *lxc) insertMountLXD(source, target, fstype string, flags int, mntnsPID
 	if shared.IsDir(source) {
 		tmpMount, err = os.MkdirTemp(d.ShmountsPath(), "lxdmount_")
 		if err != nil {
-			return fmt.Errorf("Failed to create shmounts path: %s", err)
+			return fmt.Errorf("Failed to create shmounts path: %w", err)
 		}
 	} else {
 		f, err := os.CreateTemp(d.ShmountsPath(), "lxdmount_")
 		if err != nil {
-			return fmt.Errorf("Failed to create shmounts path: %s", err)
+			return fmt.Errorf("Failed to create shmounts path: %w", err)
 		}
 
 		tmpMount = f.Name()
@@ -7624,7 +7658,7 @@ func (d *lxc) insertMountLXD(source, target, fstype string, flags int, mntnsPID
 	// Mount the filesystem
 	err = unix.Mount(source, tmpMount, fstype, uintptr(flags), "")
 	if err != nil {
-		return fmt.Errorf("Failed to setup temporary mount: %s", err)
+		return fmt.Errorf("Failed to setup temporary mount: %w", err)
 	}
 
 	defer func() { _ = unix.Unmount(tmpMount, unix.MNT_DETACH) }()
@@ -7866,7 +7900,7 @@ func (d *lxc) InsertSeccompUnixDevice(prefix string, m deviceConfig.Device, pid
 
 	dev, err := device.UnixDeviceCreate(d.state, idmapSet, d.DevicesPath(), prefix, m, true)
 	if err != nil {
-		return fmt.Errorf("Failed to setup device: %s", err)
+		return fmt.Errorf("Failed to setup device: %w", err)
 	}
 
 	devPath := dev.HostPath