feat: keep track of volumes created together with a cluster and clean…

… up on delete

pkg/client/cluster.go

@@ -309,9 +309,11 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
 	if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
 		return fmt.Errorf("failed to create image volume '%s' for cluster '%s': %w", imageVolumeName, cluster.Name, err)
 	}
+	l.Log().Infof("Created image volume %s", imageVolumeName)
 
 	clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
 	cluster.ImageVolume = imageVolumeName
+	cluster.Volumes = append(cluster.Volumes, imageVolumeName)
 
 	// attach volume to nodes
 	for _, node := range cluster.Nodes {
@@ -640,11 +642,12 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 		}
 	}
 
-	// delete image volume
-	if cluster.ImageVolume != "" {
-		l.Log().Infof("Deleting image volume '%s'", cluster.ImageVolume)
-		if err := runtime.DeleteVolume(ctx, cluster.ImageVolume); err != nil {
-			l.Log().Warningf("Failed to delete image volume '%s' of cluster '%s': Try to delete it manually", cluster.ImageVolume, cluster.Name)
+	// delete managed volumes attached to this cluster
+	l.Log().Infof("Deleting %d attached volumes...", len(cluster.Volumes))
+	for _, vol := range cluster.Volumes {
+		l.Log().Debugf("Deleting volume %s...", vol)
+		if err := runtime.DeleteVolume(ctx, vol); err != nil {
+			l.Log().Warningf("Failed to delete volume '%s' of cluster '%s': %v -> Try to delete it manually", cluster.ImageVolume, err, cluster.Name)
 		}
 	}
 
@@ -806,6 +809,12 @@ func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster
 		}
 	}
 
+	vols, err := runtime.GetVolumesByLabel(ctx, map[string]string{types.LabelClusterName: cluster.Name})
+	if err != nil {
+		return nil, err
+	}
+	cluster.Volumes = append(cluster.Volumes, vols...)
+
 	if err := populateClusterFieldsFromLabels(cluster); err != nil {
 		l.Log().Warnf("Failed to populate cluster fields from node labels: %v", err)
 	}

pkg/config/validate.go

@@ -82,7 +82,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
 		// volumes have to be either an existing path on the host or a named runtime volume
 		for _, volume := range node.Volumes {
 
-			if err := runtimeutil.ValidateVolumeMount(runtime, volume); err != nil {
+			if err := runtimeutil.ValidateVolumeMount(ctx, runtime, volume, &config.Cluster); err != nil {
 				return fmt.Errorf("failed to validate volume mount '%s': %w", volume, err)
 			}
 		}

pkg/runtimes/docker/volume.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
-	l "github.com/rancher/k3d/v5/pkg/logger"
+	runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
@@ -55,11 +55,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string
 		volumeCreateOptions.Labels[k] = v
 	}
 
-	vol, err := docker.VolumeCreate(ctx, volumeCreateOptions)
+	_, err = docker.VolumeCreate(ctx, volumeCreateOptions)
 	if err != nil {
 		return fmt.Errorf("failed to create volume '%s': %w", name, err)
 	}
-	l.Log().Infof("Created volume '%s'", vol.Name)
 	return nil
 }
 
@@ -110,9 +109,40 @@ func (d Docker) GetVolume(name string) (string, error) {
 		return "", fmt.Errorf("docker failed to list volumes: %w", err)
 	}
 	if len(volumeList.Volumes) < 1 {
-		return "", fmt.Errorf("failed to find named volume '%s'", name)
+		return "", fmt.Errorf("failed to find named volume '%s': %w", name, runtimeErrors.ErrRuntimeVolumeNotExists)
 	}
 
 	return volumeList.Volumes[0].Name, nil
 
 }
+
+func (d Docker) GetVolumesByLabel(ctx context.Context, labels map[string]string) ([]string, error) {
+	var volumes []string
+	// (0) create new docker client
+	docker, err := GetDockerClient()
+	if err != nil {
+		return volumes, fmt.Errorf("failed to get docker client: %w", err)
+	}
+	defer docker.Close()
+
+	// (1) list containers which have the default k3d labels attached
+	filters := filters.NewArgs()
+	for k, v := range k3d.DefaultRuntimeLabels {
+		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
+	}
+	for k, v := range labels {
+		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
+	}
+
+	volumeList, err := docker.VolumeList(ctx, filters)
+	if err != nil {
+		return volumes, fmt.Errorf("docker failed to list volumes: %w", err)
+	}
+
+	for _, v := range volumeList.Volumes {
+		volumes = append(volumes, v.Name)
+	}
+
+	return volumes, nil
+
+}

pkg/runtimes/errors/errors.go

@@ -37,3 +37,8 @@ var (
 
 // Container Filesystem Errors
 var ErrRuntimeFileNotFound = errors.New("file not found")
+
+// Runtime Volume Errors
+var (
+	ErrRuntimeVolumeNotExists = errors.New("volume does not exist")
+)

pkg/runtimes/runtime.go

@@ -65,6 +65,7 @@ type Runtime interface {
 	CreateVolume(context.Context, string, map[string]string) error
 	DeleteVolume(context.Context, string) error
 	GetVolume(string) (string, error)
+	GetVolumesByLabel(context.Context, map[string]string) ([]string, error) // @param context, labels - @return volumes, error
 	GetImageStream(context.Context, []string) (io.ReadCloser, error)
 	GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup
 	ExecInNode(context.Context, *k3d.Node, []string) error

pkg/runtimes/util/volumes.go

@@ -22,35 +22,53 @@ THE SOFTWARE.
 package util
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
 	rt "runtime"
 	"strings"
 
 	"github.com/rancher/k3d/v5/pkg/runtimes"
+	runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 
 	l "github.com/rancher/k3d/v5/pkg/logger"
 )
 
 // ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path
 // - SRC: source directory/file -> tests: must exist
 // - DEST: source directory/file -> tests: must be absolute path
-func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error {
+func ValidateVolumeMount(ctx context.Context, runtime runtimes.Runtime, volumeMount string, cluster *k3d.Cluster) error {
 	src, dest, err := ReadVolumeMount(volumeMount)
 	if err != nil {
 		return err
 	}
 
 	// verify that the source exists
 	if src != "" {
-		// a) named volume
-		isNamedVolume := true
-		if err := verifyNamedVolume(runtime, src); err != nil {
-			isNamedVolume = false
-		}
-		if !isNamedVolume {
+		// directory/file: path containing / or \ (not allowed in named volumes)
+		if strings.ContainsAny(src, "/\\") {
 			if _, err := os.Stat(src); err != nil {
-				l.Log().Warnf("failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount)
+				l.Log().Warnf("failed to stat file/directory '%s' volume mount '%s': please make sure it exists", src, volumeMount)
+			}
+		} else {
+			err := verifyNamedVolume(runtime, src)
+			if err != nil {
+				l.Log().Traceln(err)
+				if errors.Is(err, runtimeErrors.ErrRuntimeVolumeNotExists) {
+					if strings.HasPrefix(src, "k3d-") {
+						if err := runtime.CreateVolume(ctx, src, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
+							return fmt.Errorf("failed to create named volume '%s': %v", src, err)
+						}
+						cluster.Volumes = append(cluster.Volumes, src)
+						l.Log().Infof("Created named volume '%s'", src)
+					} else {
+						l.Log().Infof("No named volume '%s' found. The runtime will create it automatically.", src)
+					}
+				} else {
+					l.Log().Warnf("failed to get named volume: %v", err)
+				}
 			}
 		}
 	}
@@ -101,12 +119,9 @@ func ReadVolumeMount(volumeMount string) (string, string, error) {
 
 // verifyNamedVolume checks whether a named volume exists in the runtime
 func verifyNamedVolume(runtime runtimes.Runtime, volumeName string) error {
-	foundVolName, err := runtime.GetVolume(volumeName)
+	_, err := runtime.GetVolume(volumeName)
 	if err != nil {
 		return fmt.Errorf("runtime failed to get volume '%s': %w", volumeName, err)
 	}
-	if foundVolName == "" {
-		return fmt.Errorf("failed to find named volume '%s'", volumeName)
-	}
 	return nil
 }

pkg/types/types.go

@@ -234,6 +234,7 @@ type Cluster struct {
 	KubeAPI            *ExposureOpts      `yaml:"kubeAPI" json:"kubeAPI,omitempty"`
 	ServerLoadBalancer *Loadbalancer      `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"`
 	ImageVolume        string             `yaml:"imageVolume" json:"imageVolume,omitempty"`
+	Volumes            []string           `yaml:"volumes,omitempty" json:"volumes,omitempty"` // k3d-managed volumes attached to this cluster
 }
 
 // ServerCountRunning returns the number of server nodes running in the cluster and the total number