[Feature] Volume Shortcuts and k3d-managed volumes (#916)

- volume-mount destination shortcuts, e.g. `k3s-storage`
- non-existing named volumes starting with `k3d-` created/handled by k3d
- removed unused volume validation logic in `cmd/util`

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## v5.3.0
+
+**Note:** Now trying to follow a standard scheme defined by <https://keepachangelog.com/en/1.0.0/>
+
+### Added
+
+- Volumemount-Shortcuts (#916)
+  - Use some destination shortcuts with the `--volume/-v` flag that k3d automatically expands
+    - `k3s-storage` -> `/var/lib/rancher/k3s/storage`
+    - `k3s-manifests` -> `/var/lib/rancher/k3s/server/manifests`
+    - `k3s-manifests-custom` -> `/var/lib/rancher/k3s/server/manifests/custom` (not K3s default: this is just some sub-directory inside the auto-deploy manifests directory which will also be parsed)
+    - `k3s-containerd` -> `/var/lib/rancher/k3s/agent/etc/containerd/config.toml` (use with caution, K3s generates this file!)
+    - `k3s-containerd-tmpl` -> `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl` (used by K3s to generate the real config above)
+    - `k3s-registry-config` -> `/etc/rancher/k3s/registries.yaml` (or just use `--registry-config`)
+- k3d-managed volumes (#916)
+  - non-existing named volumes starting with a `k3d-` prefix will now be created and managed by `k3d`
+
+### Removed
+
+- unused volume validation functionality in `cmd/util`, does not affect the CLI (#916)
+
 ## v5.2.2
 
 ### Fixes

pkg/client/cluster.go

@@ -309,9 +309,11 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
 	if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
 		return fmt.Errorf("failed to create image volume '%s' for cluster '%s': %w", imageVolumeName, cluster.Name, err)
 	}
+	l.Log().Infof("Created image volume %s", imageVolumeName)
 
 	clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
 	cluster.ImageVolume = imageVolumeName
+	cluster.Volumes = append(cluster.Volumes, imageVolumeName)
 
 	// attach volume to nodes
 	for _, node := range cluster.Nodes {
@@ -640,11 +642,12 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 		}
 	}
 
-	// delete image volume
-	if cluster.ImageVolume != "" {
-		l.Log().Infof("Deleting image volume '%s'", cluster.ImageVolume)
-		if err := runtime.DeleteVolume(ctx, cluster.ImageVolume); err != nil {
-			l.Log().Warningf("Failed to delete image volume '%s' of cluster '%s': Try to delete it manually", cluster.ImageVolume, cluster.Name)
+	// delete managed volumes attached to this cluster
+	l.Log().Infof("Deleting %d attached volumes...", len(cluster.Volumes))
+	for _, vol := range cluster.Volumes {
+		l.Log().Debugf("Deleting volume %s...", vol)
+		if err := runtime.DeleteVolume(ctx, vol); err != nil {
+			l.Log().Warningf("Failed to delete volume '%s' of cluster '%s': %v -> Try to delete it manually", cluster.ImageVolume, err, cluster.Name)
 		}
 	}
 
@@ -806,6 +809,12 @@ func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster
 		}
 	}
 
+	vols, err := runtime.GetVolumesByLabel(ctx, map[string]string{types.LabelClusterName: cluster.Name})
+	if err != nil {
+		return nil, err
+	}
+	cluster.Volumes = append(cluster.Volumes, vols...)
+
 	if err := populateClusterFieldsFromLabels(cluster); err != nil {
 		l.Log().Warnf("Failed to populate cluster fields from node labels: %v", err)
 	}

pkg/config/process.go

@@ -23,9 +23,13 @@ THE SOFTWARE.
 package config
 
 import (
+	"strings"
+
 	conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
 	l "github.com/rancher/k3d/v5/pkg/logger"
+	runtimeutil "github.com/rancher/k3d/v5/pkg/runtimes/util"
 	k3d "github.com/rancher/k3d/v5/pkg/types"
+	"github.com/rancher/k3d/v5/pkg/types/k3s"
 )
 
 // ProcessSimpleConfig applies processing to the simple config, sanitizing it and doing some modifications
@@ -56,5 +60,18 @@ func ProcessClusterConfig(clusterConfig conf.ClusterConfig) (*conf.ClusterConfig
 		clusterConfig.ClusterCreateOpts.DisableLoadBalancer = true
 	}
 
+	for _, node := range clusterConfig.Cluster.Nodes {
+		for vIndex, volume := range node.Volumes {
+			_, dest, err := runtimeutil.ReadVolumeMount(volume)
+			if err != nil {
+				return nil, err
+			}
+			if path, ok := k3s.K3sPathShortcuts[dest]; ok {
+				l.Log().Tracef("[node: %s] expanding volume shortcut %s to %s", node.Name, dest, path)
+				node.Volumes[vIndex] = strings.Replace(volume, dest, path, 1)
+			}
+		}
+	}
+
 	return &clusterConfig, nil
 }

pkg/config/process_test.go

@@ -24,10 +24,12 @@ package config
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
 	"github.com/rancher/k3d/v5/pkg/runtimes"
+	"github.com/rancher/k3d/v5/pkg/types/k3s"
 	"github.com/spf13/viper"
 	"gotest.tools/assert"
 )
@@ -51,11 +53,20 @@ func TestProcessClusterConfig(t *testing.T) {
 		t.Error(err)
 	}
 
+	// append some volume to test K3s volume shortcut expansion
+	clusterCfg.Cluster.Nodes[0].Volumes = append(clusterCfg.Cluster.Nodes[0].Volumes, "/tmp/testexpansion:k3s-storage:rw")
+
 	t.Logf("\n========== Process Cluster Config (non-host network) ==========\n%+v\n=================================\n", cfg)
 
 	clusterCfg, err = ProcessClusterConfig(*clusterCfg)
 	assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == false, "The load balancer should be enabled")
 
+	for _, v := range clusterCfg.Cluster.Nodes[0].Volumes {
+		if strings.HasPrefix(v, "/tmp/testexpansion") {
+			assert.Assert(t, strings.Contains(v, k3s.K3sPathStorage), "volume path shortcut expansion of k3s-storage didn't work")
+		}
+	}
+
 	t.Logf("\n===== Resulting Cluster Config (non-host network) =====\n%+v\n===============\n", clusterCfg)
 
 	t.Logf("\n========== Process Cluster Config (host network) ==========\n%+v\n=================================\n", cfg)
@@ -65,5 +76,6 @@ func TestProcessClusterConfig(t *testing.T) {
 	assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == true, "The load balancer should be disabled")
 
 	t.Logf("\n===== Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg)
+	t.Logf("\n===== First Node in Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg.Cluster.Nodes[0])
 
 }

pkg/config/validate.go

@@ -82,7 +82,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
 		// volumes have to be either an existing path on the host or a named runtime volume
 		for _, volume := range node.Volumes {
 
-			if err := runtimeutil.ValidateVolumeMount(runtime, volume); err != nil {
+			if err := runtimeutil.ValidateVolumeMount(ctx, runtime, volume, &config.Cluster); err != nil {
 				return fmt.Errorf("failed to validate volume mount '%s': %w", volume, err)
 			}
 		}

pkg/runtimes/docker/volume.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
-	l "github.com/rancher/k3d/v5/pkg/logger"
+	runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
@@ -55,11 +55,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string
 		volumeCreateOptions.Labels[k] = v
 	}
 
-	vol, err := docker.VolumeCreate(ctx, volumeCreateOptions)
+	_, err = docker.VolumeCreate(ctx, volumeCreateOptions)
 	if err != nil {
 		return fmt.Errorf("failed to create volume '%s': %w", name, err)
 	}
-	l.Log().Infof("Created volume '%s'", vol.Name)
 	return nil
 }
 
@@ -110,9 +109,40 @@ func (d Docker) GetVolume(name string) (string, error) {
 		return "", fmt.Errorf("docker failed to list volumes: %w", err)
 	}
 	if len(volumeList.Volumes) < 1 {
-		return "", fmt.Errorf("failed to find named volume '%s'", name)
+		return "", fmt.Errorf("failed to find named volume '%s': %w", name, runtimeErrors.ErrRuntimeVolumeNotExists)
 	}
 
 	return volumeList.Volumes[0].Name, nil
 
 }
+
+func (d Docker) GetVolumesByLabel(ctx context.Context, labels map[string]string) ([]string, error) {
+	var volumes []string
+	// (0) create new docker client
+	docker, err := GetDockerClient()
+	if err != nil {
+		return volumes, fmt.Errorf("failed to get docker client: %w", err)
+	}
+	defer docker.Close()
+
+	// (1) list containers which have the default k3d labels attached
+	filters := filters.NewArgs()
+	for k, v := range k3d.DefaultRuntimeLabels {
+		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
+	}
+	for k, v := range labels {
+		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
+	}
+
+	volumeList, err := docker.VolumeList(ctx, filters)
+	if err != nil {
+		return volumes, fmt.Errorf("docker failed to list volumes: %w", err)
+	}
+
+	for _, v := range volumeList.Volumes {
+		volumes = append(volumes, v.Name)
+	}
+
+	return volumes, nil
+
+}

pkg/runtimes/errors/errors.go

@@ -37,3 +37,8 @@ var (
 
 // Container Filesystem Errors
 var ErrRuntimeFileNotFound = errors.New("file not found")
+
+// Runtime Volume Errors
+var (
+	ErrRuntimeVolumeNotExists = errors.New("volume does not exist")
+)

pkg/runtimes/runtime.go

@@ -65,6 +65,7 @@ type Runtime interface {
 	CreateVolume(context.Context, string, map[string]string) error
 	DeleteVolume(context.Context, string) error
 	GetVolume(string) (string, error)
+	GetVolumesByLabel(context.Context, map[string]string) ([]string, error) // @param context, labels - @return volumes, error
 	GetImageStream(context.Context, []string) (io.ReadCloser, error)
 	GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup
 	ExecInNode(context.Context, *k3d.Node, []string) error