diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b87949d7..e9ff1b6dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## v5.3.0 + +**Note:** Now trying to follow a standard scheme defined by + +### Added + +- Volumemount-Shortcuts (#916) + - Use some destination shortcuts with the `--volume/-v` flag that k3d automatically expands + - `k3s-storage` -> `/var/lib/rancher/k3s/storage` + - `k3s-manifests` -> `/var/lib/rancher/k3s/server/manifests` + - `k3s-manifests-custom` -> `/var/lib/rancher/k3s/server/manifests/custom` (not K3s default: this is just some sub-directory inside the auto-deploy manifests directory which will also be parsed) + - `k3s-containerd` -> `/var/lib/rancher/k3s/agent/etc/containerd/config.toml` (use with caution, K3s generates this file!) + - `k3s-containerd-tmpl` -> `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl` (used by K3s to generate the real config above) + - `k3s-registry-config` -> `/etc/rancher/k3s/registries.yaml` (or just use `--registry-config`) +- k3d-managed volumes (#916) + - non-existing named volumes starting with a `k3d-` prefix will now be created and managed by `k3d` + +### Removed + +- unused volume validation functionality in `cmd/util`, does not affect the CLI (#916) + ## v5.2.2 ### Fixes diff --git a/cmd/util/volumes.go b/cmd/util/volumes.go deleted file mode 100644 index 7238aa2f0..000000000 --- a/cmd/util/volumes.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright © 2020-2021 The k3d Author(s) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -package util - -import ( - "fmt" - "os" - rt "runtime" - "strings" - - "github.com/rancher/k3d/v5/pkg/runtimes" - - l "github.com/rancher/k3d/v5/pkg/logger" -) - -// ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path -// - SRC: source directory/file -> tests: must exist -// - DEST: source directory/file -> tests: must be absolute path -// ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path -// - SRC: source directory/file -> tests: must exist -// - DEST: source directory/file -> tests: must be absolute path -func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) (string, error) { - src := "" - dest := "" - - // validate 'SRC[:DEST]' substring - split := strings.Split(volumeMount, ":") - // a volume mapping can have 3 parts seperated by a ':' followed by a node filter - // [SOURCE:]DEST[:OPT[,OPT]][@NODEFILTER[;NODEFILTER...]] - // On Windows the source path needs to be an absolute path which means the path starts with - // a drive designator and will also have a ':' in it. So for Windows the maxParts is increased by one. - maxParts := 3 - if rt.GOOS == "windows" { - maxParts++ - } - if len(split) < 1 { - return "", fmt.Errorf("No volume/path specified") - } - if len(split) > maxParts { - return "", fmt.Errorf("Invalid volume mount '%s': maximal %d ':' allowed", volumeMount, maxParts-1) - } - - // we only have SRC specified -> DEST = SRC - // On windows the first part of the SRC is the drive letter, so we need to concat the first and second parts to get the path. - if len(split) == 1 { - src = split[0] - dest = src - } else if rt.GOOS == "windows" { - src = split[0] + ":" + split[1] - dest = split[2] - } else { - src = split[0] - dest = split[1] - } - - // verify that the source exists - if src != "" { - // a) named volume - isNamedVolume := true - if err := verifyNamedVolume(runtime, src); err != nil { - isNamedVolume = false - } - if !isNamedVolume { - if _, err := os.Stat(src); err != nil { - l.Log().Warnf("Failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount) - } - } - } - - // verify that the destination is an absolute path - if !strings.HasPrefix(dest, "/") { - return "", fmt.Errorf("Volume mount destination doesn't appear to be an absolute path: '%s' in '%s'", dest, volumeMount) - } - - return volumeMount, nil -} - -// verifyNamedVolume checks whether a named volume exists in the runtime -func verifyNamedVolume(runtime runtimes.Runtime, volumeName string) error { - volumeName, err := runtime.GetVolume(volumeName) - if err != nil { - return fmt.Errorf("Failed to verify named volume: %w", err) - } - if volumeName == "" { - return fmt.Errorf("Failed to find named volume '%s'", volumeName) - } - return nil -} diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 160a197ca..ebfee099a 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -309,9 +309,11 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil { return fmt.Errorf("failed to create image volume '%s' for cluster '%s': %w", imageVolumeName, cluster.Name, err) } + l.Log().Infof("Created image volume %s", imageVolumeName) clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName cluster.ImageVolume = imageVolumeName + cluster.Volumes = append(cluster.Volumes, imageVolumeName) // attach volume to nodes for _, node := range cluster.Nodes { @@ -640,11 +642,12 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus } } - // delete image volume - if cluster.ImageVolume != "" { - l.Log().Infof("Deleting image volume '%s'", cluster.ImageVolume) - if err := runtime.DeleteVolume(ctx, cluster.ImageVolume); err != nil { - l.Log().Warningf("Failed to delete image volume '%s' of cluster '%s': Try to delete it manually", cluster.ImageVolume, cluster.Name) + // delete managed volumes attached to this cluster + l.Log().Infof("Deleting %d attached volumes...", len(cluster.Volumes)) + for _, vol := range cluster.Volumes { + l.Log().Debugf("Deleting volume %s...", vol) + if err := runtime.DeleteVolume(ctx, vol); err != nil { + l.Log().Warningf("Failed to delete volume '%s' of cluster '%s': %v -> Try to delete it manually", cluster.ImageVolume, err, cluster.Name) } } @@ -806,6 +809,12 @@ func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster } } + vols, err := runtime.GetVolumesByLabel(ctx, map[string]string{types.LabelClusterName: cluster.Name}) + if err != nil { + return nil, err + } + cluster.Volumes = append(cluster.Volumes, vols...) + if err := populateClusterFieldsFromLabels(cluster); err != nil { l.Log().Warnf("Failed to populate cluster fields from node labels: %v", err) } diff --git a/pkg/config/process.go b/pkg/config/process.go index 5b9135c03..ea777af44 100644 --- a/pkg/config/process.go +++ b/pkg/config/process.go @@ -23,9 +23,13 @@ THE SOFTWARE. package config import ( + "strings" + conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3" l "github.com/rancher/k3d/v5/pkg/logger" + runtimeutil "github.com/rancher/k3d/v5/pkg/runtimes/util" k3d "github.com/rancher/k3d/v5/pkg/types" + "github.com/rancher/k3d/v5/pkg/types/k3s" ) // ProcessSimpleConfig applies processing to the simple config, sanitizing it and doing some modifications @@ -56,5 +60,18 @@ func ProcessClusterConfig(clusterConfig conf.ClusterConfig) (*conf.ClusterConfig clusterConfig.ClusterCreateOpts.DisableLoadBalancer = true } + for _, node := range clusterConfig.Cluster.Nodes { + for vIndex, volume := range node.Volumes { + _, dest, err := runtimeutil.ReadVolumeMount(volume) + if err != nil { + return nil, err + } + if path, ok := k3s.K3sPathShortcuts[dest]; ok { + l.Log().Tracef("[node: %s] expanding volume shortcut %s to %s", node.Name, dest, path) + node.Volumes[vIndex] = strings.Replace(volume, dest, path, 1) + } + } + } + return &clusterConfig, nil } diff --git a/pkg/config/process_test.go b/pkg/config/process_test.go index 47b1fc772..1c9399c60 100644 --- a/pkg/config/process_test.go +++ b/pkg/config/process_test.go @@ -24,10 +24,12 @@ package config import ( "context" + "strings" "testing" conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3" "github.com/rancher/k3d/v5/pkg/runtimes" + "github.com/rancher/k3d/v5/pkg/types/k3s" "github.com/spf13/viper" "gotest.tools/assert" ) @@ -51,11 +53,20 @@ func TestProcessClusterConfig(t *testing.T) { t.Error(err) } + // append some volume to test K3s volume shortcut expansion + clusterCfg.Cluster.Nodes[0].Volumes = append(clusterCfg.Cluster.Nodes[0].Volumes, "/tmp/testexpansion:k3s-storage:rw") + t.Logf("\n========== Process Cluster Config (non-host network) ==========\n%+v\n=================================\n", cfg) clusterCfg, err = ProcessClusterConfig(*clusterCfg) assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == false, "The load balancer should be enabled") + for _, v := range clusterCfg.Cluster.Nodes[0].Volumes { + if strings.HasPrefix(v, "/tmp/testexpansion") { + assert.Assert(t, strings.Contains(v, k3s.K3sPathStorage), "volume path shortcut expansion of k3s-storage didn't work") + } + } + t.Logf("\n===== Resulting Cluster Config (non-host network) =====\n%+v\n===============\n", clusterCfg) t.Logf("\n========== Process Cluster Config (host network) ==========\n%+v\n=================================\n", cfg) @@ -65,5 +76,6 @@ func TestProcessClusterConfig(t *testing.T) { assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == true, "The load balancer should be disabled") t.Logf("\n===== Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg) + t.Logf("\n===== First Node in Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg.Cluster.Nodes[0]) } diff --git a/pkg/config/validate.go b/pkg/config/validate.go index ec6c49c70..9d23fdd65 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -82,7 +82,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config // volumes have to be either an existing path on the host or a named runtime volume for _, volume := range node.Volumes { - if err := runtimeutil.ValidateVolumeMount(runtime, volume); err != nil { + if err := runtimeutil.ValidateVolumeMount(ctx, runtime, volume, &config.Cluster); err != nil { return fmt.Errorf("failed to validate volume mount '%s': %w", volume, err) } } diff --git a/pkg/runtimes/docker/volume.go b/pkg/runtimes/docker/volume.go index 2026dd085..d0eb4dfb3 100644 --- a/pkg/runtimes/docker/volume.go +++ b/pkg/runtimes/docker/volume.go @@ -27,7 +27,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" - l "github.com/rancher/k3d/v5/pkg/logger" + runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors" k3d "github.com/rancher/k3d/v5/pkg/types" ) @@ -55,11 +55,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string volumeCreateOptions.Labels[k] = v } - vol, err := docker.VolumeCreate(ctx, volumeCreateOptions) + _, err = docker.VolumeCreate(ctx, volumeCreateOptions) if err != nil { return fmt.Errorf("failed to create volume '%s': %w", name, err) } - l.Log().Infof("Created volume '%s'", vol.Name) return nil } @@ -110,9 +109,40 @@ func (d Docker) GetVolume(name string) (string, error) { return "", fmt.Errorf("docker failed to list volumes: %w", err) } if len(volumeList.Volumes) < 1 { - return "", fmt.Errorf("failed to find named volume '%s'", name) + return "", fmt.Errorf("failed to find named volume '%s': %w", name, runtimeErrors.ErrRuntimeVolumeNotExists) } return volumeList.Volumes[0].Name, nil } + +func (d Docker) GetVolumesByLabel(ctx context.Context, labels map[string]string) ([]string, error) { + var volumes []string + // (0) create new docker client + docker, err := GetDockerClient() + if err != nil { + return volumes, fmt.Errorf("failed to get docker client: %w", err) + } + defer docker.Close() + + // (1) list containers which have the default k3d labels attached + filters := filters.NewArgs() + for k, v := range k3d.DefaultRuntimeLabels { + filters.Add("label", fmt.Sprintf("%s=%s", k, v)) + } + for k, v := range labels { + filters.Add("label", fmt.Sprintf("%s=%s", k, v)) + } + + volumeList, err := docker.VolumeList(ctx, filters) + if err != nil { + return volumes, fmt.Errorf("docker failed to list volumes: %w", err) + } + + for _, v := range volumeList.Volumes { + volumes = append(volumes, v.Name) + } + + return volumes, nil + +} diff --git a/pkg/runtimes/errors/errors.go b/pkg/runtimes/errors/errors.go index 4997d3b54..22723f9b9 100644 --- a/pkg/runtimes/errors/errors.go +++ b/pkg/runtimes/errors/errors.go @@ -37,3 +37,8 @@ var ( // Container Filesystem Errors var ErrRuntimeFileNotFound = errors.New("file not found") + +// Runtime Volume Errors +var ( + ErrRuntimeVolumeNotExists = errors.New("volume does not exist") +) diff --git a/pkg/runtimes/runtime.go b/pkg/runtimes/runtime.go index 0234be4ce..3740f8313 100644 --- a/pkg/runtimes/runtime.go +++ b/pkg/runtimes/runtime.go @@ -65,6 +65,7 @@ type Runtime interface { CreateVolume(context.Context, string, map[string]string) error DeleteVolume(context.Context, string) error GetVolume(string) (string, error) + GetVolumesByLabel(context.Context, map[string]string) ([]string, error) // @param context, labels - @return volumes, error GetImageStream(context.Context, []string) (io.ReadCloser, error) GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup ExecInNode(context.Context, *k3d.Node, []string) error diff --git a/pkg/runtimes/util/volumes.go b/pkg/runtimes/util/volumes.go index 0262501a9..2dbac3fa6 100644 --- a/pkg/runtimes/util/volumes.go +++ b/pkg/runtimes/util/volumes.go @@ -22,12 +22,16 @@ THE SOFTWARE. package util import ( + "context" + "errors" "fmt" "os" rt "runtime" "strings" "github.com/rancher/k3d/v5/pkg/runtimes" + runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors" + k3d "github.com/rancher/k3d/v5/pkg/types" l "github.com/rancher/k3d/v5/pkg/logger" ) @@ -35,7 +39,49 @@ import ( // ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path // - SRC: source directory/file -> tests: must exist // - DEST: source directory/file -> tests: must be absolute path -func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error { +func ValidateVolumeMount(ctx context.Context, runtime runtimes.Runtime, volumeMount string, cluster *k3d.Cluster) error { + src, dest, err := ReadVolumeMount(volumeMount) + if err != nil { + return err + } + + // verify that the source exists + if src != "" { + // directory/file: path containing / or \ (not allowed in named volumes) + if strings.ContainsAny(src, "/\\") { + if _, err := os.Stat(src); err != nil { + l.Log().Warnf("failed to stat file/directory '%s' volume mount '%s': please make sure it exists", src, volumeMount) + } + } else { + err := verifyNamedVolume(runtime, src) + if err != nil { + l.Log().Traceln(err) + if errors.Is(err, runtimeErrors.ErrRuntimeVolumeNotExists) { + if strings.HasPrefix(src, "k3d-") { + if err := runtime.CreateVolume(ctx, src, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil { + return fmt.Errorf("failed to create named volume '%s': %v", src, err) + } + cluster.Volumes = append(cluster.Volumes, src) + l.Log().Infof("Created named volume '%s'", src) + } else { + l.Log().Infof("No named volume '%s' found. The runtime will create it automatically.", src) + } + } else { + l.Log().Warnf("failed to get named volume: %v", err) + } + } + } + } + + // verify that the destination is an absolute path + if !strings.HasPrefix(dest, "/") { + return fmt.Errorf("volume mount destination doesn't appear to be an absolute path: '%s' in '%s'", dest, volumeMount) + } + + return nil +} + +func ReadVolumeMount(volumeMount string) (string, string, error) { src := "" dest := "" @@ -50,10 +96,10 @@ func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error { maxParts++ } if len(split) < 1 { - return fmt.Errorf("No volume/path specified") + return src, dest, fmt.Errorf("no volume/path specified") } if len(split) > maxParts { - return fmt.Errorf("Invalid volume mount '%s': maximal %d ':' allowed", volumeMount, maxParts-1) + return src, dest, fmt.Errorf("invalid volume mount '%s': maximal %d ':' allowed", volumeMount, maxParts-1) } // we only have SRC specified -> DEST = SRC @@ -68,37 +114,14 @@ func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error { src = split[0] dest = split[1] } - - // verify that the source exists - if src != "" { - // a) named volume - isNamedVolume := true - if err := verifyNamedVolume(runtime, src); err != nil { - isNamedVolume = false - } - if !isNamedVolume { - if _, err := os.Stat(src); err != nil { - l.Log().Warnf("Failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount) - } - } - } - - // verify that the destination is an absolute path - if !strings.HasPrefix(dest, "/") { - return fmt.Errorf("Volume mount destination doesn't appear to be an absolute path: '%s' in '%s'", dest, volumeMount) - } - - return nil + return src, dest, nil } // verifyNamedVolume checks whether a named volume exists in the runtime func verifyNamedVolume(runtime runtimes.Runtime, volumeName string) error { - foundVolName, err := runtime.GetVolume(volumeName) + _, err := runtime.GetVolume(volumeName) if err != nil { return fmt.Errorf("runtime failed to get volume '%s': %w", volumeName, err) } - if foundVolName == "" { - return fmt.Errorf("failed to find named volume '%s'", volumeName) - } return nil } diff --git a/pkg/types/k3s/paths.go b/pkg/types/k3s/paths.go new file mode 100644 index 000000000..6e7f6de8f --- /dev/null +++ b/pkg/types/k3s/paths.go @@ -0,0 +1,40 @@ +/* +Copyright © 2020-2021 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package k3s + +const ( + K3sPathStorage = "/var/lib/rancher/k3s/storage" + K3sPathManifests = "/var/lib/rancher/k3s/server/manifests" + K3sPathManifestsCustom = "/var/lib/rancher/k3s/server/manifests/custom" // custom subfolder + K3sPathContainerdConfig = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml" + K3sPathContainerdConfigTmpl = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" + K3sPathRegistryConfig = "/etc/rancher/k3s/registries.yaml" +) + +var K3sPathShortcuts = map[string]string{ + "k3s-storage": K3sPathStorage, + "k3s-manifests": K3sPathManifests, + "k3s-manifests-custom": K3sPathManifestsCustom, + "k3s-containerd": K3sPathContainerdConfig, + "k3s-containerd-tmpl": K3sPathContainerdConfigTmpl, + "k3s-registry-config": K3sPathRegistryConfig, +} diff --git a/pkg/types/types.go b/pkg/types/types.go index bb0f5d903..191bf219c 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -234,6 +234,7 @@ type Cluster struct { KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI,omitempty"` ServerLoadBalancer *Loadbalancer `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"` ImageVolume string `yaml:"imageVolume" json:"imageVolume,omitempty"` + Volumes []string `yaml:"volumes,omitempty" json:"volumes,omitempty"` // k3d-managed volumes attached to this cluster } // ServerCountRunning returns the number of server nodes running in the cluster and the total number