use container volume to mount the embedded manifests

a cleaner implementation which doesn't leave tmp files behind

Signed-off-by: Ahmed AbouZaid <[email protected]>

cmd/cluster/clusterCreate.go

@@ -123,7 +123,7 @@ func NewCmdClusterCreate() *cobra.Command {
 				l.Log().Fatalf("error processing/sanitizing simple config: %v", err)
 			}
 
-			clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, simpleCfg, configFile)
+			clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, simpleCfg)
 			if err != nil {
 				l.Log().Fatalln(err)
 			}

pkg/client/cluster.go

@@ -76,10 +76,9 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 	/*
 	 * Step 2: Pre-Start Configuration
 	 */
-	// Gather Environment information, e.g. the host gateway address
-	envInfo, err := GatherEnvironmentInfo(ctx, runtime, &clusterConfig.Cluster)
+	envInfo, err := ClusterPreStartConfiguration(ctx, runtime, &clusterConfig.Cluster)
 	if err != nil {
-		return fmt.Errorf("failed to gather environment information used for cluster creation: %w", err)
+		return fmt.Errorf("Failed Pre-Start Configuration: %+v", err)
 	}
 
 	/*
@@ -147,6 +146,12 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 		}
 	}
 
+	if clusterConfig.Manifests != nil {
+		if err := ClusterPrepManifestVolume(ctx, runtime, &clusterConfig.Cluster, &clusterConfig.ClusterCreateOpts); err != nil {
+			return fmt.Errorf("Failed Manifests Volume Preparation: %+v", err)
+		}
+	}
+
 	/*
 	 * Step 3: Registries
 	 */
@@ -577,6 +582,33 @@ ClusterCreatOpts:
 	return nil
 }
 
+func ClusterPreStartConfiguration(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) (*k3d.EnvironmentInfo, error) {
+	// Ensure that the tools node is working
+	toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
+	if err != nil {
+		return nil, err
+	}
+
+	// Gather Environment information, e.g. the host gateway address
+	envInfo, err := GatherEnvironmentInfo(ctx, runtime, cluster)
+	if err != nil {
+		return nil, fmt.Errorf("failed to gather environment information used for cluster creation: %w", err)
+	}
+
+	// Copy embedded manifests to manifests volume which will be auto-deployed on the servers
+	err = CopyManifetsToVolume(ctx, runtime, cluster, toolsNode)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy embedded manifests to manifest volume: %w", err)
+	}
+
+	// Delete the tools node after finishing all pre-start configuration
+	if err := NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
+		l.Log().Warnf("Failed to delete tools node '%s'. This is not critical, but may lead to errors down the road. Error: %v", toolsNode.Name, err)
+	}
+
+	return envInfo, nil
+}
+
 // ClusterDelete deletes an existing cluster
 func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, opts k3d.ClusterDeleteOpts) error {
 	l.Log().Infof("Deleting cluster '%s'", cluster.Name)

pkg/client/environment.go

@@ -25,7 +25,6 @@ import (
 	"context"
 	"fmt"
 
-	l "github.com/k3d-io/k3d/v5/pkg/logger"
 	"github.com/k3d-io/k3d/v5/pkg/runtimes"
 
 	k3d "github.com/k3d-io/k3d/v5/pkg/types"
@@ -40,15 +39,6 @@ func GatherEnvironmentInfo(ctx context.Context, runtime runtimes.Runtime, cluste
 	}
 	envInfo.RuntimeInfo = *rtimeInfo
 
-	l.Log().Infof("Using the k3d-tools node to gather environment information")
-	toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
-	if err != nil {
-		return nil, err
-	}
-	if err := NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
-		l.Log().Warnf("Failed to delete tools node '%s'. This is not critical, but may lead to errors down the road. Error: %v", toolsNode.Name, err)
-	}
-
 	if cluster.Network.Name != "host" {
 		hostIP, err := GetHostIP(ctx, runtime, cluster)
 		if err != nil {

pkg/client/manifests.go

@@ -0,0 +1,88 @@
+/*
+Copyright © 2020-2024 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package client
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	l "github.com/k3d-io/k3d/v5/pkg/logger"
+	"github.com/k3d-io/k3d/v5/pkg/runtimes"
+	k3drt "github.com/k3d-io/k3d/v5/pkg/runtimes"
+	k3d "github.com/k3d-io/k3d/v5/pkg/types"
+	"github.com/k3d-io/k3d/v5/pkg/types/k3s"
+	"github.com/k3d-io/k3d/v5/pkg/util"
+)
+
+func ClusterPrepManifestVolume(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterCreateOpts *k3d.ClusterCreateOpts) error {
+	/*
+	 * Server Manifests volume
+	 * - manifest volume (for auto-deploy manifests)
+	 */
+	manifestVolumeName := fmt.Sprintf("%s-%s-manifests", k3d.DefaultObjectNamePrefix, cluster.Name)
+	if err := runtime.CreateVolume(ctx, manifestVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
+		return fmt.Errorf("failed to create manifest volume '%s' for cluster '%s': %w", manifestVolumeName, cluster.Name, err)
+	}
+	l.Log().Infof("Created manifest volume %s", manifestVolumeName)
+
+	clusterCreateOpts.GlobalLabels[k3d.LabelManifestVolume] = manifestVolumeName
+	cluster.ManifestVolume = manifestVolumeName
+	cluster.Volumes = append(cluster.Volumes, manifestVolumeName)
+
+	// Attach volume to server nodes only
+	filteredNodes, err := util.FilterNodes(cluster.Nodes, []string{"server:*"})
+	if err != nil {
+		return fmt.Errorf("failed to filter nodes: %w", err)
+	}
+	for _, node := range filteredNodes {
+		node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s", manifestVolumeName, k3s.K3sPathManifestsEmbedded))
+	}
+	l.Log().Debugf("Attached manifest volume to server nodes")
+
+	return nil
+}
+
+func CopyManifetsToVolume(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster, toolsNode *k3d.Node) error {
+	for _, manifest := range cluster.Manifests {
+		onContainerManifestFileName := k3s.K3sPathManifestsEmbedded + "/" + manifest.Name
+		onHostManifestFile, err := os.CreateTemp("", manifest.Name)
+		if err != nil {
+			return fmt.Errorf("failed to create manifest temporary file: %w", err)
+		}
+		l.Log().Debugf("Created temporary local manifest file: %s", onHostManifestFile.Name())
+
+		if _, err = onHostManifestFile.WriteString(manifest.Manifest); err != nil {
+			return fmt.Errorf("Failed to write to output file: %+v", err)
+		}
+
+		if err := runtime.CopyToNode(ctx, onHostManifestFile.Name(), onContainerManifestFileName, toolsNode); err != nil {
+			return fmt.Errorf("failed to copy manifestFile tar '%s' to tools node! Error below:\n%+v",
+				onHostManifestFile.Name(), err)
+		}
+
+		os.Remove(onHostManifestFile.Name())
+		l.Log().Debugf("Removed temporary local manifest file: %s", onHostManifestFile.Name())
+	}
+
+	return nil
+}

pkg/client/tools.go

@@ -37,6 +37,7 @@ import (
 	l "github.com/k3d-io/k3d/v5/pkg/logger"
 	"github.com/k3d-io/k3d/v5/pkg/runtimes"
 	k3d "github.com/k3d-io/k3d/v5/pkg/types"
+	"github.com/k3d-io/k3d/v5/pkg/types/k3s"
 )
 
 // ImageImportIntoClusterMulti starts up a k3d tools container for the selected cluster and uses it to export
@@ -424,17 +425,25 @@ func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d
 			cluster.ImageVolume = imageVolume
 		}
 
+		toolsNodeVolumes := []string{
+			fmt.Sprintf("%s:%s", cluster.ImageVolume, k3d.DefaultImageVolumeMountPath),
+			fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()),
+		}
+
+		if cluster.Manifests != nil {
+			toolsNodeVolumes = append(toolsNodeVolumes,
+				fmt.Sprintf("%s:%s", cluster.ManifestVolume, k3s.K3sPathManifestsEmbedded))
+		}
+
 		// start tools node
 		l.Log().Infoln("Starting new tools node...")
 		toolsNode, err = runToolsNode(
 			ctx,
 			runtime,
 			cluster,
 			cluster.Network.Name,
-			[]string{
-				fmt.Sprintf("%s:%s", cluster.ImageVolume, k3d.DefaultImageVolumeMountPath),
-				fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()),
-			})
+			toolsNodeVolumes)
+
 		if err != nil {
 			l.Log().Errorf("Failed to run tools container for cluster '%s'", cluster.Name)
 		}

pkg/config/config_test.go

@@ -56,11 +56,10 @@ func TestReadSimpleConfig(t *testing.T) {
 				NodeFilters: []string{"all"},
 			},
 		},
-		Manifests: []conf.ManifestWithNodeFilters{
+		Manifests: []conf.Manifest{
 			{
-				Name:        "my-manifest.yaml",
-				Manifest:    "---\n# This is the start of an empty YAML manifest.\n",
-				NodeFilters: []string{"all"},
+				Name:     "my-manifest.yaml",
+				Manifest: "---\n# This is the start of an empty YAML manifest.\n",
 			},
 		},
 		Ports: []conf.PortWithNodeFilters{

pkg/config/transform.go

@@ -28,7 +28,6 @@ import (
 	"io"
 	"net/netip"
 	"os"
-	"path/filepath"
 	"strings"
 
 	wharfie "github.com/rancher/wharfie/pkg/registries"
@@ -43,13 +42,12 @@ import (
 	l "github.com/k3d-io/k3d/v5/pkg/logger"
 	"github.com/k3d-io/k3d/v5/pkg/runtimes"
 	k3d "github.com/k3d-io/k3d/v5/pkg/types"
-	"github.com/k3d-io/k3d/v5/pkg/types/k3s"
 	"github.com/k3d-io/k3d/v5/pkg/util"
 	"github.com/k3d-io/k3d/v5/version"
 )
 
 // TransformSimpleToClusterConfig transforms a simple configuration to a full-fledged cluster configuration
-func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtime, simpleConfig conf.SimpleConfig, configFileName string) (*conf.ClusterConfig, error) {
+func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtime, simpleConfig conf.SimpleConfig) (*conf.ClusterConfig, error) {
 	// set default cluster name
 	if simpleConfig.Name == "" {
 		simpleConfig.Name = k3d.DefaultClusterName
@@ -177,6 +175,14 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 	nodeCount := len(newCluster.Nodes)
 	nodeList := newCluster.Nodes
 
+	// -> MANIFESTS
+	for _, cfgManifest := range simpleConfig.Manifests {
+		newCluster.Manifests = append(newCluster.Manifests, k3d.Manifest{
+			Name:     cfgManifest.Name,
+			Manifest: cfgManifest.Manifest,
+		})
+	}
+
 	// -> VOLUMES
 	for _, volumeWithNodeFilters := range simpleConfig.Volumes {
 		nodes, err := util.FilterNodes(nodeList, volumeWithNodeFilters.NodeFilters)
@@ -189,31 +195,6 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		}
 	}
 
-	// -> MANIFESTS
-	for _, manifestWithNodeFilters := range simpleConfig.Manifests {
-		nodes, err := util.FilterNodes(nodeList, manifestWithNodeFilters.NodeFilters)
-		if err != nil {
-			return nil, fmt.Errorf("failed to filter nodes for manifest mapping '%s': %w", manifestWithNodeFilters.Manifest, err)
-		}
-
-		manifestFileName := filepath.Join(filepath.Dir(configFileName), manifestWithNodeFilters.Name)
-		manifestFile, err := os.Create(manifestFileName)
-		if err != nil {
-			l.Log().Fatalln(err)
-		}
-		l.Log().Debugf("Created temporary local manifest file: %s", manifestFile.Name())
-
-		if _, err = manifestFile.WriteString(manifestWithNodeFilters.Manifest); err != nil {
-			l.Log().Fatalf("Failed to write to output file: %+v", err)
-		}
-
-		manifestVolume := fmt.Sprintf("%s:%s/%s", manifestFile.Name(), k3s.K3sPathManifests, manifestWithNodeFilters.Name)
-
-		for _, node := range nodes {
-			node.Volumes = append(node.Volumes, manifestVolume)
-		}
-	}
-
 	// -> PORTS
 	if err := client.TransformPorts(ctx, runtime, &newCluster, simpleConfig.Ports); err != nil {
 		return nil, fmt.Errorf("failed to transform ports: %w", err)

pkg/config/v1alpha5/schema.json

@@ -109,9 +109,6 @@
           },
           "manifest": {
             "type": "string"
-          },
-          "nodeFilters": {
-            "$ref": "#/definitions/nodeFilters"
           }
         },
         "additionalProperties": false

pkg/config/v1alpha5/types.go

@@ -58,14 +58,13 @@ var DefaultConfig = fmt.Sprintf(
 	fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.K3sVersion),
 )
 
-type VolumeWithNodeFilters struct {
-	Volume      string   `mapstructure:"volume" json:"volume,omitempty"`
-	NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
+type Manifest struct {
+	Name     string `mapstructure:"name" json:"name,omitempty"`
+	Manifest string `mapstructure:"manifest" json:"manifest,omitempty"`
 }
 
-type ManifestWithNodeFilters struct {
-	Name        string   `mapstructure:"name" json:"name,omitempty"`
-	Manifest    string   `mapstructure:"manifest" json:"manifest,omitempty"`
+type VolumeWithNodeFilters struct {
+	Volume      string   `mapstructure:"volume" json:"volume,omitempty"`
 	NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
 }
 
@@ -155,20 +154,20 @@ type SimpleConfigRegistries struct {
 type SimpleConfig struct {
 	config.TypeMeta   `mapstructure:",squash"`
 	config.ObjectMeta `mapstructure:"metadata" json:"metadata,omitempty"`
-	Servers           int                       `mapstructure:"servers" json:"servers,omitempty"` //nolint:lll    // default 1
-	Agents            int                       `mapstructure:"agents" json:"agents,omitempty"`   //nolint:lll    // default 0
-	ExposeAPI         SimpleExposureOpts        `mapstructure:"kubeAPI" json:"kubeAPI,omitempty"`
-	Image             string                    `mapstructure:"image" json:"image,omitempty"`
-	Network           string                    `mapstructure:"network" json:"network,omitempty"`
-	Subnet            string                    `mapstructure:"subnet" json:"subnet,omitempty"`
-	ClusterToken      string                    `mapstructure:"token" json:"clusterToken,omitempty"` // default: auto-generated
-	Manifests         []ManifestWithNodeFilters `mapstructure:"manifests" json:"manifests,omitempty"`
-	Volumes           []VolumeWithNodeFilters   `mapstructure:"volumes" json:"volumes,omitempty"`
-	Ports             []PortWithNodeFilters     `mapstructure:"ports" json:"ports,omitempty"`
-	Options           SimpleConfigOptions       `mapstructure:"options" json:"options,omitempty"`
-	Env               []EnvVarWithNodeFilters   `mapstructure:"env" json:"env,omitempty"`
-	Registries        SimpleConfigRegistries    `mapstructure:"registries" json:"registries,omitempty"`
-	HostAliases       []k3d.HostAlias           `mapstructure:"hostAliases" json:"hostAliases,omitempty"`
+	Servers           int                     `mapstructure:"servers" json:"servers,omitempty"` //nolint:lll    // default 1
+	Agents            int                     `mapstructure:"agents" json:"agents,omitempty"`   //nolint:lll    // default 0
+	ExposeAPI         SimpleExposureOpts      `mapstructure:"kubeAPI" json:"kubeAPI,omitempty"`
+	Image             string                  `mapstructure:"image" json:"image,omitempty"`
+	Network           string                  `mapstructure:"network" json:"network,omitempty"`
+	Subnet            string                  `mapstructure:"subnet" json:"subnet,omitempty"`
+	ClusterToken      string                  `mapstructure:"token" json:"clusterToken,omitempty"` // default: auto-generated
+	Manifests         []Manifest              `mapstructure:"manifests" json:"manifests,omitempty"`
+	Volumes           []VolumeWithNodeFilters `mapstructure:"volumes" json:"volumes,omitempty"`
+	Ports             []PortWithNodeFilters   `mapstructure:"ports" json:"ports,omitempty"`
+	Options           SimpleConfigOptions     `mapstructure:"options" json:"options,omitempty"`
+	Env               []EnvVarWithNodeFilters `mapstructure:"env" json:"env,omitempty"`
+	Registries        SimpleConfigRegistries  `mapstructure:"registries" json:"registries,omitempty"`
+	HostAliases       []k3d.HostAlias         `mapstructure:"hostAliases" json:"hostAliases,omitempty"`
 }
 
 // SimpleExposureOpts provides a simplified syntax compared to the original k3d.ExposureOpts

pkg/types/k3s/paths.go

@@ -24,7 +24,8 @@ package k3s
 const (
 	K3sPathStorage              = "/var/lib/rancher/k3s/storage"
 	K3sPathManifests            = "/var/lib/rancher/k3s/server/manifests"
-	K3sPathManifestsCustom      = "/var/lib/rancher/k3s/server/manifests/custom" // custom subfolder
+	K3sPathManifestsCustom      = "/var/lib/rancher/k3s/server/manifests/custom"   // custom subfolder
+	K3sPathManifestsEmbedded    = "/var/lib/rancher/k3s/server/manifests/embedded" // custom subfolder
 	K3sPathContainerdConfig     = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"
 	K3sPathContainerdConfigTmpl = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl"
 	K3sPathRegistryConfig       = "/etc/rancher/k3s/registries.yaml"