add auto-deploy custom manifest support

pkg/apis/config/v1alpha4/types.go

@@ -81,6 +81,10 @@ type Cluster struct {
 	// in the order listed.
 	// These should be YAML or JSON formatting RFC 6902 JSON patches
 	ContainerdConfigPatchesJSON6902 []string `yaml:"containerdConfigPatchesJSON6902,omitempty" json:"containerdConfigPatchesJSON6902,omitempty"`
+
+	// Custom manifests are applied to the cluster in the order listed.
+	// These should be inline YAML as a string or paths to filenames.
+	CustomManifests []interface{} `yaml:"customManifests,omitempty" json:"customManifests,omitempty"`
 }
 
 // TypeMeta partially copies apimachinery/pkg/apis/meta/v1.TypeMeta

pkg/cluster/internal/create/actions/installcustom/custom.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package installcustom implements the an action to install custom
+// manifests
+package installcustom
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions"
+	"sigs.k8s.io/kind/pkg/cluster/nodes"
+	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
+	"sigs.k8s.io/kind/pkg/errors"
+)
+
+type action struct{}
+
+// NewAction returns a new action for installing custom manifests
+func NewAction() actions.Action {
+	return &action{}
+}
+
+// Execute runs the action
+func (a *action) Execute(ctx *actions.ActionContext) error {
+	// skip if there are no custom manifests
+	if len(ctx.Config.CustomManifests) == 0 {
+		return nil
+	}
+
+	ctx.Status.Start("Installing custom manifests 📃")
+	defer ctx.Status.End(false)
+
+	allNodes, err := ctx.Nodes()
+	if err != nil {
+		return err
+	}
+
+	// get the target node for this task
+	controlPlanes, err := nodeutils.ControlPlaneNodes(allNodes)
+	if err != nil {
+		return err
+	}
+	node := controlPlanes[0] // kind expects at least one always
+
+	// add the custom manifests
+	if err := addCustomManifests(node, &ctx.Config.CustomManifests); err != nil {
+		return errors.Wrap(err, "failed to add default storage class")
+	}
+
+	// mark success
+	ctx.Status.End(true)
+	return nil
+}
+
+// run kubectl apply on control plane node, and can be overriden for testing
+var runApplyCustomManifest = func(controlPlane nodes.Node, path string, stdin string) error {
+	var in *strings.Reader = nil
+	// only create if we have stdin
+	if len(stdin) > 0 {
+		in = strings.NewReader(stdin)
+		path = "-"
+	}
+
+	cmd := controlPlane.Command(
+		"kubectl",
+		"--kubeconfig=/etc/kubernetes/admin.conf", "apply", "-f", path,
+	)
+
+	// only close if we had stdin
+	if in != nil {
+		cmd.SetStdin(in)
+	}
+
+	return cmd.Run()
+}
+
+func addCustomManifests(controlPlane nodes.Node, customManifests *[]interface{}) (err error) {
+	for index, customManifest := range *customManifests {
+		var manifestList map[string]string
+
+		// perform conversion to map[string]string
+		switch t := (customManifest).(type) {
+		// handle file or URL
+		case string:
+			if strings.HasPrefix(t, "http") {
+				// URL is a special case - set contents to empty
+				manifestList = map[string]string{t: ""}
+			} else {
+				// read file in
+				var manifest []byte
+				if manifest, err = os.ReadFile(t); os.IsNotExist(err) {
+					err = fmt.Errorf("customManifests[%d]: '%s' does not exist", index, t)
+					return
+				}
+				manifestList = map[string]string{t: string(manifest)}
+			}
+		// convert map[string]interface{} to map[string]string
+		case map[string]interface{}:
+			manifestList = make(map[string]string)
+			for manifestName, manifestContents := range t {
+				switch manifestContentsString := (manifestContents).(type) {
+				case string:
+					manifestList[manifestName] = manifestContentsString
+				default:
+					err = fmt.Errorf("customManifests[%s]: incorrect type (map[string]%T) expected string or map[string]string", manifestName, manifestContentsString)
+					return
+				}
+			}
+		case map[string]string:
+			manifestList = t
+		default:
+			err = fmt.Errorf("customManifests[%d]: incorrect type (%T) expected string or map[string]string", index, t)
+			return
+		}
+
+		// apply all manifest in current array member
+		if err == nil {
+			for manifestName, manifest := range manifestList {
+				path := "-"
+				// handle special cases (URL) where content is empty
+				if len(manifest) == 0 {
+					path = manifestName
+				}
+				err = runApplyCustomManifest(controlPlane, path, manifest)
+				if err != nil {
+					err = fmt.Errorf("customManifest[%d][%s]: %w", index, manifestName, err)
+					return
+				}
+			}
+		}
+	}
+
+	return
+}

pkg/cluster/internal/create/actions/installcustom/custom_test.go

@@ -0,0 +1,160 @@
+package installcustom
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"sigs.k8s.io/kind/pkg/cluster/nodes"
+	"sigs.k8s.io/kind/pkg/internal/assert"
+)
+
+func TestAddCustomManifests(t *testing.T) {
+	cases := []struct {
+		TestName       string
+		CustomManifest []interface{}
+		CreateFiles    map[string]string
+		ExpectError    string
+		ExpectOutput   []map[string]string
+		OutputError    string
+	}{
+		{
+			TestName: "Correct inline manifest",
+			CustomManifest: []interface{}{
+				map[string]string{
+					"test2.yaml": "test2: test",
+				},
+				map[string]interface{}{
+					"test3.yaml": "test3: test",
+				},
+			},
+			ExpectOutput: []map[string]string{
+				{"-": "test2: test"},
+				{"-": "test3: test"},
+			},
+		},
+		{
+			TestName: "Correct file manifest",
+			CustomManifest: []interface{}{
+				"correct_manifest1.yaml",
+			},
+			CreateFiles: map[string]string{
+				"correct_manifest1.yaml": "test: test",
+			},
+			ExpectOutput: []map[string]string{
+				{"-": "test: test"},
+			},
+		},
+		{
+			TestName: "Remote http file manifest",
+			CustomManifest: []interface{}{
+				"https://test.local/test.yaml",
+			},
+			ExpectOutput: []map[string]string{
+				{"https://test.local/test.yaml": ""},
+			},
+		},
+		{
+			TestName: "kubectl error",
+			CustomManifest: []interface{}{
+				map[string]string{
+					"test2.yaml": "test2: test",
+				},
+			},
+			ExpectError: "customManifest[0][test2.yaml]: error",
+			OutputError: "error",
+			ExpectOutput: []map[string]string{
+				{"-": "test2: test"},
+			},
+		},
+		{
+			TestName: "Non existent file manifest",
+			CustomManifest: []interface{}{
+				"no_file.yaml",
+			},
+			ExpectError: "customManifests[0]: 'no_file.yaml' does not exist",
+		},
+		{
+			TestName: "Incorrect manifest type map[string]int",
+			CustomManifest: []interface{}{
+				map[string]int{
+					"test2.yaml": 5,
+				},
+			},
+			ExpectError: "customManifests[0]: incorrect type (map[string]int) expected string or map[string]string",
+		},
+		{
+			TestName: "Incorrect manifest type map[string]interface",
+			CustomManifest: []interface{}{
+				map[string]interface{}{
+					"test2.yaml": 5,
+				},
+			},
+			ExpectError: "customManifests[test2.yaml]: incorrect type (map[string]int) expected string or map[string]string",
+		},
+		{
+			TestName: "Incorrect manifest type multiple",
+			CustomManifest: []interface{}{
+				5,
+				map[int]string{
+					6: "hello",
+				},
+			},
+			ExpectError: "customManifests[0]: incorrect type (int) expected string or map[string]string",
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc //capture loop variable
+		t.Run(tc.TestName, func(t *testing.T) {
+			// create files for test if required
+			if tc.CreateFiles != nil && len(tc.CreateFiles) > 0 {
+				for fileName, contents := range tc.CreateFiles {
+					err := os.WriteFile(fileName, []byte(contents), 0644)
+					if err != nil {
+						t.Errorf("unexpected error in creating file %s: %v", fileName, err)
+					}
+				}
+			}
+
+			// override apply manifest function to capture output for test expectations
+			expectedOuputIndex := 0
+			runApplyCustomManifest = func(controlPlane nodes.Node, path string, stdin string) error {
+				expectedStdin, ok := tc.ExpectOutput[expectedOuputIndex][path]
+				assert.BoolEqual(t, true, ok)
+				assert.StringEqual(t, expectedStdin, stdin)
+				expectedOuputIndex += 1
+
+				if tc.OutputError != "" {
+					return fmt.Errorf("%s", tc.OutputError)
+				}
+
+				return nil
+			}
+
+			err := addCustomManifests(nil, &tc.CustomManifest)
+
+			// check all expected output was produced
+			if expectedOuputIndex != len(tc.ExpectOutput) {
+				t.Errorf("Test failed, did not reach expected number of outputs, got %d and expected %d", expectedOuputIndex, len(tc.ExpectOutput))
+			}
+
+			// the error can be:
+			// - nil, in which case we should expect no errors or fail
+			if err == nil && len(tc.ExpectError) > 0 {
+				t.Errorf("Test failed, unexpected error: %s", tc.ExpectError)
+			}
+
+			if err != nil && err.Error() != tc.ExpectError {
+				t.Errorf("Test failed, error: %s expected error: %s", err, tc.ExpectError)
+			}
+
+			// remove any created test files
+			if tc.CreateFiles != nil && len(tc.CreateFiles) > 0 {
+				for fileName := range tc.CreateFiles {
+					os.Remove(fileName)
+				}
+			}
+		})
+	}
+}

pkg/cluster/internal/create/create.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions"
 	configaction "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/config"
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions/installcni"
+	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions/installcustom"
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions/installstorage"
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions/kubeadminit"
 	"sigs.k8s.io/kind/pkg/cluster/internal/create/actions/kubeadmjoin"
@@ -124,6 +125,7 @@ func Cluster(logger log.Logger, p providers.Provider, opts *ClusterOptions) erro
 		// add remaining steps
 		actionsToRun = append(actionsToRun,
 			installstorage.NewAction(),                // install StorageClass
+			installcustom.NewAction(),                 // install customManifests
 			kubeadmjoin.NewAction(),                   // run kubeadm join
 			waitforready.NewAction(opts.WaitForReady), // wait for cluster readiness
 		)

pkg/internal/apis/config/convert_v1alpha4.go

@@ -32,6 +32,7 @@ func Convertv1alpha4(in *v1alpha4.Cluster) *Cluster {
 		KubeadmConfigPatchesJSON6902:    make([]PatchJSON6902, len(in.KubeadmConfigPatchesJSON6902)),
 		ContainerdConfigPatches:         in.ContainerdConfigPatches,
 		ContainerdConfigPatchesJSON6902: in.ContainerdConfigPatchesJSON6902,
+		CustomManifests:                 in.CustomManifests,
 	}
 
 	for i := range in.Nodes {

pkg/internal/apis/config/types.go

@@ -69,6 +69,10 @@ type Cluster struct {
 	// in the order listed.
 	// These should be YAML or JSON formatting RFC 6902 JSON patches
 	ContainerdConfigPatchesJSON6902 []string
+
+	// Custom manifests are applied to the cluster in the order listed.
+	// These should be inline YAML as a string or paths to filenames.
+	CustomManifests []interface{}
 }
 
 // Node contains settings for a node in the `kind` Cluster.