[v3] Add render generator

pkg/skaffold/deploy/kustomize/kustomize.go

@@ -41,9 +41,9 @@ import (
 
 var (
 	DefaultKustomizePath = "."
-	kustomizeFilePaths   = []string{"kustomization.yaml", "kustomization.yml", "Kustomization"}
+	KustomizeFilePaths   = []string{"kustomization.yaml", "kustomization.yml", "Kustomization"}
 	basePath             = "base"
-	kustomizeBinaryCheck = kustomizeBinaryExists // For testing
+	KustomizeBinaryCheck = kustomizeBinaryExists // For testing
 )
 
 // kustomization is the content of a kustomization.yaml file.
@@ -112,7 +112,7 @@ func NewDeployer(cfg kubectl.Config, labels map[string]string, d *latestV1.Kusto
 
 	kubectl := kubectl.NewCLI(cfg, d.Flags, defaultNamespace)
 	// if user has kustomize binary, prioritize that over kubectl kustomize
-	useKubectlKustomize := !kustomizeBinaryCheck() && kubectlVersionCheck(kubectl)
+	useKubectlKustomize := !KustomizeBinaryCheck() && kubectlVersionCheck(kubectl)
 
 	return &Deployer{
 		KustomizeDeploy:     d,
@@ -306,7 +306,7 @@ func IsKustomizationBase(path string) bool {
 
 func IsKustomizationPath(path string) bool {
 	filename := filepath.Base(path)
-	for _, candidate := range kustomizeFilePaths {
+	for _, candidate := range KustomizeFilePaths {
 		if filename == candidate {
 			return true
 		}

pkg/skaffold/deploy/kustomize/kustomize_test.go

@@ -167,7 +167,7 @@ func TestKustomizeDeploy(t *testing.T) {
 		testutil.Run(t, test.description, func(t *testutil.T) {
 			t.SetEnvs(test.envs)
 			t.Override(&util.DefaultExecCommand, test.commands)
-			t.Override(&kustomizeBinaryCheck, func() bool { return test.kustomizeCmdPresent })
+			t.Override(&KustomizeBinaryCheck, func() bool { return test.kustomizeCmdPresent })
 			t.NewTempDir().
 				Chdir()
 
@@ -246,7 +246,7 @@ func TestKustomizeCleanup(t *testing.T) {
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
 			t.Override(&util.DefaultExecCommand, test.commands)
-			t.Override(&kustomizeBinaryCheck, func() bool { return true })
+			t.Override(&KustomizeBinaryCheck, func() bool { return true })
 
 			k, err := NewDeployer(&kustomizeConfig{
 				workingDir: tmpDir.Root(),

pkg/skaffold/deploy/kustomize/util.go

@@ -118,7 +118,7 @@ func DependenciesForKustomization(dir string) ([]string, error) {
 // A Kustomization config must be at the root of the directory. Kustomize will
 // error if more than one of these files exists so order doesn't matter.
 func FindKustomizationConfig(dir string) (string, error) {
-	for _, candidate := range kustomizeFilePaths {
+	for _, candidate := range KustomizeFilePaths {
 		if local, _ := pathExistsLocally(candidate, dir); local {
 			return filepath.Join(dir, candidate), nil
 		}

pkg/skaffold/render/generate/generate.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package generate
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/kustomize"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest"
+	latestV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v2"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+)
+
+// NewGenerator instantiates a Generator object.
+func NewGenerator(workingDir string, config latestV2.Generate) *Generator {
+	return &Generator{
+		workingDir: workingDir,
+		config:     config,
+	}
+}
+
+// Generator provides the functions for the manifest sources (raw manifests, helm charts, kustomize configs and remote packages).
+type Generator struct {
+	workingDir string
+	config     latestV2.Generate
+}
+
+// Generate parses the config resources from the paths in .Generate.Manifests. This path can be the path to raw manifest,
+// kustomize manifests, helm charts or kpt function configs. All should be file-watched.
+func (g *Generator) Generate(ctx context.Context) (manifest.ManifestList, error) {
+	// exclude remote url.
+	var paths []string
+	for _, path := range g.config.Manifests {
+		switch {
+		case util.IsURL(path):
+			// TODO(yuwenma): remote URL should be changed to use kpt package management approach, via API Schema
+			//  `render.generate.remotePackages`
+		case strings.HasPrefix(path, "gs://"):
+			// TODO(yuwenma): handle GS packages.
+		default:
+			paths = append(paths, path)
+		}
+	}
+
+	// expend the glob paths.
+	expanded, err := util.ExpandPathsGlob(g.workingDir, paths)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse kustomize manifests and non-kustomize manifests.  We may also want to parse (and exclude) kpt function manifests later.
+	// TODO: Update `kustomize build` to kustomize kpt-fn once https://github.com/GoogleContainerTools/kpt/issues/1447 is fixed.
+	kustomizePathMap := make(map[string]bool)
+	var nonKustomizePaths []string
+	for _, path := range expanded {
+		if dir, ok := isKustomizeDir(path); ok {
+			kustomizePathMap[dir] = true
+		}
+	}
+	for _, path := range expanded {
+		kustomizeDirDup := false
+		for kPath := range kustomizePathMap {
+			// Before kustomize kpt-fn can provide a way to parse the kustomize content, we assume the users do not place non-kustomize manifests under the kustomization.yaml directory.
+			if strings.HasPrefix(path, kPath) {
+				kustomizeDirDup = true
+				break
+			}
+		}
+		if !kustomizeDirDup {
+			nonKustomizePaths = append(nonKustomizePaths, path)
+		}
+	}
+
+	var manifests manifest.ManifestList
+	for kPath := range kustomizePathMap {
+		// TODO:  support kustomize buildArgs (shall we support it in kpt-fn)?
+		cmd := exec.CommandContext(ctx, "kustomize", "build", kPath)
+		out, err := util.RunCmdOut(cmd)
+		if err != nil {
+			return nil, err
+		}
+		if len(out) == 0 {
+			continue
+		}
+		manifests.Append(out)
+	}
+	for _, nkPath := range nonKustomizePaths {
+		if !kubernetes.HasKubernetesFileExtension(nkPath) {
+			if !util.StrSliceContains(g.config.Manifests, nkPath) {
+				logrus.Infof("refusing to deploy/delete non {json, yaml} file %s", nkPath)
+				logrus.Info("If you still wish to deploy this file, please specify it directly, outside a glob pattern.")
+				continue
+			}
+		}
+		manifestFileContent, err := ioutil.ReadFile(nkPath)
+		if err != nil {
+			return nil, err
+		}
+		manifests.Append(manifestFileContent)
+	}
+	// TODO(yuwenma): helm resources. `render.generate.helmCharts`
+	return manifests, nil
+}
+
+// isKustomizeDir checks if the path is managed by kustomize. A more reliable approach is parsing the kustomize content
+// resources, bases, overlays. However, this switches the manifests parsing from kustomize/kpt to skaffold. To avoid
+// skaffold render.generate mis-use, we expect the users do not place non-kustomize manifests under the kustomization.yaml directory, so as the kpt manifests.
+func isKustomizeDir(path string) (string, bool) {
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return "", false
+	}
+	var dir string
+	switch mode := fileInfo.Mode(); {
+	// TODO: Check if regular file contains kpt functions. if so, we may want to abstract that info as well.
+	case mode.IsDir():
+		dir = path
+	case mode.IsRegular():
+		dir = filepath.Dir(path)
+	}
+
+	for _, base := range kustomize.KustomizeFilePaths {
+		if _, err := os.Stat(filepath.Join(dir, base)); os.IsNotExist(err) {
+			continue
+		}
+		return dir, true
+	}
+	return "", false
+}

pkg/skaffold/render/generate/generate_test.go

@@ -0,0 +1,185 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package generate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest"
+	latestV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v2"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+	"github.com/GoogleContainerTools/skaffold/testutil"
+)
+
+const (
+	// Test file under <tmp>/pod.yaml
+	podYaml = `apiVersion: v1
+kind: Pod
+metadata:
+  name: leeroy-web
+spec:
+  containers:
+  - name: leeroy-web
+    image: leeroy-web
+`
+
+	// Test file under <tmp>/pods.yaml. This file contains multiple config object.
+	podsYaml = `apiVersion: v1
+kind: Pod
+metadata:
+  name: leeroy-web2
+spec:
+  containers:
+  - name: leeroy-web2
+    image: leeroy-web2
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: leeroy-web3
+spec:
+  containers:
+  - name: leeroy-web3
+    image: leeroy-web3
+`
+
+	// Test file under <tmp>/base/patch.yaml
+	patchYaml = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-test
+spec:
+  template:
+    spec:
+      containers:
+      - name: kustomize-test
+        image: index.docker.io/library/busybox
+        command:
+          - sleep
+          - "3600"
+`
+	// Test file under <tmp>/base/deployment.yaml
+	kustomizeDeploymentYaml = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-test
+  labels:
+    app: kustomize-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kustomize-test
+  template:
+    metadata:
+      labels:
+        app: kustomize-test
+    spec:
+      containers:
+      - name: kustomize-test
+        image: not/a/valid/image
+`
+	// The kustomize build result from kustomizeYaml file.
+	kustomizePatchedOutput = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: kustomize-test
+  name: kustomize-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kustomize-test
+  template:
+    metadata:
+      labels:
+        app: kustomize-test
+    spec:
+      containers:
+      - command:
+        - sleep
+        - "3600"
+        image: index.docker.io/library/busybox
+        name: kustomize-test
+`
+	// Test file under <tmp>/base/kustomization.yaml
+	kustomizeYaml = `resources:
+  - deployment.yaml
+patches:
+  - patch.yaml
+`
+)
+
+func TestGenerate(t *testing.T) {
+	tests := []struct {
+		description    string
+		generateConfig latestV2.Generate
+		expected       manifest.ManifestList
+	}{
+		{
+			description: "render raw manifests",
+			generateConfig: latestV2.Generate{
+				Manifests: []string{"pod.yaml"},
+			},
+			expected: manifest.ManifestList{[]byte(podYaml)},
+		},
+		{
+			description: "render glob raw manifests",
+			generateConfig: latestV2.Generate{
+				Manifests: []string{"*.yaml"},
+			},
+			expected: manifest.ManifestList{[]byte(podYaml), []byte(podsYaml)},
+		},
+		{
+			description: "render kustomize manifests",
+			generateConfig: latestV2.Generate{
+				Manifests: []string{"base"},
+			},
+			expected: manifest.ManifestList{[]byte(kustomizePatchedOutput)},
+		},
+
+		{
+			description: "render mixed raw and kustomize manifests",
+			generateConfig: latestV2.Generate{
+				Manifests: []string{"*"},
+			},
+			expected: manifest.ManifestList{[]byte(kustomizePatchedOutput), []byte(podYaml), []byte(podsYaml)},
+		},
+	}
+	for _, test := range tests {
+		testutil.Run(t, test.description, func(t *testutil.T) {
+
+			fakeCmd := testutil.CmdRunOut("kustomize build base", kustomizePatchedOutput)
+			t.Override(&util.DefaultExecCommand, fakeCmd)
+
+			t.NewTempDir().
+				Write("pod.yaml", podYaml).
+				Write("pods.yaml", podsYaml).
+				Write("base/kustomization.yaml", kustomizeYaml).
+				Write("base/patch.yaml", patchYaml).
+				Write("base/deployment.yaml", kustomizeDeploymentYaml).
+				Touch("empty.ignored").
+				Chdir()
+
+			g := NewGenerator(".", test.generateConfig)
+			actual, err := g.Generate(context.Background())
+			t.CheckNoError(err)
+			t.CheckDeepEqual(actual.String(), test.expected.String())
+		})
+	}
+}