Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add fallback to kubectl kustomize if kustomize binary isn't present #4484

Merged
merged 11 commits into from
Sep 21, 2020
2 changes: 1 addition & 1 deletion pkg/skaffold/deploy/kpt.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ func (k *KptDeployer) kustomizeBuild(ctx context.Context) error {
return nil
}

cmd := exec.CommandContext(ctx, "kustomize", buildCommandArgs([]string{"-o", filepath.Join(pipeline, k.Dir)}, k.Dir)...)
cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, buildCommandArgs([]string{"-o", filepath.Join(pipeline, k.Dir)}, k.Dir)...)...)
if _, err := util.RunCmdOut(cmd); err != nil {
return err
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/skaffold/deploy/kubectl/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,11 @@ func (c *CLI) Apply(ctx context.Context, out io.Writer, manifests ManifestList)
return nil
}

// Kustomize runs `kubectl kustomize` with the provided args
func (c *CLI) Kustomize(ctx context.Context, args []string) ([]byte, error) {
return c.RunOut(ctx, "kustomize", c.args(nil, args...)...)
}

type getResult struct {
Items []struct {
Metadata struct {
Expand Down
56 changes: 43 additions & 13 deletions pkg/skaffold/deploy/kustomize.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ var (
DefaultKustomizePath = "."
kustomizeFilePaths = []string{"kustomization.yaml", "kustomization.yml", "Kustomization"}
basePath = "base"
kustomizeBinaryCheck = kustomizeBinaryExists // For testing
)

// kustomization is the content of a kustomization.yaml file.
Expand Down Expand Up @@ -92,11 +93,11 @@ type secretGenerator struct {
// KustomizeDeployer deploys workflows using kustomize CLI.
type KustomizeDeployer struct {
*latest.KustomizeDeploy

kubectl deploy.CLI
insecureRegistries map[string]bool
labels map[string]string
globalConfig string
kubectl deploy.CLI
insecureRegistries map[string]bool
labels map[string]string
globalConfig string
useKubectlKustomize bool
}

func NewKustomizeDeployer(cfg Config, labels map[string]string) (*KustomizeDeployer, error) {
Expand All @@ -109,15 +110,37 @@ func NewKustomizeDeployer(cfg Config, labels map[string]string) (*KustomizeDeplo
}
}

kubectl := deploy.NewCLI(cfg, cfg.Pipeline().Deploy.KustomizeDeploy.Flags, defaultNamespace)
// if user has kustomize binary, prioritize that over kubectl kustomize
useKubectlKustomize := !kustomizeBinaryCheck() && kubectlVersionCheck(kubectl)

return &KustomizeDeployer{
KustomizeDeploy: cfg.Pipeline().Deploy.KustomizeDeploy,
kubectl: deploy.NewCLI(cfg, cfg.Pipeline().Deploy.KustomizeDeploy.Flags, defaultNamespace),
insecureRegistries: cfg.GetInsecureRegistries(),
globalConfig: cfg.GlobalConfig(),
labels: labels,
KustomizeDeploy: cfg.Pipeline().Deploy.KustomizeDeploy,
kubectl: kubectl,
insecureRegistries: cfg.GetInsecureRegistries(),
globalConfig: cfg.GlobalConfig(),
labels: labels,
useKubectlKustomize: useKubectlKustomize,
}, nil
}

// Check for existence of kustomize binary in user's PATH
func kustomizeBinaryExists() bool {
_, err := exec.LookPath("kustomize")

return err == nil
}

// Check that kubectl version is valid to use kubectl kustomize
func kubectlVersionCheck(kubectl deploy.CLI) bool {
gt, err := kubectl.CompareVersionTo(context.Background(), 1, 14)
if err != nil {
return false
}

return gt == 1
}

// Deploy runs `kubectl apply` on the manifest generated by kustomize.
func (k *KustomizeDeployer) Deploy(ctx context.Context, out io.Writer, builds []build.Artifact) ([]string, error) {
manifests, err := k.renderManifests(ctx, out, builds)
Expand Down Expand Up @@ -354,8 +377,16 @@ func pathExistsLocally(filename string, workingDir string) (bool, os.FileMode) {
func (k *KustomizeDeployer) readManifests(ctx context.Context) (deploy.ManifestList, error) {
var manifests deploy.ManifestList
for _, kustomizePath := range k.KustomizePaths {
cmd := exec.CommandContext(ctx, "kustomize", buildCommandArgs(k.BuildArgs, kustomizePath)...)
out, err := util.RunCmdOut(cmd)
var out []byte
var err error

if k.useKubectlKustomize {
out, err = k.kubectl.Kustomize(ctx, buildCommandArgs(k.BuildArgs, kustomizePath))
} else {
cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, buildCommandArgs(k.BuildArgs, kustomizePath)...)...)
out, err = util.RunCmdOut(cmd)
}

if err != nil {
return nil, fmt.Errorf("kustomize build: %w", err)
}
Expand All @@ -370,7 +401,6 @@ func (k *KustomizeDeployer) readManifests(ctx context.Context) (deploy.ManifestL

func buildCommandArgs(buildArgs []string, kustomizePath string) []string {
var args []string
args = append(args, "build")

if len(buildArgs) > 0 {
for _, v := range buildArgs {
Expand Down
63 changes: 46 additions & 17 deletions pkg/skaffold/deploy/kustomize_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func TestKustomizeDeploy(t *testing.T) {
shouldErr bool
forceDeploy bool
skipSkaffoldNamespaceOption bool
kustomizeCmdPresent bool
envs map[string]string
}{
{
Expand All @@ -50,24 +51,26 @@ func TestKustomizeDeploy(t *testing.T) {
KustomizePaths: []string{"."},
},
commands: testutil.
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
CmdRunOut("kubectl version --client -ojson", kubectlVersion118).
AndRunOut("kustomize build .", ""),
kustomizeCmdPresent: true,
},
{
description: "deploy success",
kustomize: latest.KustomizeDeploy{
KustomizePaths: []string{"."},
},
commands: testutil.
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
CmdRunOut("kubectl version --client -ojson", kubectlVersion118).
AndRunOut("kustomize build .", deploymentWebYAML).
AndRunInputOut("kubectl --context kubecontext --namespace testNamespace get -f - --ignore-not-found -ojson", deploymentWebYAMLv1, "").
AndRun("kubectl --context kubecontext --namespace testNamespace apply -f - --force --grace-period=0"),
builds: []build.Artifact{{
ImageName: "leeroy-web",
Tag: "leeroy-web:v1",
}},
forceDeploy: true,
forceDeploy: true,
kustomizeCmdPresent: true,
},
{
description: "deploy success (default namespace)",
Expand All @@ -86,6 +89,7 @@ func TestKustomizeDeploy(t *testing.T) {
}},
forceDeploy: true,
skipSkaffoldNamespaceOption: true,
kustomizeCmdPresent: true,
},
{
description: "deploy success (default namespace with env template)",
Expand All @@ -107,14 +111,15 @@ func TestKustomizeDeploy(t *testing.T) {
envs: map[string]string{
"MYENV": "Namesp",
},
kustomizeCmdPresent: true,
},
{
description: "deploy success with multiple kustomizations",
kustomize: latest.KustomizeDeploy{
KustomizePaths: []string{"a", "b"},
},
commands: testutil.
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
CmdRunOut("kubectl version --client -ojson", kubectlVersion118).
AndRunOut("kustomize build a", deploymentWebYAML).
AndRunOut("kustomize build b", deploymentAppYAML).
AndRunInputOut("kubectl --context kubecontext --namespace testNamespace get -f - --ignore-not-found -ojson", deploymentWebYAMLv1+"\n---\n"+deploymentAppYAMLv1, "").
Expand All @@ -129,13 +134,39 @@ func TestKustomizeDeploy(t *testing.T) {
Tag: "leeroy-app:v1",
},
},
forceDeploy: true,
kustomizeCmdPresent: true,
},
{
description: "built-in kubectl kustomize",
kustomize: latest.KustomizeDeploy{
KustomizePaths: []string{"a", "b"},
},
commands: testutil.
CmdRunOut("kubectl version --client -ojson", kubectlVersion118).
AndRunOut("kubectl --context kubecontext --namespace testNamespace kustomize a", deploymentWebYAML).
AndRunOut("kubectl --context kubecontext --namespace testNamespace kustomize b", deploymentAppYAML).
AndRunInputOut("kubectl --context kubecontext --namespace testNamespace get -f - --ignore-not-found -ojson", deploymentWebYAMLv1+"\n---\n"+deploymentAppYAMLv1, "").
AndRun("kubectl --context kubecontext --namespace testNamespace apply -f - --force --grace-period=0"),
builds: []build.Artifact{
{
ImageName: "leeroy-web",
Tag: "leeroy-web:v1",
},
{
ImageName: "leeroy-app",
Tag: "leeroy-app:v1",
},
},
forceDeploy: true,
},
}

for _, test := range tests {
testutil.Run(t, test.description, func(t *testutil.T) {
t.SetEnvs(test.envs)
t.Override(&util.DefaultExecCommand, test.commands)
t.Override(&kustomizeBinaryCheck, func() bool { return test.kustomizeCmdPresent })
t.NewTempDir().
Chdir()

Expand Down Expand Up @@ -207,17 +238,15 @@ func TestKustomizeCleanup(t *testing.T) {
kustomize: latest.KustomizeDeploy{
KustomizePaths: []string{tmpDir.Root()},
},
commands: testutil.CmdRunOutErr(
"kustomize build "+tmpDir.Root(),
"",
errors.New("BUG"),
),
commands: testutil.
CmdRunOutErr("kustomize build "+tmpDir.Root(), "", errors.New("BUG")),
shouldErr: true,
},
}
for _, test := range tests {
testutil.Run(t, test.description, func(t *testutil.T) {
t.Override(&util.DefaultExecCommand, test.commands)
t.Override(&kustomizeBinaryCheck, func() bool { return true })

k, err := NewKustomizeDeployer(&kustomizeConfig{
workingDir: tmpDir.Root(),
Expand Down Expand Up @@ -451,49 +480,49 @@ func TestKustomizeBuildCommandArgs(t *testing.T) {
description: "no BuildArgs, empty KustomizePaths ",
buildArgs: []string{},
kustomizePath: "",
expectedArgs: []string{"build"},
expectedArgs: nil,
},
{
description: "One BuildArg, empty KustomizePaths",
buildArgs: []string{"--foo"},
kustomizePath: "",
expectedArgs: []string{"build", "--foo"},
expectedArgs: []string{"--foo"},
},
{
description: "no BuildArgs, non-empty KustomizePaths",
buildArgs: []string{},
kustomizePath: "foo",
expectedArgs: []string{"build", "foo"},
expectedArgs: []string{"foo"},
},
{
description: "One BuildArg, non-empty KustomizePaths",
buildArgs: []string{"--foo"},
kustomizePath: "bar",
expectedArgs: []string{"build", "--foo", "bar"},
expectedArgs: []string{"--foo", "bar"},
},
{
description: "Multiple BuildArg, empty KustomizePaths",
buildArgs: []string{"--foo", "--bar"},
kustomizePath: "",
expectedArgs: []string{"build", "--foo", "--bar"},
expectedArgs: []string{"--foo", "--bar"},
},
{
description: "Multiple BuildArg with spaces, empty KustomizePaths",
buildArgs: []string{"--foo bar", "--baz"},
kustomizePath: "",
expectedArgs: []string{"build", "--foo", "bar", "--baz"},
expectedArgs: []string{"--foo", "bar", "--baz"},
},
{
description: "Multiple BuildArg with spaces, non-empty KustomizePaths",
buildArgs: []string{"--foo bar", "--baz"},
kustomizePath: "barfoo",
expectedArgs: []string{"build", "--foo", "bar", "--baz", "barfoo"},
expectedArgs: []string{"--foo", "bar", "--baz", "barfoo"},
},
{
description: "Multiple BuildArg no spaces, non-empty KustomizePaths",
buildArgs: []string{"--foo", "bar", "--baz"},
kustomizePath: "barfoo",
expectedArgs: []string{"build", "--foo", "bar", "--baz", "barfoo"},
expectedArgs: []string{"--foo", "bar", "--baz", "barfoo"},
},
}

Expand Down