From ade908d90fbfc77bad0cd66df4280c971557a071 Mon Sep 17 00:00:00 2001 From: Yuwen Ma Date: Wed, 23 Sep 2020 13:48:32 -0700 Subject: [PATCH] [kpt deployer] Customize the manipulated resource directory. * Add new config .kpt.fn.sinkDir * Remove unnecessary tmp dir if user does not want to export the resource output. --- docs/content/en/schemas/v2beta8.json | 8 ++- pkg/skaffold/deploy/kpt/kpt.go | 71 +++++++++++--------- pkg/skaffold/deploy/kpt/kpt_test.go | 96 ++++++++++++++-------------- pkg/skaffold/schema/latest/config.go | 3 + 4 files changed, 99 insertions(+), 79 deletions(-) diff --git a/docs/content/en/schemas/v2beta8.json b/docs/content/en/schemas/v2beta8.json index 58193eee051..4674dc459e9 100755 --- a/docs/content/en/schemas/v2beta8.json +++ b/docs/content/en/schemas/v2beta8.json @@ -1768,6 +1768,11 @@ "type": "string", "description": "docker network name to run the kpt function containers (default \"bridge\").", "x-intellij-html-description": "docker network name to run the kpt function containers (default "bridge")." + }, + "sinkDir": { + "type": "string", + "description": "directory to where the manipulated resource output is stored.", + "x-intellij-html-description": "directory to where the manipulated resource output is stored." } }, "preferredOrder": [ @@ -1776,7 +1781,8 @@ "networkName", "globalScope", "network", - "mount" + "mount", + "sinkDir" ], "additionalProperties": false, "description": "adds additional configurations used when calling `kpt fn`.", diff --git a/pkg/skaffold/deploy/kpt/kpt.go b/pkg/skaffold/deploy/kpt/kpt.go index cc1b96a7138..4eefc82fa4d 100644 --- a/pkg/skaffold/deploy/kpt/kpt.go +++ b/pkg/skaffold/deploy/kpt/kpt.go @@ -45,7 +45,7 @@ import ( const ( inventoryTemplate = "inventory-template.yaml" kptHydrated = ".kpt-hydrated" - pipeline = ".pipeline" + tmpSinkDir = ".tmp-sink-dir" kptFnAnnotation = "config.kubernetes.io/function" kptFnLocalConfig = "config.kubernetes.io/local-config" ) @@ -72,7 +72,12 @@ func NewDeployer(cfg types.Config, labels map[string]string) *Deployer { // outputs them to the applyDir, and runs `kpt live apply` against applyDir to create resources in the cluster. // `kpt live apply` supports automated pruning declaratively via resources in the applyDir. func (k *Deployer) Deploy(ctx context.Context, out io.Writer, builds []build.Artifact) ([]string, error) { - manifests, err := k.renderManifests(ctx, out, builds) + flags, err := k.getKptFnRunArgs() + if err != nil { + return []string{}, err + } + + manifests, err := k.renderManifests(ctx, out, builds, flags) if err != nil { return nil, err } @@ -93,6 +98,12 @@ func (k *Deployer) Deploy(ctx context.Context, out io.Writer, builds []build.Art } manifest.Write(manifests.String(), filepath.Join(applyDir, "resources.yaml"), out) + // Hydrated config is stored in applyDir, no need to dup the config in temporary sink dir. + if k.Fn.SinkDir == tmpSinkDir { + if err := os.RemoveAll(tmpSinkDir); err != nil { + return nil, fmt.Errorf("deleting temporary directory %s: %w", k.Fn.SinkDir, err) + } + } cmd := exec.CommandContext(ctx, "kpt", kptCommandArgs(applyDir, []string{"live", "apply"}, k.getKptLiveApplyArgs(), nil)...) cmd.Stdout = out @@ -149,7 +160,12 @@ func (k *Deployer) Cleanup(ctx context.Context, out io.Writer) error { // Render hydrates manifests using both kustomization and kpt functions. func (k *Deployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, _ bool, filepath string) error { - manifests, err := k.renderManifests(ctx, out, builds) + flags, err := k.getKptFnRunArgs() + if err != nil { + return err + } + + manifests, err := k.renderManifests(ctx, out, builds, flags) if err != nil { return err } @@ -160,21 +176,21 @@ func (k *Deployer) Render(ctx context.Context, out io.Writer, builds []build.Art // renderManifests handles a majority of the hydration process for manifests. // This involves reading configs from a source directory, running kustomize build, running kpt pipelines, // adding image digests, and adding run-id labels. -func (k *Deployer) renderManifests(ctx context.Context, _ io.Writer, builds []build.Artifact) (manifest.ManifestList, error) { +func (k *Deployer) renderManifests(ctx context.Context, _ io.Writer, builds []build.Artifact, + flags []string) (manifest.ManifestList, error) { debugHelpersRegistry, err := config.GetDebugHelpersRegistry(k.globalConfig) if err != nil { return nil, fmt.Errorf("retrieving debug helpers registry: %w", err) } + if k.Fn.SinkDir == "" { + k.Fn.SinkDir = tmpSinkDir + } - // .pipeline is a temp dir used to store output between steps of the desired workflow - // This can be removed once kpt can fully support the desired workflow independently. - if err := os.RemoveAll(filepath.Join(pipeline, k.Dir)); err != nil { - return nil, fmt.Errorf("deleting temporary directory %s: %w", filepath.Join(pipeline, k.Dir), err) + if err := os.RemoveAll(filepath.Join(k.Fn.SinkDir, k.Dir)); err != nil { + return nil, fmt.Errorf("deleting temporary directory %s: %w", filepath.Join(k.Fn.SinkDir, k.Dir), err) } - // 0755 is a permission setting where the owner can read, write, and execute. - // Others can read and execute but not modify the directory. - if err := os.MkdirAll(filepath.Join(pipeline, k.Dir), 0755); err != nil { - return nil, fmt.Errorf("creating temporary directory %s: %w", filepath.Join(pipeline, k.Dir), err) + if err := os.MkdirAll(filepath.Join(k.Fn.SinkDir, k.Dir), os.ModePerm); err != nil { + return nil, fmt.Errorf("creating temporary directory %s: %w", filepath.Join(k.Fn.SinkDir, k.Dir), err) } if err := k.readConfigs(ctx); err != nil { @@ -185,7 +201,7 @@ func (k *Deployer) renderManifests(ctx context.Context, _ io.Writer, builds []bu return nil, fmt.Errorf("kustomize build: %w", err) } - manifests, err := k.kptFnRun(ctx) + manifests, err := k.kptFnRun(ctx, flags) if err != nil { return nil, fmt.Errorf("running kpt functions: %w", err) } @@ -213,7 +229,7 @@ func (k *Deployer) renderManifests(ctx context.Context, _ io.Writer, builds []bu } // readConfigs uses `kpt fn source` to read config manifests from k.Dir -// and uses `kpt fn sink` to output those manifests to .pipeline. +// and uses `kpt fn sink` to output those manifests to sinkDir. func (k *Deployer) readConfigs(ctx context.Context) error { cmd := exec.CommandContext(ctx, "kpt", kptCommandArgs(k.Dir, []string{"fn", "source"}, nil, nil)...) b, err := util.RunCmdOut(cmd) @@ -221,7 +237,7 @@ func (k *Deployer) readConfigs(ctx context.Context) error { return err } - cmd = exec.CommandContext(ctx, "kpt", kptCommandArgs(filepath.Join(pipeline, k.Dir), []string{"fn", "sink"}, nil, nil)...) + cmd = exec.CommandContext(ctx, "kpt", kptCommandArgs(filepath.Join(k.Fn.SinkDir, k.Dir), []string{"fn", "sink"}, nil, nil)...) cmd.Stdin = bytes.NewBuffer(b) if _, err := util.RunCmdOut(cmd); err != nil { return err @@ -237,7 +253,8 @@ func (k *Deployer) kustomizeBuild(ctx context.Context) error { return nil } - cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, kustomize.BuildCommandArgs([]string{"-o", filepath.Join(pipeline, k.Dir)}, k.Dir)...)...) + cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, + kustomize.BuildCommandArgs([]string{"-o", filepath.Join(k.Fn.SinkDir, k.Dir)}, k.Dir)...)...) if _, err := util.RunCmdOut(cmd); err != nil { return err } @@ -247,9 +264,9 @@ func (k *Deployer) kustomizeBuild(ctx context.Context) error { return fmt.Errorf("finding kustomization dependencies: %w", err) } - // Kustomize build outputs hydrated configs to .pipeline, so the dry configs must be removed. + // Kustomize build outputs hydrated configs to sinkDir, so the dry configs must be removed. for _, v := range deps { - if err := os.RemoveAll(filepath.Join(pipeline, v)); err != nil { + if err := os.RemoveAll(filepath.Join(k.Fn.SinkDir, v)); err != nil { return err } } @@ -257,18 +274,12 @@ func (k *Deployer) kustomizeBuild(ctx context.Context) error { return nil } -// kptFnRun does a dry run with the specified kpt functions (fn-path XOR image) against .pipeline. -// If neither fn-path nor image are specified, functions will attempt to be discovered in .pipeline. +// kptFnRun does a dry run with the specified kpt functions (fn-path XOR image) against sinkDir. +// If neither fn-path nor image are specified, functions will attempt to be discovered in sinkDir. // An error occurs if both fn-path and image are specified. -func (k *Deployer) kptFnRun(ctx context.Context) (manifest.ManifestList, error) { +func (k *Deployer) kptFnRun(ctx context.Context, flags []string) (manifest.ManifestList, error) { var manifests manifest.ManifestList - - flags, err := k.getKptFnRunArgs() - if err != nil { - return nil, fmt.Errorf("getting kpt fn run args: %w", err) - } - - cmd := exec.CommandContext(ctx, "kpt", kptCommandArgs(pipeline, []string{"fn", "run"}, flags, nil)...) + cmd := exec.CommandContext(ctx, "kpt", kptCommandArgs(k.Fn.SinkDir, []string{"fn", "run"}, flags, nil)...) out, err := util.RunCmdOut(cmd) if err != nil { return nil, err @@ -282,7 +293,7 @@ func (k *Deployer) kptFnRun(ctx context.Context) (manifest.ManifestList, error) } // excludeKptFn adds an annotation "config.kubernetes.io/local-config: 'true'" to kpt function. -// This will exclude kpt functions from deployed to the cluster in kpt live apply. +// This will exclude kpt functions from deployed to the cluster in `kpt live apply`. func (k *Deployer) excludeKptFn(originalManifest manifest.ManifestList) (manifest.ManifestList, error) { var newManifest manifest.ManifestList for _, yByte := range originalManifest { @@ -332,7 +343,7 @@ func (k *Deployer) getApplyDir(ctx context.Context) (string, error) { // 0755 is a permission setting where the owner can read, write, and execute. // Others can read and execute but not modify the directory. - if err := os.MkdirAll(kptHydrated, 0755); err != nil { + if err := os.MkdirAll(kptHydrated, os.ModePerm); err != nil { return "", fmt.Errorf("applyDir was unspecified. creating applyDir: %w", err) } diff --git a/pkg/skaffold/deploy/kpt/kpt_test.go b/pkg/skaffold/deploy/kpt/kpt_test.go index 603342dbdd5..d413a2ff0ec 100644 --- a/pkg/skaffold/deploy/kpt/kpt_test.go +++ b/pkg/skaffold/deploy/kpt/kpt_test.go @@ -62,8 +62,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", ``), }, { description: "invalid manifest", @@ -72,8 +72,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", `foo`), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", `foo`), shouldErr: true, }, { @@ -88,8 +88,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output), shouldErr: true, }, { @@ -107,9 +107,9 @@ spec: - foo.yaml`}, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kustomize build -o .pipeline .", ``). - AndRunOut("kpt fn run .pipeline --dry-run --fn-path kpt-func.yaml", output). + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kustomize build -o .tmp-sink-dir .", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --fn-path kpt-func.yaml", output). AndRun("kpt live apply valid_path"), expected: []string{"default"}, }, @@ -120,8 +120,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output). + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output). AndRunOut("kpt live init .kpt-hydrated", ``). AndRunErr("kpt live apply .kpt-hydrated", errors.New("BUG")), shouldErr: true, @@ -142,8 +142,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output). + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output). AndRun("kpt live apply valid_path --poll-period 5s --reconcile-timeout 2m"), }, { @@ -162,8 +162,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output). + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output). AndRun("kpt live apply valid_path --poll-period foo --reconcile-timeout bar"), }, { @@ -182,8 +182,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output). + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output). AndRun("kpt live apply valid_path --prune-propagation-policy Orphan --prune-timeout 2m"), }, { @@ -202,8 +202,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output). + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output). AndRun("kpt live apply valid_path --prune-propagation-policy foo --prune-timeout bar"), }, } @@ -476,8 +476,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output1), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output1), expected: `apiVersion: v1 kind: Pod metadata: @@ -507,8 +507,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source test", ``). - AndRunOut(fmt.Sprintf("kpt fn sink %s", filepath.Join(".pipeline", "test")), ``). - AndRunOut("kpt fn run .pipeline --dry-run --fn-path kpt-func.yaml", output3), + AndRunOut(fmt.Sprintf("kpt fn sink %s", filepath.Join(".tmp-sink-dir", "test")), ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --fn-path kpt-func.yaml", output3), expected: `apiVersion: v1 kind: Pod metadata: @@ -550,8 +550,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", output2), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", output2), expected: `apiVersion: v1 kind: Pod metadata: @@ -578,8 +578,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", ``), expected: "\n", }, { @@ -592,7 +592,7 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``), shouldErr: true, }, { @@ -608,9 +608,9 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kustomize build -o .pipeline .", ``). - AndRunOut("kpt fn run .pipeline --dry-run", output1), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kustomize build -o .tmp-sink-dir .", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output1), kustomizations: map[string]string{"kustomization.yaml": `resources: - foo.yaml`}, expected: `apiVersion: v1 @@ -630,8 +630,8 @@ spec: }, commands: testutil. CmdRunOutErr("kpt fn source .", ``, errors.New("BUG")). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run", "invalid pipeline"), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", "invalid pipeline"), shouldErr: true, }, { @@ -641,8 +641,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOutErr("kpt fn sink .pipeline", ``, errors.New("BUG")). - AndRunOut("kpt fn run .pipeline --dry-run", "invalid pipeline"), + AndRunOutErr("kpt fn sink .tmp-sink-dir", ``, errors.New("BUG")). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", "invalid pipeline"), shouldErr: true, }, { @@ -658,9 +658,9 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOutErr("kustomize build -o .pipeline .", ``, errors.New("BUG")). - AndRunOut("kpt fn run .pipeline --dry-run", output1), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOutErr("kustomize build -o .tmp-sink-dir .", ``, errors.New("BUG")). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run", output1), kustomizations: map[string]string{"kustomization.yaml": `resources: - foo.yaml`}, shouldErr: true, @@ -672,8 +672,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOutErr("kpt fn run .pipeline --dry-run", "invalid pipeline", errors.New("BUG")), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOutErr("kpt fn run .tmp-sink-dir --dry-run", "invalid pipeline", errors.New("BUG")), shouldErr: true, }, { @@ -687,8 +687,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run --global-scope --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --global-scope --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), expected: "\n", }, { @@ -702,8 +702,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run --mount type=bind,src=$(pwd),dst=/source --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --mount type=bind,src=$(pwd),dst=/source --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), expected: "\n", }, { @@ -717,8 +717,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run --mount foo,,bar --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --mount foo,,bar --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), expected: "\n", }, { @@ -733,8 +733,8 @@ spec: }, commands: testutil. CmdRunOut("kpt fn source .", ``). - AndRunOut("kpt fn sink .pipeline", ``). - AndRunOut("kpt fn run .pipeline --dry-run --network --network-name foo --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), + AndRunOut("kpt fn sink .tmp-sink-dir", ``). + AndRunOut("kpt fn run .tmp-sink-dir --dry-run --network --network-name foo --image gcr.io/example.com/my-fn:v1.0.0 -- foo=bar", ``), expected: "\n", }, } diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go index 9e0cb09d2a8..c1f8b7336d1 100644 --- a/pkg/skaffold/schema/latest/config.go +++ b/pkg/skaffold/schema/latest/config.go @@ -572,6 +572,9 @@ type KptFn struct { // Mount is a list of storage options to mount to the fn image. Mount []string `yaml:"mount,omitempty"` + + // SinkDir is the directory to where the manipulated resource output is stored. + SinkDir string `yaml:"sinkDir,omitempty"` } // KptLive adds additional configurations used when calling `kpt live`.