From 7d1f3cc9efd40f5ca32992d48b30a01adecd2bc6 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 14 Dec 2023 19:41:38 -0600 Subject: [PATCH] feat: add `zarf dev deploy` for quickly testing packages and restructure `zarf prepare` (#2170) ## Description Adds a `zarf dev deploy` command that will create + deploy a local package in one shot. ## Related Issue Fixes #2169 Fixes #2098 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Other (security config, docs update, etc) ## Checklist before merging - [x] Test, docs, adr added or updated as needed - [x] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow) followed --------- Signed-off-by: razzle Co-authored-by: Wayne Starr --- README.md | 2 +- adr/0022-dev-cmd.md | 42 + docs/0-zarf-overview.md | 2 +- docs/2-the-zarf-cli/100-cli-commands/zarf.md | 2 +- .../{zarf_prepare.md => zarf_dev.md} | 17 +- .../100-cli-commands/zarf_dev_deploy.md | 41 + ...find-images.md => zarf_dev_find-images.md} | 10 +- ...-config.md => zarf_dev_generate-config.md} | 6 +- ...{zarf_prepare_lint.md => zarf_dev_lint.md} | 6 +- ...are_patch-git.md => zarf_dev_patch-git.md} | 6 +- ...are_sha256sum.md => zarf_dev_sha256sum.md} | 6 +- docs/2-the-zarf-cli/2-zarf-config-files.md | 2 +- docs/2-the-zarf-cli/index.md | 2 +- .../1-zarf-packages.md | 1 + docs/3-create-a-zarf-package/10-dev.md | 57 ++ .../2-zarf-components.md | 4 +- .../3-zarf-init-package.md | 2 +- .../6-package-sboms.md | 1 + docs/3-create-a-zarf-package/8-vscode.md | 1 + docs/3-create-a-zarf-package/index.md | 4 +- .../0-creating-a-zarf-package.md | 10 +- src/cmd/common/viper.go | 4 + src/cmd/{prepare.go => dev.go} | 94 ++- src/cmd/internal.go | 7 +- src/cmd/package.go | 1 + src/config/lang/english.go | 13 +- src/pkg/packager/create.go | 703 +--------------- src/pkg/packager/create_stages.go | 761 ++++++++++++++++++ src/pkg/packager/deploy.go | 16 +- src/pkg/packager/dev.go | 97 +++ src/pkg/packager/publish.go | 60 +- src/test/e2e/99_yolo_test.go | 14 + src/types/runtime.go | 6 +- 33 files changed, 1177 insertions(+), 823 deletions(-) create mode 100644 adr/0022-dev-cmd.md rename docs/2-the-zarf-cli/100-cli-commands/{zarf_prepare.md => zarf_dev.md} (60%) create mode 100644 docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md rename docs/2-the-zarf-cli/100-cli-commands/{zarf_prepare_find-images.md => zarf_dev_find-images.md} (85%) rename docs/2-the-zarf-cli/100-cli-commands/{zarf_prepare_generate-config.md => zarf_dev_generate-config.md} (90%) rename docs/2-the-zarf-cli/100-cli-commands/{zarf_prepare_lint.md => zarf_dev_lint.md} (89%) rename docs/2-the-zarf-cli/100-cli-commands/{zarf_prepare_patch-git.md => zarf_dev_patch-git.md} (90%) rename docs/2-the-zarf-cli/100-cli-commands/{zarf_prepare_sha256sum.md => zarf_dev_sha256sum.md} (89%) create mode 100644 docs/3-create-a-zarf-package/10-dev.md rename src/cmd/{prepare.go => dev.go} (64%) create mode 100644 src/pkg/packager/create_stages.go create mode 100644 src/pkg/packager/dev.go diff --git a/README.md b/README.md index a5b3518f51..54f61185e1 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Zarf eliminates the [complexity of air gap software delivery](https://www.itopst - Built-in Docker registry - Builtin [K9s Dashboard](https://k9scli.io/) for managing a cluster from the terminal - [Mutating Webhook](adr/0005-mutating-webhook.md) to automatically update Kubernetes pod's image path and pull secrets as well as [Flux Git Repository](https://fluxcd.io/docs/components/source/gitrepositories/) URLs and secret references -- Builtin [command to find images](https://docs.zarf.dev/docs/the-zarf-cli/cli-commands/zarf_prepare_find-images) and resources from a Helm chart +- Builtin [command to find images](https://docs.zarf.dev/docs/the-zarf-cli/cli-commands/zarf_dev_find-images) and resources from a Helm chart - Tunneling capability to [connect to Kubernetes resources](https://docs.zarf.dev/docs/the-zarf-cli/cli-commands/zarf_connect) without network routing, DNS, TLS or Ingress configuration required ## 🛠️ Configurable Features diff --git a/adr/0022-dev-cmd.md b/adr/0022-dev-cmd.md new file mode 100644 index 0000000000..5939aea526 --- /dev/null +++ b/adr/0022-dev-cmd.md @@ -0,0 +1,42 @@ +# 22. Introduce `dev` command + +Date: 2023-12-03 + +## Status + +Accepted + +## Context + +> Feature request: + +The current package development lifecycle is: + +1. Create a `zarf.yaml` file and add components +2. Create a package with `zarf package create ` +3. Debug any create errors and resolve by editing `zarf.yaml` and repeating step 2 +4. Run `zarf init` to initialize the cluster +5. Deploy the package with `zarf package deploy ` +6. Debug any deploy errors and resolve by editing `zarf.yaml` and repeating step 2 or 5 + +If there are deployment errors, the common pattern is to reset the cluster (ex: `k3d cluster delete && k3d cluster create`) and repeat steps 4-6. Re-initializing the cluster, recreating the package, and redeploying the package is tedious and time consuming; especially when the package is large or the change was small. + +`zarf package create` is designed around air-gapped environments where the package is created in one environment and deployed in another. Due to this architecture, a package's dependencies _must_ be retrieved and assembled _each_ and _every_ time. + +There already exists the concept of [`YOLO` mode](0010-yolo-mode.md), which can build + deploy a package without the need for `zarf init`, and builds without fetching certain heavy dependencies (like Docker images). However, `YOLO` mode is not exposed via CLI flags, and is meant to develop and deploy packages in fully connected environments. + +## Decision + +Introduce a `dev deploy` command that will combine the lifecycle of `package create` and `package deploy` into a single command. This command will: + +- Not result in a re-usable tarball / OCI artifact +- Not have any interactive prompts +- Not require `zarf init` to be run by default (override-able with a CLI flag) +- Be able to create+deploy a package in either YOLO mode (default) or prod mode (exposed via a CLI flag) +- Only build + deploy components that _will_ be deployed (contrasting with `package create` which builds _all_ components regardless of whether they will be deployed) + +## Consequences + +The `dev deploy` command will make it easier to develop and deploy packages in connected **development** environments. It will also make it easier to debug deployment errors by allowing the user to iterate on the package without having to re-initialize the cluster, re-build and re-deploy the package each cycle. + +There is a purpose to placing this functionality behind a new command, rather than adding it to `package create`. Commands under `dev` are meant to be used in **development** environments, and are **not** meant to be used in **production** environments. There is still the possibility that a user will use `dev deploy` in a production environment, but the command name and documentation will make it clear that this is not the intended use case. diff --git a/docs/0-zarf-overview.md b/docs/0-zarf-overview.md index d9234c1d50..98d2b5f97a 100644 --- a/docs/0-zarf-overview.md +++ b/docs/0-zarf-overview.md @@ -148,7 +148,7 @@ In this use case, you configure Zarf to initialize a cluster that already exists - Builtin Docker registry - Builtin [K9s Dashboard](https://k9scli.io/) for managing a cluster from the terminal - [Mutating Webhook](adr/0005-mutating-webhook.md) to automatically update Kubernetes pod's image path and pull secrets as well as [Flux Git Repository](https://fluxcd.io/docs/components/source/gitrepositories/) URLs and secret references -- Builtin [command to find images](./2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md) and resources from a Helm chart +- Builtin [command to find images](./2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md) and resources from a Helm chart - Tunneling capability to [connect to Kuberenetes resources](./2-the-zarf-cli/100-cli-commands/zarf_connect.md) without network routing, DNS, TLS or Ingress configuration required ### 🛠️ Configurable Features diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf.md b/docs/2-the-zarf-cli/100-cli-commands/zarf.md index 5fa846e461..f2794aa98e 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf.md @@ -31,8 +31,8 @@ zarf COMMAND [flags] * [zarf completion](zarf_completion.md) - Generate the autocompletion script for the specified shell * [zarf connect](zarf_connect.md) - Accesses services or pods deployed in the cluster * [zarf destroy](zarf_destroy.md) - Tears down Zarf and removes its components from the environment +* [zarf dev](zarf_dev.md) - Commands useful for developing packages * [zarf init](zarf_init.md) - Prepares a k8s cluster for the deployment of Zarf packages * [zarf package](zarf_package.md) - Zarf package commands for creating, deploying, and inspecting packages -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging * [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier * [zarf version](zarf_version.md) - Shows the version of the running Zarf binary diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md similarity index 60% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index 837e4c03ba..ae76b14632 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -1,12 +1,12 @@ -# zarf prepare +# zarf dev -Tools to help prepare assets for packaging +Commands useful for developing packages ## Options ``` - -h, --help help for prepare + -h, --help help for dev ``` ## Options inherited from parent commands @@ -25,9 +25,10 @@ Tools to help prepare assets for packaging ## SEE ALSO * [zarf](zarf.md) - DevSecOps for Airgap -* [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests -* [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Verifies the package schema -* [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: +* [zarf dev deploy](zarf_dev_deploy.md) - [beta] Creates and deploys a Zarf package from a given directory +* [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests +* [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf +* [zarf dev lint](zarf_dev_lint.md) - Verifies the package schema +* [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. -* [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file +* [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md new file mode 100644 index 0000000000..8b1700f426 --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md @@ -0,0 +1,41 @@ +# zarf dev deploy + + +[beta] Creates and deploys a Zarf package from a given directory + +## Synopsis + +[beta] Creates and deploys a Zarf package from a given directory, setting options like YOLO mode for faster iteration. + +``` +zarf dev deploy [flags] +``` + +## Options + +``` + --components string Comma-separated list of components to install. Adding this flag will skip the init prompts for which components to install + --create-set stringToString Specify package variables to set on the command line (KEY=value) (default []) + --deploy-set stringToString Specify deployment variables to set on the command line (KEY=value) (default []) + -f, --flavor string The flavor of components to include in the resulting package (i.e. have a matching or empty "only.flavor" key) + -h, --help help for deploy + --no-yolo Disable the YOLO mode default override and create / deploy the package as-defined + --registry-override stringToString Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default []) +``` + +## Options inherited from parent commands + +``` + -a, --architecture string Architecture for OCI images and Zarf packages + --insecure Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture. + -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") + --no-color Disable colors in output + --no-log-file Disable log file creation + --no-progress Disable fancy UI progress bars, spinners, logos, etc + --tmpdir string Specify the temporary directory to use for intermediate files + --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") +``` + +## SEE ALSO + +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md similarity index 85% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md index 6e2a1d84e4..01e43dfbcb 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md @@ -1,16 +1,16 @@ -# zarf prepare find-images +# zarf dev find-images -Evaluates components in a zarf file to identify images specified in their helm charts and manifests +Evaluates components in a Zarf file to identify images specified in their helm charts and manifests ## Synopsis -Evaluates components in a zarf file to identify images specified in their helm charts and manifests. +Evaluates components in a Zarf file to identify images specified in their helm charts and manifests. Components that have repos that host helm charts can be processed by providing the --repo-chart-path. ``` -zarf prepare find-images [ PACKAGE ] [flags] +zarf dev find-images [ PACKAGE ] [flags] ``` ## Options @@ -37,4 +37,4 @@ zarf prepare find-images [ PACKAGE ] [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_generate-config.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_generate-config.md similarity index 90% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_generate-config.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_generate-config.md index 6a12b68380..4005e20995 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_generate-config.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_generate-config.md @@ -1,4 +1,4 @@ -# zarf prepare generate-config +# zarf dev generate-config Generates a config file for Zarf @@ -13,7 +13,7 @@ Accepted extensions are json, toml, yaml. NOTE: This file must not already exist. If no filename is provided, the config will be written to the current working directory as zarf-config.toml. ``` -zarf prepare generate-config [ FILENAME ] [flags] +zarf dev generate-config [ FILENAME ] [flags] ``` ## Options @@ -37,4 +37,4 @@ zarf prepare generate-config [ FILENAME ] [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md similarity index 89% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md index bb36206a1d..209b7f1b4a 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md @@ -1,4 +1,4 @@ -# zarf prepare lint +# zarf dev lint Verifies the package schema @@ -8,7 +8,7 @@ Verifies the package schema Verifies the package schema and warns the user if they have variables that won't be evaluated ``` -zarf prepare lint [ DIRECTORY ] [flags] +zarf dev lint [ DIRECTORY ] [flags] ``` ## Options @@ -32,4 +32,4 @@ zarf prepare lint [ DIRECTORY ] [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_patch-git.md similarity index 90% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_patch-git.md index 022e93eb72..808df2ba07 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_patch-git.md @@ -1,11 +1,11 @@ -# zarf prepare patch-git +# zarf dev patch-git Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. ``` -zarf prepare patch-git HOST FILE [flags] +zarf dev patch-git HOST FILE [flags] ``` ## Options @@ -30,4 +30,4 @@ zarf prepare patch-git HOST FILE [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_sha256sum.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_sha256sum.md similarity index 89% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_sha256sum.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_sha256sum.md index 4b64ade810..4be9cbcd30 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_sha256sum.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_sha256sum.md @@ -1,10 +1,10 @@ -# zarf prepare sha256sum +# zarf dev sha256sum Generates a SHA256SUM for the given file ``` -zarf prepare sha256sum { FILE | URL } [flags] +zarf dev sha256sum { FILE | URL } [flags] ``` ## Options @@ -29,4 +29,4 @@ zarf prepare sha256sum { FILE | URL } [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/2-zarf-config-files.md b/docs/2-the-zarf-cli/2-zarf-config-files.md index 9aa9852fd7..650abaed3a 100644 --- a/docs/2-the-zarf-cli/2-zarf-config-files.md +++ b/docs/2-the-zarf-cli/2-zarf-config-files.md @@ -8,7 +8,7 @@ import FetchFileCodeBlock from '@site/src/components/FetchFileCodeBlock'; Users can use a config file to easily control flags for `zarf init`, `zarf package create`, and `zarf package deploy` commands, as well as global flags (excluding `--confirm`), enabling a more straightforward and declarative workflow. -Zarf supports config files written in common configuration file formats including `toml`, `json`, `yaml`, `ini` and `props`, and by default Zarf will look for a file called `zarf-config` with one of these filenames in the current working directory. To generate a blank config file you can run `zarf prepare generate-config` with an optional output filename/format. For example, to create an empty config file with the `my-cool-env` in the yaml format, you can use `zarf prepare generate-config my-cool-env.yaml`. +Zarf supports config files written in common configuration file formats including `toml`, `json`, `yaml`, `ini` and `props`, and by default Zarf will look for a file called `zarf-config` with one of these filenames in the current working directory. To generate a blank config file you can run `zarf dev generate-config` with an optional output filename/format. For example, to create an empty config file with the `my-cool-env` in the yaml format, you can use `zarf dev generate-config my-cool-env.yaml`. To use a custom config filename, set the `ZARF_CONFIG` environment variable to the config file's path. For example, to use the `my-cool-env.yaml` config file in the current working directory, you can set the `ZARF_CONFIG` environment variable to `my-cool-env.yaml`. The `ZARF_CONFIG` environment variable can be set either in the shell or in a `.env` file in the current working directory. Note that the `ZARF_CONFIG` environment variable takes precedence over the default config file path. diff --git a/docs/2-the-zarf-cli/index.md b/docs/2-the-zarf-cli/index.md index 3992d368e8..0e774c88fb 100644 --- a/docs/2-the-zarf-cli/index.md +++ b/docs/2-the-zarf-cli/index.md @@ -34,6 +34,6 @@ The `zarf package create` command is used to create a Zarf package from a `zarf. :::tip -When deploying and managing packages you may find the sub-commands under `zarf prepare` useful to find resources and manipulate package definitions as needed. +When developing packages you may find the sub-commands under `zarf dev` useful to find resources and manipulate package definitions. ::: diff --git a/docs/3-create-a-zarf-package/1-zarf-packages.md b/docs/3-create-a-zarf-package/1-zarf-packages.md index 6595ac1bef..610b3c14d7 100644 --- a/docs/3-create-a-zarf-package/1-zarf-packages.md +++ b/docs/3-create-a-zarf-package/1-zarf-packages.md @@ -68,6 +68,7 @@ When executed, the `zarf package create` command locates the `zarf.yaml` file in The process of defining and creating a package is also elaborated on in detail in the [Creating a Zarf Package Tutorial](../5-zarf-tutorials/0-creating-a-zarf-package.md). ### Creating Differential Packages + If you already have a Zarf package and you want to create an updated package you would normally have to re-create the entire package from scratch, including things that might not have changed. Depending on your workflow, you may want to create a package that only contains the artifacts that have changed since the last time you built your package. This can be achieved by using the `--differential` flag while running the `zarf package create` command. You can use this flag to point to an already built package you have locally or to a package that has been previously [published](../5-zarf-tutorials/7-publish-and-deploy.md#publish-package) to a registry. ## Inspecting a Created Package diff --git a/docs/3-create-a-zarf-package/10-dev.md b/docs/3-create-a-zarf-package/10-dev.md new file mode 100644 index 0000000000..9f7b132887 --- /dev/null +++ b/docs/3-create-a-zarf-package/10-dev.md @@ -0,0 +1,57 @@ +# Developing Zarf Packages + +## `dev` Commands + +Zarf contains many commands that are useful while developing a Zarf package to iterate on configuration, discover resources and more! Below are explanations of some of these commands with the full list discoverable with `zarf dev --help`. + +:::caution + +The `dev` commands are meant to be used in **development** environments / workflows. They are **not** meant to be used in **production** environments / workflows. + +::: + +### `dev deploy` + +The `dev deploy` command will combine the lifecycle of `package create` and `package deploy` into a single command. This command will: + +- Not result in a re-usable tarball / OCI artifact +- Not have any interactive prompts +- Not require `zarf init` to be run (by default, but _is required_ if `--no-yolo` is not set) +- Be able to create+deploy a package in either YOLO mode (default) or prod mode (exposed via `--no-yolo` flag) +- Only build + deploy components that _will_ be deployed (contrasting with `package create` which builds _all_ components regardless of whether they will be deployed) + +```bash +# Create and deploy dos-games in yolo mode +$ zarf dev deploy examples/dos-games +``` + +```bash +# If deploying a package in prod mode, `zarf init` must be run first +$ zarf init --confirm +# Create and deploy dos-games in prod mode +$ zarf dev deploy examples/dos-games --no-yolo +``` + +### `dev find-images` + +Evaluates components in a `zarf.yaml` to identify images specified in their helm charts and manifests. + +Components that have `git` repositories that host helm charts can be processed by providing the `--repo-chart-path`. + +```bash +$ zarf dev find-images examples/wordpress + +components: + + - name: wordpress + images: + - docker.io/bitnami/apache-exporter:0.13.3-debian-11-r2 + - docker.io/bitnami/mariadb:10.11.2-debian-11-r21 + - docker.io/bitnami/wordpress:6.2.0-debian-11-r18 +``` + +### Misc `dev` Commands + +Not all `dev` commands have been mentioned here. + +Further `dev` commands can be discovered by running `zarf dev --help`. diff --git a/docs/3-create-a-zarf-package/2-zarf-components.md b/docs/3-create-a-zarf-package/2-zarf-components.md index df26966412..d10acf2368 100644 --- a/docs/3-create-a-zarf-package/2-zarf-components.md +++ b/docs/3-create-a-zarf-package/2-zarf-components.md @@ -116,11 +116,11 @@ Kustomizations are handled a bit differently than normal manifests in that Zarf -Images can either be discovered manually, or automatically by using [`zarf prepare find-images`](../2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md). +Images can either be discovered manually, or automatically by using [`zarf dev find-images`](../2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md). :::note -`zarf prepare find-images` will find images for most standard manifests, kustomizations, and helm charts, however some images cannot be discovered this way as some upstream resources (like operators) may bury image definitions inside. For these images, `zarf prepare find-images` also offers support for the draft [Helm Improvement Proposal 15](https://github.com/helm/community/blob/main/hips/hip-0015.md) which allows chart creators to annotate any hidden images in their charts along with the [values conditions](https://github.com/helm/community/issues/277) that will cause those images to be used. +`zarf dev find-images` will find images for most standard manifests, kustomizations, and helm charts, however some images cannot be discovered this way as some upstream resources (like operators) may bury image definitions inside. For these images, `zarf dev find-images` also offers support for the draft [Helm Improvement Proposal 15](https://github.com/helm/community/blob/main/hips/hip-0015.md) which allows chart creators to annotate any hidden images in their charts along with the [values conditions](https://github.com/helm/community/issues/277) that will cause those images to be used. ::: diff --git a/docs/3-create-a-zarf-package/3-zarf-init-package.md b/docs/3-create-a-zarf-package/3-zarf-init-package.md index f402349786..f405193893 100644 --- a/docs/3-create-a-zarf-package/3-zarf-init-package.md +++ b/docs/3-create-a-zarf-package/3-zarf-init-package.md @@ -72,7 +72,7 @@ There are two ways to deploy these optional components. First, you can provide a The `k3s` component included in Zarf differs from the default `k3s` install in that it disables the installation of `traefik` out of the box. This was done so that people could more intentionally choose if they wanted `traefik` or another ingress provider (or no ingress at all) depending on their needs. If you would like to return `k3s` to its defaults, you can set the `K3S_ARGS` zarf variable to an empty string: -``` +```text root@machine ~ # zarf init --components k3s --set K3S_ARGS="" --confirm ``` diff --git a/docs/3-create-a-zarf-package/6-package-sboms.md b/docs/3-create-a-zarf-package/6-package-sboms.md index cc84e4244a..f9826f0374 100644 --- a/docs/3-create-a-zarf-package/6-package-sboms.md +++ b/docs/3-create-a-zarf-package/6-package-sboms.md @@ -29,6 +29,7 @@ Zarf uses the `file:` Syft SBOM scheme even if given a directory as the `files` Given the Syft CLI is vendored into Zarf you can run these commands with the Zarf binary as well: ```bash +# Syft is vendored as `zarf tools sbom` $ zarf tools sbom packages file:path/to/yourproject/file -o json > my-sbom.json ``` diff --git a/docs/3-create-a-zarf-package/8-vscode.md b/docs/3-create-a-zarf-package/8-vscode.md index 79e10ce663..896c898d3c 100644 --- a/docs/3-create-a-zarf-package/8-vscode.md +++ b/docs/3-create-a-zarf-package/8-vscode.md @@ -15,6 +15,7 @@ Zarf uses the [Zarf package schema](https://github.com/defenseunicorns/zarf/blob "https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json": "zarf.yaml" } ``` + :::note When successfully installed, the `yaml.schema` line will match the color of the other lines within the settings. diff --git a/docs/3-create-a-zarf-package/index.md b/docs/3-create-a-zarf-package/index.md index 692f37a9db..ce5f4f6ff8 100644 --- a/docs/3-create-a-zarf-package/index.md +++ b/docs/3-create-a-zarf-package/index.md @@ -16,13 +16,13 @@ To learn more about creating a Zarf package, you can check out the following res - [The Package Create Lifecycle](./5-package-create-lifecycle.md): An overview of the lifecycle of `zarf package create`. - [Creating a Zarf Package Tutorial](../5-zarf-tutorials/0-creating-a-zarf-package.md): A tutorial covering how to take an application and create a package for it. -## Typical Creation Workflow: +## Typical Creation Workflow The general flow of a Zarf package deployment on an existing initialized cluster is as follows: ```shell # Before creating your package you can lint your zarf.yaml -$ zarf prepare lint +$ zarf dev lint # To create a package run the following: $ zarf package create diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index f311a4d387..e398df6933 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -2,7 +2,7 @@ ## Introduction -In this tutorial, we will demonstrate the process to create a Zarf package for an application from defining a `zarf.yaml`, finding resources with `zarf prepare` commands and finally building the package with `zarf package create`. +In this tutorial, we will demonstrate the process to create a Zarf package for an application from defining a `zarf.yaml`, finding resources with `zarf dev` commands and finally building the package with `zarf package create`. When creating a Zarf package, you must have a network connection so that Zarf can fetch all of the dependencies and resources necessary to build the package. If your package is using images from a private registry or is referencing repositories in a private repository, you will need to have your credentials configured on your machine for Zarf to be able to fetch the resources. @@ -37,7 +37,7 @@ metadata: :::tip If you are using an Integrated Development Environment (such as [VS Code](../3-create-a-zarf-package/8-vscode.md)) to create and edit the `zarf.yaml` file, you can install or reference the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) file to get error checking and autocomplete. -Additionally, you can run `zarf prepare lint ` to validate aginst the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) +Additionally, you can run `zarf dev lint ` to validate aginst the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) ::: @@ -83,7 +83,7 @@ service: :::note -We create any `values.yaml` file(s) at this stage because the `zarf prepare find-images` command we will use next will template out this chart to look only for the images we need. +We create any `values.yaml` file(s) at this stage because the `zarf dev find-images` command we will use next will template out this chart to look only for the images we need. ::: @@ -95,7 +95,7 @@ Note that we are explicitly defining the `wordpress` namespace for this deployme ### Finding the Images -Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf prepare find-images`. Running this command in the directory of your zarf.yaml will result in the following output: +Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf dev find-images`. Running this command in the directory of your zarf.yaml will result in the following output: @@ -109,7 +109,7 @@ Due to the way some applications are deployed, Zarf might not be able to find al :::tip -Zarf has more `prepare` commands you can learn about on the [prepare CLI docs page](../2-the-zarf-cli/100-cli-commands/zarf_prepare.md). +Zarf has more `dev` commands you can learn about on the [dev CLI docs page](../3-create-a-zarf-package/10-dev.md). ::: diff --git a/src/cmd/common/viper.go b/src/cmd/common/viper.go index 4b040179b9..34d6d0888b 100644 --- a/src/cmd/common/viper.go +++ b/src/cmd/common/viper.go @@ -93,6 +93,10 @@ const ( // Package pull config keys VPkgPullOutputDir = "package.pull.output_directory" + + // Dev deploy config keys + + VDevDeployNoYolo = "dev.deploy.no_yolo" ) var ( diff --git a/src/cmd/prepare.go b/src/cmd/dev.go similarity index 64% rename from src/cmd/prepare.go rename to src/cmd/dev.go index fcd5e113c8..0b76ee55f8 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/dev.go @@ -23,17 +23,52 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/mholt/archiver/v3" "github.com/spf13/cobra" + "github.com/spf13/viper" ) var extractPath string -var prepareCmd = &cobra.Command{ - Use: "prepare", - Aliases: []string{"prep"}, - Short: lang.CmdPrepareShort, +var devCmd = &cobra.Command{ + Use: "dev", + Aliases: []string{"prepare", "prep"}, + Short: lang.CmdDevShort, } -var prepareTransformGitLinks = &cobra.Command{ +var devDeployCmd = &cobra.Command{ + Use: "deploy", + Args: cobra.MaximumNArgs(1), + Short: lang.CmdDevDeployShort, + Long: lang.CmdDevDeployLong, + Run: func(cmd *cobra.Command, args []string) { + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + var err error + pkgConfig.CreateOpts.BaseDir, err = os.Getwd() + if err != nil { + message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) + } + } + + v := common.GetViper() + pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) + + pkgConfig.PkgOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgDeploySet), pkgConfig.PkgOpts.SetVariables, strings.ToUpper) + + // Configure the packager + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + + // Create the package + if err := pkgClient.DevDeploy(); err != nil { + message.Fatalf(err, lang.CmdDevDeployErr, err.Error()) + } + }, +} + +var devTransformGitLinksCmd = &cobra.Command{ Use: "patch-git HOST FILE", Aliases: []string{"p"}, Short: lang.CmdPreparePatchGitShort, @@ -76,7 +111,7 @@ var prepareTransformGitLinks = &cobra.Command{ }, } -var prepareComputeFileSha256sum = &cobra.Command{ +var devSha256SumCmd = &cobra.Command{ Use: "sha256sum { FILE | URL }", Aliases: []string{"s"}, Short: lang.CmdPrepareSha256sumShort, @@ -151,7 +186,7 @@ var prepareComputeFileSha256sum = &cobra.Command{ }, } -var prepareFindImages = &cobra.Command{ +var devFindImagesCmd = &cobra.Command{ Use: "find-images [ PACKAGE ]", Aliases: []string{"f"}, Args: cobra.MaximumNArgs(1), @@ -185,7 +220,7 @@ var prepareFindImages = &cobra.Command{ }, } -var prepareGenerateConfigFile = &cobra.Command{ +var devGenConfigFileCmd = &cobra.Command{ Use: "generate-config [ FILENAME ]", Aliases: []string{"gc"}, Args: cobra.MaximumNArgs(1), @@ -206,7 +241,7 @@ var prepareGenerateConfigFile = &cobra.Command{ }, } -var lintCmd = &cobra.Command{ +var devLintCmd = &cobra.Command{ Use: "lint [ DIRECTORY ]", Args: cobra.MaximumNArgs(1), Aliases: []string{"l"}, @@ -235,22 +270,39 @@ var lintCmd = &cobra.Command{ } func init() { - v := common.InitViper() + v := common.GetViper() + rootCmd.AddCommand(devCmd) - rootCmd.AddCommand(prepareCmd) - prepareCmd.AddCommand(prepareTransformGitLinks) - prepareCmd.AddCommand(prepareComputeFileSha256sum) - prepareCmd.AddCommand(prepareFindImages) - prepareCmd.AddCommand(prepareGenerateConfigFile) - prepareCmd.AddCommand(lintCmd) + devCmd.AddCommand(devDeployCmd) + devCmd.AddCommand(devTransformGitLinksCmd) + devCmd.AddCommand(devSha256SumCmd) + devCmd.AddCommand(devFindImagesCmd) + devCmd.AddCommand(devGenConfigFileCmd) + devCmd.AddCommand(devLintCmd) - prepareComputeFileSha256sum.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdPrepareFlagExtractPath) + bindDevDeployFlags(v) - prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) + devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdPrepareFlagExtractPath) + + devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables - prepareFindImages.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPrepareFlagSet) + devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPrepareFlagSet) // allow for the override of the default helm KubeVersion - prepareFindImages.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) + devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) + + devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) +} + +func bindDevDeployFlags(v *viper.Viper) { + devDeployFlags := devDeployCmd.Flags() + + devDeployFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "create-set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) + devDeployFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(common.VPkgCreateRegistryOverride), lang.CmdPackageCreateFlagRegistryOverride) + devDeployFlags.StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor) + + devDeployFlags.StringToStringVar(&pkgConfig.PkgOpts.SetVariables, "deploy-set", v.GetStringMapString(common.VPkgDeploySet), lang.CmdPackageDeployFlagSet) + + devDeployFlags.StringVar(&pkgConfig.PkgOpts.OptionalComponents, "components", v.GetString(common.VPkgDeployComponents), lang.CmdPackageDeployFlagComponents) - prepareTransformGitLinks.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) + devDeployFlags.BoolVar(&pkgConfig.CreateOpts.NoYOLO, "no-yolo", v.GetBool(common.VDevDeployNoYolo), lang.CmdDevDeployFlagNoYolo) } diff --git a/src/cmd/internal.go b/src/cmd/internal.go index 3dd40230f9..bb2981d13e 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -24,10 +24,9 @@ import ( ) var internalCmd = &cobra.Command{ - Use: "internal", - Aliases: []string{"dev"}, - Hidden: true, - Short: lang.CmdInternalShort, + Use: "internal", + Hidden: true, + Short: lang.CmdInternalShort, } var agentCmd = &cobra.Command{ diff --git a/src/cmd/package.go b/src/cmd/package.go index 42fc9fbc34..c626412beb 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -226,6 +226,7 @@ var packagePublishCmd = &cobra.Command{ if utils.IsDir(pkgConfig.PkgOpts.PackageSource) { pkgConfig.CreateOpts.BaseDir = pkgConfig.PkgOpts.PackageSource + pkgConfig.CreateOpts.IsSkeleton = true } pkgConfig.PublishOpts.PackageDestination = ref.String() diff --git a/src/config/lang/english.go b/src/config/lang/english.go index d14bf0b23d..47563cf656 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -328,8 +328,13 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPackageClusterSourceFallback = "%q does not satisfy any current sources, assuming it is a package deployed to a cluster" CmdPackageInvalidSource = "Unable to identify source from %q: %s" - // zarf prepare - CmdPrepareShort = "Tools to help prepare assets for packaging" + // zarf dev (prepare is an alias for dev) + CmdDevShort = "Commands useful for developing packages" + + CmdDevDeployShort = "[beta] Creates and deploys a Zarf package from a given directory" + CmdDevDeployLong = "[beta] Creates and deploys a Zarf package from a given directory, setting options like YOLO mode for faster iteration." + CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" + CmdDevDeployErr = "Failed to dev deploy: %s" CmdPreparePatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + "This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook." @@ -342,8 +347,8 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPrepareSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." CmdPrepareSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" - CmdPrepareFindImagesShort = "Evaluates components in a zarf file to identify images specified in their helm charts and manifests" - CmdPrepareFindImagesLong = "Evaluates components in a zarf file to identify images specified in their helm charts and manifests.\n\n" + + CmdPrepareFindImagesShort = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests" + CmdPrepareFindImagesLong = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests.\n\n" + "Components that have repos that host helm charts can be processed by providing the --repo-chart-path." CmdPrepareFindImagesErr = "Unable to find images: %s" diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index f39947bffc..02f13b240b 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -5,102 +5,29 @@ package packager import ( - "errors" "fmt" "os" - "path/filepath" - "strconv" - "strings" - "time" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" - "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/images" - "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver/v3" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { - if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - - // Load the images and repos from the 'reference' package - if err := p.loadDifferentialData(); err != nil { - return err - } - cwd, err := os.Getwd() if err != nil { return err } - if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { - return fmt.Errorf("unable to access directory '%s': %w", p.cfg.CreateOpts.BaseDir, err) - } - message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { return err } - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, p.arch) - if err != nil { - return err - } - err = p.setOCIRemote(ref) - if err != nil { - return err - } - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { + if err := p.load(); err != nil { return err } - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := p.removeCopiesFromDifferentialPackage(); err != nil { - return err - } - } - // Perform early package validation. if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) @@ -110,632 +37,14 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - componentSBOMs := map[string]*layout.ComponentSBOM{} - var combinedImageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - isSkeleton := false - if err := p.addComponent(idx, component, isSkeleton); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - onFailure() - return fmt.Errorf("unable to create component SBOM: %w", err) - } - - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - combinedImageList = append(combinedImageList, refInfo) - } - } - - imageList := helpers.Unique(combinedImageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after 3 attempts: %w", err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if there the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - // Process the component directories into compressed tarballs - // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range p.cfg.Pkg.Components { - // Make the component a tar archive - if err := p.layout.Components.Archive(component, true); err != nil { - return fmt.Errorf("unable to archive component: %s", err.Error()) - } - } - - // Calculate all the checksums - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for the package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - // Save the transformed config. - if err := p.writeYaml(); err != nil { - return fmt.Errorf("unable to write zarf.yaml: %w", err) - } - - // cd back - if err := os.Chdir(cwd); err != nil { + if err := p.assemble(); err != nil { return err } - // Sign the config file if a key was provided - if p.cfg.CreateOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { - return err - } - } - - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - err := p.remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) - if err != nil { - return fmt.Errorf("unable to publish package: %w", err) - } - message.HorizontalRule() - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Title("To inspect/deploy/pull:", "") - message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) - message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) - message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) - } else { - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Create the package tarball. - if err := p.archivePackage(packageName); err != nil { - return fmt.Errorf("unable to archive package: %w", err) - } - } - - // Output the SBOM files into a directory if specified. - if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := p.cfg.CreateOpts.SBOMOutputDir - var sbomDir string - if err := p.layout.SBOMs.Unarchive(); err != nil { - return fmt.Errorf("unable to unarchive SBOMs: %w", err) - } - sbomDir = p.layout.SBOMs.Path - - if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) - if err != nil { - return err - } - sbomDir = out - } - - if p.cfg.CreateOpts.ViewSBOM { - sbom.ViewSBOMFiles(sbomDir) - } - } - - return nil -} - -func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return nil, err - } - // Create an struct to hold the SBOM information for this component. - componentSBOM := &layout.ComponentSBOM{ - Files: []string{}, - Component: componentPaths, - } - - appendSBOMFiles := func(path string) { - if utils.IsDir(path) { - files, _ := utils.RecursiveFileList(path, nil, false) - componentSBOM.Files = append(componentSBOM.Files, files...) - } else { - componentSBOM.Files = append(componentSBOM.Files, path) - } - } - - for filesIdx, file := range component.Files { - path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - appendSBOMFiles(path) - } - - for dataIdx, data := range component.DataInjections { - path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - - appendSBOMFiles(path) - } - - return componentSBOM, nil -} - -func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkeleton bool) error { - message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - - componentPaths, err := p.layout.Components.Create(component) - if err != nil { + // cd back for output + if err := os.Chdir(cwd); err != nil { return err } - if isSkeleton && component.DeprecatedCosignKeyPath != "" { - dst := filepath.Join(componentPaths.Base, "cosign.pub") - err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) - if err != nil { - return err - } - p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" - } - - // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. - if isSkeleton { - component.Actions.OnCreate.Defaults.Dir = "" - resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { - for idx := range actions { - actions[idx].Dir = nil - } - return actions - } - component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) - component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) - component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) - component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) - } - - onCreate := component.Actions.OnCreate - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { - return fmt.Errorf("unable to run component before action: %w", err) - } - } - - // If any helm charts are defined, process them. - for chartIdx, chart := range component.Charts { - - helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) - - if isSkeleton { - if chart.LocalPath != "" { - rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - err := utils.CreatePathAndCopy(chart.LocalPath, dst) - if err != nil { - return err - } - - p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel - } - - for valuesIdx, path := range chart.ValuesFiles { - if helpers.IsURL(path) { - continue - } - - rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) - p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel - - if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { - return fmt.Errorf("unable to copy chart values file %s: %w", path, err) - } - } - } else { - err := helmCfg.PackageChart(componentPaths.Charts) - if err != nil { - return err - } - } - } - - for filesIdx, file := range component.Files { - message.Debugf("Loading %#v", file) - - rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - dst := filepath.Join(componentPaths.Base, rel) - destinationDir := filepath.Dir(dst) - - if helpers.IsURL(file.Source) { - if isSkeleton { - continue - } - - if file.ExtractPath != "" { - - // get the compressedFileName from the source - compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) - if err != nil { - return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) - } - - compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) - - // If the file is an archive, download it to the componentPath.Temp - if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - - err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) - if err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) - } - - } else { - if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - } - - } else { - if file.ExtractPath != "" { - if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) - } - } - - } - - if file.ExtractPath != "" { - // Make sure dst reflects the actual file or directory. - updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) - if updatedExtractedFileOrDir != dst { - if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { - return fmt.Errorf(lang.ErrWritingFile, dst, err) - } - } - } - - if isSkeleton { - // Change the source to the new relative source directory (any remote files will have been skipped above) - p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel - // Remove the extractPath from a skeleton since it will already extract it - p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" - } - - // Abort packaging on invalid shasum (if one is specified). - if file.Shasum != "" { - if err := utils.SHAsMatch(dst, file.Shasum); err != nil { - return err - } - } - - if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, 0700) - } else { - _ = os.Chmod(dst, 0600) - } - } - - if len(component.DataInjections) > 0 { - spinner := message.NewProgressSpinner("Loading data injections") - defer spinner.Stop() - - for dataIdx, data := range component.DataInjections { - spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) - - rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - dst := filepath.Join(componentPaths.Base, rel) - - if helpers.IsURL(data.Source) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { - return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) - } - if isSkeleton { - p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel - } - } - } - spinner.Success() - } - - if len(component.Manifests) > 0 { - // Get the proper count of total manifests to add. - manifestCount := 0 - - for _, manifest := range component.Manifests { - manifestCount += len(manifest.Files) - manifestCount += len(manifest.Kustomizations) - } - - spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) - defer spinner.Stop() - - // Iterate over all manifests. - for manifestIdx, manifest := range component.Manifests { - for fileIdx, path := range manifest.Files { - rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - // Copy manifests without any processing. - spinner.Updatef("Copying manifest %s", path) - if helpers.IsURL(path) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, path, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(path, dst); err != nil { - return fmt.Errorf("unable to copy manifest %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel - } - } - } - - for kustomizeIdx, path := range manifest.Kustomizations { - // Generate manifests from kustomizations and place in the package. - spinner.Updatef("Building kustomization for %s", path) - - kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) - rel := filepath.Join(layout.ManifestsDir, kname) - dst := filepath.Join(componentPaths.Base, rel) - - if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return fmt.Errorf("unable to build kustomization %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) - } - } - if isSkeleton { - // remove kustomizations - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil - } - } - spinner.Success() - } - - // Load all specified git repos. - if len(component.Repos) > 0 && !isSkeleton { - spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) - defer spinner.Stop() - - for _, url := range component.Repos { - // Pull all the references if there is no `@` in the string. - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { - return fmt.Errorf("unable to pull git repo %s: %w", url, err) - } - } - spinner.Success() - } - - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { - return fmt.Errorf("unable to run component after action: %w", err) - } - } - - return nil -} - -// generateChecksum walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksum returns a SHA256 checksum of the checksums.txt file. -func (p *Packager) generatePackageChecksums() (string, error) { - var checksumsData string - - // Loop over the "loaded" files - for rel, abs := range p.layout.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData += fmt.Sprintf("%s %s\n", sum, rel) - } - - // Create the checksums file - checksumsFilePath := p.layout.Checksums - if err := utils.WriteFile(checksumsFilePath, []byte(checksumsData)); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} - -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package -func (p *Packager) loadDifferentialData() error { - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - err := p.setOCIRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - if err != nil { - return err - } - pkg, err := p.remote.FetchZarfYAML() - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil + return p.output() } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go new file mode 100644 index 0000000000..8df927996e --- /dev/null +++ b/src/pkg/packager/create_stages.go @@ -0,0 +1,761 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/images" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" + "github.com/defenseunicorns/zarf/src/internal/packager/sbom" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mholt/archiver/v3" +) + +func (p *Packager) cdToBaseDir(base string, cwd string) error { + if err := os.Chdir(base); err != nil { + return fmt.Errorf("unable to access directory %q: %w", base, err) + } + message.Note(fmt.Sprintf("Using build directory %s", base)) + + // differentials are relative to the current working directory + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + } + return nil +} + +func (p *Packager) load() error { + if err := p.readZarfYAML(layout.ZarfYAML); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + if p.isInitConfig() { + p.cfg.Pkg.Metadata.Version = config.CLIVersion + } + + // Compose components into a single zarf.yaml file + if err := p.composeComponents(); err != nil { + return err + } + + if p.cfg.CreateOpts.IsSkeleton { + return nil + } + + // After components are composed, template the active package. + if err := p.fillActiveTemplate(); err != nil { + return fmt.Errorf("unable to fill values in template: %s", err.Error()) + } + + // After templates are filled process any create extensions + if err := p.processExtensions(); err != nil { + return err + } + + // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + // Load the images and repos from the 'reference' package + if err := p.loadDifferentialData(); err != nil { + return err + } + // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building + // If the package versions are the same return an error + if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { + return errors.New(lang.PkgCreateErrDifferentialSameVersion) + } + if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { + return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + } + + // Handle any potential differential images/repos before going forward + if err := p.removeCopiesFromDifferentialPackage(); err != nil { + return err + } + } + + return nil +} + +func (p *Packager) assemble() error { + componentSBOMs := map[string]*layout.ComponentSBOM{} + var imageList []transform.Image + for idx, component := range p.cfg.Pkg.Components { + onCreate := component.Actions.OnCreate + onFailure := func() { + if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + message.Debugf("unable to run component failure action: %s", err.Error()) + } + } + if err := p.addComponent(idx, component); err != nil { + onFailure() + return fmt.Errorf("unable to add component %q: %w", component.Name, err) + } + + if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + onFailure() + return fmt.Errorf("unable to run component success action: %w", err) + } + + if !p.cfg.CreateOpts.SkipSBOM { + componentSBOM, err := p.getFilesToSBOM(component) + if err != nil { + return fmt.Errorf("unable to create component SBOM: %w", err) + } + if componentSBOM != nil && len(componentSBOM.Files) > 0 { + componentSBOMs[component.Name] = componentSBOM + } + } + + // Combine all component images into a single entry for efficient layer reuse. + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + imageList = append(imageList, refInfo) + } + } + + imageList = helpers.Unique(imageList) + var sbomImageList []transform.Image + + // Images are handled separately from other component assets. + if len(imageList) > 0 { + message.HeaderInfof("📦 PACKAGE IMAGES") + + p.layout = p.layout.AddImages() + + var pulled []images.ImgInfo + var err error + + doPull := func() error { + imgConfig := images.ImageConfig{ + ImagesPath: p.layout.Images.Base, + ImageList: imageList, + Insecure: config.CommonOptions.Insecure, + Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, + RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, + } + + pulled, err = imgConfig.PullAll() + return err + } + + if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + } + + for _, imgInfo := range pulled { + if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { + return err + } + if imgInfo.HasImageLayers { + sbomImageList = append(sbomImageList, imgInfo.RefInfo) + } + } + } + + // Ignore SBOM creation if the flag is set. + if p.cfg.CreateOpts.SkipSBOM { + message.Debug("Skipping image SBOM processing per --skip-sbom flag") + } else { + p.layout = p.layout.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { + return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) + } + } + + return nil +} + +func (p *Packager) assembleSkeleton() error { + if err := p.skeletonizeExtensions(); err != nil { + return err + } + for _, warning := range p.warnings { + message.Warn(warning) + } + for idx, component := range p.cfg.Pkg.Components { + if err := p.addComponent(idx, component); err != nil { + return err + } + + if err := p.layout.Components.Archive(component, false); err != nil { + return err + } + } + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + return p.writeYaml() +} + +// output assumes it is running from cwd, not the build directory +func (p *Packager) output() error { + // Process the component directories into compressed tarballs + // NOTE: This is purposefully being done after the SBOM cataloging + for _, component := range p.cfg.Pkg.Components { + // Make the component a tar archive + if err := p.layout.Components.Archive(component, true); err != nil { + return fmt.Errorf("unable to archive component: %s", err.Error()) + } + } + + // Calculate all the checksums + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for the package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + // Save the transformed config. + if err := p.writeYaml(); err != nil { + return fmt.Errorf("unable to write zarf.yaml: %w", err) + } + + // Sign the config file if a key was provided + if p.cfg.CreateOpts.SigningKeyPath != "" { + if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { + return err + } + } + + // Create a remote ref + client for the package (if output is OCI) + // then publish the package to the remote. + if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { + ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, p.arch) + if err != nil { + return err + } + err = p.setOCIRemote(ref) + if err != nil { + return err + } + + err = p.remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) + if err != nil { + return fmt.Errorf("unable to publish package: %w", err) + } + message.HorizontalRule() + flags := "" + if config.CommonOptions.Insecure { + flags = "--insecure" + } + message.Title("To inspect/deploy/pull:", "") + message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) + message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) + message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) + } else { + // Use the output path if the user specified it. + packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) + + // Try to remove the package if it already exists. + _ = os.Remove(packageName) + + // Create the package tarball. + if err := p.archivePackage(packageName); err != nil { + return fmt.Errorf("unable to archive package: %w", err) + } + } + + // Output the SBOM files into a directory if specified. + if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { + outputSBOM := p.cfg.CreateOpts.SBOMOutputDir + var sbomDir string + if err := p.layout.SBOMs.Unarchive(); err != nil { + return fmt.Errorf("unable to unarchive SBOMs: %w", err) + } + sbomDir = p.layout.SBOMs.Path + + if outputSBOM != "" { + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) + if err != nil { + return err + } + sbomDir = out + } + + if p.cfg.CreateOpts.ViewSBOM { + sbom.ViewSBOMFiles(sbomDir) + } + } + return nil +} + +func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { + componentPaths, err := p.layout.Components.Create(component) + if err != nil { + return nil, err + } + // Create an struct to hold the SBOM information for this component. + componentSBOM := &layout.ComponentSBOM{ + Files: []string{}, + Component: componentPaths, + } + + appendSBOMFiles := func(path string) { + if utils.IsDir(path) { + files, _ := utils.RecursiveFileList(path, nil, false) + componentSBOM.Files = append(componentSBOM.Files, files...) + } else { + componentSBOM.Files = append(componentSBOM.Files, path) + } + } + + for filesIdx, file := range component.Files { + path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + appendSBOMFiles(path) + } + + for dataIdx, data := range component.DataInjections { + path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + + appendSBOMFiles(path) + } + + return componentSBOM, nil +} + +func (p *Packager) addComponent(index int, component types.ZarfComponent) error { + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + + isSkeleton := p.cfg.CreateOpts.IsSkeleton + + componentPaths, err := p.layout.Components.Create(component) + if err != nil { + return err + } + + if isSkeleton && component.DeprecatedCosignKeyPath != "" { + dst := filepath.Join(componentPaths.Base, "cosign.pub") + err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) + if err != nil { + return err + } + p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" + } + + // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. + if isSkeleton { + component.Actions.OnCreate.Defaults.Dir = "" + resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for idx := range actions { + actions[idx].Dir = nil + } + return actions + } + component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + } + + onCreate := component.Actions.OnCreate + if !isSkeleton { + if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { + return fmt.Errorf("unable to run component before action: %w", err) + } + } + + // If any helm charts are defined, process them. + for chartIdx, chart := range component.Charts { + + helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) + + if isSkeleton { + if chart.LocalPath != "" { + rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + err := utils.CreatePathAndCopy(chart.LocalPath, dst) + if err != nil { + return err + } + + p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel + } + + for valuesIdx, path := range chart.ValuesFiles { + if helpers.IsURL(path) { + continue + } + + rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) + p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel + + if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { + return fmt.Errorf("unable to copy chart values file %s: %w", path, err) + } + } + } else { + err := helmCfg.PackageChart(componentPaths.Charts) + if err != nil { + return err + } + } + } + + for filesIdx, file := range component.Files { + message.Debugf("Loading %#v", file) + + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + dst := filepath.Join(componentPaths.Base, rel) + destinationDir := filepath.Dir(dst) + + if helpers.IsURL(file.Source) { + if isSkeleton { + continue + } + + if file.ExtractPath != "" { + + // get the compressedFileName from the source + compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) + if err != nil { + return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) + } + + compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) + + // If the file is an archive, download it to the componentPath.Temp + if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + + err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) + if err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) + } + + } else { + if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + } + + } else { + if file.ExtractPath != "" { + if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } + } + + } + + if file.ExtractPath != "" { + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { + return fmt.Errorf(lang.ErrWritingFile, dst, err) + } + } + } + + if isSkeleton { + // Change the source to the new relative source directory (any remote files will have been skipped above) + p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel + // Remove the extractPath from a skeleton since it will already extract it + p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" + } + + // Abort packaging on invalid shasum (if one is specified). + if file.Shasum != "" { + if err := utils.SHAsMatch(dst, file.Shasum); err != nil { + return err + } + } + + if file.Executable || utils.IsDir(dst) { + _ = os.Chmod(dst, 0700) + } else { + _ = os.Chmod(dst, 0600) + } + } + + if len(component.DataInjections) > 0 { + spinner := message.NewProgressSpinner("Loading data injections") + defer spinner.Stop() + + for dataIdx, data := range component.DataInjections { + spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) + + rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + dst := filepath.Join(componentPaths.Base, rel) + + if helpers.IsURL(data.Source) { + if isSkeleton { + continue + } + if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { + return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + } + if isSkeleton { + p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel + } + } + } + spinner.Success() + } + + if len(component.Manifests) > 0 { + // Get the proper count of total manifests to add. + manifestCount := 0 + + for _, manifest := range component.Manifests { + manifestCount += len(manifest.Files) + manifestCount += len(manifest.Kustomizations) + } + + spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) + defer spinner.Stop() + + // Iterate over all manifests. + for manifestIdx, manifest := range component.Manifests { + for fileIdx, path := range manifest.Files { + rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + // Copy manifests without any processing. + spinner.Updatef("Copying manifest %s", path) + if helpers.IsURL(path) { + if isSkeleton { + continue + } + if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, path, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(path, dst); err != nil { + return fmt.Errorf("unable to copy manifest %s: %w", path, err) + } + if isSkeleton { + p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel + } + } + } + + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + if isSkeleton { + p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) + } + } + if isSkeleton { + // remove kustomizations + p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil + } + } + spinner.Success() + } + + // Load all specified git repos. + if len(component.Repos) > 0 && !isSkeleton { + spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) + defer spinner.Stop() + + for _, url := range component.Repos { + // Pull all the references if there is no `@` in the string. + gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) + if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { + return fmt.Errorf("unable to pull git repo %s: %w", url, err) + } + } + spinner.Success() + } + + if !isSkeleton { + if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { + return fmt.Errorf("unable to run component after action: %w", err) + } + } + + return nil +} + +// generateChecksum walks through all of the files starting at the base path and generates a checksum file. +// Each file within the basePath represents a layer within the Zarf package. +// generateChecksum returns a SHA256 checksum of the checksums.txt file. +func (p *Packager) generatePackageChecksums() (string, error) { + var checksumsData string + + // Loop over the "loaded" files + for rel, abs := range p.layout.Files() { + if rel == layout.ZarfYAML || rel == layout.Checksums { + continue + } + + sum, err := utils.GetSHA256OfFile(abs) + if err != nil { + return "", err + } + checksumsData += fmt.Sprintf("%s %s\n", sum, rel) + } + + // Create the checksums file + checksumsFilePath := p.layout.Checksums + if err := utils.WriteFile(checksumsFilePath, []byte(checksumsData)); err != nil { + return "", err + } + + // Calculate the checksum of the checksum file + return utils.GetSHA256OfFile(checksumsFilePath) +} + +// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package +func (p *Packager) loadDifferentialData() error { + // Save the fact that this is a differential build into the build data of the package + p.cfg.Pkg.Build.Differential = true + + tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + defer os.RemoveAll(tmpDir) + + // Load the package spec of the package we're using as a 'reference' for the differential build + if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { + err := p.setOCIRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + if err != nil { + return err + } + pkg, err := p.remote.FetchZarfYAML() + if err != nil { + return err + } + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + if err != nil { + return err + } + } else { + if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { + return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + } + } + + var differentialZarfConfig types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { + return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + } + + // Generate a map of all the images and repos that are included in the provided package + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap + p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap + p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + + return nil +} + +// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package +func (p *Packager) removeCopiesFromDifferentialPackage() error { + // If a differential build was not requested, continue on as normal + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { + return nil + } + + // Loop through all of the components to determine if any of them are using already included images or repos + componentMap := make(map[int]types.ZarfComponent) + for idx, component := range p.cfg.Pkg.Components { + newImageList := []string{} + newRepoList := []string{} + // Generate a list of all unique images for this component + for _, img := range component.Images { + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + // Only include new images or images that have a commonly overwritten tag + imgTag := imgRef.TagOrDigest + useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } else { + message.Debugf("Image %s is already included in the differential package", img) + } + } + + // Generate a list of all unique repos for this component + for _, repoURL := range component.Repos { + // Split the remote url and the zarf reference + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return err + } + + var ref plumbing.ReferenceName + // Parse the ref from the git URL. + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + // Only include new repos or repos that were not referenced by a specific commit sha or tag + useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } else { + message.Debugf("Repo %s is already included in the differential package", repoURL) + } + } + + // Update the component with the unique lists of repos and images + component.Images = newImageList + component.Repos = newRepoList + componentMap[idx] = component + } + + // Update the package with the new component list + for idx, component := range componentMap { + p.cfg.Pkg.Components[idx] = component + } + + return nil +} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 8835e0f92e..655900a9fb 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -30,6 +30,14 @@ import ( corev1 "k8s.io/api/core/v1" ) +func (p *Packager) resetRegistryHPA() { + if p.isConnectedToCluster() && p.hpaModified { + if err := p.cluster.EnableRegHPAScaleDown(); err != nil { + message.Debugf("unable to reenable the registry HPA scale down: %s", err.Error()) + } + } +} + // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { @@ -61,13 +69,7 @@ func (p *Packager) Deploy() (err error) { p.hpaModified = false p.connectStrings = make(types.ConnectStrings) // Reset registry HPA scale down whether an error occurs or not - defer func() { - if p.isConnectedToCluster() && p.hpaModified { - if err := p.cluster.EnableRegHPAScaleDown(); err != nil { - message.Debugf("unable to reenable the registry HPA scale down: %s", err.Error()) - } - } - }() + defer p.resetRegistryHPA() // Filter out components that are not compatible with this system p.filterComponents() diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go new file mode 100644 index 0000000000..a48d3fdc31 --- /dev/null +++ b/src/pkg/packager/dev.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "fmt" + "os" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// DevDeploy creates + deploys a package in one shot +func (p *Packager) DevDeploy() error { + config.CommonOptions.Confirm = true + p.cfg.CreateOpts.SkipSBOM = !p.cfg.CreateOpts.NoYOLO + + cwd, err := os.Getwd() + if err != nil { + return err + } + + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + return err + } + + if err := p.load(); err != nil { + return err + } + + // Filter out components that are not compatible with this system + p.filterComponents() + + // Also filter out components that are not required, nor requested via --components + // This is different from the above filter, as it is not based on the system, but rather + // the user's selection and the component's `required` field + // This is also different from regular package creation, where we still assemble and package up + // all components and their dependencies, regardless of whether they are required or not + p.cfg.Pkg.Components = p.getValidComponents() + + if err := validate.Run(p.cfg.Pkg); err != nil { + return fmt.Errorf("unable to validate package: %w", err) + } + + // If building in yolo mode, strip out all images and repos + if !p.cfg.CreateOpts.NoYOLO { + for idx := range p.cfg.Pkg.Components { + p.cfg.Pkg.Components[idx].Images = []string{} + p.cfg.Pkg.Components[idx].Repos = []string{} + } + } + + if err := p.assemble(); err != nil { + return err + } + + message.HeaderInfof("📦 PACKAGE DEPLOY %s", p.cfg.Pkg.Metadata.Name) + + // Set variables and prompt if --confirm is not set + if err := p.setVariableMapInConfig(); err != nil { + return fmt.Errorf("unable to set the active variables: %w", err) + } + + p.connectStrings = make(types.ConnectStrings) + + if !p.cfg.CreateOpts.NoYOLO { + p.cfg.Pkg.Metadata.YOLO = true + } else { + p.hpaModified = false + // Reset registry HPA scale down whether an error occurs or not + defer p.resetRegistryHPA() + } + + // Get a list of all the components we are deploying and actually deploy them + deployedComponents, err := p.deployComponents() + if err != nil { + return err + } + if len(deployedComponents) == 0 { + message.Warn("No components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") + } + + // Notify all the things about the successful deployment + message.Successf("Zarf dev deployment complete") + + message.HorizontalRule() + message.Title("Next steps:", "") + + message.ZarfCommand("package inspect %s", p.cfg.Pkg.Metadata.Name) + + // cd back + return os.Chdir(cwd) +} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 2eacd46f6b..5c654d0825 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" @@ -67,12 +66,21 @@ func (p *Packager) Publish() (err error) { } var referenceSuffix string - if p.cfg.CreateOpts.BaseDir != "" { + if p.cfg.CreateOpts.IsSkeleton { referenceSuffix = oci.SkeletonSuffix - err := p.loadSkeleton() + cwd, err := os.Getwd() if err != nil { return err } + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + return err + } + if err := p.load(); err != nil { + return err + } + if err := p.assembleSkeleton(); err != nil { + return err + } } else { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) @@ -124,49 +132,3 @@ func (p *Packager) Publish() (err error) { } return nil } - -func (p *Packager) loadSkeleton() (err error) { - if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { - return err - } - if err = p.readZarfYAML(layout.ZarfYAML); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - err = p.composeComponents() - if err != nil { - return err - } - - err = p.skeletonizeExtensions() - if err != nil { - return err - } - - for _, warning := range p.warnings { - message.Warn(warning) - } - - for idx, component := range p.cfg.Pkg.Components { - isSkeleton := true - if err := p.addComponent(idx, component, isSkeleton); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} diff --git a/src/test/e2e/99_yolo_test.go b/src/test/e2e/99_yolo_test.go index 1f10634bc9..a4044c53a9 100644 --- a/src/test/e2e/99_yolo_test.go +++ b/src/test/e2e/99_yolo_test.go @@ -47,3 +47,17 @@ func TestYOLOMode(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "remove", "yolo", "--confirm") require.NoError(t, err, stdOut, stdErr) } + +func TestDevDeploy(t *testing.T) { + // Don't run this test in appliance mode + if e2e.ApplianceMode { + return + } + e2e.SetupWithCluster(t) + + stdOut, stdErr, err := e2e.Zarf("dev", "deploy", "examples/dos-games") + require.NoError(t, err, stdOut, stdErr) + + stdOut, stdErr, err = e2e.Zarf("package", "remove", "dos-games", "--confirm") + require.NoError(t, err, stdOut, stdErr) +} diff --git a/src/types/runtime.go b/src/types/runtime.go index ee41ec8bc2..3878bb0aef 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -4,7 +4,9 @@ // Package types contains all the types used by Zarf. package types -import "time" +import ( + "time" +) const ( // RawVariableType is the default type for a Zarf package variable @@ -109,6 +111,8 @@ type ZarfCreateOptions struct { DifferentialData DifferentialData `json:"differential" jsonschema:"description=A package's differential images and git repositories from a referenced previously built package"` RegistryOverrides map[string]string `json:"registryOverrides" jsonschema:"description=A map of domains to override on package create when pulling images"` Flavor string `json:"flavor" jsonschema:"description=An optional variant that controls which components will be included in a package"` + IsSkeleton bool `json:"isSkeleton" jsonschema:"description=Whether to create a skeleton package"` + NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` } // ZarfSplitPackageData contains info about a split package.