diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acd2a13c8e..9758e718af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,9 @@ on: jobs: push-resources: runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index 158bb1a3e9..1fe763e78c 100644 --- a/Makefile +++ b/Makefile @@ -176,8 +176,9 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster cd src/test/upgrade-test && go test -failfast -v -timeout 30m .PHONY: test-unit -test-unit: ensure-ui-build-dir ## Run unit tests within the src/pkg directory +test-unit: ensure-ui-build-dir ## Run unit tests within the src/pkg and the bigbang extension directory cd src/pkg && go test ./... -failfast -v -timeout 30m + cd src/extensions/bigbang && go test ./. -failfast -v timeout 30m .PHONY: test-built-ui test-built-ui: ## Run the Zarf UI E2E tests (requires `make build-ui` first) diff --git a/README.md b/README.md index c3431c5893..a99b1c9ad1 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ To try Zarf out for yourself, visit the ["Try It Now"](https://zarf.dev/install) From the docs you can learn more about [installation](https://docs.zarf.dev/docs/operator-manual/set-up-and-install), [using the CLI](https://docs.zarf.dev/docs/user-guide/the-zarf-cli/), [making packages](https://docs.zarf.dev/docs/user-guide/zarf-packages/), and the [Zarf package schema](https://docs.zarf.dev/docs/user-guide/zarf-schema). -Using Zarf in Github workflows? Check out the [setup-zarf](https://github.com/defenseunicorns/setup-zarf)](https://github.com/defenseunicorns/setup-zarf) action. Install any version of Zarf and its `init` package with zero added dependencies. +Using Zarf in Github workflows? Check out the [setup-zarf](https://github.com/defenseunicorns/setup-zarf) action. Install any version of Zarf and its `init` package with zero added dependencies. ## Developing diff --git a/adr/0015-artifact-server-support.md b/adr/0015-artifact-server-support.md new file mode 100644 index 0000000000..9c1ee2aac1 --- /dev/null +++ b/adr/0015-artifact-server-support.md @@ -0,0 +1,24 @@ +# 15. Artifact Server Support + +Date: 2023-04-05 + +## Status + +Accepted + +## Context + +Zarf currently supports git servers and container registries within the airgap to host dependencies for applications, and while this works well for most production deployments, it doesn't cater as well to airgap development where you may need artifacts and libraries for various coding languages to compile and develop software. Zarf's support of `git` is also somewhat lacking in that it only supports flux `GitRepository` objects and does not support more generic use cases where clients try to reach out to upstream `git` hosts in a more native way. + +## Decision + +Since we already had an artifact registry available to us in Gitea, it was decided to utilize that as the default provider for this functionality in addition to matching the external support for `git` and `registry` servers with a new `artifact` server specification on `init`. From here, to access the configured server we had two main options: + +1. Transform any artifact references statically (i.e. swap upstream URLs in build scripts before bringing them into the environment) +2. Transform any artifact references dynamically (i.e. swap upstream URLs in an active proxy that any build scripts could pickup through DNS) + +It was decided to go with #2 since this would allow us to support a wider array of build technologies (including those that wrap existing commands like Buck/Bazel/Make) as well as support builds that may be coming from resources not brought in by Zarf. This allows for more flexibility in how Zarf could transform URLs while allowing these commands to run as they would on the internet side without any modification. + +## Consequences + +For now this should be an internal-only feature while we work through issues and continue to experiment with this functionality given that it may be confusing to use or have rough edges for a while. This also ties us to building specific http request matching logic for various artifact technologies which currently is done first via User Agent, then by parsing the URL to look for the artifact protocol specific path information. While this works, it must be created per artifact technology and right now only supports git (http), pip, npm, and generic repositories and registries in Gitea and Gitlab. diff --git a/docs/0-zarf-overview.md b/docs/0-zarf-overview.md index e0bf5e94b1..a0d415c897 100644 --- a/docs/0-zarf-overview.md +++ b/docs/0-zarf-overview.md @@ -81,7 +81,7 @@ Zarf can pull from various places like Docker Hub, Iron Bank, GitHub, and local ### (1) Create a Package -This part of the process requires access to the internet. The `zarf` binary is presented with a `zarf.yaml`, it then begins downloading, packing, and compressing the software that you requested. It then outputs a single, ready-to-move distributable called "a package". +This part of the process requires access to the internet. The `zarf` binary is presented with a `zarf.yaml`, it then begins downloading, packing, and compressing the software that you requested. It then outputs a single, ready-to-move distributable called "a package". For additional information, see the [Building a package](./13-walkthroughs/0-using-zarf-package-create.md) section. @@ -162,7 +162,7 @@ This quick start requires you to already have: - [Homebrew](https://brew.sh/) package manager installed on your machine. - [Docker](https://www.docker.com/) installed and running on your machine. - + For more install options please visit our [Getting Started page](3-getting-started.md). @@ -171,7 +171,7 @@ For more install options please visit our [Getting Started page](3-getting-start ```bash # To install Zarf -brew tap defenseunicorns/tap brew install zarf +brew tap defenseunicorns/tap && brew install zarf # Next, you will need a Kubernetes cluster. This example uses KIND. brew install kind && kind delete cluster && kind create cluster @@ -200,7 +200,7 @@ This quick start requires you to already have: - [Homebrew](https://brew.sh/) package manager installed on your machine. - [Docker](https://www.docker.com/) installed and running on your machine. - + For more install options please visit our [Getting Started page](3-getting-started.md). @@ -209,7 +209,7 @@ For more install options please visit our [Getting Started page](3-getting-start ```bash # To install Zarf -brew tap defenseunicorns/tap brew install zarf +brew tap defenseunicorns/tap && brew install zarf # Next, you will need a Kubernetes cluster. This example uses KIND. brew install kind && kind delete cluster && kind create cluster diff --git a/docs/13-walkthroughs/5-big-bang.md b/docs/13-walkthroughs/5-big-bang.md index 218747ea53..bc6b67f332 100644 --- a/docs/13-walkthroughs/5-big-bang.md +++ b/docs/13-walkthroughs/5-big-bang.md @@ -110,7 +110,7 @@ The `valuesFiles` are applied from top to bottom and will apply the last value t :::note -This extension works best with Big Bang version `1.54.0` or later. Version `1.53.0` requires manual patches to images to function correctly. +This extension requires Big Bang version `1.54.0` or later. ::: diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md index 6b8d9fd8bf..02281f8ddb 100644 --- a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md @@ -44,6 +44,9 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur ## Options ``` + --artifact-push-token string API Token for the push-user to access the artifact registry + --artifact-push-username string Username to access to the artifact registry Zarf is configured to use. User must be able to upload package artifacts. + --artifact-url string External artifact registry url to use for this Zarf cluster --components string Specify which optional components to install. E.g. --components=git-server,logging --confirm Confirm the install without prompting --git-pull-password string Password for the pull-only user to access the git server diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_clear-cache.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_clear-cache.md index 043b1c5b19..e434035607 100644 --- a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_clear-cache.md +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_clear-cache.md @@ -11,7 +11,7 @@ zarf tools clear-cache [flags] ``` -h, --help help for clear-cache - --zarf-cache string Specify the location of the Zarf artifact cache (images and git repositories) (default "~/.zarf-cache") + --zarf-cache string Specify the location of the Zarf artifact cache (images and git repositories) (default "~/.zarf-cache") ``` ## Options inherited from parent commands diff --git a/docs/4-user-guide/2-zarf-packages/1-zarf-packages.md b/docs/4-user-guide/2-zarf-packages/1-zarf-packages.md index 98e83afd98..412801a462 100644 --- a/docs/4-user-guide/2-zarf-packages/1-zarf-packages.md +++ b/docs/4-user-guide/2-zarf-packages/1-zarf-packages.md @@ -4,72 +4,73 @@ sidebar_position: 1 # Understanding Zarf Packages -A Zarf package is a single tarball archive that contains everything you need to deploy a system or capability while fully disconnected. Zarf packages are defined by a `zarf.yaml` file. +Zarf offers a comprehensive solution for deploying system software or capabilities while fully disconnected. This is accomplished through the use of a single tarball archive, which includes all necessary components and is defined by a `zarf.yaml` file. -Zarf packages are built while 'online' and connected to whatever is hosting the dependencies your package definition defined. When being built, all these defined dependencies are downloaded and stored within the archive. Because all the dependencies are now within the tarball, the package can be deployed to disconnected systems that don't have a connection to the outside world. +Zarf Packages are created while you are connected to internet or intranet resources defined in a Zarf package configuration. This allows these resources to be pulled and stored within a package archive and be deployed on disconnected systems without requiring a connection to the outside world. -The `zarf.yaml` file, which the package builds from, defines declarative instructions on how the capabilities of the package should be deployed. The declarative nature of the package means everything is represented by code and automatically runs as it is configured, instead of having to give manual steps that might not be reproducible on all systems. +The `zarf.yaml` file also defines declarative instructions for the deployment of package capabilities, which are automatically executed on package deployment and are represented in code. This ensures reproducibility across different systems without the need for manual configuration. -Zarf Packages are made up of functionality blocks called components which are described more on the [Zarf Components page](./2-zarf-components.md). These components can be optional, giving more flexibility to how packages can be used. +Zarf Packages consist of functional blocks, known as components. Components can be optional, providing greater flexibility in package usage. Further details on components can be found on the [Zarf Components](./2-zarf-components.md) page. -## Deploying on to Airgapped Systems +## Deploying onto Air-gapped Systems -Zarf packages are built with all the dependencies necessary being included within the package itself, this is important when deploying to air-gapped systems. Since there is no need for an outbound connection to the internet, these packages become highly distributable and can be run on the edge, embedded systems, secure cloud, data centers, or even in a local environment. When deploying a package onto a cluster, the dependencies of the cluster (which were included in the package itself when it was created) are pushed into a docker registry and git server that Zarf stands up on the air-gapped system. This way later steps can use the dependencies as they are needed. +Zarf Packages are built to include all necessary dependencies within the package itself, making it particularly useful for deploying to air-gapped systems. This eliminates the need for outbound internet connectivity, making the packages easily distributable and executable on a variety of systems, including edge, embedded systems, secure cloud, data centers, or local environments. + +When deploying a package onto a cluster, the dependencies contained in each component are automatically pushed into a Docker registry and/or Git server created by or known to Zarf on the air-gapped system. This enables any later steps to utilize these dependencies as needed. ## Types of Zarf Packages -There are two types of Zarf packages, a `ZarfInitConfig` and a `ZarfPackageConfig`. The package type is defined by the `kind:` field in the `zarf.yaml` file. +There are two types of Zarf Packages, the `ZarfInitConfig` and the `ZarfPackageConfig`, which are distinguished by the `kind:` field and specified in the `zarf.yaml` file. -For the remainder of the docs, we will often refer to the `ZarfInitConfig` as an `init config` package or `init` package and the `ZarfPackageConfig` as any package. +Throughout the rest of the documentation, we will refer to the `ZarfInitConfig` as an `init config` package or `init` package, and to the `ZarfPackageConfig` as simply a "package". ### ZarfInitConfig -The init package is the package you use to initialize your cluster to be ready to deploy other Zarf packages. Because the init package is special, we have more documentation on the Zarf ['init' package page](./3-the-zarf-init-package.md) if you're still curious after reading this section. +The init package is used to initialize a cluster, making it ready for deployment of other Zarf Packages. It must be executed once on each cluster that you want to deploy another package onto, even if multiple clusters share the same host. For additional information on the init package, we provide detailed documentation on the Zarf ['init' package page](./3-the-zarf-init-package.md). + +If there is no running cluster, the init package can be used to create one. It has a deployable K3s cluster component that can be optionally deployed on your machine. Usually, an init package is the first Zarf Package to be deployed on a cluster as other packages often depend on the services installed or configured by the init package. If you want to install a K8s cluster with Zarf, but you don’t want to use K3s as your cluster, you will need to create or find another Zarf Package that will stand up your cluster before you run the zarf init command. -**The init package needs to be run once on every cluster you want to deploy another package onto, even if the clusters share the same host.** +:::note -If you don't have a cluster running yet, the init package can help with that too! The init package has a deployable k3s cluster as a component that can optionally be deployed onto your machine. An init package will almost always be the first Zarf package you deploy onto a cluster since other packages will often depend on the services the package installs onto your cluster. +To clarify, in most cases, the first Zarf Package you deploy onto a cluster should be the init package since other packages often depend on the services it installs onto your cluster. However, if you don't want to use the K3s distribution included in the init package or if you already have a preferred K8s distribution, you can deploy the distribution package first, followed by the init package, and then any other packages you want. This only applies if you don't have a K8s cluster yet. -> Note: The only exception where you wouldn't deploy an init package first is when you don't have a k8s cluster yet, you don't want to deploy with the k3s distribution built into the init package and you have a package that deploys your preferred distribution. In those situations, you can deploy the distribution package first, then the init package, and then whatever other packages you want.) +::: -While initializing, Zarf will seed your cluster with a container registry so it can have a place to push images that other packages will need. The init package will also optionally deploy other functionality to your cluster, such as a git server for your repositories, or a simple PLG logging stack so you can monitor the things running on your cluster. +During the initialization process, Zarf will seed your cluster with a container registry to store images that other packages may require. Additionally, the init package has the option to deploy other features to your cluster, such as a Git server to manage your repositories or a PLG logging stack that allows you to monitor the applications running on your cluster. #### Using the init-package -You initialize your cluster by running the command `zarf init`, which will search your current working directory for a file that matches the name `zarf-init-{ARCHITECTURE}-{VERSION}.tar.zst` where the `ARCHITECTURE` matches the architecture of the host you are running on. If the machine you are deploying onto has a different machine architecture, you will have to specify the name of the architecture you are deploying onto. For example, if you are on an arm64 machine but are deploying on an amd64 machine, you will run `zarf init zarf-init-amd64-v0.24.0.tar.zst` +To initialize your cluster, you need to run the command `zarf init`. This command will search for a file with the specific naming convention: `zarf-init-{ARCHITECTURE}-{VERSION}.tar.zst`. The architecture must match that of the cluster you are deploying to. If you are deploying to a cluster with a different architecture, you will need to specify the name of the architecture you are deploying on with the `-a` flag. For example, if you are on an arm64 machine but are deploying on an amd64 machine, you will run `zarf init -a amd64`. -At the end of the day, init packages are just like other packages, meaning they can also be run with `zarf package deploy zarf-init-{ARCHITECTURE}-{VERSION}.tar.zst` +Init packages can also be run with `zarf package deploy zarf-init-{ARCHITECTURE}-{VERSION}.tar.zst`. -Init configs are not something you will have to create yourself unless you want to customize how your cluster is installed/configured (i.e. if you wanted to use the init process to install a specifically configured k3s cluster onto your host machine), and even then it is often easier to create a specific package to do that before your run the init package. +You do not need to create init configs by yourself unless you want to customize how your cluster is installed/configured. For example, if you want to use the init process to install a specifically configured K3s cluster onto your host machine, you can create a specific package to do that before running the init package. ### ZarfPackageConfig -`ZarfPackageConfigs` is any package that isn't an init package. These packages define named capabilities that you want to deploy onto your already initialized cluster. +`ZarfPackageConfig` refers to any package that is not an init package and is used to define specific capabilities that you want to deploy onto your initialized cluster. -You can deploy a Zarf package with the command `zarf package deploy` which will bring up a prompt listing all of the files in your current path that match the name `zarf-package-*.tar.zst` so that you can select which package you want to deploy. If you already know which package you want to deploy, you can do that easily with the command `zarf package deploy {PACKAGE_NAME}`. +To deploy a Zarf Package, you can use the command `zarf package deploy`. This will prompt you to select from all of the files in your current directory that match the name `zarf-package-*.tar.zst`. Alternatively, if you already know which package you want to deploy, you can simply use the command `zarf package deploy {PACKAGE_NAME}`. -When Zarf is deploying the package, it will use the infrastructure that was created when doing the 'init' process (such as the docker registry and git server) to push all of the images and repositories that the package needs to operate. +During the deployment process, Zarf will leverage the infrastructure created during the 'init' process (such as the Docker registry and Git server) to push all the necessary images and repositories required for the package to operate. -## What Makes Up A Package +## Package Components -Zarf packages are split into smaller chunks called 'components'. These components are defined more on the [Zarf Components page](./2-zarf-components.md) but, in short, components are the named capabilities that packages provide. The schema of a `zarf.yaml` package is available here: [ZarfPackage Schema Docs](../3-zarf-schema.md) +Zarf Packages consist of smaller units known as 'components'. These components are further defined on the [Zarf Components page](./2-zarf-components.md), but to summarize, they represent the named capabilities that packages offer. For additional information regarding the `zarf.yaml` package structure, please refer to the [Zarf Package Schema](../3-zarf-schema.md) documentation. -## Building A Zarf Package +## Creating a Zarf Package -:::info +The following list outlines the dependencies for creating a Zarf Package: -**Dependencies** for Building a Zarf Package +- A local K8s cluster to work with ([K3s](https://k3s.io/)/[k3d](https://k3d.io/v5.4.1/)/[Kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation)). +- A Zarf CLI ([downloaded](https://github.com/defenseunicorns/zarf/releases) or [manually built](../1-the-zarf-cli/1-building-your-own-cli.md)). +- A Zarf init package ([downloaded](https://github.com/defenseunicorns/zarf/releases) or [manually built](../1-the-zarf-cli/1-building-your-own-cli.md)). -- A local k8s cluster to work with ([k3s](https://k3s.io/)/[k3d](https://k3d.io/v5.4.1/)/[Kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation)) -- A Zarf CLI ([downloaded](https://github.com/defenseunicorns/zarf/releases) or [manually built](../1-the-zarf-cli/1-building-your-own-cli.md)) -- A Zarf init package ([downloaded](https://github.com/defenseunicorns/zarf/releases) or [manually built](../1-the-zarf-cli/1-building-your-own-cli.md)) - -::: +The process of defining a package is elaborated in detail on the [Creating a Package](../../13-walkthroughs/0-using-zarf-package-create.md) page. Once a package has been defined, building it is a relatively straightforward task. -The process of defining a package is covered on the [Creating a Package](../../13-walkthroughs/0-using-zarf-package-create.md) page. Assuming you have a package already defined, building the package itself is fairly simple. +The `zarf package create` command locates the `zarf.yaml` file in the current directory and constructs the package from that file. The command utilizes internet or intranet resources to retrieve all the required assets and stores them in a temporary directory. After the required resources have been obtained, Zarf generates a tarball of the temporary directory and performs necessary cleanup actions. -`zarf package create` will look for a `zarf.yaml` file in the current directory and build the package from that file. Behind the scenes, this is pulling down all the resources it needs from the internet and placing them in a temporary directory, once all the necessary resources of retrieved, Zarf will create the tarball of the temp directory and clean up the temp directory. -## Inspecting a Built Package +## Inspecting a Created Package -`zarf package inspect ./path/to/package.tar.zst` will look at the contents of the package and print out the contents of the `zarf.yaml` file that defined it. +To inspect the contents of a Zarf Package, you can use the command `zarf package inspect` followed by the path to the package file. This will print out the contents of the `zarf.yaml` file that defines the package. For example, if your package is located at `./path/to/package.tar.zst`, you can run `zarf package inspect ./path/to/package.tar.zst` to view the contents of the `zarf.yaml` file. diff --git a/docs/4-user-guide/2-zarf-packages/3-the-zarf-init-package.md b/docs/4-user-guide/2-zarf-packages/3-the-zarf-init-package.md index df814619b4..0b52c4cf50 100644 --- a/docs/4-user-guide/2-zarf-packages/3-the-zarf-init-package.md +++ b/docs/4-user-guide/2-zarf-packages/3-the-zarf-init-package.md @@ -4,36 +4,47 @@ sidebar_position: 3 # The Zarf 'init' Package -The init package is the `zarf.yaml` file that lives at the [root of the Zarf repository](https://github.com/defenseunicorns/zarf/blob/main/zarf.yaml). It is defined via composed components that all offer value for future packages to utilize. When the init package is deployed, it will create a `zarf` namespace within your k8s cluster and deploy various pods, services, and secrets to that namespace, depending on which optional components you choose to deploy. +The init package is the `zarf.yaml` file that lives at the [root of the Zarf repository](https://github.com/defenseunicorns/zarf/blob/main/zarf.yaml). +It is defined by composed components which provide a foundation for future packages to utilize. Upon deployment, the init package generates a `zarf` namespace within your K8s cluster and deploys pods, services, and secrets to that namespace based on the optional components selected for deployment. -## Mandatory Components +## Required Component -Zarf's work necessitates that some components in the [init package](https://github.com/defenseunicorns/zarf/blob/main/zarf.yaml) are "always on" (a.k.a. required & cannot be disabled). These components are always deployed whenever you perform a `zarf init` command. Those include: +Zarf's capabilities require that the [`zarf-agent`](https://docs.zarf.dev/docs/faq#what-is-the-zarf-agent) component of the init package is constantly active, meaning that it cannot be disabled and is always on. This component is automatically deployed whenever a `zarf init` command is executed. -| | Description | +| Component | Description | | ----------------------- | -------------------------------------------------------------------------------------------------------------------- | -| zarf-injector | Adds a Rust and Go binary to the working directory to use during the registry bootstrapping. -| container-registry-seed | Adds a container registry so Zarf can bootstrap itself into the cluster. | -| container-registry | Adds a container registry service—[docker registry](https://docs.docker.com/registry/)—into the cluster. | +| zarf-agent | A Kubernetes mutating webhook installed during `zarf init` that converts Pod specs and Flux GitRepository objects to match their air gap equivalents. -## Additional Components +## Core Components -In addition to those that are always installed, Zarf's optional components provide additional functionality and can be enabled as & when you need them. +In addition to the required `zarf-agent` component, Zarf also offers components that provide additional functionality and can be enabled as needed based on your desired end-state. -These optional components for the init package are listed below along with the "magic strings" you pass to `zarf init --components` to pull them in: +In most scenarios, Zarf will also deploy an internal registry using the components described below. However, Zarf can be configured to use an already existing registry with the `--registry-*` flags when running `zarf init` (detailed information on all `zarf init` command flags can be found in the [zarf init CLI](https://docs.zarf.dev/docs/user-guide/the-zarf-cli/cli-commands/zarf_init) section). This option skips the injector and seed process, and will not deploy a registry to the cluster. Instead, it uploads any images to the externally configured registry. -| components | Description | +| Components | Description +| ----------------------- | -------------------------------------------------------------------------------------------------------------------- | +| zarf-injector | Adds a Rust and Go binary to the working directory to use during the registry bootstrapping. | +| zarf-seed-registry | Adds a temporary container registry so Zarf can bootstrap itself into the cluster. | +| zarf-registry | Adds a long-lived container registry service—[docker registry](https://docs.docker.com/registry/)—into the cluster. | + +Additionally, below are the fully-optional components available for the init package, along with their respective component names that can be passed to `zarf init --components` to deploy them to the cluster: + +| Components | Description | | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| k3s | REQUIRES ROOT. Installs a lightweight Kubernetes Cluster on the local host—[k3s](https://k3s.io/)—and configures it to start up on boot. | -| logging | Adds a log monitoring stack—[promtail / loki / graphana (a.k.a. PLG)](https://github.com/grafana/loki)—into the cluster. | +| k3s | REQUIRES ROOT. Installs a lightweight Kubernetes Cluster on the local host—[K3s](https://k3s.io/)—and configures it to start up on boot. | +| logging | Adds a log monitoring stack—[promtail/loki/graphana (aka PLG)](https://github.com/grafana/loki)—into the cluster. | | git-server | Adds a [GitOps](https://www.cloudbees.com/gitops/what-is-gitops)-compatible source control service—[Gitea](https://gitea.io/en-us/)—into the cluster. | -There are two ways to deploy optional components, you can either pass a comma separated list of components to the `--components` flag such as `zarf init --components k3s,git-server --confirm` or you can exclude the flags and say yes/no as each optional component gets prompted to you. +There are two ways to deploy optional components. Firstly, you can provide a comma-separated list of components to the `--components` flag, such as `zarf init --components k3s,git-server --confirm`. Alternatively, you can choose to exclude the `--components` and `--confirm` flags and respond with a yes or no for each optional component when prompted. + +:::note -> Note: The 'k3s' component requires root access when deploying as it will modify your host machine to install the cluster. +Deploying the 'k3s' component will require root access (not just sudo), as it modifies your host machine to install the cluster. + +::: ## What Makes the Init Package Special -Deploying onto air-gapped environments is a [hard problem](../../1-understand-the-basics.md#what-is-the-air-gap), especially when the k8s environment you're deploying to doesn't have a container registry running for you to put your images into. This leads to a classic 'chicken or the egg' problem since the container registry image needs to make its way into the cluster but there is no container registry running on the cluster to push to yet because the image isn't in the cluster yet. To remain distro agnostic, we had to come up with a unique solution to seed the container registry into the cluster. +Deploying onto air-gapped environments is a [hard problem](../../1-understand-the-basics.md#what-is-the-air-gap), particularly when the K8s environment doesn't have a container registry for you to store images. This results in a dilemma where the container registry image must be introduced to the cluster, but there is no container registry to push it to as the image is not yet in the cluster. To ensure that our approach is distro-agnostic, we developed a unique solution to seed the container registry into the cluster. -The `zarf-injector` [component](https://github.com/defenseunicorns/zarf/blob/main/packages/zarf-injector/zarf.yaml) within the init-package solves this problem by injecting a single rust binary (statically compiled) and a series of configmap chunks of a `registry:2` image into an ephemeral pod based on an existing image in the cluster. +To address this problem, we use the `zarf-injector` [component](https://github.com/defenseunicorns/zarf/blob/main/packages/zarf-injector/zarf.yaml) within the init-package. This resolves the issue by injecting a single rust binary (statically compiled) and a series of configmap chunks of a `registry:2` image into an ephemeral pod that is based on an existing image in the cluster. diff --git a/docs/4-user-guide/3-zarf-schema.md b/docs/4-user-guide/3-zarf-schema.md index dae2d2ae0f..f9ad04f416 100644 --- a/docs/4-user-guide/3-zarf-schema.md +++ b/docs/4-user-guide/3-zarf-schema.md @@ -93,7 +93,7 @@ Must be one of:  
-**Description:** Generic string to track the package version by a package author +**Description:** Generic string set by a package author to track the package version (Note: ZarfInitConfigs will always be versioned to the CLIVersion they were created with) | | | | -------- | -------- | diff --git a/examples/big-bang/zarf.yaml b/examples/big-bang/zarf.yaml index 7e06d039fb..46f9b25d49 100644 --- a/examples/big-bang/zarf.yaml +++ b/examples/big-bang/zarf.yaml @@ -2,7 +2,7 @@ kind: ZarfPackageConfig metadata: name: big-bang-example description: "Deploy Big Bang Core" - version: 1.56.0 + version: 1.57.1 url: https://p1.dso.mil/products/big-bang # Big Bang / Iron Bank are only amd64 architecture: amd64 @@ -17,7 +17,7 @@ components: required: true extensions: bigbang: - version: 1.56.0 + version: 1.57.1 valuesFiles: # Istio configs - config/ingress.yaml diff --git a/examples/git-data/README.md b/examples/git-data/README.md index 6204398fdb..bdf360f542 100644 --- a/examples/git-data/README.md +++ b/examples/git-data/README.md @@ -28,7 +28,7 @@ A SHA-based clone only mirrors the SHA hash defined in the Zarf definition. The ## Git Reference-Based Git Repository Clone -If you need even more control, Zarf also supports providing full `git` [refspecs](https://git-scm.com/book/en/v2/Git-Internals-The-Refspec), as seen in `https://repo1.dso.mil/big-bang/bigbang.git@refs/heads/release-1.53.x`. This allows you to pull specific tags or branches by using this standard. The branch name used by zarf on deploy will depend on the kind of ref specified, branches will use the upstream branch name, whereas other refs (namely tags) will use the `zarf-ref-*` branch name. +If you need even more control, Zarf also supports providing full `git` [refspecs](https://git-scm.com/book/en/v2/Git-Internals-The-Refspec), as seen in `https://repo1.dso.mil/big-bang/bigbang.git@refs/heads/release-1.54.x`. This allows you to pull specific tags or branches by using this standard. The branch name used by zarf on deploy will depend on the kind of ref specified, branches will use the upstream branch name, whereas other refs (namely tags) will use the `zarf-ref-*` branch name. ## Git Repository Full Clone diff --git a/examples/git-data/zarf.yaml b/examples/git-data/zarf.yaml index d060f2f98e..096a5e4c76 100644 --- a/examples/git-data/zarf.yaml +++ b/examples/git-data/zarf.yaml @@ -31,7 +31,7 @@ components: required: true repos: # Do a branch-provided Git Repo mirror - - "https://github.com/DoD-Platform-One/big-bang.git@refs/heads/release-1.53.x" + - "https://github.com/DoD-Platform-One/big-bang.git@refs/heads/release-1.54.x" - name: specific-hash required: true diff --git a/package-lock.json b/package-lock.json index 56b233adb2..791c1c649b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "devDependencies": { "@playwright/test": "1.32.2", "@sveltejs/adapter-static": "2.0.1", - "@sveltejs/kit": "1.15.1", + "@sveltejs/kit": "1.15.2", "@sveltejs/package": "2.0.2", "@testing-library/svelte": "3.2.2", "@tsconfig/svelte": "4.0.1", @@ -1308,9 +1308,9 @@ } }, "node_modules/@sveltejs/kit": { - "version": "1.15.1", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-1.15.1.tgz", - "integrity": "sha512-Wexy3N+COoClTuRawVJRbLoH5HFxNrXG3uoHt/Yd5IGx8WAcJM9Nj/CcBLw/tjCR9uDDYMnx27HxuPy3YIYQUA==", + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-1.15.2.tgz", + "integrity": "sha512-rLNxZrjbrlPf8AWW8GAU4L/Vvu17e9v8EYl7pUip7x72lTft7RcxeP3z7tsrHpMSBBxC9o4XdKzFvz1vMZyXZw==", "dev": true, "hasInstallScript": true, "dependencies": { @@ -6550,9 +6550,9 @@ "requires": {} }, "@sveltejs/kit": { - "version": "1.15.1", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-1.15.1.tgz", - "integrity": "sha512-Wexy3N+COoClTuRawVJRbLoH5HFxNrXG3uoHt/Yd5IGx8WAcJM9Nj/CcBLw/tjCR9uDDYMnx27HxuPy3YIYQUA==", + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-1.15.2.tgz", + "integrity": "sha512-rLNxZrjbrlPf8AWW8GAU4L/Vvu17e9v8EYl7pUip7x72lTft7RcxeP3z7tsrHpMSBBxC9o4XdKzFvz1vMZyXZw==", "dev": true, "requires": { "@sveltejs/vite-plugin-svelte": "^2.0.0", diff --git a/package.json b/package.json index 446c915a7e..123406a066 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "devDependencies": { "@playwright/test": "1.32.2", "@sveltejs/adapter-static": "2.0.1", - "@sveltejs/kit": "1.15.1", + "@sveltejs/kit": "1.15.2", "@sveltejs/package": "2.0.2", "@testing-library/svelte": "3.2.2", "@tsconfig/svelte": "4.0.1", diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index df0a17e98e..aa6d1ef31d 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -53,3 +53,6 @@ components: - cmd: "./zarf internal create-read-only-gitea-user" maxRetries: 3 maxTotalSeconds: 60 + - cmd: "./zarf internal create-artifact-registry-token" + maxRetries: 3 + maxTotalSeconds: 60 diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index 3674516f5c..0f15f7e9ed 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -145,12 +145,19 @@ func validateInitFlags() error { } } - //If 'registry-url' is provided, make sure they provided values for the username and password of the push user + // If 'registry-url' is provided, make sure they provided values for the username and password of the push user if pkgConfig.InitOpts.RegistryInfo.Address != "" { if pkgConfig.InitOpts.RegistryInfo.PushUsername == "" || pkgConfig.InitOpts.RegistryInfo.PushPassword == "" { return fmt.Errorf(lang.CmdInitErrValidateRegistry) } } + + // If 'artifact-url' is provided, make sure they provided values for the username and password of the push user + if pkgConfig.InitOpts.ArtifactServer.Address != "" { + if pkgConfig.InitOpts.ArtifactServer.PushUsername == "" || pkgConfig.InitOpts.ArtifactServer.PushToken == "" { + return fmt.Errorf(lang.CmdInitErrValidateArtifact) + } + } return nil } @@ -203,5 +210,10 @@ func init() { initCmd.Flags().StringVar(&pkgConfig.InitOpts.RegistryInfo.PullPassword, "registry-pull-password", v.GetString(V_INIT_REGISTRY_PULL_PASS), lang.CmdInitFlagRegPullPass) initCmd.Flags().StringVar(&pkgConfig.InitOpts.RegistryInfo.Secret, "registry-secret", v.GetString(V_INIT_REGISTRY_SECRET), lang.CmdInitFlagRegSecret) + // Flags for using an external artifact server + initCmd.Flags().StringVar(&pkgConfig.InitOpts.ArtifactServer.Address, "artifact-url", v.GetString(V_INIT_ARTIFACT_URL), lang.CmdInitFlagArtifactURL) + initCmd.Flags().StringVar(&pkgConfig.InitOpts.ArtifactServer.PushUsername, "artifact-push-username", v.GetString(V_INIT_ARTIFACT_PUSH_USER), lang.CmdInitFlagArtifactPushUser) + initCmd.Flags().StringVar(&pkgConfig.InitOpts.ArtifactServer.PushToken, "artifact-push-token", v.GetString(V_INIT_ARTIFACT_PUSH_TOKEN), lang.CmdInitFlagArtifactPushToken) + initCmd.Flags().SortFlags = true } diff --git a/src/cmd/internal.go b/src/cmd/internal.go index 6eb66ec905..57611aeeb0 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -38,6 +38,17 @@ var agentCmd = &cobra.Command{ }, } +var httpProxyCmd = &cobra.Command{ + Use: "http-proxy", + Short: "Runs the zarf agent http proxy", + Long: "[EXPERIMENTAL] NOTE: This command is a hidden command and generally shouldn't be run by a human.\n" + + "This command starts up a http proxy that can be used by running pods to transform queries " + + "that conform to Gitea server URLs in the airgap", + Run: func(cmd *cobra.Command, args []string) { + agent.StartHTTPProxy() + }, +} + var generateCLIDocs = &cobra.Command{ Use: "generate-cli-docs", Short: lang.CmdInternalGenerateCliDocsShort, @@ -104,6 +115,33 @@ var createReadOnlyGiteaUser = &cobra.Command{ }, } +var createPackageRegistryToken = &cobra.Command{ + Use: "create-artifact-registry-token", + Short: "Creates an artifact registry token for Gitea", + Long: "Creates an artifact registry token in Gitea using the Gitea API. " + + "This is called internally by the supported Gitea package component.", + Run: func(cmd *cobra.Command, args []string) { + // Load the state so we can get the credentials for the admin git user + cluster := cluster.NewClusterOrDie() + state, err := cluster.LoadZarfState() + if err != nil { + message.Error(err, "Unable to load the Zarf state") + } + + // If we are setup to use an internal artifact server, create the artifact registry token + if state.ArtifactServer.InternalServer { + token, err := git.New(state.GitServer).CreatePackageRegistryToken() + if err != nil { + message.Error(err, "Unable to create an artifact registry token for the Gitea service.") + } + + state.ArtifactServer.PushToken = token.Sha1 + + cluster.SaveZarfState(state) + } + }, +} + var uiCmd = &cobra.Command{ Use: "ui", Short: lang.CmdInternalUIShort, @@ -127,10 +165,12 @@ func init() { rootCmd.AddCommand(internalCmd) internalCmd.AddCommand(agentCmd) + internalCmd.AddCommand(httpProxyCmd) internalCmd.AddCommand(generateCLIDocs) internalCmd.AddCommand(configSchemaCmd) internalCmd.AddCommand(apiSchemaCmd) internalCmd.AddCommand(createReadOnlyGiteaUser) + internalCmd.AddCommand(createPackageRegistryToken) internalCmd.AddCommand(uiCmd) internalCmd.AddCommand(isValidHostname) } diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 9fb3a751d5..ce55d3ef2d 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -13,9 +13,9 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/spf13/cobra" ) @@ -45,8 +45,7 @@ var prepareTransformGitLinks = &cobra.Command{ // Perform git url transformation via regex text := string(content) - gitCfg := git.New(pkgConfig.InitOpts.GitServer) - processedText := gitCfg.MutateGitURLsInText(text) + processedText := transform.MutateGitURLsInText(pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) // Ask the user before this destructive action confirm := false diff --git a/src/cmd/viper.go b/src/cmd/viper.go index 1f2c1d69e8..0fddd9d0d4 100644 --- a/src/cmd/viper.go +++ b/src/cmd/viper.go @@ -44,6 +44,11 @@ const ( V_INIT_REGISTRY_PULL_USER = "init.registry.pull_username" V_INIT_REGISTRY_PULL_PASS = "init.registry.pull_password" + // Init Package config keys + V_INIT_ARTIFACT_URL = "init.artifact.url" + V_INIT_ARTIFACT_PUSH_USER = "init.artifact.push_username" + V_INIT_ARTIFACT_PUSH_TOKEN = "init.artifact.push_token" + // Package create config keys V_PKG_CREATE_SET = "package.create.set" V_PKG_CREATE_OUTPUT_DIR = "package.create.output_directory" diff --git a/src/config/config.go b/src/config/config.go index 7e454069e9..e536792541 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -31,6 +31,7 @@ const ( ZarfMaxChartNameLength = 40 ZarfGitPushUser = "zarf-git-user" ZarfGitReadUser = "zarf-git-read-user" + ZarfArtifactTokenName = "zarf-artifact-registry-token" ZarfRegistryPushUser = "zarf-push" ZarfRegistryPullUser = "zarf-pull" ZarfImagePullSecretName = "private-registry" @@ -55,7 +56,8 @@ const ( ZarfInClusterContainerRegistryNodePort = 31999 - ZarfInClusterGitServiceURL = "http://zarf-gitea-http.zarf.svc.cluster.local:3000" + ZarfInClusterGitServiceURL = "http://zarf-gitea-http.zarf.svc.cluster.local:3000" + ZarfInClusterArtifactServiceURL = ZarfInClusterGitServiceURL + "/api/packages/" + ZarfGitPushUser ZarfSeedImage = "registry" ZarfSeedTag = "2.8.1" diff --git a/src/config/lang/english.go b/src/config/lang/english.go index f914dfdf30..2f80ff9af0 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -120,7 +120,8 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdInitErrFlags = "Invalid command flags were provided." CmdInitErrDownload = "failed to download the init package: %s" CmdInitErrValidateGit = "the 'git-push-username' and 'git-push-password' flags must be provided if the 'git-url' flag is provided" - CmdInitErrValidateRegistry = "the 'registry-push-username' and 'registry-push-password' flags must be provided if the 'registry-url' flag is provided " + CmdInitErrValidateRegistry = "the 'registry-push-username' and 'registry-push-password' flags must be provided if the 'registry-url' flag is provided" + CmdInitErrValidateArtifact = "the 'artifact-push-username' and 'artifact-push-token' flags must be provided if the 'artifact-url' flag is provided" CmdInitErrUnableCreateCache = "Unable to create the cache directory: %s" CmdInitDownloadAsk = "It seems the init package could not be found locally, but can be downloaded from %s" @@ -149,6 +150,10 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdInitFlagRegPullPass = "Password for the pull-only user to access the registry" CmdInitFlagRegSecret = "Registry secret value" + CmdInitFlagArtifactURL = "External artifact registry url to use for this Zarf cluster" + CmdInitFlagArtifactPushUser = "Username to access to the artifact registry Zarf is configured to use. User must be able to upload package artifacts." + CmdInitFlagArtifactPushToken = "API Token for the push-user to access the artifact registry" + // zarf internal CmdInternalShort = "Internal tools used by zarf" @@ -276,7 +281,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdToolsClearCacheShort = "Clears the configured git and image cache directory." CmdToolsClearCacheErr = "Unable to clear the cache directory %s" CmdToolsClearCacheSuccess = "Successfully cleared the cache from %s" - CmdToolsClearCacheFlagCachePath = "Specify the location of the Zarf artifact cache (images and git repositories)" + CmdToolsClearCacheFlagCachePath = "Specify the location of the Zarf artifact cache (images and git repositories)" CmdToolsGenPkiShort = "Generates a Certificate Authority and PKI chain of trust for the given host" CmdToolsGenPkiSuccess = "Successfully created a chain of trust for %s" @@ -331,6 +336,7 @@ const ( AgentErrNilReq = "malformed admission review: request is nil" AgentErrShutdown = "unable to properly shutdown the web server" AgentErrStart = "Failed to start the web server" + AgentErrUnableTransform = "unable to transform the provided request; see zarf http proxy logs for more details" ) // src/internal/packager/validate. diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 58b688578c..eeab77575f 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -7,10 +7,9 @@ package bigbang import ( "fmt" "path" - "strconv" - "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -24,8 +23,9 @@ import ( // Default location for pulling Big Bang. const ( - bb = "bigbang" - bbRepo = "https://repo1.dso.mil/big-bang/bigbang.git" + bb = "bigbang" + bbRepo = "https://repo1.dso.mil/big-bang/bigbang.git" + bbMinRequiredVersion = "1.54.0" ) var tenMins = metav1.Duration{ @@ -43,9 +43,15 @@ func Run(tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfCompon cfg := c.Extensions.BigBang manifests := []types.ZarfManifest{} + err, validVersionResponse := isValidVersion(cfg.Version) + + if err != nil { + return c, fmt.Errorf("invalid Big Bang version: %s, parsing issue %s", cfg.Version, err) + } + // Make sure the version is valid. - if !isValidVersion(cfg.Version) { - return c, fmt.Errorf("invalid Big Bang version: %s, must be at least 1.53.0", cfg.Version) + if !validVersionResponse { + return c, fmt.Errorf("invalid Big Bang version: %s, must be at least %s", cfg.Version, bbMinRequiredVersion) } // Print the banner for Big Bang. @@ -142,9 +148,9 @@ func Run(tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfCompon } // In Big Bang the metrics-server is a special case that only deploy if needed. - // The check it, we need to look for the APIService to be exist instead of the HelmRelease, which + // The check it, we need to look for the existence of APIService instead of the HelmRelease, which // may not ever be created. See links below for more details. - // https://repo1.dso.mil/big-bang/bigbang/-/blob/1.53.0/chart/templates/metrics-server/helmrelease.yaml + // https://repo1.dso.mil/big-bang/bigbang/-/blob/1.54.0/chart/templates/metrics-server/helmrelease.yaml if hr.Metadata.Name == "metrics-server" { action.Description = "K8s metric server to exist or be deployed by Big Bang" action.Wait.Cluster = &types.ZarfComponentActionWaitCluster{ @@ -226,22 +232,21 @@ func Run(tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfCompon return c, nil } -// isValidVersion check if the version is 1.53.0 or greater. -func isValidVersion(version string) bool { - // Split the version string into its major, minor, and patch components - parts := strings.Split(version, ".") - if len(parts) != 3 { - return false +// isValidVersion check if the version is 1.54.0 or greater. +func isValidVersion(version string) (error, bool) { + specifiedVersion, err := semver.NewVersion(version) + + if err != nil { + return err, false } - // Parse the major and minor components as integers. - // Ignore errors because we are checking the values later. - major, _ := strconv.Atoi(parts[0]) - minor, _ := strconv.Atoi(parts[1]) + minRequiredVersion, _ := semver.NewVersion(bbMinRequiredVersion) + + // Evaluating pre-releases too + c, _ := semver.NewConstraint(fmt.Sprintf(">= %s-0", minRequiredVersion)) - // This extension requires BB 1.53.0 or greater. - // @todo: This should be updated to 1.54.0 when 1.55.0 is released. - return major >= 1 && minor >= 53 + // This extension requires BB 1.54.0 or greater. + return nil, c.Check(specifiedVersion) } // findBBResources takes a list of yaml objects (as a string) and diff --git a/src/extensions/bigbang/bigbang_test.go b/src/extensions/bigbang/bigbang_test.go new file mode 100644 index 0000000000..e6d1167089 --- /dev/null +++ b/src/extensions/bigbang/bigbang_test.go @@ -0,0 +1,36 @@ +package bigbang + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRequiredBigBangVersions(t *testing.T) { + // Support 1.54.0 and beyond + err, vv := isValidVersion("1.54.0") + assert.Equal(t, err, nil) + assert.Equal(t, vv, true) + + // Do not support earlier than 1.54.0 + err, vv = isValidVersion("1.53.0") + assert.Equal(t, err, nil) + assert.Equal(t, vv, false) + + // Support for Big Bang release candidates + err, vv = isValidVersion("1.57.0-rc.0") + assert.Equal(t, err, nil) + assert.Equal(t, vv, true) + + // Support for Big Bang 2.0.0 + err, vv = isValidVersion("2.0.0") + assert.Equal(t, err, nil) + assert.Equal(t, vv, true) + + // Fail on non-semantic versions + err, vv = isValidVersion("1.57b") + Expected := "Invalid Semantic Version" + if err.Error() != Expected { + t.Errorf("Error actual = %v, and Expected = %v.", err, Expected) + } + assert.Equal(t, vv, false) +} diff --git a/src/internal/agent/hooks/flux.go b/src/internal/agent/hooks/flux.go index d9854aa6cb..21ad215192 100644 --- a/src/internal/agent/hooks/flux.go +++ b/src/internal/agent/hooks/flux.go @@ -11,15 +11,14 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/agent/operations" - "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/internal/agent/state" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" v1 "k8s.io/api/admission/v1" ) -const zarfStatePath = "/etc/zarf-state/state" - // SecretRef contains the name used to reference a git repository secret. type SecretRef struct { Name string `json:"name"` @@ -46,7 +45,7 @@ func NewGitRepositoryMutationHook() operations.Hook { func mutateGitRepo(r *v1.AdmissionRequest) (result *operations.Result, err error) { var ( - state types.ZarfState + zarfState types.ZarfState patches []operations.PatchOperation isPatched bool @@ -54,12 +53,12 @@ func mutateGitRepo(r *v1.AdmissionRequest) (result *operations.Result, err error isUpdate = r.Operation == v1.Update ) - // Form the state.GitServer.Address from the state - if state, err = getStateFromAgentPod(zarfStatePath); err != nil { + // Form the zarfState.GitServer.Address from the zarfState + if zarfState, err = state.GetZarfStateFromAgentPod(); err != nil { return nil, fmt.Errorf(lang.AgentErrGetState, err) } - message.Debugf("Using the url of (%s) to mutate the flux repository", state.GitServer.Address) + message.Debugf("Using the url of (%s) to mutate the flux repository", zarfState.GitServer.Address) // parse to simple struct to read the git url src := &GenericGitRepo{} @@ -68,11 +67,11 @@ func mutateGitRepo(r *v1.AdmissionRequest) (result *operations.Result, err error } patchedURL := src.Spec.URL - // Check if this is an update operation and the hostname is different from what we have in the state + // Check if this is an update operation and the hostname is different from what we have in the zarfState // NOTE: We mutate on updates IF AND ONLY IF the hostname in the request is different than the hostname in the zarfState // NOTE: We are checking if the hostname is different before because we do not want to potentially mutate a URL that has already been mutated. if isUpdate { - isPatched, err = utils.DoHostnamesMatch(state.GitServer.Address, src.Spec.URL) + isPatched, err = utils.DoHostnamesMatch(zarfState.GitServer.Address, src.Spec.URL) if err != nil { return nil, fmt.Errorf(lang.AgentErrHostnameMatch, err) } @@ -81,10 +80,11 @@ func mutateGitRepo(r *v1.AdmissionRequest) (result *operations.Result, err error // Mutate the git URL if necessary if isCreate || (isUpdate && !isPatched) { // Mutate the git URL so that the hostname matches the hostname in the Zarf state - patchedURL, err = git.New(state.GitServer).TransformURL(patchedURL) + transformedURL, err := transform.GitTransformURL(zarfState.GitServer.Address, patchedURL, zarfState.GitServer.PushUsername) if err != nil { message.Warnf("Unable to transform the git url, using the original url we have: %s", patchedURL) } + patchedURL = transformedURL.String() message.Debugf("original git URL of (%s) got mutated to (%s)", src.Spec.URL, patchedURL) } diff --git a/src/internal/agent/hooks/pods.go b/src/internal/agent/hooks/pods.go index 04b7140f53..2e90709bfd 100644 --- a/src/internal/agent/hooks/pods.go +++ b/src/internal/agent/hooks/pods.go @@ -7,14 +7,13 @@ package hooks import ( "encoding/json" "fmt" - "os" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/agent/operations" + "github.com/defenseunicorns/zarf/src/internal/agent/state" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" + "github.com/defenseunicorns/zarf/src/pkg/transform" v1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" @@ -60,7 +59,7 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { zarfSecret := []corev1.LocalObjectReference{{Name: config.ZarfImagePullSecretName}} patchOperations = append(patchOperations, operations.ReplacePatchOperation("/spec/imagePullSecrets", zarfSecret)) - zarfState, err := getStateFromAgentPod(zarfStatePath) + zarfState, err := state.GetZarfStateFromAgentPod() if err != nil { return nil, fmt.Errorf(lang.AgentErrGetState, err) } @@ -69,7 +68,7 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { // update the image host for each init container for idx, container := range pod.Spec.InitContainers { path := fmt.Sprintf("/spec/initContainers/%d/image", idx) - replacement, err := utils.SwapHost(container.Image, containerRegistryURL) + replacement, err := transform.ImageTransformHost(containerRegistryURL, container.Image) if err != nil { message.Warnf(lang.AgentErrImageSwap, container.Image) continue // Continue, because we might as well attempt to mutate the other containers for this pod @@ -80,7 +79,7 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { // update the image host for each ephemeral container for idx, container := range pod.Spec.EphemeralContainers { path := fmt.Sprintf("/spec/ephemeralContainers/%d/image", idx) - replacement, err := utils.SwapHost(container.Image, containerRegistryURL) + replacement, err := transform.ImageTransformHost(containerRegistryURL, container.Image) if err != nil { message.Warnf(lang.AgentErrImageSwap, container.Image) continue // Continue, because we might as well attempt to mutate the other containers for this pod @@ -91,7 +90,7 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { // update the image host for each normal container for idx, container := range pod.Spec.Containers { path := fmt.Sprintf("/spec/containers/%d/image", idx) - replacement, err := utils.SwapHost(container.Image, containerRegistryURL) + replacement, err := transform.ImageTransformHost(containerRegistryURL, container.Image) if err != nil { message.Warnf(lang.AgentErrImageSwap, container.Image) continue // Continue, because we might as well attempt to mutate the other containers for this pod @@ -107,17 +106,3 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { PatchOps: patchOperations, }, nil } - -// Reads the state json file that was mounted into the agent pods. -func getStateFromAgentPod(zarfStatePath string) (types.ZarfState, error) { - zarfState := types.ZarfState{} - - // Read the state file - stateFile, err := os.ReadFile(zarfStatePath) - if err != nil { - return zarfState, err - } - - // Unmarshal the json file into a Go struct - return zarfState, json.Unmarshal(stateFile, &zarfState) -} diff --git a/src/internal/agent/http/handlers.go b/src/internal/agent/http/admission.go similarity index 94% rename from src/internal/agent/http/handlers.go rename to src/internal/agent/http/admission.go index 8157dd1c50..d6f3ee51d7 100644 --- a/src/internal/agent/http/handlers.go +++ b/src/internal/agent/http/admission.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package http provides a http server for the agent. +// Package http provides a http server for the webhook and proxy. package http import ( @@ -111,10 +111,3 @@ func (h *admissionHandler) Serve(hook operations.Hook) http.HandlerFunc { w.Write(jsonResponse) } } - -func healthz() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) - } -} diff --git a/src/internal/agent/http/proxy.go b/src/internal/agent/http/proxy.go new file mode 100644 index 0000000000..8c563fcc3a --- /dev/null +++ b/src/internal/agent/http/proxy.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package http provides a http server for the webhook and proxy. +package http + +import ( + "crypto/tls" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/internal/agent/state" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/transform" +) + +// ProxyHandler constructs a new httputil.ReverseProxy and returns an http handler. +func ProxyHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + err := proxyRequestTransform(r) + if err != nil { + message.Debugf("%#v", err) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(lang.AgentErrUnableTransform)) + return + } + + proxy := &httputil.ReverseProxy{Director: func(r *http.Request) {}, ModifyResponse: proxyResponseTransform} + proxy.ServeHTTP(w, r) + } +} + +func proxyRequestTransform(r *http.Request) error { + message.Debugf("Before Req %#v", r) + message.Debugf("Before Req URL %#v", r.URL) + + // We add this so that we can use it to rewrite urls in the response if needed + r.Header.Add("X-Forwarded-Host", r.Host) + + // We remove this so that go will encode and decode on our behalf (see https://pkg.go.dev/net/http#Transport DisableCompression) + r.Header.Del("Accept-Encoding") + + zarfState, err := state.GetZarfStateFromAgentPod() + if err != nil { + return err + } + + var targetURL *url.URL + + // Setup authentication for each type of service based on User Agent + switch { + case isGitUserAgent(r.UserAgent()): + r.SetBasicAuth(zarfState.GitServer.PushUsername, zarfState.GitServer.PushPassword) + case isNpmUserAgent(r.UserAgent()): + r.Header.Set("Authorization", "Bearer "+zarfState.ArtifactServer.PushToken) + default: + r.SetBasicAuth(zarfState.ArtifactServer.PushUsername, zarfState.ArtifactServer.PushToken) + } + + // Transform the URL; if we see the NoTransform prefix, strip it; otherwise, transform the URL based on User Agent + if strings.HasPrefix(r.URL.Path, transform.NoTransform) { + switch { + case isGitUserAgent(r.UserAgent()): + targetURL, err = transform.NoTransformTarget(zarfState.GitServer.Address, r.URL.Path) + default: + targetURL, err = transform.NoTransformTarget(zarfState.ArtifactServer.Address, r.URL.Path) + } + } else { + switch { + case isGitUserAgent(r.UserAgent()): + targetURL, err = transform.GitTransformURL(zarfState.GitServer.Address, getTLSScheme(r.TLS)+r.Host+r.URL.String(), zarfState.GitServer.PushUsername) + case isPipUserAgent(r.UserAgent()): + targetURL, err = transform.PipTransformURL(zarfState.ArtifactServer.Address, getTLSScheme(r.TLS)+r.Host+r.URL.String()) + case isNpmUserAgent(r.UserAgent()): + targetURL, err = transform.NpmTransformURL(zarfState.ArtifactServer.Address, getTLSScheme(r.TLS)+r.Host+r.URL.String()) + default: + targetURL, err = transform.GenTransformURL(zarfState.ArtifactServer.Address, getTLSScheme(r.TLS)+r.Host+r.URL.String()) + } + } + + if err != nil { + return err + } + + r.Host = targetURL.Host + r.URL = targetURL + r.RequestURI = getRequestURI(targetURL.Path, targetURL.RawQuery, targetURL.Fragment) + + message.Debugf("After Req %#v", r) + message.Debugf("After Req URL%#v", r.URL) + + return nil +} + +func proxyResponseTransform(resp *http.Response) error { + message.Debugf("Before Resp %#v", resp) + + // Handle redirection codes (3xx) by adding a marker to let Zarf know this has been redirected + if resp.StatusCode/100 == 3 { + message.Debugf("Before Resp Location %#v", resp.Header.Get("Location")) + + locationURL, err := url.Parse(resp.Header.Get("Location")) + message.Debugf("%#v", err) + locationURL.Path = transform.NoTransform + locationURL.Path + locationURL.Host = resp.Request.Header.Get("X-Forwarded-Host") + + resp.Header.Set("Location", locationURL.String()) + + message.Debugf("After Resp Location %#v", resp.Header.Get("Location")) + } + + contentType := resp.Header.Get("Content-Type") + + // Handle text content returns that may contain links + if strings.HasPrefix(contentType, "text") || strings.HasPrefix(contentType, "application/json") || strings.HasPrefix(contentType, "application/xml") { + err := replaceBodyLinks(resp) + + if err != nil { + message.Debugf("%#v", err) + } + } + + message.Debugf("After Resp %#v", resp) + + return nil +} + +func replaceBodyLinks(resp *http.Response) error { + message.Debugf("Resp Request: %#v", resp.Request) + + // Create the forwarded (online) and target (offline) URL prefixes to replace + forwardedPrefix := fmt.Sprintf("%s%s%s", getTLSScheme(resp.Request.TLS), resp.Request.Header.Get("X-Forwarded-Host"), transform.NoTransform) + targetPrefix := fmt.Sprintf("%s%s", getTLSScheme(resp.TLS), resp.Request.Host) + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + err = resp.Body.Close() + if err != nil { + return err + } + + bodyString := string(body) + message.Warnf("%s", bodyString) + + bodyString = strings.ReplaceAll(bodyString, targetPrefix, forwardedPrefix) + + message.Warnf("%s", bodyString) + + // Setup the new reader, and correct the content length + resp.Body = io.NopCloser(strings.NewReader(bodyString)) + resp.ContentLength = int64(len(bodyString)) + resp.Header.Set("Content-Length", fmt.Sprint(int64(len(bodyString)))) + + return nil +} + +func getTLSScheme(tls *tls.ConnectionState) string { + scheme := "https://" + + if tls == nil { + scheme = "http://" + } + + return scheme +} + +func getRequestURI(path, query, fragment string) string { + uri := path + + if query != "" { + uri += "?" + query + } + + if fragment != "" { + uri += "#" + fragment + } + + return uri +} + +func isGitUserAgent(userAgent string) bool { + return strings.HasPrefix(userAgent, "git") +} + +func isPipUserAgent(userAgent string) bool { + return strings.HasPrefix(userAgent, "pip") || strings.HasPrefix(userAgent, "twine") +} + +func isNpmUserAgent(userAgent string) bool { + return strings.HasPrefix(userAgent, "npm") || strings.HasPrefix(userAgent, "pnpm") || strings.HasPrefix(userAgent, "yarn") || strings.HasPrefix(userAgent, "bun") +} diff --git a/src/internal/agent/http/server.go b/src/internal/agent/http/server.go index 7cbe4d0ac5..7910d18f44 100644 --- a/src/internal/agent/http/server.go +++ b/src/internal/agent/http/server.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package http provides a http server for the webhook. +// Package http provides a http server for the webhook and proxy. package http import ( @@ -12,8 +12,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" ) -// NewServer creates and return a http.Server. -func NewServer(port string) *http.Server { +// NewAdmissionServer creates an http.Server for the mutating webhook admission handler. +func NewAdmissionServer(port string) *http.Server { message.Debugf("http.NewServer(%s)", port) // Instances hooks @@ -32,3 +32,24 @@ func NewServer(port string) *http.Server { Handler: mux, } } + +// NewProxyServer creates and returns an http proxy server. +func NewProxyServer(port string) *http.Server { + message.Debugf("http.NewHTTPProxy(%s)", port) + + mux := http.NewServeMux() + mux.Handle("/healthz", healthz()) + mux.Handle("/", ProxyHandler()) + + return &http.Server{ + Addr: fmt.Sprintf(":%s", port), + Handler: mux, + } +} + +func healthz() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) + } +} diff --git a/src/internal/agent/start.go b/src/internal/agent/start.go index b6cd1ea2dd..35b57e5fe2 100644 --- a/src/internal/agent/start.go +++ b/src/internal/agent/start.go @@ -30,7 +30,17 @@ const ( func StartWebhook() { message.Debug("agent.StartWebhook()") - server := agentHttp.NewServer(httpPort) + startServer(agentHttp.NewAdmissionServer(httpPort)) +} + +// StartHTTPProxy launches the zarf agent proxy in the cluster. +func StartHTTPProxy() { + message.Debug("agent.StartHttpProxy()") + + startServer(agentHttp.NewProxyServer(httpPort)) +} + +func startServer(server *http.Server) { go func() { if err := server.ListenAndServeTLS(tlsCert, tlsKey); err != nil && err != http.ErrServerClosed { message.Fatal(err, lang.AgentErrStart) diff --git a/src/internal/agent/state/state.go b/src/internal/agent/state/state.go new file mode 100644 index 0000000000..59184478ab --- /dev/null +++ b/src/internal/agent/state/state.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package state provides helpers for interacting with the Zarf agent state. +package state + +import ( + "encoding/json" + "os" + + "github.com/defenseunicorns/zarf/src/types" +) + +const zarfStatePath = "/etc/zarf-state/state" + +// GetZarfStateFromAgentPod reads the state json file that was mounted into the agent pods. +func GetZarfStateFromAgentPod() (types.ZarfState, error) { + zarfState := types.ZarfState{} + + // Read the state file + stateFile, err := os.ReadFile(zarfStatePath) + if err != nil { + return zarfState, err + } + + // Unmarshal the json file into a Go struct + return zarfState, json.Unmarshal(stateFile, &zarfState) +} diff --git a/src/internal/cluster/state.go b/src/internal/cluster/state.go index 988507864d..928c39dcd3 100644 --- a/src/internal/cluster/state.go +++ b/src/internal/cluster/state.go @@ -132,6 +132,7 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { state.GitServer = c.fillInEmptyGitServerValues(initOptions.GitServer) state.RegistryInfo = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo) + state.ArtifactServer = c.fillInEmptyArtifactServerValues(initOptions.ArtifactServer) spinner.Success() @@ -299,3 +300,19 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type return gitServer } + +// Fill in empty ArtifactServerInfo values with the defaults. +func (c *Cluster) fillInEmptyArtifactServerValues(artifactServer types.ArtifactServerInfo) types.ArtifactServerInfo { + // Set default svc url if an external registry was not provided + if artifactServer.Address == "" { + artifactServer.Address = config.ZarfInClusterArtifactServiceURL + artifactServer.InternalServer = true + } + + // Set the push username to the git push user if not specified + if artifactServer.PushUsername == "" { + artifactServer.PushUsername = config.ZarfGitPushUser + } + + return artifactServer +} diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 3815837f5b..73affbffd4 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -13,10 +13,19 @@ import ( netHttp "net/http" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" ) +// CreateTokenResponse is the response given from creating a token in Gitea +type CreateTokenResponse struct { + ID int64 `json:"id"` + Name string `json:"name"` + Sha1 string `json:"sha1"` + TokenLastEight string `json:"token_last_eight"` +} + // CreateReadOnlyUser uses the Gitea API to create a non-admin Zarf user. func (g *Git) CreateReadOnlyUser() error { message.Debugf("git.CreateReadOnlyUser()") @@ -102,24 +111,72 @@ func (g *Git) CreateReadOnlyUser() error { return err } -func (g *Git) addReadOnlyUserToRepo(tunnelURL, repo string) error { - message.Debugf("git.addReadOnlyUserToRepo()") +// CreatePackageRegistryToken uses the Gitea API to create a package registry token. +func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { + message.Debugf("git.CreatePackageRegistryToken()") - // Add the readonly user to the repo - addColabBody := map[string]string{ - "permission": "read", + // Establish a git tunnel to send the repo + tunnel, err := cluster.NewZarfTunnel() + if err != nil { + return CreateTokenResponse{}, err } - addColabData, err := json.Marshal(addColabBody) + tunnel.Connect(cluster.ZarfGit, false) + defer tunnel.Close() + + tunnelURL := tunnel.Endpoint() + + // Determine if the package token already exists + getTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) + getTokensRequest, _ := netHttp.NewRequest("GET", getTokensEndpoint, nil) + out, err := g.DoHTTPThings(getTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + message.Debugf("GET %s:\n%s", getTokensEndpoint, string(out)) if err != nil { - return err + return CreateTokenResponse{}, err } - // Send API request to add a user as a read-only collaborator to a repo - addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelURL, g.Server.PushUsername, repo, g.Server.PullUsername) - addColabRequest, _ := netHttp.NewRequest("PUT", addColabEndpoint, bytes.NewBuffer(addColabData)) - out, err := g.DoHTTPThings(addColabRequest, g.Server.PushUsername, g.Server.PushPassword) - message.Debugf("PUT %s:\n%s", addColabEndpoint, string(out)) - return err + hasPackageToken := false + var tokens []map[string]interface{} + err = json.Unmarshal(out, &tokens) + if err != nil { + return CreateTokenResponse{}, err + } + + for _, token := range tokens { + if token["name"] == config.ZarfArtifactTokenName { + hasPackageToken = true + } + } + + if hasPackageToken { + // Delete the existing token to be replaced + deleteTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens/%s", tunnelURL, g.Server.PushUsername, config.ZarfArtifactTokenName) + deleteTokensRequest, _ := netHttp.NewRequest("DELETE", deleteTokensEndpoint, nil) + out, err := g.DoHTTPThings(deleteTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + message.Debugf("DELETE %s:\n%s", deleteTokensEndpoint, string(out)) + if err != nil { + return CreateTokenResponse{}, err + } + } + + createTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) + createTokensBody := map[string]interface{}{ + "name": config.ZarfArtifactTokenName, + } + createTokensData, _ := json.Marshal(createTokensBody) + createTokensRequest, _ := netHttp.NewRequest("POST", createTokensEndpoint, bytes.NewBuffer(createTokensData)) + out, err = g.DoHTTPThings(createTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + message.Debugf("POST %s:\n%s", createTokensEndpoint, string(out)) + if err != nil { + return CreateTokenResponse{}, err + } + + createTokenResponse := CreateTokenResponse{} + err = json.Unmarshal(out, &createTokenResponse) + if err != nil { + return CreateTokenResponse{}, err + } + + return createTokenResponse, nil } // DoHTTPThings adds http request boilerplate and perform the request, checking for a successful response. @@ -147,3 +204,23 @@ func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([ return responseBody, nil } + +func (g *Git) addReadOnlyUserToRepo(tunnelURL, repo string) error { + message.Debugf("git.addReadOnlyUserToRepo()") + + // Add the readonly user to the repo + addColabBody := map[string]string{ + "permission": "read", + } + addColabData, err := json.Marshal(addColabBody) + if err != nil { + return err + } + + // Send API request to add a user as a read-only collaborator to a repo + addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelURL, g.Server.PushUsername, repo, g.Server.PullUsername) + addColabRequest, _ := netHttp.NewRequest("PUT", addColabEndpoint, bytes.NewBuffer(addColabData)) + out, err := g.DoHTTPThings(addColabRequest, g.Server.PushUsername, g.Server.PushPassword) + message.Debugf("PUT %s:\n%s", addColabEndpoint, string(out)) + return err +} diff --git a/src/internal/packager/git/pull.go b/src/internal/packager/git/pull.go index ca52584d96..44e6a687fb 100644 --- a/src/internal/packager/git/pull.go +++ b/src/internal/packager/git/pull.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/go-git/go-git/v5/plumbing" ) @@ -37,15 +38,12 @@ func (g *Git) DownloadRepoToTemp(gitURL string) error { func (g *Git) Pull(gitURL, targetFolder string) error { g.Spinner.Updatef("Processing git repo %s", gitURL) - // Find the Zarf-specific repo name from the git URL. - get, err := utils.MatchRegex(gitURLRegex, gitURL) + // Split the remote url and the zarf reference + gitURLNoRef, refPlain, err := transform.GitTransformURLSplitRef(gitURL) if err != nil { - return fmt.Errorf("unable to parse git url (%s): %w", gitURL, err) + return err } - // Setup the reference for this repository - refPlain := get("ref") - var ref plumbing.ReferenceName // Parse the ref from the git URL. @@ -54,11 +52,12 @@ func (g *Git) Pull(gitURL, targetFolder string) error { } // Construct a path unique to this git repo - repoFolder := fmt.Sprintf("%s-%d", get("repo"), utils.GetCRCHash(gitURL)) - g.GitPath = path.Join(targetFolder, repoFolder) + repoFolder, err := transform.GitTransformURLtoFolderName(gitURL) + if err != nil { + return err + } - // Construct the remote URL without the reference - gitURLNoRef := fmt.Sprintf("%s%s/%s%s", get("proto"), get("hostPath"), get("repo"), get("git")) + g.GitPath = path.Join(targetFolder, repoFolder) // Clone the git repository. err = g.clone(gitURLNoRef, ref) diff --git a/src/internal/packager/git/push.go b/src/internal/packager/git/push.go index 2afc4205c9..518281ac70 100644 --- a/src/internal/packager/git/push.go +++ b/src/internal/packager/git/push.go @@ -11,7 +11,7 @@ import ( "path" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/go-git/go-git/v5" goConfig "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing/transport" @@ -23,20 +23,17 @@ func (g *Git) PushRepo(srcURL, targetFolder string) error { spinner := message.NewProgressSpinner("Processing git repo %s", srcURL) defer spinner.Stop() - // Parse the git URL. - get, err := utils.MatchRegex(gitURLRegex, srcURL) + // Setup git paths, including a unique name for the repo based on the hash of the git URL to avoid conflicts. + repoFolder, err := transform.GitTransformURLtoFolderName(srcURL) if err != nil { return fmt.Errorf("unable to parse git url (%s): %w", srcURL, err) } - - // Setup git paths, including a unique name for the repo based on the hash of the git URL to avoid conflicts. - repoFolder := fmt.Sprintf("%s-%d", get("repo"), utils.GetCRCHash(srcURL)) repoPath := path.Join(targetFolder, repoFolder) // Check that this package is using the new repo format (if not fallback to the format from <= 0.24.x) _, err = os.Stat(repoPath) if os.IsNotExist(err) { - repoFolder, err = g.TransformURLtoRepoName(srcURL) + repoFolder, err = transform.GitTransformURLtoRepoName(srcURL) if err != nil { return fmt.Errorf("unable to parse git url (%s): %w", srcURL, err) } @@ -52,7 +49,7 @@ func (g *Git) PushRepo(srcURL, targetFolder string) error { } if err := g.push(repo, spinner); err != nil { - spinner.Warnf("Unable to push the git repo %s (%s). Retrying....", get("repo"), err.Error()) + spinner.Warnf("Unable to push the git repo %s (%s). Retrying....", repoFolder, err.Error()) return err } @@ -65,7 +62,7 @@ func (g *Git) PushRepo(srcURL, targetFolder string) error { return err } remoteURL := remote.Config().URLs[0] - repoName, err := g.TransformURLtoRepoName(remoteURL) + repoName, err := transform.GitTransformURLtoRepoName(remoteURL) if err != nil { message.Warnf("Unable to add the read-only user to the repo: %s\n", repoName) return err @@ -96,7 +93,7 @@ func (g *Git) prepRepoForPush() (*git.Repository, error) { } remoteURL := remote.Config().URLs[0] - targetURL, err := g.TransformURL(remoteURL) + targetURL, err := transform.GitTransformURL(g.Server.Address, remoteURL, g.Server.PushUsername) if err != nil { return nil, fmt.Errorf("unable to transform the git url: %w", err) } @@ -106,7 +103,7 @@ func (g *Git) prepRepoForPush() (*git.Repository, error) { _, err = repo.CreateRemote(&goConfig.RemoteConfig{ Name: offlineRemoteName, - URLs: []string{targetURL}, + URLs: []string{targetURL.String()}, }) if err != nil { return nil, fmt.Errorf("failed to create offline remote: %w", err) diff --git a/src/internal/packager/git/url.go b/src/internal/packager/git/url.go deleted file mode 100644 index 9ed957c1a4..0000000000 --- a/src/internal/packager/git/url.go +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "fmt" - "regexp" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" -) - -// For further explanation: https://regex101.com/r/xx8NQe/1. -var gitURLRegex = regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)\/(?P[\w\-\.]+?)(?P\.git)?(?P@(?P\+)?(?P[\/\+\w\-\.]+))?$`) - -// MutateGitURLsInText changes the gitURL hostname to use the repository Zarf is configured to use. -func (g *Git) MutateGitURLsInText(text string) string { - extractPathRegex := regexp.MustCompilePOSIX(`https?://[^/]+/(.*\.git)`) - output := extractPathRegex.ReplaceAllStringFunc(text, func(match string) string { - output, err := g.TransformURL(match) - if err != nil { - message.Warnf("Unable to transform the git url, using the original url we have: %s", match) - } - return output - }) - return output -} - -// TransformURLtoRepoName takes a git url and returns a Zarf-compatible repo name. -func (g *Git) TransformURLtoRepoName(url string) (string, error) { - get, err := utils.MatchRegex(gitURLRegex, url) - - if err != nil { - // Unable to find a substring match for the regex - return "", fmt.Errorf("unable to get extract the repoName from the url %s", url) - } - - repoName := get("repo") - // NOTE: We remove the .git and protocol so that https://zarf.dev/repo.git and http://zarf.dev/repo - // resolve to the same repp (as they would in real life) - sanitizedURL := fmt.Sprintf("%s/%s", get("hostPath"), repoName) - - // Add crc32 hash of the repoName to the end of the repo - checksum := utils.GetCRCHash(sanitizedURL) - - newRepoName := fmt.Sprintf("%s-%d", repoName, checksum) - - return newRepoName, nil -} - -// TransformURL takes a git url and returns a Zarf-compatible url. -func (g *Git) TransformURL(url string) (string, error) { - repoName, err := g.TransformURLtoRepoName(url) - if err != nil { - return url, err - } - output := fmt.Sprintf("%s/%s/%s", g.Server.Address, g.Server.PushUsername, repoName) - message.Debugf("Rewrite git URL: %s -> %s", url, output) - return output, nil -} diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index 241d52ef08..dbd9c9c239 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -8,7 +8,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/logs" ) @@ -74,7 +74,7 @@ func (i *ImgConfig) PushToZarfRegistry() error { // If this is not a no checksum image push it for use with the Zarf agent if !i.NoChecksum { - offlineNameCRC, err := utils.SwapHost(src, registryURL) + offlineNameCRC, err := transform.ImageTransformHost(registryURL, src) if err != nil { return err } @@ -88,7 +88,7 @@ func (i *ImgConfig) PushToZarfRegistry() error { // To allow for other non-zarf workloads to easily see the images upload a non-checksum version // (this may result in collisions but this is acceptable for this use case) - offlineName, err := utils.SwapHostWithoutChecksum(src, registryURL) + offlineName, err := transform.ImageTransformHostWithoutChecksum(registryURL, src) if err != nil { return err } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index a55bf87465..3945c5878e 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -461,7 +461,7 @@ func (p *Packager) validatePackageSignature(publicKeyPath string) error { // Validate the signature of the package if publicKeyPath == "" { - return fmt.Errorf("package is signed but no key was provided, using signed packages requires a --key or --insecure flag to continue") + return fmt.Errorf("package is signed but no key was provided - add a key with the --key flag or use the --insecure flag and run the command again") } // Validate the signature with the key we were provided diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 5f90ad7f9a..22ed3d10b1 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -48,6 +48,7 @@ func (p *Packager) Create(baseDir string) error { } if p.cfg.Pkg.Kind == "ZarfInitConfig" { + p.cfg.Pkg.Metadata.Version = config.CLIVersion p.cfg.IsInitConfig = true } diff --git a/src/pkg/transform/artifact.go b/src/pkg/transform/artifact.go new file mode 100644 index 0000000000..105a6f9f50 --- /dev/null +++ b/src/pkg/transform/artifact.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package transform provides helper functions to transform URLs to airgap equivalents +package transform + +import ( + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/defenseunicorns/zarf/src/pkg/utils" +) + +const ( + // NoTransform is the URL prefix added to HTTP 3xx or text URLs that instructs Zarf not to transform on a subsequent request. + NoTransform = "/zarf-3xx-no-transform" +) + +// NoTransformTarget takes an address that Zarf should not transform, and removes the NoTransform prefix. +func NoTransformTarget(address string, path string) (*url.URL, error) { + targetURL, err := url.Parse(address) + if err != nil { + return nil, err + } + + targetURL.Path = strings.TrimPrefix(path, NoTransform) + + return targetURL, nil +} + +// NpmTransformURL finds the npm API path on a given URL and transforms that to align with the offline registry. +func NpmTransformURL(targetBaseURL string, sourceURL string) (*url.URL, error) { + // For further explanation: https://regex101.com/r/RRyazc/3 + // This regex was created with information from https://github.com/go-gitea/gitea/blob/0e58201d1a8247561809d832eb8f576e05e5d26d/routers/api/packages/api.go#L210 + npmURLRegex := regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)` + + `(?P(\/(@[\w\.\-\~]+(\/|%2[fF]))?[\w\.\-\~]+(\/-\/([\w\.\-\~]+\/)?[\w\.\-\~]+\.[\w]+)?(\/-rev\/.+)?)|(\/-\/(npm|v1|user|package)\/.+))$`) + + return transformRegistryPath(targetBaseURL, sourceURL, npmURLRegex, "npmPath", "npm") +} + +// PipTransformURL finds the pip API path on a given URL and transforms that to align with the offline registry. +func PipTransformURL(targetBaseURL string, sourceURL string) (*url.URL, error) { + // For further explanation: https://regex101.com/r/lreZiD/2 + // This regex was created with information from https://github.com/go-gitea/gitea/blob/0e58201d1a8247561809d832eb8f576e05e5d26d/routers/api/packages/api.go#L267 + pipURLRegex := regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)(?P\/((simple|files\/)[\/\w\-\.\?\=&%#]*?)?)?$`) + + return transformRegistryPath(targetBaseURL, sourceURL, pipURLRegex, "pipPath", "pypi") +} + +// GenTransformURL finds the generic API path on a given URL and transforms that to align with the offline registry. +func GenTransformURL(targetBaseURL string, sourceURL string) (*url.URL, error) { + // For further explanation: https://regex101.com/r/bwMkCm/5 + // This regex was created with information from https://www.rfc-editor.org/rfc/rfc3986#section-2 + genURLRegex := regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P[a-zA-Z0-9\-\.]+)(?P:[0-9]+?)?(?P\/[\w\-\.+~%]+?\/[\w\-\.+~%]+?)?(?P\/.+?)??(?P\/[\w\-\.+~%]+?)??(?P\/[\w\-\.+~%]*)?(?P[\w\-\.\?\=,;+~!$'*&%#()\[\]]*?)?$`) + + matches := genURLRegex.FindStringSubmatch(sourceURL) + idx := genURLRegex.SubexpIndex + + if len(matches) == 0 { + // Unable to find a substring match for the regex + return nil, fmt.Errorf("unable to extract the genericPath from the url %s", sourceURL) + } + + fileName := strings.ReplaceAll(matches[idx("fileName")], "/", "") + if fileName == "" { + fileName = matches[idx("host")] + } + + // NOTE: We remove the protocol, port and file name so that https://zarf.dev:443/package/package1.zip and http://zarf.dev/package/package2.zip + // resolve to the same "folder" (as they would in real life) + sanitizedURL := fmt.Sprintf("%s%s%s", matches[idx("host")], matches[idx("startPath")], matches[idx("midPath")]) + + packageName := strings.ReplaceAll(matches[idx("startPath")], "/", "") + if packageName == "" { + packageName = fileName + } + // Add crc32 hash of the url to the end of the package name + packageNameGlobal := fmt.Sprintf("%s-%d", packageName, utils.GetCRCHash(sanitizedURL)) + + version := strings.ReplaceAll(matches[idx("version")], "/", "") + if version == "" { + version = fileName + } + + // Rebuild the generic URL + transformedURL := fmt.Sprintf("%s/generic/%s/%s/%s", targetBaseURL, packageNameGlobal, version, fileName) + + url, err := url.Parse(transformedURL) + if err != nil { + return url, err + } + + // Drop the RawQuery and Fragment to avoid them being interpreted for generic packages + url.RawQuery = "" + url.Fragment = "" + + return url, err +} + +// transformRegistryPath transforms a given request path using a new base URL and regex. +// - pathGroup specifies the named group for the registry's URL path inside the regex (i.e. pipPath) and registryType specifies the registry type (i.e. pypi). +func transformRegistryPath(targetBaseURL string, sourceURL string, regex *regexp.Regexp, pathGroup string, registryType string) (*url.URL, error) { + matches := regex.FindStringSubmatch(sourceURL) + idx := regex.SubexpIndex + + if len(matches) == 0 { + // Unable to find a substring match for the regex + return nil, fmt.Errorf("unable to extract the %s from the url %s", pathGroup, sourceURL) + } + + // Rebuild the URL based on registry type + transformedURL := fmt.Sprintf("%s/%s%s", targetBaseURL, registryType, matches[idx(pathGroup)]) + + return url.Parse(transformedURL) +} diff --git a/src/pkg/transform/artifact_test.go b/src/pkg/transform/artifact_test.go new file mode 100644 index 0000000000..bd3cee0522 --- /dev/null +++ b/src/pkg/transform/artifact_test.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package transform provides helper functions to transform URLs to airgap equivalents +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNoTransformTarget(t *testing.T) { + // Properly removes the NoTransform target + newURL, err := NoTransformTarget("https://gitlab.com", NoTransform+"/some-path/without-query") + assert.NoError(t, err) + assert.Equal(t, "https://gitlab.com/some-path/without-query", newURL.String()) + + // Passes through without the NoTransform target + newURL, err = NoTransformTarget("https://gitlab.com", "/some-path/without-query") + assert.NoError(t, err) + assert.Equal(t, "https://gitlab.com/some-path/without-query", newURL.String()) + + // Returns an error when given a bad base url + _, err = NoTransformTarget("https*://gitlab.com", "/some-path/without-query") + assert.Error(t, err) +} + +func TestNpmTransformURL(t *testing.T) { + protocolPaths := []string{ + "/npm", + "/npm/-rev/undefined", + "/npm/-/8.19.2/npm-8.19.2.tgz", + "/npm/-/8.19.2/npm-8.19.2.tgz/-rev/undefined", + "/lodash", + "/lodash/-rev/1", + "/lodash/-/4.17.21/lodash-4.17.21.tgz", + "/lodash/-/4.17.21/lodash-4.17.21.tgz/-rev/1", + "/@types/node", + "/@types%2fnode", + "/@types%2Fnode", + "/@types/node/-rev/100", + "/@types%2fnode/-rev/100", + "/@types%2Fnode/-rev/100", + "/@types/node/-/18.11.2/types-node-18.11.2.tgz", + "/@types%2fnode/-/18.11.2/types-node-18.11.2.tgz", + "/@types%2Fnode/-/18.11.2/types-node-18.11.2.tgz", + "/@types/node/-/18.11.2/types-node-18.11.2.tgz/-rev/100", + "/-/package/lodash/dist-tags", + "/-/package/lodash/dist-tags/latest", + "/-/package/@types/node/dist-tags", + "/-/package/@types/node/dist-tags/release", + "/-/npm/v1/security/advisories/bulk", + "/-/npm/v1/security/audits/quick", + "/-/user/org.couchdb.user:username", + "/-/v1/login", + "/-/v1/search", + } + + protocolHosts := []string{ + "https://git.privatemirror.com/api/packages/zarf-mirror-user/npm", + "https://registry.npmjs.org", + } + + for _, host := range protocolHosts { + for _, path := range protocolPaths { + newURL, err := NpmTransformURL("https://gitlab.com/project", host+path) + assert.NoError(t, err) + // For each host/path swap them and add `npm` for compatibility with Gitea/Gitlab + assert.Equal(t, "https://gitlab.com/project/npm"+path, newURL.String()) + } + } + + // Returns an error when given a bad base url + _, err := NpmTransformURL("https*://gitlab.com/project", "https://registry.npmjs.org/npm") + assert.Error(t, err) +} + +func TestPipTransformURL(t *testing.T) { + protocolPaths := []string{ + "", + "/", + "/simple", + "/simple/", + "/simple/numpy", + "/simple/numpy/", + "/files/numpy/1.23.4/numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl#sha256-4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962", + } + + protocolHosts := []string{ + "https://git.privatemirror.com/api/packages/zarf-mirror-user/pip", + "https://pypi.org", + } + + for _, host := range protocolHosts { + for _, path := range protocolPaths { + newURL, err := PipTransformURL("https://gitlab.com/project", host+path) + assert.NoError(t, err) + // For each host/path swap them and add `pypi` for compatibility with Gitea/Gitlab + assert.Equal(t, "https://gitlab.com/project/pypi"+path, newURL.String()) + } + } + + // Returns an error when given a bad base url + _, err := PipTransformURL("https*://gitlab.com/project", "https://pypi.org/simple") + assert.Error(t, err) +} + +func TestGenTransformURL(t *testing.T) { + urls := []string{ + "https://git.example.com/api/packages/zarf-git-user/generic", + "https://git.example.com/api/packages/zarf-git-user/generic/packageVersion/packageName", + "https://git.example.com/api/packages/zarf-git-user/generic/package+name/packageVersion/filename", + "https://git.example.com/api/packages/zarf-git-user/generic/some%20generic%20package/0.0.1/superGeneric.tar.gz", + "https://git.example.com/archive.zip", + "https://git.example.com:443/archive.zip", + "https://git.example.com/api/packages/username/generic/some-package-name/file.1.4.4.tar.gz", + "https://git.example.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz", + "https://some.host.com/mirror/some-package-release/github.com/baselbuild/bazel-toolchains/archive/someshasum9318490392.tar.gz", + "https://some.host.com/mirror/some-package-release/some-org/some-library/archive/refs/tags/file.zip", + "https://i.am.a.weird.php.thing:8080/~/and/rfc/3986/loves/me?array[]=1&array[]=2&pie=(1)", + "http://why.microsoft/did/you/do/this/Foo.aspx?id=1,2,3,4", + "http://this.is.legal.too/according/to/spec/?world=!$;,*'", + "http://i.end.in.nothing.com", + "http://i.end.in.slash.com/", + } + + expectedURLs := []string{ + // We want most of these to exist in the form of /project/generic/packageName/version/filename + "https://gitlab.com/project/generic/apipackages-3151594639/zarf-git-user/generic", + "https://gitlab.com/project/generic/apipackages-2561175711/packageVersion/packageName", + "https://gitlab.com/project/generic/apipackages-2265319408/packageVersion/filename", + "https://gitlab.com/project/generic/apipackages-4040139506/0.0.1/superGeneric.tar.gz", + "https://gitlab.com/project/generic/archive.zip-2052577494/archive.zip/archive.zip", + "https://gitlab.com/project/generic/archive.zip-2052577494/archive.zip/archive.zip", + "https://gitlab.com/project/generic/apipackages-764706626/some-package-name/file.1.4.4.tar.gz", + "https://gitlab.com/project/generic/facebookzstd-1475713874/v1.4.4/zstd-1.4.4.tar.gz", + "https://gitlab.com/project/generic/mirrorsome-package-release-1448769245/archive/someshasum9318490392.tar.gz", + "https://gitlab.com/project/generic/mirrorsome-package-release-2849990407/tags/file.zip", + "https://gitlab.com/project/generic/~and-1138178232/loves/me", + "https://gitlab.com/project/generic/didyou-2239567090/this/Foo.aspx", + "https://gitlab.com/project/generic/accordingto-29577836/spec/this.is.legal.too", + "https://gitlab.com/project/generic/i.end.in.nothing.com-2766891503/i.end.in.nothing.com/i.end.in.nothing.com", + "https://gitlab.com/project/generic/i.end.in.slash.com-1566625415/i.end.in.slash.com/i.end.in.slash.com", + } + + for idx, url := range urls { + newURL, err := GenTransformURL("https://gitlab.com/project", url) + assert.NoError(t, err) + // For each host/path swap them and add `pypi` for compatibility with Gitea/Gitlab + assert.Equal(t, expectedURLs[idx], newURL.String()) + } + + // Returns an error when given a bad base url + _, err := GenTransformURL("https*://gitlab.com/project", "http://i.end.in.nothing.com") + assert.Error(t, err) +} diff --git a/src/pkg/transform/git.go b/src/pkg/transform/git.go new file mode 100644 index 0000000000..076f4b081b --- /dev/null +++ b/src/pkg/transform/git.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package transform provides helper functions to transform URLs to airgap equivalents +package transform + +import ( + "fmt" + "net/url" + "regexp" + + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" +) + +// For further explanation: https://regex101.com/r/YxpfhC/3 +var gitURLRegex = regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)\/(?P[\w\-\.]+?)(?P\.git)?(?P@(?P\+)?(?P[\/\+\w\-\.]+))?(?P\/(?Pinfo\/.*|git-upload-pack|git-receive-pack))?$`) + +// MutateGitURLsInText changes the gitURL hostname to use the repository Zarf is configured to use. +func MutateGitURLsInText(targetBaseURL string, text string, pushUser string) string { + extractPathRegex := regexp.MustCompilePOSIX(`https?://[^/]+/(.*\.git)`) + output := extractPathRegex.ReplaceAllStringFunc(text, func(match string) string { + output, err := GitTransformURL(targetBaseURL, match, pushUser) + if err != nil { + message.Warnf("Unable to transform the git url, using the original url we have: %s", match) + return match + } + return output.String() + }) + return output +} + +// GitTransformURLSplitRef takes a git url and returns a separated source url and zarf reference. +func GitTransformURLSplitRef(sourceURL string) (string, string, error) { + get, err := utils.MatchRegex(gitURLRegex, sourceURL) + + if err != nil { + return "", "", fmt.Errorf("unable to get extract the source url and ref from the url %s", sourceURL) + } + + gitURLNoRef := fmt.Sprintf("%s%s/%s%s", get("proto"), get("hostPath"), get("repo"), get("git")) + refPlain := get("ref") + + return gitURLNoRef, refPlain, nil +} + +// GitTransformURLtoFolderName takes a git url and returns the folder name for the repo in the Zarf package. +func GitTransformURLtoFolderName(sourceURL string) (string, error) { + get, err := utils.MatchRegex(gitURLRegex, sourceURL) + + if err != nil { + // Unable to find a substring match for the regex + return "", fmt.Errorf("unable to get extract the folder name from the url %s", sourceURL) + } + + repoName := get("repo") + // NOTE: For folders we use the full URL (without any protocol stuff) so that different refs are kept in different folders on disk to avoid conflicts + // Add crc32 hash of the repoName to the end of the repo + gitURL := fmt.Sprintf("%s%s/%s%s%s", get("proto"), get("hostPath"), get("repo"), get("git"), get("atRef")) + + checksum := utils.GetCRCHash(gitURL) + + newRepoName := fmt.Sprintf("%s-%d", repoName, checksum) + + return newRepoName, nil +} + +// GitTransformURLtoRepoName takes a git url and returns the name of the repo in the remote airgap repository. +func GitTransformURLtoRepoName(sourceURL string) (string, error) { + get, err := utils.MatchRegex(gitURLRegex, sourceURL) + + if err != nil { + // Unable to find a substring match for the regex + return "", fmt.Errorf("unable to get extract the repo name from the url %s", sourceURL) + } + + repoName := get("repo") + // NOTE: We remove the .git and protocol so that https://zarf.dev/repo.git and http://zarf.dev/repo + // resolve to the same repo (as they would in real life) + sanitizedURL := fmt.Sprintf("%s/%s", get("hostPath"), repoName) + + // Add crc32 hash of the repoName to the end of the repo + checksum := utils.GetCRCHash(sanitizedURL) + + newRepoName := fmt.Sprintf("%s-%d", repoName, checksum) + + return newRepoName, nil +} + +// GitTransformURL takes a base URL, a source url and a username and returns a Zarf-compatible url. +func GitTransformURL(targetBaseURL string, sourceURL string, pushUser string) (*url.URL, error) { + repoName, err := GitTransformURLtoRepoName(sourceURL) + if err != nil { + return nil, err + } + + // Get the full path + matches := gitURLRegex.FindStringSubmatch(sourceURL) + idx := gitURLRegex.SubexpIndex + + if len(matches) == 0 { + // Unable to find a substring match for the regex + return nil, fmt.Errorf("unable to extract the airgap target url from the url %s", sourceURL) + } + + output := fmt.Sprintf("%s/%s/%s%s%s", targetBaseURL, pushUser, repoName, matches[idx("git")], matches[idx("gitPath")]) + message.Debugf("Rewrite git URL: %s -> %s", sourceURL, output) + return url.Parse(output) +} diff --git a/src/pkg/transform/git_test.go b/src/pkg/transform/git_test.go new file mode 100644 index 0000000000..f7bac2c0f4 --- /dev/null +++ b/src/pkg/transform/git_test.go @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package transform provides helper functions to transform URLs to airgap equivalents +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var gitURLs = []string{ + // Normal git repos and references for pushing/pulling + "https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git", + "https://github.com/defenseunicorns/zarf.git", + "https://ghcr.io/stefanprodan/podinfo_fasd-123.git", + "git://k3d-cluster.localhost/defenseunicorns/zarf-agent", + "http://localhost:5000/some-cool-repo", + "ssh://ghcr.io/stefanprodan/podinfo@6.0.0", + "https://stefanprodan/podinfo.git@adf0fasd10.1.223124123123-asdf", + "https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git@0.0.9-bb.0", + "file:///srv/git/stefanprodan/podinfo@adf0fasd10.1.223124123123-asdf", + "https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test", + "https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test@524980951ff16e19dc25232e9aea8fd693989ba6", + "https://github.com/defenseunicorns/zarf.helm.git", + "https://github.com/defenseunicorns/zarf.git@refs/tags/v0.16.0", + "https://github.com/DoD-Platform-One/big-bang.git@refs/heads/release-1.54.x", + + // Smart Git Protocol URLs for proxying (https://www.git-scm.com/docs/http-protocol) + "https://github.com/defenseunicorns/zarf.helm.git/info/refs", + "https://github.com/defenseunicorns/zarf.helm.git/info/refs?service=git-upload-pack", + "https://github.com/defenseunicorns/zarf.helm.git/info/refs?service=git-receive-pack", + "https://github.com/defenseunicorns/zarf.helm.git/git-upload-pack", + "https://github.com/defenseunicorns/zarf.helm.git/git-receive-pack", +} + +var badGitURLs = []string{ + "i am not a url at all", + "C:\\Users\\zarf", +} + +func TestMutateGitURLsInText(t *testing.T) { + originalText := ` + # Here we handle invalid URLs (see below comment) + # We transform https://*/*.git URLs + https://github.com/defenseunicorns/zarf.git + # Even URLs with things on either side + stuffhttps://github.com/defenseunicorns/zarf.gitandthings + # But not ssh://*/*.git URLs + ssh://git@github.com/defenseunicorns/zarf.git + # Or non .git URLs + https://www.defenseunicorns.com/ + ` + + expectedText := ` + # Here we handle invalid URLs (see below comment) + # We transform https://*/*.git URLs + https://gitlab.com/repo-owner/zarf-1211668992.git + # Even URLs with things on either side + stuffhttps://gitlab.com/repo-owner/zarf-1211668992.gitandthings + # But not ssh://*/*.git URLs + ssh://git@github.com/defenseunicorns/zarf.git + # Or non .git URLs + https://www.defenseunicorns.com/ + ` + + resultingText := MutateGitURLsInText("https://gitlab.com", originalText, "repo-owner") + assert.Equal(t, expectedText, resultingText) +} + +func TestGitTransformURLSplitRef(t *testing.T) { + var expectedResult = [][]string{ + // Normal git repos and references for pushing/pulling + {"https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git", ""}, + {"https://github.com/defenseunicorns/zarf.git", ""}, + {"https://ghcr.io/stefanprodan/podinfo_fasd-123.git", ""}, + {"git://k3d-cluster.localhost/defenseunicorns/zarf-agent", ""}, + {"http://localhost:5000/some-cool-repo", ""}, + {"ssh://ghcr.io/stefanprodan/podinfo", "6.0.0"}, + {"https://stefanprodan/podinfo.git", "adf0fasd10.1.223124123123-asdf"}, + {"https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git", "0.0.9-bb.0"}, + {"file:///srv/git/stefanprodan/podinfo", "adf0fasd10.1.223124123123-asdf"}, + {"https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test", ""}, + {"https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test", "524980951ff16e19dc25232e9aea8fd693989ba6"}, + {"https://github.com/defenseunicorns/zarf.helm.git", ""}, + {"https://github.com/defenseunicorns/zarf.git", "refs/tags/v0.16.0"}, + {"https://github.com/DoD-Platform-One/big-bang.git", "refs/heads/release-1.54.x"}, + + // Smart Git Protocol URLs for proxying (https://www.git-scm.com/docs/http-protocol) + {"https://github.com/defenseunicorns/zarf.helm.git", ""}, + {"https://github.com/defenseunicorns/zarf.helm.git", ""}, + {"https://github.com/defenseunicorns/zarf.helm.git", ""}, + {"https://github.com/defenseunicorns/zarf.helm.git", ""}, + {"https://github.com/defenseunicorns/zarf.helm.git", ""}, + } + + for idx, url := range gitURLs { + gitURLNoRef, refPlain, err := GitTransformURLSplitRef(url) + assert.NoError(t, err) + assert.Equal(t, expectedResult[idx][0], gitURLNoRef) + assert.Equal(t, expectedResult[idx][1], refPlain) + } + + for _, url := range badGitURLs { + _, _, err := GitTransformURLSplitRef(url) + assert.Error(t, err) + } +} + +func TestGitTransformURLtoFolderName(t *testing.T) { + var expectedResult = []string{ + // Normal git repos and references for pushing/pulling + "twistlock-1590638614", + "zarf-3863619701", + "podinfo_fasd-123-1478387306", + "zarf-agent-802453811", + "some-cool-repo-1916670310", + "podinfo-1350532569", + "podinfo-1853010387", + "twistlock-1920149257", + "podinfo-122075437", + "zarf-public-test-612413317", + "zarf-public-test-634307705", + "zarf.helm-2570741950", + "zarf-2175050463", + "big-bang-2705706079", + + // Smart Git Protocol URLs for proxying (https://www.git-scm.com/docs/http-protocol) + "zarf.helm-2570741950", + "zarf.helm-2570741950", + "zarf.helm-2570741950", + "zarf.helm-2570741950", + "zarf.helm-2570741950", + } + + for idx, url := range gitURLs { + repoFolder, err := GitTransformURLtoFolderName(url) + assert.NoError(t, err) + assert.Equal(t, expectedResult[idx], repoFolder) + } + + for _, url := range badGitURLs { + _, err := GitTransformURLtoFolderName(url) + assert.Error(t, err) + } +} + +func TestGitTransformURLtoRepoName(t *testing.T) { + var expectedResult = []string{ + // Normal git repos and references for pushing/pulling + "twistlock-97328248", + "zarf-1211668992", + "podinfo_fasd-123-84577122", + "zarf-agent-3633494462", + "some-cool-repo-926913879", + "podinfo-2985051089", + "podinfo-2197246515", + "twistlock-97328248", + "podinfo-1175499642", + "zarf-public-test-2170732467", + "zarf-public-test-2170732467", + "zarf.helm-842267124", + "zarf-1211668992", + "big-bang-2366614037", + + // Smart Git Protocol URLs for proxying (https://www.git-scm.com/docs/http-protocol) + "zarf.helm-842267124", + "zarf.helm-842267124", + "zarf.helm-842267124", + "zarf.helm-842267124", + "zarf.helm-842267124", + } + + for idx, url := range gitURLs { + repoName, err := GitTransformURLtoRepoName(url) + assert.NoError(t, err) + assert.Equal(t, expectedResult[idx], repoName) + } + + for _, url := range badGitURLs { + _, err := GitTransformURLtoRepoName(url) + assert.Error(t, err) + } +} + +func TestGitTransformURL(t *testing.T) { + var expectedResult = []string{ + // Normal git repos and references for pushing/pulling + "https://gitlab.com/repo-owner/twistlock-97328248.git", + "https://gitlab.com/repo-owner/zarf-1211668992.git", + "https://gitlab.com/repo-owner/podinfo_fasd-123-84577122.git", + "https://gitlab.com/repo-owner/zarf-agent-3633494462", + "https://gitlab.com/repo-owner/some-cool-repo-926913879", + "https://gitlab.com/repo-owner/podinfo-2985051089", + "https://gitlab.com/repo-owner/podinfo-2197246515.git", + "https://gitlab.com/repo-owner/twistlock-97328248.git", + "https://gitlab.com/repo-owner/podinfo-1175499642", + "https://gitlab.com/repo-owner/zarf-public-test-2170732467", + "https://gitlab.com/repo-owner/zarf-public-test-2170732467", + "https://gitlab.com/repo-owner/zarf.helm-842267124.git", + "https://gitlab.com/repo-owner/zarf-1211668992.git", + "https://gitlab.com/repo-owner/big-bang-2366614037.git", + + // Smart Git Protocol URLs for proxying (https://www.git-scm.com/docs/http-protocol) + "https://gitlab.com/repo-owner/zarf.helm-842267124.git/info/refs", + "https://gitlab.com/repo-owner/zarf.helm-842267124.git/info/refs?service=git-upload-pack", + "https://gitlab.com/repo-owner/zarf.helm-842267124.git/info/refs?service=git-receive-pack", + "https://gitlab.com/repo-owner/zarf.helm-842267124.git/git-upload-pack", + "https://gitlab.com/repo-owner/zarf.helm-842267124.git/git-receive-pack", + } + + for idx, url := range gitURLs { + repoURL, err := GitTransformURL("https://gitlab.com", url, "repo-owner") + assert.NoError(t, err) + assert.Equal(t, expectedResult[idx], repoURL.String()) + } + + for _, url := range badGitURLs { + _, err := GitTransformURL("https://gitlab.com", url, "repo-owner") + assert.Error(t, err) + } +} diff --git a/src/pkg/utils/image.go b/src/pkg/transform/image.go similarity index 61% rename from src/pkg/utils/image.go rename to src/pkg/transform/image.go index 3441be714c..127fd91a63 100644 --- a/src/pkg/utils/image.go +++ b/src/pkg/transform/image.go @@ -1,12 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. -package utils +// Package transform provides helper functions to transform URLs to airgap equivalents +package transform import ( "fmt" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/distribution/distribution/reference" ) @@ -21,30 +22,30 @@ type Image struct { TagOrDigest string } -// SwapHost Perform base url replacement and adds a crc32 of the original url to the end of the src. -func SwapHost(src string, targetHost string) (string, error) { - image, err := parseImageURL(src) +// ImageTransformHost replaces the base url for an image and adds a crc32 of the original url to the end of the src (note image refs are not full URLs). +func ImageTransformHost(targetHost, srcReference string) (string, error) { + image, err := parseImageURL(srcReference) if err != nil { return "", err } // Generate a crc32 hash of the image host + name - checksum := GetCRCHash(image.Name) + checksum := utils.GetCRCHash(image.Name) return fmt.Sprintf("%s/%s-%d%s", targetHost, image.Path, checksum, image.TagOrDigest), nil } -// SwapHostWithoutChecksum Perform base url replacement but avoids adding a checksum of the original url. -func SwapHostWithoutChecksum(src string, targetHost string) (string, error) { - image, err := parseImageURL(src) +// ImageTransformHostWithoutChecksum replaces the base url for an image but avoids adding a checksum of the original url (note image refs are not full URLs). +func ImageTransformHostWithoutChecksum(targetHost, srcReference string) (string, error) { + image, err := parseImageURL(srcReference) if err != nil { return "", err } return fmt.Sprintf("%s/%s%s", targetHost, image.Path, image.TagOrDigest), nil } -func parseImageURL(src string) (out Image, err error) { - ref, err := reference.ParseAnyReference(src) +func parseImageURL(srcReference string) (out Image, err error) { + ref, err := reference.ParseAnyReference(srcReference) if err != nil { return out, err } @@ -56,7 +57,7 @@ func parseImageURL(src string) (out Image, err error) { out.Host = reference.Domain(named) out.Reference = ref.String() } else { - return out, fmt.Errorf("unable to parse image name from %s", src) + return out, fmt.Errorf("unable to parse image name from %s", srcReference) } // Parse the tag and add it to digestOrReference diff --git a/src/pkg/transform/image_test.go b/src/pkg/transform/image_test.go new file mode 100644 index 0000000000..b8d017d39c --- /dev/null +++ b/src/pkg/transform/image_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package transform provides helper functions to transform URLs to airgap equivalents +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var imageRefs = []string{ + "nginx:1.23.3", + "ghcr.io/stefanprodan/podinfo:6.3.3", + "registry1.dso.mil/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", +} + +var badImageRefs = []string{ + "i am not a ref at all", + "C:\\Users\\zarf", + "http://urls.are/not/refs", +} + +func TestImageTransformHost(t *testing.T) { + var expectedResult = []string{ + // Normal git repos and references for pushing/pulling + "gitlab.com/project/library/nginx-3793515731:1.23.3", + "gitlab.com/project/stefanprodan/podinfo-2985051089:6.3.3", + "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent-2003217571:v0.25.0", + } + + for idx, ref := range imageRefs { + newRef, err := ImageTransformHost("gitlab.com/project", ref) + assert.NoError(t, err) + assert.Equal(t, expectedResult[idx], newRef) + } + + for _, ref := range badImageRefs { + _, err := ImageTransformHost("gitlab.com/project", ref) + assert.Error(t, err) + } +} + +func TestImageTransformHostWithoutChecksum(t *testing.T) { + var expectedResult = []string{ + "gitlab.com/project/library/nginx:1.23.3", + "gitlab.com/project/stefanprodan/podinfo:6.3.3", + "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", + } + + for idx, ref := range imageRefs { + newRef, err := ImageTransformHostWithoutChecksum("gitlab.com/project", ref) + assert.NoError(t, err) + assert.Equal(t, expectedResult[idx], newRef) + } + + for _, ref := range badImageRefs { + _, err := ImageTransformHostWithoutChecksum("gitlab.com/project", ref) + assert.Error(t, err) + } +} diff --git a/src/test/e2e/07_create_git_test.go b/src/test/e2e/07_create_git_test.go index 2fd1e32050..e3c1021564 100644 --- a/src/test/e2e/07_create_git_test.go +++ b/src/test/e2e/07_create_git_test.go @@ -57,15 +57,15 @@ func TestCreateGit(t *testing.T) { require.Equal(t, "v0.16.0\n", stdOut) // Verify a repo with a branch. - gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-branch/repos/big-bang-2752174810/.git", extractDir) + gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-branch/repos/big-bang-2705706079/.git", extractDir) stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "log", "HEAD^..HEAD", "--oneline", "--decorate") require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdOut, "(HEAD -> release-1.53.x, tag: 1.53.0-rc.1, tag: 1.53.0, online-upstream/release-1.53.x)") + require.Contains(t, stdOut, "(HEAD -> release-1.54.x, tag: 1.54.0-rc.0, tag: 1.54.0, online-upstream/release-1.54.x)") // Verify a repo with a branch only has one branch. stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "branch") require.NoError(t, err, stdOut, stdErr) - require.Equal(t, "* release-1.53.x\n", stdOut) + require.Equal(t, "* release-1.54.x\n", stdOut) // Verify a repo with a commit hash. gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-hash/repos/zarf-1356873667/.git", extractDir) diff --git a/src/types/k8s.go b/src/types/k8s.go index bb69e015fc..57aa85b281 100644 --- a/src/types/k8s.go +++ b/src/types/k8s.go @@ -14,9 +14,10 @@ type ZarfState struct { StorageClass string `json:"storageClass" jsonschema:"Default StorageClass value Zarf uses for variable templating"` AgentTLS k8s.GeneratedPKI `json:"agentTLS" jsonschema:"PKI certificate information for the agent pods Zarf manages"` - GitServer GitServerInfo `json:"gitServer" jsonschema:"description=Information about the repository Zarf is configured to use"` - RegistryInfo RegistryInfo `json:"registryInfo" jsonschema:"description=Information about the registry Zarf is configured to use"` - LoggingSecret string `json:"loggingSecret" jsonschema:"description=Secret value that the internal Grafana server was seeded with"` + GitServer GitServerInfo `json:"gitServer" jsonschema:"description=Information about the repository Zarf is configured to use"` + RegistryInfo RegistryInfo `json:"registryInfo" jsonschema:"description=Information about the container registry Zarf is configured to use"` + ArtifactServer ArtifactServerInfo `json:"artifactServer" jsonschema:"description=Information about the artifact registry Zarf is configured to use"` + LoggingSecret string `json:"loggingSecret" jsonschema:"description=Secret value that the internal Grafana server was seeded with"` } // DeployedPackage contains information about a Zarf Package that has been deployed to a cluster @@ -52,6 +53,15 @@ type GitServerInfo struct { InternalServer bool `json:"internalServer" jsonschema:"description=Indicates if we are using a git server that Zarf is directly managing"` } +// ArtifactServerInfo contains information Zarf uses to communicate with a artifact registry to push/pull repositories to. +type ArtifactServerInfo struct { + PushUsername string `json:"pushUsername" jsonschema:"description=Username of a user with push access to the artifact registry"` + PushToken string `json:"pushPassword" jsonschema:"description=Password of a user with push access to the artifact registry"` + + Address string `json:"address" jsonschema:"description=URL address of the artifact registry"` + InternalServer bool `json:"internalServer" jsonschema:"description=Indicates if we are using a artifact registry that Zarf is directly managing"` +} + // RegistryInfo contains information Zarf uses to communicate with a container registry to push/pull images. type RegistryInfo struct { PushUsername string `json:"pushUsername" jsonschema:"description=Username of a user with push access to the registry"` diff --git a/src/types/package.go b/src/types/package.go index 3eac7762d2..6ccb3c613f 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -18,7 +18,7 @@ type ZarfPackage struct { type ZarfMetadata struct { Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]+$"` Description string `json:"description,omitempty" jsonschema:"description=Additional information about this package"` - Version string `json:"version,omitempty" jsonschema:"description=Generic string to track the package version by a package author"` + Version string `json:"version,omitempty" jsonschema:"description=Generic string set by a package author to track the package version (Note: ZarfInitConfigs will always be versioned to the CLIVersion they were created with)"` URL string `json:"url,omitempty" jsonschema:"description=Link to package information when online"` Image string `json:"image,omitempty" jsonschema:"description=An image URL to embed in this package (Reserved for future use in Zarf UI)"` Uncompressed bool `json:"uncompressed,omitempty" jsonschema:"description=Disable compression of this package"` diff --git a/src/types/runtime.go b/src/types/runtime.go index 67cb40d895..9149aeeec8 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -51,10 +51,10 @@ type ZarfInitOptions struct { // Zarf init is installing the k3s component ApplianceMode bool `json:"applianceMode" jsonschema:"description=Indicates if Zarf was initialized while deploying its own k8s cluster"` - // Using a remote git server - GitServer GitServerInfo `json:"gitServer" jsonschema:"description=Information about the repository Zarf is going to be using"` - - RegistryInfo RegistryInfo `json:"registryInfo" jsonschema:"description=Information about the registry Zarf is going to be using"` + // Using alternative services + GitServer GitServerInfo `json:"gitServer" jsonschema:"description=Information about the repository Zarf is going to be using"` + RegistryInfo RegistryInfo `json:"registryInfo" jsonschema:"description=Information about the container registry Zarf is going to be using"` + ArtifactServer ArtifactServerInfo `json:"artifactServer" jsonschema:"description=Information about the artifact registry Zarf is going to be using"` StorageClass string `json:"storageClass" jsonschema:"description=StorageClass of the k8s cluster Zarf is initializing"` } diff --git a/src/ui/lib/api-types.ts b/src/ui/lib/api-types.ts index ea205db8b7..15bdf0041f 100644 --- a/src/ui/lib/api-types.ts +++ b/src/ui/lib/api-types.ts @@ -59,12 +59,16 @@ export interface ZarfInitOptions { * Indicates if Zarf was initialized while deploying its own k8s cluster */ applianceMode: boolean; + /** + * Information about the artifact registry Zarf is going to be using + */ + artifactServer: ArtifactServerInfo; /** * Information about the repository Zarf is going to be using */ gitServer: GitServerInfo; /** - * Information about the registry Zarf is going to be using + * Information about the container registry Zarf is going to be using */ registryInfo: RegistryInfo; /** @@ -73,6 +77,30 @@ export interface ZarfInitOptions { storageClass: string; } +/** + * Information about the artifact registry Zarf is going to be using + * + * Information about the artifact registry Zarf is configured to use + */ +export interface ArtifactServerInfo { + /** + * URL address of the artifact registry + */ + address: string; + /** + * Indicates if we are using a artifact registry that Zarf is directly managing + */ + internalServer: boolean; + /** + * Password of a user with push access to the artifact registry + */ + pushPassword: string; + /** + * Username of a user with push access to the artifact registry + */ + pushUsername: string; +} + /** * Information about the repository Zarf is going to be using * @@ -108,9 +136,9 @@ export interface GitServerInfo { } /** - * Information about the registry Zarf is going to be using + * Information about the container registry Zarf is going to be using * - * Information about the registry Zarf is configured to use + * Information about the container registry Zarf is configured to use */ export interface RegistryInfo { /** @@ -820,7 +848,8 @@ export interface ZarfMetadata { */ vendor?: string; /** - * Generic string to track the package version by a package author + * Generic string set by a package author to track the package version (Note: + * ZarfInitConfigs will always be versioned to the CLIVersion they were created with) */ version?: string; /** @@ -872,6 +901,10 @@ export interface ZarfState { * Machine architecture of the k8s node(s) */ architecture: string; + /** + * Information about the artifact registry Zarf is configured to use + */ + artifactServer: ArtifactServerInfo; /** * K8s distribution of the cluster Zarf was deployed to */ @@ -885,7 +918,7 @@ export interface ZarfState { */ loggingSecret: string; /** - * Information about the registry Zarf is configured to use + * Information about the container registry Zarf is configured to use */ registryInfo: RegistryInfo; storageClass: string; @@ -1177,10 +1210,17 @@ const typeMap: any = { ], false), "ZarfInitOptions": o([ { json: "applianceMode", js: "applianceMode", typ: true }, + { json: "artifactServer", js: "artifactServer", typ: r("ArtifactServerInfo") }, { json: "gitServer", js: "gitServer", typ: r("GitServerInfo") }, { json: "registryInfo", js: "registryInfo", typ: r("RegistryInfo") }, { json: "storageClass", js: "storageClass", typ: "" }, ], false), + "ArtifactServerInfo": o([ + { json: "address", js: "address", typ: "" }, + { json: "internalServer", js: "internalServer", typ: true }, + { json: "pushPassword", js: "pushPassword", typ: "" }, + { json: "pushUsername", js: "pushUsername", typ: "" }, + ], false), "GitServerInfo": o([ { json: "address", js: "address", typ: "" }, { json: "internalServer", js: "internalServer", typ: true }, @@ -1393,6 +1433,7 @@ const typeMap: any = { "ZarfState": o([ { json: "agentTLS", js: "agentTLS", typ: r("GeneratedPKI") }, { json: "architecture", js: "architecture", typ: "" }, + { json: "artifactServer", js: "artifactServer", typ: r("ArtifactServerInfo") }, { json: "distro", js: "distro", typ: "" }, { json: "gitServer", js: "gitServer", typ: r("GitServerInfo") }, { json: "loggingSecret", js: "loggingSecret", typ: "" }, diff --git a/zarf.schema.json b/zarf.schema.json index bf31a5c60c..f786474c9a 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -753,7 +753,7 @@ }, "version": { "type": "string", - "description": "Generic string to track the package version by a package author" + "description": "Generic string set by a package author to track the package version (Note: ZarfInitConfigs will always be versioned to the CLIVersion they were created with)" }, "url": { "type": "string",