Remove stage 2 and replace w/ a static docker registry implementation in rust

.github/workflows/build-rust-injector.yml

@@ -15,6 +15,11 @@ jobs:
   build-injector:
     runs-on: ubuntu-latest
     steps:
+      - name: "Checkout Repo"
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.branchName }}
+
       - name: Install tools
         uses: ./.github/actions/install-tools
 
@@ -24,20 +29,15 @@ jobs:
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
           sudo apt install build-essential -y
 
-      - name: "Checkout Repo"
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.inputs.branchName }}
-
       - name: "Build Rust Binary for x86_64"
-        working-directory: src/injector/stage1
+        working-directory: src/injector
         run: |
           rustup target add x86_64-unknown-linux-musl
           cargo build --target x86_64-unknown-linux-musl --release
           strip target/x86_64-unknown-linux-musl/release/zarf-injector
 
       - name: "Build Rust Binary for aarch64"
-        working-directory: src/injector/stage1
+        working-directory: src/injector
         run: |
           rustup target add aarch64-unknown-linux-musl
           curl https://zarf-public.s3-us-gov-west-1.amazonaws.com/pipelines/aarch64-linux-musl-cross.tgz | tar -xz
@@ -52,7 +52,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: "Upload Binaries To DockerHub"
-        working-directory: src/injector/stage1/target
+        working-directory: src/injector/target
         run: |
           cosign upload blob -f x86_64-unknown-linux-musl/release/zarf-injector defenseunicorns/zarf-injector:amd64-${{ github.event.inputs.versionTag }}
           cosign upload blob -f aarch64-unknown-linux-musl/release/zarf-injector defenseunicorns/zarf-injector:arm64-${{ github.event.inputs.versionTag }}

Makefile

@@ -1,7 +1,14 @@
-# Figure out which Zarf binary we should use based on the operating system we are on
-ZARF_BIN := ./build/zarf
 # Provide a default value for the operating system architecture used in tests, e.g. " APPLIANCE_MODE=true|false make test-e2e ARCH=arm64"
 ARCH ?= amd64
+# The image tag used for the zarf agent, defaults to a dev image tag
+AGENT_IMAGE ?= dev-agent:e32f41ab50f994302614adf62ab6f13a7ecfbb25
+# The zarf injector registry binary to use
+INJECTOR_VERSION := pr-948-e699899
+
+######################################################################################
+
+# Figure out which Zarf binary we should use based on the operating system we are on
+ZARF_BIN := ./build/zarf
 ifeq ($(OS),Windows_NT)
 	ZARF_BIN := $(addsuffix .exe,$(ZARF_BIN))
 else
@@ -20,8 +27,6 @@ else
 	endif
 endif
 
-AGENT_IMAGE ?= dev-agent:e32f41ab50f994302614adf62ab6f13a7ecfbb25
-
 CLI_VERSION := $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion")
 BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)'
 .DEFAULT_GOAL := help
@@ -68,34 +73,28 @@ build-ui: ## Build the Zarf UI
 	npm ci
 	npm run build
 
-build-cli-linux-amd: build-injector-registry-amd check-ui ## Build the Zarf CLI for Linux on AMD64
+build-cli-linux-amd: check-ui ## Build the Zarf CLI for Linux on AMD64
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf main.go
 
-build-cli-linux-arm: build-injector-registry-arm check-ui ## Build the Zarf CLI for Linux on ARM
+build-cli-linux-arm: check-ui ## Build the Zarf CLI for Linux on ARM
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf-arm main.go
 
-build-cli-mac-intel: build-injector-registry-amd check-ui ## Build the Zarf CLI for macOS on AMD64
+build-cli-mac-intel: check-ui ## Build the Zarf CLI for macOS on AMD64
 	GOOS=darwin GOARCH=amd64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf-mac-intel main.go
 
-build-cli-mac-apple: build-injector-registry-arm check-ui ## Build the Zarf CLI for macOS on ARM
+build-cli-mac-apple: check-ui ## Build the Zarf CLI for macOS on ARM
 	GOOS=darwin GOARCH=arm64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf-mac-apple main.go
 
-build-cli-windows-amd: build-injector-registry-amd check-ui ## Build the Zarf CLI for Windows on AMD64
+build-cli-windows-amd: check-ui ## Build the Zarf CLI for Windows on AMD64
 	GOOS=windows GOARCH=amd64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf.exe main.go ## Build the Zarf CLI for Windows on AMD64
 
-build-cli-windows-arm: build-injector-registry-amd check-ui ## Build the Zarf CLI for Windows on ARM
+build-cli-windows-arm: check-ui ## Build the Zarf CLI for Windows on ARM
 	GOOS=windows GOARCH=arm64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf-arm.exe main.go ## Build the Zarf CLI for Windows on ARM
 
 build-cli-linux: build-cli-linux-amd build-cli-linux-arm ## Build the Zarf CLI for Linux on AMD64 and ARM
 
 build-cli: build-cli-linux-amd build-cli-linux-arm build-cli-mac-intel build-cli-mac-apple build-cli-windows-amd build-cli-windows-arm ## Build the CLI
 
-build-injector-registry-amd: ## Build the Zarf Injector Stage 2 for Linux on AMD64
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o build/zarf-registry-amd64 src/injector/stage2/registry.go
-
-build-injector-registry-arm: ## Build the Zarf Injector Stage 2 for Linux on ARM
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o build/zarf-registry-arm64 src/injector/stage2/registry.go
-
 docs-and-schema: ensure-ui-build-dir ## Generate the Zarf Documentation and Schema
 	go run main.go internal generate-cli-docs
 	.hooks/create-zarf-schema.sh
@@ -119,7 +118,7 @@ dev-agent-image: ## Create a new agent image and inject it into a currently init
 
 init-package: ## Create the zarf init package (must `brew install coreutils` on macOS first)
 	@test -s $(ZARF_BIN) || $(MAKE) build-cli
-	$(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm .
+	$(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --set INJECTOR_TAG=$(ARCH)-$(INJECTOR_VERSION) --confirm .
 
 ci-release: init-package
 
@@ -152,7 +151,6 @@ build-examples: ## Build all of the example packages
 ## Requires an existing cluster for the env var APPLIANCE_MODE=true
 .PHONY: test-e2e
 test-e2e: build-examples ## Run all of the core Zarf CLI E2E tests
-	@test -s ./build/zarf-init-$(ARCH)-$(CLI_VERSION).tar.zst || $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm .
 	@test -s ./build/zarf-init-$(ARCH)-$(CLI_VERSION).tar.zst || $(MAKE) init-package
 	cd src/test/e2e && go test -failfast -v -timeout 30m
 

adr/0003-image-injection-into-remote-clusters-without-native-support.md

@@ -6,6 +6,8 @@ Date: 2022-03-04
 
 Accepted
 
+Amended by [7. Use Rust Binary for Both Injection Stages](0007-use-rust-binary-for-both-injection-stages.md)
+
 ## Context
 
 In order to create any workloads in K8s, an image has to exist on the node or be pulled in from an OCI Distribution server (docker registry). Local K8s distros such as KIND, K3D, K3S, Microk8s have CLI support for injecting or pushing images into the CRIs of cluster nodes. No standard or tool exists to do this generically in K8s as the CRI sits outside the problem of K8s beyond just communicating with it. For Zarf to push images into a K8s cluster that does not have a CLI option to inject the image, we have to establish some mechanism to create a temporary registry, use an existing one, or inject the image in the cluster. Zarf must also support unknown environments with no other dependencies; we cannot assume a registry exists. Typically when K8s pulls images from a remote registry, it sends the request to the CRI that then does the pull outside of the K8s context. The CRI runs at the host level on a per-node basis, so to access a private registry, the TLS trust needs to be modified on any host/node that would attempt to pull the image. The two primary ways to do this are modifying the node's root certificate authorities or the CRIs configuration if it has an option for TLS root CA. Lastly, as this is per node, all nodes in the cluster would need to be modified or some affinity/taint to force the pod to use a single node during bootstrapping.

adr/0007-use-rust-binary-for-both-injection-stages.md

@@ -0,0 +1,30 @@
+# 7. Use Rust Binary for Both Injection Stages
+
+Date: 2022-10-29
+
+## Status
+
+Accepted
+
+Amends [3. Image injection into remote clusters without native support](0003-image-injection-into-remote-clusters-without-native-support.md)
+
+## Context
+
+In ADR 3, the decision was made to create a rust binary (`stage1`) that would re-assemble a `registry:2` image and a go registry binary (`stage2`) from a series of configmaps. While this solution works, it is overkill for the operations that `stage2` performs. The `stage2` binary is only responsible for 1. starting a docker registry in `rw` mode, 2. pushing the `registry:2` crane tarball into said registry, 3. starting the docker registry in `r` mode. This `registry:2` image is then immedietely consumed by the `zarf-registry` package, creating a true in-cluster docker registry. The injector pod is then destroyed. The overhead this operation creates:
+
+- having to keep track of another binary (making the total number 3 for the zarf ecosystem)
+- nearly doubling the amount of configmaps loaded into the cluster (makes init slower)
+- having to compile the binary for each platform (adds to the build time + ci)
+- using a full-featured docker registry to host a single image (wasteful)
+
+## Decision
+
+The rust binary is already being injected via configmap and unpacking the tarball. There is little need to bring everything but the kitchen sink to just serve a single image. Therefore the decision is to use the rust binary to perform the entire injection process. This required a few changes to the rust binary:
+
+- not only re-assemble + unpack the tarball, but also unpack the `registry:2` image (stored as a [crane tarball format](https://github.com/google/go-containerregistry/tree/main/pkg/v1/tarball))
+- transform the `registry:2` crane manifest to a docker v2 manifest
+- spin up an HTTP server compliant with the v2 docker registry API to serve the `registry:2` image
+
+## Consequences
+
+The removal of the `stage2` binary makes the build process much simpler, and cuts down on make targets, as well as considerations needed during the build process. Additionally, the init process is faster, as the `stage2` binary is not injected anymore and the `registry:2` image is directly served instead of being pushed to an ephemeral registry, then served. There is a current risk to the new size of the `stage1` binary. Using stable compiler optimizations, the smallest it is able to come to is 868kb. While successfully tested on `k3s`, `kind`, and `k3d`, it is possible that the binary is too large for some Kubernetes distros (due to the potential interpretation of the configmap size limit). Additionally, the docker API implementation only serves a v2 manifest (most likely OCI in a future iteration) / image pull flow. This also greatly increases the lines of rust code and logic used within this repo, and future changes to this code will require more rust knowledge and experience, especially understanding the docker registry API and docker manifest formats.

docs/6-developer-guide/3-nerd-notes.md

@@ -1,28 +1,26 @@
-# Zarf Nerd Notes 
+# Zarf Nerd Notes
 
 :::caution Hard Hat Area
 This page is still being developed. More content will be added soon!
 :::
 
-
-Zarf is written entirely in [go](https://go.dev/), except for a single 400Kb binary for the injector system written in [rust](https://www.rust-lang.org/), so we can fit it in a [configmap](https://kubernetes.io/docs/concepts/configuration/configmap/). All assets are bundled together into a single [zstd](https://facebook.github.io/zstd/) tarball on each `zarf package create` operation. On the air gap / offline side, `zarf package deploy` extracts the various assets and places them on the filesystem or installs them in the cluster, depending on what the zarf package says to do. Some important ideas behind Zarf:
+Zarf is written entirely in [go](https://go.dev/), except for a single 868Kb binary for the injector system written in [rust](https://www.rust-lang.org/), so we can fit it in a [configmap](https://kubernetes.io/docs/concepts/configuration/configmap/). All assets are bundled together into a single [zstd](https://facebook.github.io/zstd/) tarball on each `zarf package create` operation. On the air gap / offline side, `zarf package deploy` extracts the various assets and places them on the filesystem or installs them in the cluster, depending on what the zarf package says to do. Some important ideas behind Zarf:
 
 - All workloads are installed in the cluster via the [Helm SDK](https://helm.sh/docs/topics/advanced/#go-sdk)
 - The OCI Registries used are both from [Docker](https://github.com/distribution/distribution)
 - Currently, the Registry and Git servers _are not HA_, see [#375](https://github.com/defenseunicorns/zarf/issues/376) and [#376](https://github.com/defenseunicorns/zarf/issues/376) for discussion on this
 - To avoid TLS issues, Zarf binds to `127.0.0.1:31999` on each node as a [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) to allow all nodes to access the pod(s) in the cluster
 - Until [#306](https://github.com/defenseunicorns/zarf/pull/306) is merged, during helm install/upgrade a [Helm PostRender](https://helm.sh/docs/topics/advanced/#post-rendering) function is called to mutate images and [ImagePullSecrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) so the deployed resources use the NodePort binding
-- Zarf uses a custom injector system to bootstrap a new cluster. See the PR [#329](https://github.com/defenseunicorns/zarf/pull/329) and [ADR](https://github.com/defenseunicorns/zarf/blob/master/docs/adr/0003-image-injection-into-remote-clusters-without-native-support.md) for more details on how we came to this solution.  The general steps are listed below:
+- Zarf uses a custom injector system to bootstrap a new cluster. See the PR [#329](https://github.com/defenseunicorns/zarf/pull/329) and [ADR](https://github.com/defenseunicorns/zarf/blob/master/adr/0003-image-injection-into-remote-clusters-without-native-support.md) for more details on how we came to this solution.  The general steps are listed below:
   - Get a list of images in the cluster
   - Attempt to create an ephemeral pod using an image from the list
-  - A small rust binary that is compiled using [musl](https://www.musl-libc.org/) to keep the size the max binary size of ~ 672 KBs is injected into the pod
-  - The mini zarf registry binary and `docker:2` images are put in a tar archive and split into 512 KB chunks; larger sizes tended to cause latency issues on low-resource control planes
+  - A small rust binary that is compiled using [musl](https://www.musl-libc.org/) to keep the max binary size as minimal as possible
+  - The `registry:2` image is placed in a tar archive and split into 512 KB chunks; larger sizes tended to cause latency issues on low-resource control planes
   - An init container runs the rust binary to re-assemble and extract the zarf binary and registry image
-  - The container then starts and runs the zarf binary to host the registry image in an embedded docker registry
+  - The container then starts and runs the rust binary to host the registry image in an static docker registry
   - After this, the main docker registry chart is deployed, pulls the image from the ephemeral pod, and finally destroys the created configmaps, pod, and service
 
-<br />
-
 <!-- TODO: @JPERRY I think this svg is out of date now that the 'Zarf agent' PR got merged.. -->
-### Zarf Architecture
+## Zarf Architecture
+
 ![Architecture Diagram](../.images/architecture.drawio.svg)