Native arm64 docker build with buildkit (#1601)

.github/workflows/ci.yaml

@@ -134,6 +134,8 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Critical for correct image detection in Makefile
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       - name: Build and push fleet-manager-tools image to quay.io
         if: github.event_name == 'push'
         env:
@@ -142,4 +144,4 @@ jobs:
       - name: Build and push fleetshard-operator image to quay.io
         run: make image/push/fleetshard-operator
       - name: Build and push fleet-manager image to quay.io
-        run: make image/push/fleet-manager
+        run: make image/push/fleet-manager IMAGE_PLATFORM=linux/amd64,linux/arm64

.openshift-ci/e2e-runtime/Dockerfile

@@ -11,7 +11,7 @@ COPY --from=quay.io/operator-framework/operator-sdk:v1.25 /usr/local/bin/operato
 
 ENV GOPATH=/go
 ENV GOROOT=/usr/local/go
-ENV PATH="/usr/local/go/bin:${PATH}"
+ENV PATH="${GOROOT}/bin:${PATH}"
 
 RUN ln -s /usr/bin/oc /usr/bin/kubectl
 

.openshift-ci/e2e-runtime/e2e_dockerized.sh

@@ -22,7 +22,6 @@ AWS_SESSION_TOKEN=$(aws configure get aws_session_token --profile=saml)
 FLEET_MANAGER_IMAGE=$(make -s -C "$GITROOT" full-image-tag)
 
 # Run the necessary docker actions out of the container
-preload_dependency_images
 ensure_fleet_manager_image_exists
 
 docker build -t acscs-e2e -f "$GITROOT/.openshift-ci/e2e-runtime/Dockerfile" "${GITROOT}"

Dockerfile

@@ -1,21 +1,25 @@
-FROM registry.ci.openshift.org/openshift/release:golang-1.20 AS build
+FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi8/go-toolset:1.20 AS build
 
+USER root
 RUN mkdir /src /rds_ca
 ADD https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem /rds_ca/aws-rds-ca-global-bundle.pem
-
 WORKDIR /src
+
+RUN go env -w GOCACHE=/go/.cache; \
+    go env -w GOMODCACHE=/go/pkg/mod
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
      --mount=type=bind,source=go.sum,target=go.sum \
      --mount=type=bind,source=go.mod,target=go.mod \
       go mod download -x
 
 COPY . ./
 
-ARG GOARCH
+ARG TARGETARCH
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/go/.cache/ \
-    make binary GOOS=linux GOARCH=${GOARCH}
+    make binary GOOS=linux GOARCH=${TARGETARCH}
 
 FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9 as standard
 

Dockerfile.tools

@@ -1,5 +1,5 @@
-FROM registry.ci.openshift.org/openshift/release:golang-1.20 AS build
-
+FROM registry.access.redhat.com/ubi8/go-toolset:1.20 AS build
+USER root
 RUN mkdir /src
 WORKDIR /src
 COPY . ./

Makefile

@@ -62,8 +62,9 @@ else
 GOBIN=$(shell $(GO) env GOBIN)
 endif
 
-# Used for local builds to support arm64
-goarch=$(shell go env GOARCH)
+ifeq ($(IMAGE_PLATFORM),)
+IMAGE_PLATFORM=linux/$(shell $(GO) env GOARCH)
+endif
 
 LOCAL_BIN_PATH := ${PROJECT_PATH}/bin
 # Add the project-level bin directory into PATH. Needed in order
@@ -496,12 +497,9 @@ docker/login/internal:
 	$(DOCKER) login -u kubeadmin --password-stdin <<< $(shell oc whoami -t) $(shell oc get route default-route -n openshift-image-registry -o jsonpath="{.spec.host}")
 .PHONY: docker/login/internal
 
-# Build the image using by specifying a specific image target within the Dockerfile.
-image/build: GOARCH?=amd64
-image/build: IMAGE_REF="$(external_image_registry)/$(image_repository):$(image_tag)"
+# Build the image
 image/build:
-	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) build -t $(IMAGE_REF) --build-arg GOARCH=$(GOARCH) .
-	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) tag $(IMAGE_REF) $(SHORT_IMAGE_REF)
+	DOCKER_CONFIG=${DOCKER_CONFIG} DOCKER_BUILDKIT=1 $(DOCKER) build -t $(SHORT_IMAGE_REF) .
 	@echo "New image tag: $(SHORT_IMAGE_REF). You might want to"
 	@echo "export FLEET_MANAGER_IMAGE=$(SHORT_IMAGE_REF)"
 ifeq ("$(CLUSTER_TYPE)","kind")
@@ -536,8 +534,8 @@ image/push: image/push/fleet-manager image/push/probe
 .PHONY: image/push
 
 image/push/fleet-manager: IMAGE_REF="$(external_image_registry)/$(image_repository):$(image_tag)"
-image/push/fleet-manager: image/build
-	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) push $(IMAGE_REF)
+image/push/fleet-manager:
+	DOCKER_CONFIG=${DOCKER_CONFIG} $(DOCKER) buildx build -t $(IMAGE_REF) --platform $(IMAGE_PLATFORM) --push .
 	@echo
 	@echo "Image was pushed as $(IMAGE_REF). You might want to"
 	@echo "export FLEET_MANAGER_IMAGE=$(IMAGE_REF)"
@@ -846,15 +844,12 @@ deploy/bootstrap:
 .PHONY: deploy/bootstrap
 
 # Deploy local images fast for development
-deploy/dev-fast: GOARCH=$(goarch)
 deploy/dev-fast: image/build deploy/dev-fast/fleet-manager deploy/dev-fast/fleetshard-sync
 
-deploy/dev-fast/fleet-manager: GOARCH=$(goarch)
 deploy/dev-fast/fleet-manager: image/build
 	kubectl -n $(ACSCS_NAMESPACE) set image deploy/fleet-manager fleet-manager=$(SHORT_IMAGE_REF) db-migrate=$(SHORT_IMAGE_REF)
 	kubectl -n $(ACSCS_NAMESPACE) delete pod -l application=fleet-manager
 
-deploy/dev-fast/fleetshard-sync: GOARCH=$(goarch)
 deploy/dev-fast/fleetshard-sync: image/build
 	kubectl -n $(ACSCS_NAMESPACE) set image deploy/fleetshard-sync fleetshard-sync=$(SHORT_IMAGE_REF)
 	kubectl -n $(ACSCS_NAMESPACE) delete pod -l application=fleetshard-sync

dev/env/scripts/docker.sh

@@ -61,7 +61,7 @@ ensure_fleet_manager_image_exists() {
                 # Attempt to build this image.
                 if [[ "$FLEET_MANAGER_IMAGE" == "$(make -s -C "${GITROOT}" full-image-tag)" ]]; then
                         log "Building local image..."
-                        make -C "${GITROOT}" image/build GOARCH="$(go env GOARCH)" # supports arm64
+                        make -C "${GITROOT}" image/build
                 else
                     die "Cannot find image '${FLEET_MANAGER_IMAGE}' and don't know how to build it"
                 fi

probe/Dockerfile

@@ -1,5 +1,5 @@
-FROM registry.ci.openshift.org/openshift/release:golang-1.20 AS build
-
+FROM registry.access.redhat.com/ubi8/go-toolset:1.20 AS build
+USER root
 ENV GOFLAGS="-mod=mod"
 
 RUN mkdir /src