From f24ea1fec164935c55e1345507970e806e7f8988 Mon Sep 17 00:00:00 2001
From: Maya Rashish <mrashish@redhat.com>
Date: Wed, 3 Aug 2022 19:34:51 +0300
Subject: [PATCH] Use buildah instead of docker, support multi-arch builds
 (#115)

* Instead of passing environment variables, export them in Makefile

This passes them to all subprocesses, but they can still be overridden
due to the ?= construct.

Signed-off-by: Maya Rashish <mrashish@redhat.com>

* Build sanity.test statically

Avoid failure when the binary is built with Fedora 36 and thus
requires newer glibc symbols than the container.

Signed-off-by: Maya Rashish <mrashish@redhat.com>

* Switch to buildah & podman, enable multi-arch builds

Now we can run:
make clean && \
GOARCH=arm64 make manifest && GOARCH=amd64 make manifest && \
make manifest-push

And spit out a manifest for both arm64 and amd64, in the same image.

Caveats:
- We have a special 'manifest-clean' target, as we can add arbitrarily
  many images to a manifest and don't want the old ones.
  Delete old image in case a regular non-manifest image exists by the
  same name, too.
- The push and image/manifest creation are split, so we can run the
  image creation for more than one architecture and push the combined
  manifest including both.
- We keep `make push` behaving the same to avoid breaking CI.
- Full DOCKER_REPO name is used, as podman-like tools have odd behavior
  with short names.

Signed-off-by: Maya Rashish <mrashish@redhat.com>

* Tolerate docker instead of podman

Signed-off-by: Maya Rashish <mrashish@redhat.com>

* Only add --tls-verify=false if the registry matches localhost*

Add a message about this, too

Signed-off-by: Maya Rashish <mrashish@redhat.com>
---
 Makefile             | 63 +++++++++++++++++++++++++++++++-------------
 cluster-sync/sync.sh |  6 ++++-
 hack/k8s-e2e.sh      |  6 ++++-
 hack/sanity.sh       |  6 ++---
 4 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/Makefile b/Makefile
index 377d8d3e..a706c6c1 100644
--- a/Makefile
+++ b/Makefile
@@ -18,57 +18,84 @@ KUBEVIRT_PROVIDER?=k8s-1.23
 HPP_IMAGE?=hostpath-provisioner
 HPP_CSI_IMAGE?=hostpath-csi-driver
 TAG?=latest
-DOCKER_REPO?=kubevirt
+DOCKER_REPO?=quay.io/kubevirt
 ARTIFACTS_PATH?=_out
 GOLANG_VER?=1.18.2
+GOOS?=linux
+GOARCH?=amd64
+BUILDAH_PLATFORM_FLAG?=--platform $(GOOS)/$(GOARCH)
+OCI_BIN ?= $(shell if podman ps >/dev/null 2>&1; then echo podman; elif docker ps >/dev/null 2>&1; then echo docker; fi)
+
+export GOLANG_VER
+export KUBEVIRT_PROVIDER
+export DOCKER_REPO
+export GOOS
+export GOARCH
+export OCI_BIN
 
 all: controller hostpath-provisioner
 
 hostpath-provisioner:
-	GOLANG_VER=${GOLANG_VER} ./hack/build-provisioner.sh
+	./hack/build-provisioner.sh
 
 hostpath-csi-driver:
-	GOLANG_VER=${GOLANG_VER} ./hack/build-csi.sh
+	./hack/build-csi.sh
 
 image: image-controller image-csi
 
-push: push-controller push-csi
+push: clean manifest manifest-push
 
-push-controller: hostpath-provisioner image
-	docker push $(DOCKER_REPO)/$(HPP_IMAGE):$(TAG)
+manifest: manifest-controller manifest-csi
+
+manifest-push: push-csi push-controller
 
 image-controller: hostpath-provisioner
-	docker build -t $(DOCKER_REPO)/$(HPP_IMAGE):$(TAG) -f Dockerfile.controller .
+	buildah build $(BUILDAH_PLATFORM_FLAG) -t $(DOCKER_REPO)/$(HPP_IMAGE):$(GOARCH) -f Dockerfile.controller .
 
 image-csi: hostpath-csi-driver
-	docker build -t $(DOCKER_REPO)/$(HPP_CSI_IMAGE):$(TAG) -f Dockerfile.csi .
+	buildah build $(BUILDAH_PLATFORM_FLAG) -t $(DOCKER_REPO)/$(HPP_CSI_IMAGE):$(GOARCH) -f Dockerfile.csi .
+
+manifest-controller: image-controller
+	-buildah manifest create $(DOCKER_REPO)/$(HPP_IMAGE):local
+	buildah manifest add --arch $(GOARCH) $(DOCKER_REPO)/$(HPP_IMAGE):local containers-storage:$(DOCKER_REPO)/$(HPP_IMAGE):$(GOARCH)
+
+manifest-csi: image-csi
+	-buildah manifest create $(DOCKER_REPO)/$(HPP_CSI_IMAGE):local
+	buildah manifest add --arch $(GOARCH) $(DOCKER_REPO)/$(HPP_CSI_IMAGE):local containers-storage:$(DOCKER_REPO)/$(HPP_CSI_IMAGE):$(GOARCH)
 
-push-csi: hostpath-csi-driver image-csi
-	docker push $(DOCKER_REPO)/$(HPP_CSI_IMAGE):$(TAG)
+push-csi:
+	buildah manifest push $(BUILDAH_PUSH_FLAGS) --all $(DOCKER_REPO)/$(HPP_CSI_IMAGE):local docker://$(DOCKER_REPO)/$(HPP_CSI_IMAGE):$(TAG)
 
-clean:
+push-controller:
+	buildah manifest push $(BUILDAH_PUSH_FLAGS) --all $(DOCKER_REPO)/$(HPP_IMAGE):local docker://$(DOCKER_REPO)/$(HPP_IMAGE):$(TAG)
+
+clean: manifest-clean
 	rm -rf _out
 
+manifest-clean:
+	-buildah manifest rm $(DOCKER_REPO)/$(HPP_IMAGE):local
+	-buildah manifest rm $(DOCKER_REPO)/$(HPP_CSI_IMAGE):local
+
 build: clean hostpath-provisioner hostpath-csi-driver
 
 cluster-up:
-	KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER} ./cluster-up/up.sh
+	./cluster-up/up.sh
 
 cluster-down: 
-	KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER} ./cluster-up/down.sh
+	./cluster-up/down.sh
 
 cluster-sync: cluster-clean
-	KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER} ./cluster-sync/sync.sh
+	./cluster-sync/sync.sh
 
 cluster-clean:
-	KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER} ./cluster-sync/clean.sh
+	./cluster-sync/clean.sh
 
 test:
-	GOLANG_VER=${GOLANG_VER} ./hack/run-unit-test.sh
+	./hack/run-unit-test.sh
 	hack/language.sh
 
 test-functional:
-	KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER} gotestsum --format short-verbose --junitfile ${ARTIFACTS_PATH}/junit.functest.xml -- ./tests/... -kubeconfig="../_ci-configs/$(KUBEVIRT_PROVIDER)/.kubeconfig"
+	gotestsum --format short-verbose --junitfile ${ARTIFACTS_PATH}/junit.functest.xml -- ./tests/... -kubeconfig="../_ci-configs/$(KUBEVIRT_PROVIDER)/.kubeconfig"
 
 test-sanity:
-	GOLANG_VER=${GOLANG_VER} DOCKER_REPO=${DOCKER_REPO} hack/sanity.sh
+	hack/sanity.sh
diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh
index c6d28d9e..5a649637 100755
--- a/cluster-sync/sync.sh
+++ b/cluster-sync/sync.sh
@@ -27,7 +27,11 @@ for i in $(seq 1 ${KUBEVIRT_NUM_NODES}); do
 done
 
 registry=${IMAGE_REGISTRY:-localhost:$(_port registry)}
-DOCKER_REPO=${registry} make push
+if [[ ${registry} == localhost* ]]; then
+  echo "not verifying tls, registry contains localhost"
+  export BUILDAH_PUSH_FLAGS="--tls-verify=false"
+fi
+DOCKER_REPO=${registry} make manifest manifest-push
 
 if [ ! -z $UPGRADE_FROM ]; then
   _kubectl apply -f https://github.com/kubevirt/hostpath-provisioner-operator/releases/download/$UPGRADE_FROM/namespace.yaml
diff --git a/hack/k8s-e2e.sh b/hack/k8s-e2e.sh
index 5ab03e3b..767eafa7 100755
--- a/hack/k8s-e2e.sh
+++ b/hack/k8s-e2e.sh
@@ -62,7 +62,11 @@ fi
 echo "install hpp"
 registry=${IMAGE_REGISTRY:-localhost:$(_port registry)}
 echo "registry: ${registry}"
-DOCKER_REPO=${registry} make push
+if [[ ${registry} == localhost* ]]; then
+  echo "not verifying tls, registry contains localhost"
+  export BUILDAH_PUSH_FLAGS="--tls-verify=false"
+fi
+DOCKER_REPO=${registry} make manifest manifest-push
 
 #install hpp
 _kubectl apply -f https://raw.githubusercontent.com/kubevirt/hostpath-provisioner-operator/main/deploy/namespace.yaml
diff --git a/hack/sanity.sh b/hack/sanity.sh
index 529e2321..a85b567d 100755
--- a/hack/sanity.sh
+++ b/hack/sanity.sh
@@ -19,8 +19,8 @@ source "${script_dir}"/common.sh
 setGoInProw $GOLANG_VER
 
 echo "docker repo: [$DOCKER_REPO]"
-go test -o _out/sanity.test -c -v ./sanity/...
-docker build -t ${DOCKER_REPO}/sanity:test -f ./sanity/Dockerfile .
+CGO_ENABLED=0 go test -o _out/sanity.test -c -v ./sanity/...
+$OCI_BIN build -t ${DOCKER_REPO}/sanity:test -f ./sanity/Dockerfile .
 # Need privileged so we can bind mount inside container, and hostpath capacity cannot change, so skipping that test
-docker run --privileged ${DOCKER_REPO}/sanity:test -ginkgo.noColor -ginkgo.skip="should fail when requesting to create a volume with already existing name and different capacity"
+$OCI_BIN run --privileged ${DOCKER_REPO}/sanity:test -ginkgo.noColor -ginkgo.skip="should fail when requesting to create a volume with already existing name and different capacity"