Upgrade libgit2 and fix static builds

* Bump to golang-with-libgit2:1.1.1.6 to speed up build time when cross compiling. Previous version was compiling in emulation mode instead, which added +10x overhead.
* Ensure that make test is executed against the exact same libraries that will be shipped on the built image.
* Simplify Makefile to reduce its complexity.
* Libgit2 behaviour:
    linux-amd64 download static libraries from the official container image.
    linux-arm64 on top of the above, requires static musl tool chain (automatically downloaded).
    darwin-amd64 and darwin-arm64 download universal static libraries for darwin from https://github.com/fluxcd/golang-with-libgit2 releases.

Co-authored-by: Paulo Gomes <[email protected]>
Signed-off-by: Sanskar Jaiswal <[email protected]>

.github/workflows/build.yaml

@@ -30,3 +30,48 @@ jobs:
       run: make test
     - name: Verify
       run: make verify
+
+  kind-linux-arm64:
+    # Hosted on Equinix
+    # Docs: https://github.com/fluxcd/flux2/tree/main/.github/runners
+    runs-on: [self-hosted, Linux, ARM64, equinix]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.x
+      - name: Run tests
+        run: |
+          mkdir tmp-download; cd tmp-download; go mod init go-download;
+          GOBIN="${GITHUB_WORKSPACE}/build/gobin" go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+          GOBIN="${GITHUB_WORKSPACE}/build/gobin" go install sigs.k8s.io/controller-tools/cmd/[email protected]
+          GOBIN="${GITHUB_WORKSPACE}/build/gobin" go install github.com/ahmetb/[email protected]
+          cd ..; rm -rf tmp-download
+          make test
+      - name: Verify
+        run: make verify
+
+  # Runs 'make test' on macos-10.15 to assure development environment for 
+  # contributors using MacOS.
+  darwin-amd64:
+    runs-on: macos-10.15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.x
+      - name: Restore Go cache
+        uses: actions/cache@v1
+        with:
+          path: /home/runner/work/_temp/_github_home/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Run tests
+        run: make test
+      - name: Verify
+        run: make verify

.gitignore

@@ -30,5 +30,4 @@ testbin
 *~
 
 # Exclude all libgit2 related files
-hack/libgit2/
 build/

Dockerfile

@@ -2,58 +2,98 @@ ARG BASE_VARIANT=alpine
 ARG GO_VERSION=1.17
 ARG XX_VERSION=1.1.0
 
-ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2
-ARG LIBGIT2_TAG=libgit2-1.1.1-4
+ARG LIBGIT2_IMG
+ARG LIBGIT2_TAG
 
-FROM --platform=linux/amd64 ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build-amd64
-FROM --platform=linux/arm64 ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build-arm64
-FROM --platform=linux/arm/v7 ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build-armv7
+FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
 
-FROM --platform=$BUILDPLATFORM build-$TARGETARCH$TARGETVARIANT AS build
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+
+FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+
+FROM gostable AS go-linux
+
+# Build-base consists of build platform dependencies and xx.
+# These will be used at current arch to yield execute the cross compilations.
+FROM go-${TARGETOS} AS build-base
+
+RUN apk add --no-cache clang lld pkgconfig
+
+COPY --from=xx / /
+
+# build-go-mod can still be cached at build platform architecture.
+FROM build-base as build-go-mod
 
 # Configure workspace
 WORKDIR /workspace
 
-# This has its own go.mod, which needs to be present so go mod
-# download works.
+# Copy api submodule
 COPY api/ api/
 
 # Copy modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
 
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
+# Cache modules
 RUN go mod download
 
-RUN apk add clang lld pkgconfig ca-certificates
+# The musl-tool-chain layer is an adhoc solution
+# for the problem in which xx gets confused during compilation
+# and a) looks for gold linker and then b) cannot find musl's dynamic linker.
+FROM --platform=$BUILDPLATFORM alpine as musl-tool-chain
+
+COPY --from=xx / /
+
+RUN apk add bash curl tar
+
+WORKDIR /workspace
+COPY hack/download-musl.sh .
 
-ENV CGO_ENABLED=1
 ARG TARGETPLATFORM
+ARG TARGETARCH
+RUN ROOT_DIR="$(pwd)" TARGET_ARCH="$(xx-info alpine-arch)" ENV_FILE=true \
+        ./download-musl.sh
 
-RUN xx-apk add --no-cache \
-        musl-dev gcc lld binutils-gold
+# Build stage install per target platform
+# dependency and effectively cross compile the application.
+FROM build-go-mod as build
+
+ARG TARGETPLATFORM
+
+COPY --from=libgit2-libs /usr/local/ /usr/local/
+
+# Some dependencies have to installed 
+# for the target platform: https://github.com/tonistiigi/xx#go--cgo
+RUN xx-apk add musl-dev gcc lld
+
+WORKDIR /workspace
+
+# Copy source code
+COPY main.go main.go
+COPY controllers/ controllers/
+COPY pkg/ pkg/
+
+COPY --from=musl-tool-chain /workspace/build /workspace/build
+
+ARG TARGETPLATFORM
+ARG TARGETARCH
+ENV CGO_ENABLED=1
 
 # Performance related changes:
 # - Use read-only bind instead of copying go source files.
 # - Cache go packages.
-RUN --mount=target=. \
-    --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg \
-    export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \
+RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \
+    export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \
     export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \
-	export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \
-    CGO_LDFLAGS="${FLAGS} -static" \
-	xx-go build \
+    export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2) -static" && \
+    GOARCH=$TARGETARCH go build  \
         -ldflags "-s -w" \
         -tags 'netgo,osusergo,static_build' \
-        -o /image-automation-controller -trimpath \
-    main.go
+        -o /image-automation-controller -trimpath main.go;
 
 # Ensure that the binary was cross-compiled correctly to the target platform.
 RUN xx-verify --static /image-automation-controller
 
-
 FROM alpine:3.15
 
 ARG TARGETPLATFORM

Makefile

@@ -8,7 +8,7 @@ CRD_OPTIONS ?= crd:crdVersions=v1
 
 # Base image used to build the Go binary
 LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2
-LIBGIT2_TAG ?= libgit2-1.1.1-4
+LIBGIT2_TAG ?= libgit2-1.1.1-6
 
 # Allows for defining additional Docker buildx arguments,
 # e.g. '--push'.
@@ -29,30 +29,72 @@ REFLECTOR_VER ?= v0.16.0
 
 # Repository root based on Git metadata.
 REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel)
+BUILD_DIR := $(REPOSITORY_ROOT)/build
 
-LIBGIT2_PATH := $(REPOSITORY_ROOT)/build/libgit2
+# Other dependency versions
+ENVTEST_BIN_VERSION ?= 1.19.2
+
+# Caches libgit2 versions per tag, "forcing" rebuild only when needed.
+LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
 LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
 LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64
 LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
+MUSL-CC =
 
 export CGO_ENABLED=1
-export LIBRARY_PATH=$(LIBGIT2_LIB_PATH):$(LIBGIT2_LIB64_PATH)
-export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig:$(LIBGIT2_LIB64_PATH)/pkgconfig
-export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include
+export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
+export LIBRARY_PATH=$(LIBGIT2_LIB_PATH)
+export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include -I$(LIBGIT2_PATH)/include/openssl
+
 
+# The pkg-config command will yield warning messages until libgit2 is downloaded.
 ifeq ($(shell uname -s),Darwin)
-	export CGO_LDFLAGS=-L$(LIBGIT2_LIB_PATH) -lssh2 -lssl -lcrypto -lgit2
+export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2 2>/dev/null)
+GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build'
+else
+export PKG_CONFIG_PATH:=$(PKG_CONFIG_PATH):$(LIBGIT2_LIB64_PATH)/pkgconfig
+export LIBRARY_PATH:=$(LIBRARY_PATH):$(LIBGIT2_LIB64_PATH)
+export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2 2>/dev/null)
+endif
+
+ifeq ($(shell uname -s),Linux)
+ifeq ($(shell uname -m),x86_64)
+# Linux x86_64 seem to be able to cope with the static libraries 
+# by having only musl-dev installed, without the need of using musl toolchain.
+	GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build'
 else
-	export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2)
+	MUSL-PREFIX=$(BUILD_DIR)/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl
+	MUSL-CC=$(MUSL-PREFIX)-gcc
+	export CC=$(MUSL-PREFIX)-gcc
+	export CXX=$(MUSL-PREFIX)-g++
+	export AR=$(MUSL-PREFIX)-ar
+	GO_STATIC_FLAGS=-ldflags "-s -w -extldflags \"-static\"" -tags 'netgo,osusergo,static_build'
 endif
+endif
+
+# API (doc) generation utilities
+CONTROLLER_GEN_VERSION ?= v0.7.0
+GEN_API_REF_DOCS_VERSION ?= v0.3.0
 
-# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+# If gobin not set, create one on ./build and add to path.
 ifeq (,$(shell go env GOBIN))
-GOBIN=$(shell go env GOPATH)/bin
+export GOBIN=$(BUILD_DIR)/gobin
 else
-GOBIN=$(shell go env GOBIN)
+export GOBIN=$(shell go env GOBIN)
 endif
+export PATH:=${GOBIN}:${PATH}
 
+# Architecture to use envtest with
+ifeq ($(shell uname -m),x86_64)
+ENVTEST_ARCH ?= amd64
+else
+ENVTEST_ARCH ?= arm64
+endif
+
+ifeq ($(shell uname -s),Darwin)
+# Envtest only supports darwin-amd64
+ENVTEST_ARCH=amd64
+endif
 
 TEST_CRDS := controllers/testdata/crds
 
@@ -88,13 +130,15 @@ ${CACHE}/imagepolicies_${REFLECTOR_VER}.yaml:
 	curl -s --fail https://raw.githubusercontent.com/fluxcd/image-reflector-controller/${REFLECTOR_VER}/config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml \
 		-o ${CACHE}/imagepolicies_${REFLECTOR_VER}.yaml
 
+check-deps:
+ifeq ($(shell uname -s),Darwin)
+	if ! command -v pkg-config &> /dev/null; then echo "pkg-config is required"; exit 1; fi
+endif
+
 KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
 test: $(LIBGIT2) test-api test_deps generate fmt vet manifests api-docs	install-envtest ## Run tests
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
-	go test ./... \
-		-ldflags "-s -w" \
-		-coverprofile cover.out \
-	 	-tags 'netgo,osusergo,static_build'
+	go test $(GO_STATIC_FLAGS) ./... -coverprofile cover.out
 
 test-api:	## Run api tests
 	cd api; go test ./... -coverprofile cover.out
@@ -104,7 +148,7 @@ manager: $(LIBGIT2) generate fmt vet	## Build manager binary
 
 
 run: $(LIBGIT2) generate fmt vet manifests	# Run against the configured Kubernetes cluster in ~/.kube/config
-	go run ./main.go --log-level=${LOG_LEVEL} --log-encoding=console
+	go run $(GO_STATIC_FLAGS) ./main.go --log-level=${LOG_LEVEL} --log-encoding=console
 
 install: manifests	## Install CRDs into a cluster
 	kustomize build config/crd | kubectl apply -f -
@@ -148,7 +192,7 @@ docker-build:  ## Build the Docker image
 	docker buildx build \
 		--build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \
 		--build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \
-		--platform=$(BUILD_PLATFORMS) \
+		--platform=linux/amd64 \
 		-t $(IMG):$(TAG) \
 		$(BUILD_ARGS) .
 
@@ -159,43 +203,48 @@ docker-deploy:	## Set the Docker image in-cluster
 	kubectl -n flux-system set image deployment/image-automation-controller manager=$(IMG):$(TAG)
 
 # Find or download controller-gen
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+CONTROLLER_GEN = $(GOBIN)/controller-gen
 .PHONY: controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
 	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
 
 libgit2: $(LIBGIT2)  ## Detect or download libgit2 library
 
-$(LIBGIT2):
-	IMG_TAG=$(LIBGIT2_IMG):$(LIBGIT2_TAG) ./hack/extract-libraries.sh
+$(LIBGIT2): $(MUSL-CC)
+	IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) ./hack/install-libraries.sh
+
+$(MUSL-CC):
+ifneq ($(shell uname -s),Darwin)
+	./hack/download-musl.sh
+endif
 
 # Find or download gen-crd-api-reference-docs
-GEN_CRD_API_REFERENCE_DOCS = $(shell pwd)/bin/gen-crd-api-reference-docs
+GEN_CRD_API_REFERENCE_DOCS = $(GOBIN)/gen-crd-api-reference-docs
 .PHONY: gen-crd-api-reference-docs
 gen-crd-api-reference-docs:
 	$(call go-install-tool,$(GEN_CRD_API_REFERENCE_DOCS),github.com/ahmetb/[email protected])
 
-ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+ENVTEST_ASSETS_DIR=$(BUILD_DIR)/testbin
 ENVTEST_KUBERNETES_VERSION?=latest
 install-envtest: setup-envtest
 	mkdir -p ${ENVTEST_ASSETS_DIR}
 	$(ENVTEST) use $(ENVTEST_KUBERNETES_VERSION) --arch=$(ENVTEST_ARCH) --bin-dir=$(ENVTEST_ASSETS_DIR)
+	chmod -R u+w $(BUILD_DIR)
 
-ENVTEST = $(shell pwd)/bin/setup-envtest
+ENVTEST = $(GOBIN)/setup-envtest
 .PHONY: envtest
 setup-envtest: ## Download envtest-setup locally if necessary.
 	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
 # go-install-tool will 'go install' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 define go-install-tool
 @[ -f $(1) ] || { \
 set -e ;\
 TMP_DIR=$$(mktemp -d) ;\
 cd $$TMP_DIR ;\
 go mod init tmp ;\
 echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
 endef