diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1e2f1e16 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +hack/libgit2/ diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index addded17..6e124b64 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,19 +1,35 @@ -FROM golang:1.16-buster as builder - -# Up-to-date libgit2 dependencies are only available in sid (unstable). -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y libgit2-dev/unstable zlib1g-dev/unstable libssh2-1-dev/unstable libpcre3-dev/unstable \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +ARG BASE_VARIANT=bullseye +ARG GO_VERSION=1.16.8 +ARG XX_VERSION=1.0.0-rc.2 +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG LIBGIT2_TAG=libgit2-1.1.1-1 + +FROM tonistiigi/xx:${XX_VERSION} AS xx +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 + +FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable + +# Copy the build utiltiies +COPY --from=xx / / +COPY --from=libgit2 /Makefile /libgit2/ + +# Install the libgit2 build dependencies +RUN make -C /libgit2 cmake + +RUN make -C /libgit2 dependencies + +# Compile and install libgit2 +RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 + +# Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ useradd -u 1001 --gid test --shell /bin/sh --create-home test # Run as test user USER test +# Set path to envtest binaries. +ENV PATH="/github/workspace/envtest:${PATH}" + ENTRYPOINT ["/bin/sh", "-c"] diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 71eb0a05..6a252bc9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,8 +22,11 @@ jobs: ${{ runner.os }}-go- - name: Set up kubebuilder uses: fluxcd/pkg/actions/kubebuilder@main + - name: Setup envtest + uses: fluxcd/pkg/actions/envtest@main + with: + version: "1.19.2" - name: Run tests uses: ./.github/actions/run-tests env: GOPATH: /github/home/go - KUBEBUILDER_ASSETS: ${{ github.workspace }}/kubebuilder/bin diff --git a/.gitignore b/.gitignore index eaade77c..b88e54c7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ bin *.swp *.swo *~ + +# Exclude all libgit2 related files +hack/libgit2/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f59b4ce6..8a782a71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,40 @@ to join the conversation (this will also add an invitation to your Google calendar for our [Flux meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARDeh6b70B0/edit#)). +### Installing required dependencies + +The dependency [libgit2](https://libgit2.org/) needs to be installed to be able +to run source-controller or its test-suite locally (not in a container). + +In case this dependency is not present on your system (at the expected +version), the first invocation of a `make` target that requires the +dependency will attempt to compile it locally to `hack/libgit2`. For this build +to succeed; CMake, OpenSSL 1.1 and LibSSH2 must be present on the system. + +Triggering a manual build of the dependency is possible as well by running +`make libgit2`. To enforce the build, for example if your system dependencies +match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the +`make` command. + +#### macOS + +```console +$ # Ensure libgit2 dependencies are available +$ brew install cmake openssl@1.1 libssh2 pkg-config +$ LIBGIT2_FORCE=1 make libgit2 +``` + +#### Linux + +```console +$ # Ensure libgit2 dependencies are available +$ pacman -S cmake openssl libssh2 +$ LIBGIT2_FORCE=1 make libgit2 +``` + +**Note:** Example shown is for Arch Linux, but likewise procedure can be +followed using any other package manager, e.g. `apt`. + ### How to run the test suite Prerequisites: diff --git a/Dockerfile b/Dockerfile index 28d9277c..d313634a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,49 @@ -FROM golang:1.16-buster as builder - -# Up-to-date libgit2 dependencies are only available in sid (unstable). -# The libgit2 dependencies must be listed here to be able to build on ARM64. -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y libgit2-dev/unstable zlib1g-dev/unstable libssh2-1-dev/unstable libpcre3-dev/unstable \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +ARG BASE_VARIANT=bullseye +ARG GO_VERSION=1.16.8 +ARG XX_VERSION=1.0.0-rc.2 -WORKDIR /workspace +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG LIBGIT2_TAG=libgit2-1.1.1-1 -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 + +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable +FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest + +FROM gostable AS go-linux + +FROM go-${TARGETOS} AS build-base-bullseye + +# Copy the build utiltiies +COPY --from=xx / / +COPY --from=libgit2 /Makefile /libgit2/ + +# Install the libgit2 build dependencies +RUN make -C /libgit2 cmake + +ARG TARGETPLATFORM +RUN make -C /libgit2 dependencies + +FROM build-base-${BASE_VARIANT} as libgit2-bullseye + +# Compile and install libgit2 +ARG TARGETPLATFORM +RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 + +FROM libgit2-${BASE_VARIANT} as build-bullseye + +# Configure workspace +WORKDIR /workspace # This has its own go.mod, which needs to be present so go mod # download works. COPY api/ api/ +# Copy modules manifests +COPY go.mod go.mod +COPY go.sum go.sum + # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download @@ -30,30 +53,43 @@ COPY main.go main.go COPY pkg/ pkg/ COPY controllers/ controllers/ -# Build -RUN CGO_ENABLED=1 go build -o image-automation-controller main.go +# Build the binary +ENV CGO_ENABLED=1 +ARG TARGETPLATFORM +RUN xx-go build -o image-automation-controller -trimpath \ + main.go -FROM debian:buster-slim as controller +FROM build-${BASE_VARIANT} as prepare-bullseye -LABEL org.opencontainers.image.source="https://github.com/fluxcd/image-automation-controller" +# Move libgit2 lib to generic and predictable location +ARG TARGETPLATFORM +RUN mkdir -p /libgit2/lib/ \ + && cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/ -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y ca-certificates libgit2-1.1 \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +FROM prepare-${BASE_VARIANT} as build -COPY --from=builder /workspace/image-automation-controller /usr/local/bin/ +FROM debian:${BASE_VARIANT}-slim as controller +# Configure user RUN groupadd controller && \ useradd --gid controller --shell /bin/sh --create-home controller -USER controller +# Copy libgit2 +COPY --from=build /libgit2/lib/ /usr/local/lib/ +RUN ldconfig + +# Upgrade packages and install runtime dependencies +RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \ + && echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \ + && apt update \ + && apt install --no-install-recommends -y zlib1g/sid libssl1.1/sid libssh2-1/sid \ + && apt install --no-install-recommends -y ca-certificates \ + && apt clean \ + && apt autoremove --purge -y \ + && rm -rf /var/lib/apt/lists/* + +# Copy over binary from build +COPY --from=build /workspace/image-automation-controller /usr/local/bin/ +USER controller ENTRYPOINT [ "image-automation-controller" ] diff --git a/Makefile b/Makefile index 2364d504..a63aafef 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,54 @@ # Image URL to use all building/pushing image targets -IMG ?= fluxcd/image-automation-controller:latest +IMG ?= fluxcd/image-automation-controller +# Image tag to use all building/push image targets +TAG ?= latest + # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 +# Base image used to build the Go binary +LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 +LIBGIT2_TAG ?= libgit2-1.1.1-1 + +# Allows for defining additional Docker buildx arguments, +# e.g. '--push'. +BUILD_ARGS ?= +# Architectures to build images for +BUILD_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 + # Directory with versioned, downloaded things -CACHE:=cache +CACHE := cache # Version of the source-controller from which to get the GitRepository CRD. # Change this if you bump the source-controller/api version in go.mod. -SOURCE_VER ?= v0.15.4 +SOURCE_VER ?= v0.16.0 # Version of the image-reflector-controller from which to get the ImagePolicy CRD. # Change this if you bump the image-reflector-controller/api version in go.mod. REFLECTOR_VER ?= v0.11.1 +# Version of libgit2 the controller should depend on. +LIBGIT2_VER ?= 1.1.1 + +# Repository root based on Git metadata. +REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) + +# libgit2 related magical paths +# These are used to determine if the target libgit2 version is already available on +# the system, or where they should be installed to +SYSTEM_LIBGIT2_VERSION := $(shell pkg-config --modversion libgit2 2>/dev/null) +LIBGIT2_PATH := $(REPOSITORY_ROOT)/hack/libgit2 +LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib +LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.so.$(LIBGIT2_VERSION) + +ifneq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) + LIBGIT2_FORCE ?= 1 +endif + +ifeq ($(shell uname -s),Darwin) + LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.$(LIBGIT2_VERSION).dylib +endif + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -21,10 +56,10 @@ else GOBIN=$(shell go env GOBIN) endif -TEST_CRDS:=controllers/testdata/crds +TEST_CRDS := controllers/testdata/crds # Log level for `make run` -LOG_LEVEL?=info +LOG_LEVEL ?= info all: manager @@ -52,80 +87,74 @@ ${CACHE}/imagepolicies_${REFLECTOR_VER}.yaml: curl -s --fail https://raw.githubusercontent.com/fluxcd/image-reflector-controller/${REFLECTOR_VER}/config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml \ -o ${CACHE}/imagepolicies_${REFLECTOR_VER}.yaml -# Run tests -test: test_deps generate fmt vet manifests api-docs +test: $(LIBGIT2) test-api test_deps generate fmt vet manifests api-docs ## Run tests + LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \ + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \ go test ./... -coverprofile cover.out + +test-api: ## Run api tests cd api; go test ./... -coverprofile cover.out -# Build manager binary -manager: generate fmt vet +manager: $(LIBGIT2) generate fmt vet ## Build manager binary + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \ go build -o bin/manager main.go -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests +run: $(LIBGIT2) generate fmt vet manifests # Run against the configured Kubernetes cluster in ~/.kube/config go run ./main.go --log-level=${LOG_LEVEL} --log-encoding=console -# Install CRDs into a cluster -install: manifests +install: manifests ## Install CRDs into a cluster kustomize build config/crd | kubectl apply -f - -# Uninstall CRDs from a cluster -uninstall: manifests +uninstall: manifests ## Uninstall CRDs from a cluster kustomize build config/crd | kubectl delete -f - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests - cd config/manager && kustomize edit set image fluxcd/image-automation-controller=${IMG} +deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config + cd config/manager && kustomize edit set image fluxcd/image-automation-controller=$(IMG):$(TAG) kustomize build config/default | kubectl apply -f - dev-deploy: manifests mkdir -p config/dev && cp config/default/* config/dev - cd config/dev && kustomize edit set image fluxcd/image-automation-controller=${IMG} + cd config/dev && kustomize edit set image fluxcd/image-automation-controller=$(IMG):$(TAG) kustomize build config/dev | kubectl apply -f - rm -rf config/dev -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen +manifests: controller-gen ## Generate manifests e.g. CRD, RBAC etc. cd api; $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="../config/crd/bases" -# Generate API reference documentation -api-docs: gen-crd-api-reference-docs +api-docs: gen-crd-api-reference-docs ## Generate API reference documentation $(API_REF_GEN) -api-dir=./api/v1beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/image-automation.md -# Run go mod tidy -tidy: +tidy: ## Run go mod tidy cd api; rm -f go.sum; go mod tidy rm -f go.sum; go mod tidy -# Run go fmt against code -fmt: +fmt: ## Run go fmt against code go fmt ./... cd api; go fmt ./... -# Run go vet against code -vet: +vet: $(LIBGIT2) ## Run go vet against code + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig \ go vet ./... cd api; go vet ./... -# Generate code -generate: controller-gen +generate: controller-gen ## Generate code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." -# Build the docker image -docker-build: test - docker build . -t ${IMG} +docker-build: ## Build the Docker image + docker buildx build \ + --build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \ + --build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \ + --platform=$(BUILD_PLATFORMS) \ + -t $(IMG):$(TAG) \ + $(BUILD_ARGS) . -# Push the docker image -docker-push: - docker push ${IMG} +docker-push: ## Push the Docker image + docker push $(IMG):$(TAG) -# Set the docker image in-cluster -docker-deploy: - kubectl -n flux-system set image deployment/image-automation-controller manager=${IMG} +docker-deploy: ## Set the Docker image in-cluster + kubectl -n flux-system set image deployment/image-automation-controller manager=$(IMG):$(TAG) -# find or download controller-gen -# download controller-gen if necessary -controller-gen: +controller-gen: ## Find or download controller-gen ifeq (, $(shell which controller-gen)) @{ \ set -e ;\ @@ -140,8 +169,7 @@ else CONTROLLER_GEN=$(shell which controller-gen) endif -# Find or download gen-crd-api-reference-docs -gen-crd-api-reference-docs: +gen-crd-api-reference-docs: ## Find or download gen-crd-api-reference-docs ifeq (, $(shell which gen-crd-api-reference-docs)) @{ \ set -e ;\ @@ -155,3 +183,19 @@ API_REF_GEN=$(GOBIN)/gen-crd-api-reference-docs else API_REF_GEN=$(shell which gen-crd-api-reference-docs) endif + +libgit2: $(LIBGIT2) ## Detect or download libgit2 library + +$(LIBGIT2): +ifeq (1, $(LIBGIT2_FORCE)) + @{ \ + set -e; \ + mkdir -p $(LIBGIT2_PATH); \ + curl -sL https://raw.githubusercontent.com/fluxcd/golang-with-libgit2/$(LIBGIT2_TAG)/hack/Makefile -o $(LIBGIT2_PATH)/Makefile; \ + INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH) libgit2; \ + } +endif + +.PHONY: help +help: ## Display this help menu + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/controllers/imageupdateautomation_controller.go b/controllers/imageupdateautomation_controller.go index aac3ab3a..dda6fd85 100644 --- a/controllers/imageupdateautomation_controller.go +++ b/controllers/imageupdateautomation_controller.go @@ -668,7 +668,7 @@ func push(ctx context.Context, path, branch string, access repoAccess) error { if status != "" { callbackErr = fmt.Errorf("ref %s rejected: %s", refname, status) } - return libgit2.ErrOk + return libgit2.ErrorCodeOK } err = origin.Push([]string{fmt.Sprintf("refs/heads/%s:refs/heads/%s", branch, branch)}, &libgit2.PushOptions{ RemoteCallbacks: callbacks, diff --git a/go.mod b/go.mod index 644b6b6a..c4beb4f2 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/google/go-containerregistry v0.6.0 - github.com/libgit2/git2go/v31 v31.4.14 + github.com/libgit2/git2go/v31 v31.6.1 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.15.0 github.com/otiai10/copy v1.2.0 diff --git a/go.sum b/go.sum index 20a692ce..bb25a61c 100644 --- a/go.sum +++ b/go.sum @@ -706,8 +706,9 @@ github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtB github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libgit2/git2go/v31 v31.4.14 h1:6GOd3965D9e/+gjxCwZF4eQ+vB9kKB4yKFqdQr6XZ2E= github.com/libgit2/git2go/v31 v31.4.14/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= +github.com/libgit2/git2go/v31 v31.6.1 h1:FnKHHDDBgltSsu9RpKuL4rSR8dQ1JTf9dfvFhZ1y7Aw= +github.com/libgit2/git2go/v31 v31.6.1/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=