From 21af7f448d5ed022dab457ee046208d81ed99e0d Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Sun, 31 Oct 2021 17:04:07 +0100 Subject: [PATCH] make Cluster API dev tooling to work on arm64 --- Makefile | 2 +- Tiltfile | 9 +++++---- hack/ensure-kind.sh | 6 ++++-- hack/ensure-kubectl.sh | 6 ++++-- hack/ensure-kustomize.sh | 14 ++++---------- scripts/ci-e2e-lib.sh | 7 ++++--- test/e2e/config/docker.yaml | 8 ++++---- test/framework/clusterctl/e2e_config.go | 3 +++ test/infrastructure/docker/Dockerfile | 4 ++-- test/infrastructure/docker/Makefile | 4 ++-- 10 files changed, 33 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 32bf33ba9f56..a85ed128ed44 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ KUBEADM_CONTROL_PLANE_CONTROLLER_IMG ?= $(REGISTRY)/$(KUBEADM_CONTROL_PLANE_IMAG # It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971 TAG ?= dev -ARCH ?= amd64 +ARCH ?= $(shell go env GOARCH) ALL_ARCH = amd64 arm arm64 ppc64le s390x # Allow overriding the imagePullPolicy diff --git a/Tiltfile b/Tiltfile index b59d89c1f88f..630dd2576e36 100644 --- a/Tiltfile +++ b/Tiltfile @@ -93,9 +93,9 @@ providers = { "exp", "third_party", ], - "additional_docker_helper_commands": """ -RUN wget -qO- https://dl.k8s.io/v1.21.2/kubernetes-client-linux-amd64.tar.gz | tar xvz -""", + "additional_docker_helper_commands": "RUN wget -qO- https://dl.k8s.io/v1.21.2/kubernetes-client-linux-{arch}.tar.gz | tar xvz".format( + arch = os_arch, + ), "additional_docker_build_commands": """ COPY --from=tilt-helper /go/kubernetes/client/bin/kubectl /usr/bin/kubectl """, @@ -197,8 +197,9 @@ def enable_provider(name, debug): else: gcflags = "" - build_env = "CGO_ENABLED={cgo_enabled} GOOS=linux GOARCH=amd64".format( + build_env = "CGO_ENABLED={cgo_enabled} GOOS=linux GOARCH={arch}".format( cgo_enabled = cgo_enabled, + arch = os_arch, ) build_cmd = "{build_env} go build {build_options} -gcflags '{gcflags}' -ldflags '{ldflags}' -o .tiltbuild/manager {go_main}".format( build_env = build_env, diff --git a/hack/ensure-kind.sh b/hack/ensure-kind.sh index 0d18e45b5d77..6598f5730927 100755 --- a/hack/ensure-kind.sh +++ b/hack/ensure-kind.sh @@ -22,18 +22,20 @@ set -x GOPATH_BIN="$(go env GOPATH)/bin/" MINIMUM_KIND_VERSION=v0.11.0 +goarch="$(go env GOARCH)" +goos="$(go env GOOS)" # Ensure the kind tool exists and is a viable version, or installs it verify_kind_version() { # If kind is not available on the path, get it if ! [ -x "$(command -v kind)" ]; then - if [[ "${OSTYPE}" == "linux-gnu" ]]; then + if [ "$goos" == "linux" ] || [ "$goos" == "darwin" ]; then echo 'kind not found, installing' if ! [ -d "${GOPATH_BIN}" ]; then mkdir -p "${GOPATH_BIN}" fi - curl -sLo "${GOPATH_BIN}/kind" https://github.com/kubernetes-sigs/kind/releases/download/${MINIMUM_KIND_VERSION}/kind-linux-amd64 + curl -sLo "${GOPATH_BIN}/kind" "https://github.com/kubernetes-sigs/kind/releases/download/${MINIMUM_KIND_VERSION}/kind-${goos}-${goarch}" chmod +x "${GOPATH_BIN}/kind" else echo "Missing required binary in path: kind" diff --git a/hack/ensure-kubectl.sh b/hack/ensure-kubectl.sh index 6d0a831b65da..afe62a39d943 100755 --- a/hack/ensure-kubectl.sh +++ b/hack/ensure-kubectl.sh @@ -20,18 +20,20 @@ set -o pipefail GOPATH_BIN="$(go env GOPATH)/bin/" MINIMUM_KUBECTL_VERSION=v1.19.0 +goarch="$(go env GOARCH)" +goos="$(go env GOOS)" # Ensure the kubectl tool exists and is a viable version, or installs it verify_kubectl_version() { # If kubectl is not available on the path, get it if ! [ -x "$(command -v kubectl)" ]; then - if [[ "${OSTYPE}" == "linux-gnu" ]]; then + if [ "$goos" == "linux" ] || [ "$goos" == "darwin" ]; then if ! [ -d "${GOPATH_BIN}" ]; then mkdir -p "${GOPATH_BIN}" fi echo 'kubectl not found, installing' - curl -sLo "${GOPATH_BIN}/kubectl" https://dl.k8s.io/release/${MINIMUM_KUBECTL_VERSION}/bin/linux/amd64/kubectl + curl -sLo "${GOPATH_BIN}/kubectl" "https://dl.k8s.io/release/${MINIMUM_KUBECTL_VERSION}/bin/${goos}/${goarch}/kubectl" chmod +x "${GOPATH_BIN}/kubectl" else echo "Missing required binary in path: kubectl" diff --git a/hack/ensure-kustomize.sh b/hack/ensure-kustomize.sh index 65498d7e34df..d0cba832b70b 100755 --- a/hack/ensure-kustomize.sh +++ b/hack/ensure-kustomize.sh @@ -23,15 +23,9 @@ BIN_ROOT="${KUBE_ROOT}/hack/tools/bin" kustomize_version=3.9.1 -goarch=amd64 -goos="unknown" -if [[ "${OSTYPE}" == "linux"* ]]; then - goos="linux" -elif [[ "${OSTYPE}" == "darwin"* ]]; then - goos="darwin" -fi - -if [[ "$goos" == "unknown" ]]; then +goarch="$(go env GOARCH)" +goos="$(go env GOOS)" +if [ "$goos" != "linux" ] && [ "$goos" != "darwin" ]; then echo "OS '$OSTYPE' not supported. Aborting." >&2 exit 1 fi @@ -44,7 +38,7 @@ verify_kustomize_version() { mkdir -p "${BIN_ROOT}" fi archive_name="kustomize-v${kustomize_version}.tar.gz" - curl -sLo "${BIN_ROOT}/${archive_name}" https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${kustomize_version}/kustomize_v${kustomize_version}_${goos}_${goarch}.tar.gz + curl -sLo "${BIN_ROOT}/${archive_name}" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${kustomize_version}/kustomize_v${kustomize_version}_${goos}_${goarch}.tar.gz" tar -zvxf "${BIN_ROOT}/${archive_name}" -C "${BIN_ROOT}/" chmod +x "${BIN_ROOT}/kustomize" rm "${BIN_ROOT}/${archive_name}" diff --git a/scripts/ci-e2e-lib.sh b/scripts/ci-e2e-lib.sh index d1da7cdd33c5..1bad02397212 100644 --- a/scripts/ci-e2e-lib.sh +++ b/scripts/ci-e2e-lib.sh @@ -18,13 +18,14 @@ capi:buildDockerImages () { # Configure provider images generation; # please ensure the generated image name matches image names used in the E2E_CONF_FILE + ARCH="$(go env GOARCH)" export REGISTRY=gcr.io/k8s-staging-cluster-api export TAG=dev - export ARCH=amd64 + export ARCH export PULL_POLICY=IfNotPresent ## Build all Cluster API provider images, if missing - if [[ "$(docker images -q $REGISTRY/cluster-api-controller-amd64:$TAG 2> /dev/null)" == "" ]]; then + if [[ "$(docker images -q "$REGISTRY/cluster-api-controller-$ARCH:$TAG" 2> /dev/null)" == "" ]]; then echo "+ Building CAPI images" make docker-build else @@ -32,7 +33,7 @@ capi:buildDockerImages () { fi ## Build CAPD provider images, if missing - if [[ "$(docker images -q $REGISTRY/capd-manager-amd64:$TAG 2> /dev/null)" == "" ]]; then + if [[ "$(docker images -q "$REGISTRY/capd-manager-$ARCH:$TAG" 2> /dev/null)" == "" ]]; then echo "+ Building CAPD images" make -C test/infrastructure/docker docker-build else diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index d44f8fafa31d..fcb7fc2d8e65 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -9,13 +9,13 @@ images: # Use local dev images built source tree; -- name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:dev +- name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-{ARCH}:dev loadBehavior: tryLoad -- name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:dev +- name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-{ARCH}:dev loadBehavior: tryLoad -- name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:dev +- name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-{ARCH}:dev loadBehavior: tryLoad -- name: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev +- name: gcr.io/k8s-staging-cluster-api/capd-manager-{ARCH}:dev loadBehavior: tryLoad - name: quay.io/jetstack/cert-manager-cainjector:v1.5.3 loadBehavior: tryLoad diff --git a/test/framework/clusterctl/e2e_config.go b/test/framework/clusterctl/e2e_config.go index a6d7783f6a6b..b87395b85283 100644 --- a/test/framework/clusterctl/e2e_config.go +++ b/test/framework/clusterctl/e2e_config.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "sort" "strconv" "strings" @@ -274,8 +275,10 @@ func (c *E2EConfig) Defaults() { } } } + imageReplacer := strings.NewReplacer("{OS}", runtime.GOOS, "{ARCH}", runtime.GOARCH) for i := range c.Images { containerImage := &c.Images[i] + containerImage.Name = imageReplacer.Replace(containerImage.Name) if containerImage.LoadBehavior == "" { containerImage.LoadBehavior = MustLoadImage } diff --git a/test/infrastructure/docker/Dockerfile b/test/infrastructure/docker/Dockerfile index f30bf3795f36..c141796acfc9 100644 --- a/test/infrastructure/docker/Dockerfile +++ b/test/infrastructure/docker/Dockerfile @@ -25,7 +25,7 @@ ENV GOPROXY=$goproxy # Gets additional CAPD dependencies WORKDIR /tmp -RUN curl -LO https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl && \ +RUN curl -LO https://dl.k8s.io/release/v1.22.0/bin/linux/${ARCH}/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/bin/kubectl @@ -55,7 +55,7 @@ WORKDIR /workspace/test/infrastructure/docker # Build the CAPD manager using the compiler cache folder RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o /workspace/manager main.go + CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -o /workspace/manager main.go # NOTE: CAPD can't use non-root because docker requires access to the docker socket FROM gcr.io/distroless/static:latest diff --git a/test/infrastructure/docker/Makefile b/test/infrastructure/docker/Makefile index 600494d92a46..a72e24fb5c7a 100644 --- a/test/infrastructure/docker/Makefile +++ b/test/infrastructure/docker/Makefile @@ -60,8 +60,8 @@ $(KUSTOMIZE): # Build kustomize from tools folder. REGISTRY ?= gcr.io/$(shell gcloud config get-value project) IMAGE_NAME ?= capd-manager CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -ARCH ?= amd64 -ALL_ARCH = amd64 arm arm64 +ARCH ?= $(shell go env GOARCH) +ALL_ARCH = amd64 arm arm64 ppc64le s390x STAGING_REGISTRY ?= gcr.io/k8s-staging-cluster-api STAGING_BUCKET ?= artifacts.k8s-staging-cluster-api.appspot.com