From c1f12daad5196fa60940bd478c3b6fb62e2a7ac2 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 6 Jun 2024 15:59:27 -0400 Subject: [PATCH] Multi Arch Support for Devfile Index and Devfile Index Base (#245) * add multi arch build and push script Signed-off-by: Jordan Dubrick * add ability to build for different arch to existing build scripts and update docs Signed-off-by: Jordan Dubrick * update workflows to build and push multi arch index base Signed-off-by: Jordan Dubrick * update ci dockerfile to download proper arch for yq Signed-off-by: Jordan Dubrick * pr-comment: improve log readability Signed-off-by: Jordan Dubrick * pr-comment: update ci to have jobs for each arch Signed-off-by: Jordan Dubrick --------- Signed-off-by: Jordan Dubrick --- .ci/Dockerfile | 6 ++- .github/workflows/ci.yaml | 9 ++++- .github/workflows/pushimage-next.yaml | 8 ++-- README.md | 3 ++ build_registry.sh | 15 +++++++- index/server/README.md | 7 +++- index/server/build-multi-arch.sh | 55 +++++++++++++++++++++++++++ index/server/build.sh | 17 ++++++++- 8 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 index/server/build-multi-arch.sh diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 38c66dc0..843d85ee 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -14,12 +14,16 @@ # limitations under the License. FROM golang:1.19-alpine@sha256:0ec0646e208ea58e5d29e558e39f2e59fccf39b7bda306cb53bbaff91919eca5 AS builder +# Allows for the proper yq to be downloaded +# This arg is automatically set if the dockerfile is built with --platform flag +ARG TARGETARCH=amd64 + # Install dependencies RUN apk add --no-cache git bash curl zip # Install yq ENV YQ_VERSION=v4.44.1 -RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq +RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} -o /usr/local/bin/yq && mv ./yq_linux_${TARGETARCH} /usr/local/bin/yq && chmod +x /usr/local/bin/yq RUN yq # Copy the registry build tools diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9402e69d..5c388b16 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -105,13 +105,20 @@ jobs: exit 1 fi - - name: Check if index server build is working + - name: Test index server build for linux/amd64 run: | cd index/server export GOPATH=$(go env GOPATH) go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@${{ env.OAPI_CODEGEN_VERSION }} bash ./build.sh + - name: Test index server build for linux/arm64 + run: | + cd index/server + export GOPATH=$(go env GOPATH) + go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@${{ env.OAPI_CODEGEN_VERSION }} + bash ./build.sh linux/arm64 + - name: Test index server run: cd index/server && go test ./... -coverprofile cover.out diff --git a/.github/workflows/pushimage-next.yaml b/.github/workflows/pushimage-next.yaml index b943ab65..38cb8c50 100644 --- a/.github/workflows/pushimage-next.yaml +++ b/.github/workflows/pushimage-next.yaml @@ -31,20 +31,20 @@ jobs: uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 with: go-version: 1.19 + - name: Set up QEMU # Enables arm64 image building + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - name: Login to Quay uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - name: Build the index server base image + - name: Build and push the index server base image run: | cd index/server export GOPATH=$(go env GOPATH) go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.12.4 - bash ./build.sh - - name: Push the index server base image - run: cd index/server && bash ./push.sh quay.io/devfile/devfile-index-base:next + bash ./build-multi-arch.sh dispatch: needs: indexServerBuild diff --git a/README.md b/README.md index 0cbf3e4e..37b9e5db 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,9 @@ If you want to run the build scripts with Podman, set the environment variable To build all of the components together (recommended) for dev/test, run `bash ./build_registry.sh` to build a Devfile Registry index image that is populated with the mock devfile registry data under `tests/registry/`. +By default `bash ./build_registry.sh` will build for `linux/amd64` architectures. To build for a different architecture pass in an argument to the script. +E.g. `bash ./build_registry.sh linux/arm64`. + Once the container has been pushed, you can push it to a container registry of your choosing with the following commands: ``` diff --git a/build_registry.sh b/build_registry.sh index 38b64703..17f88587 100755 --- a/build_registry.sh +++ b/build_registry.sh @@ -20,6 +20,17 @@ # and want to test all of the components together set -ex +DEFAULT_ARCH="linux/amd64" + +# Check if different architecture was passed for image build +# Will default to $DEFAULT_ARCH if unset +if [ ! -z "$1" ] + then + arch="$1" +else + arch="$DEFAULT_ARCH" +fi + # Set base registry support directory BASE_DIR=$(dirname $0) @@ -27,7 +38,7 @@ BASE_DIR=$(dirname $0) . ${BASE_DIR}/setenv.sh # Build the index server base image -ENABLE_HTTP2=${ENABLE_HTTP2} . ${BASE_DIR}/index/server/build.sh +ENABLE_HTTP2=${ENABLE_HTTP2} . ${BASE_DIR}/index/server/build.sh "${arch}" # Build the test devfile registry image -docker build -t devfile-index:latest -f ${BASE_DIR}/.ci/Dockerfile ${BASE_DIR} +docker build -t devfile-index:latest --platform "${arch}" -f ${BASE_DIR}/.ci/Dockerfile ${BASE_DIR} diff --git a/index/server/README.md b/index/server/README.md index 2c70bb3b..9bed74fe 100644 --- a/index/server/README.md +++ b/index/server/README.md @@ -39,12 +39,17 @@ See [swagger.io/docs](https://swagger.io/docs/specification/paths-and-operations ## Build To run the build scripts with `Podman` instead of `Docker` first run `export USE_PODMAN=true`. -The registry index server is built into a container image, `devfile-index-base:latest`, by running the following script: +The registry index server is built into a container image for the `linux/amd64` architecture by default named `devfile-index-base:latest`, by running the following script: ```sh bash build.sh ``` +to build for a different architecture like `linux/arm64`, pass it as an argument: +``` +bash build.sh linux/arm64 +``` + You retag it with one of the two command: **Docker CLI** diff --git a/index/server/build-multi-arch.sh b/index/server/build-multi-arch.sh new file mode 100644 index 00000000..6dc1d1fe --- /dev/null +++ b/index/server/build-multi-arch.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +# +# Copyright Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the index container for the registry +buildfolder="$(realpath $(dirname ${BASH_SOURCE[0]}))" +# Due to command differences between podman and docker we need to separate the process +# for creating and adding images to a multi-arch manifest +podman=${USE_PODMAN:-false} +# Base Repository +BASE_REPO="quay.io/devfile/devfile-index-base" +BASE_TAG="next" +DEFAULT_IMG="$BASE_REPO:$BASE_TAG" +# Platforms to build for +PLATFORMS="linux/amd64,linux/arm64" + +# Generate OpenAPI endpoint and type definitions +bash ${buildfolder}/codegen.sh + +if [ ${podman} == true ]; then + echo "Executing with podman" + + podman manifest create "$DEFAULT_IMG" + + podman build --platform="$PLATFORMS" --manifest "$DEFAULT_IMG" --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder + + podman manifest push "$DEFAULT_IMG" + + podman manifest rm "$DEFAULT_IMG" + +else + echo "Executing with docker" + + docker buildx create --name index-base-builder + + docker buildx use index-base-builder + + docker buildx build --push --platform="$PLATFORMS" --tag "$DEFAULT_IMG" --provenance=false --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder + + docker buildx rm index-base-builder + +fi \ No newline at end of file diff --git a/index/server/build.sh b/index/server/build.sh index 8691e6d6..c7e8d3cd 100755 --- a/index/server/build.sh +++ b/index/server/build.sh @@ -21,11 +21,26 @@ if [ "$0" == "$BASH_SOURCE" ]; then . ../../setenv.sh fi +DEFAULT_ARCH="linux/amd64" + +# Check if different architecture was passed for image build +# Will default to $DEFAULT_ARCH if unset +if [ ! -z "$1" ] + then + arch="$1" +else + arch="$DEFAULT_ARCH" +fi + +echo "BUILDING: devfile-index-base for ${arch}" + # Build the index container for the registry buildfolder="$(realpath $(dirname ${BASH_SOURCE[0]}))" +echo "RUNNING: bash ${buildfolders}/codegen.sh" # Generate OpenAPI endpoint and type definitions bash ${buildfolder}/codegen.sh +echo "RUNNING: docker build -t devfile-index-base:latest --platform ${arch} --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder" # Build the index server -docker build -t devfile-index-base:latest --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder +docker build -t devfile-index-base:latest --platform "${arch}" --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder