Multi Arch Support for Devfile Index and Devfile Index Base (#245)

* add multi arch build and push script

Signed-off-by: Jordan Dubrick <[email protected]>

* add ability to build for different arch to existing build scripts and update docs

Signed-off-by: Jordan Dubrick <[email protected]>

* update workflows to build and push multi arch index base

Signed-off-by: Jordan Dubrick <[email protected]>

* update ci dockerfile to download proper arch for yq

Signed-off-by: Jordan Dubrick <[email protected]>

* pr-comment: improve log readability

Signed-off-by: Jordan Dubrick <[email protected]>

* pr-comment: update ci to have jobs for each arch

Signed-off-by: Jordan Dubrick <[email protected]>

---------

Signed-off-by: Jordan Dubrick <[email protected]>

.ci/Dockerfile

@@ -14,12 +14,16 @@
 # limitations under the License.
 FROM golang:1.19-alpine@sha256:0ec0646e208ea58e5d29e558e39f2e59fccf39b7bda306cb53bbaff91919eca5 AS builder
 
+# Allows for the proper yq to be downloaded
+# This arg is automatically set if the dockerfile is built with --platform flag
+ARG TARGETARCH=amd64
+
 # Install dependencies
 RUN apk add --no-cache git bash curl zip
 
 # Install yq
 ENV YQ_VERSION=v4.44.1
-RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq
+RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} -o /usr/local/bin/yq && mv ./yq_linux_${TARGETARCH} /usr/local/bin/yq && chmod +x /usr/local/bin/yq
 RUN yq
 
 # Copy the registry build tools

.github/workflows/ci.yaml

@@ -105,13 +105,20 @@ jobs:
             exit 1
           fi
 
-      - name: Check if index server build is working
+      - name: Test index server build for linux/amd64
         run: |
           cd index/server
           export GOPATH=$(go env GOPATH)
           go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@${{ env.OAPI_CODEGEN_VERSION }}
           bash ./build.sh
 
+      - name: Test index server build for linux/arm64
+        run: |
+          cd index/server
+          export GOPATH=$(go env GOPATH)
+          go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@${{ env.OAPI_CODEGEN_VERSION }}
+          bash ./build.sh linux/arm64
+
       - name: Test index server
         run: cd index/server && go test ./... -coverprofile cover.out
 

.github/workflows/pushimage-next.yaml

@@ -31,20 +31,20 @@ jobs:
         uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
         with:
           go-version: 1.19
+      - name: Set up QEMU # Enables arm64 image building
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0
       - name: Login to Quay
         uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
-      - name: Build the index server base image
+      - name: Build and push the index server base image
         run: | 
           cd index/server
           export GOPATH=$(go env GOPATH)
           go install github.com/deepmap/oapi-codegen/cmd/[email protected]
-          bash ./build.sh
-      - name: Push the index server base image
-        run: cd index/server && bash ./push.sh quay.io/devfile/devfile-index-base:next
+          bash ./build-multi-arch.sh
 
   dispatch:
     needs: indexServerBuild

README.md

@@ -28,6 +28,9 @@ If you want to run the build scripts with Podman, set the environment variable
 
 To build all of the components together (recommended) for dev/test, run `bash ./build_registry.sh` to build a Devfile Registry index image that is populated with the mock devfile registry data under `tests/registry/`.
 
+By default `bash ./build_registry.sh` will build for `linux/amd64` architectures. To build for a different architecture pass in an argument to the script.
+E.g. `bash ./build_registry.sh linux/arm64`.
+
 Once the container has been pushed, you can push it to a container registry of your choosing with the following commands:
 
 ```

build_registry.sh

@@ -20,14 +20,25 @@
 # and want to test all of the components together
 set -ex
 
+DEFAULT_ARCH="linux/amd64"
+
+# Check if different architecture was passed for image build
+# Will default to $DEFAULT_ARCH if unset
+if [ ! -z "$1" ]
+  then
+    arch="$1"
+else
+    arch="$DEFAULT_ARCH"
+fi
+
 # Set base registry support directory
 BASE_DIR=$(dirname $0)
 
 #set the docker alias if necessary
 . ${BASE_DIR}/setenv.sh
 
 # Build the index server base image
-ENABLE_HTTP2=${ENABLE_HTTP2} . ${BASE_DIR}/index/server/build.sh
+ENABLE_HTTP2=${ENABLE_HTTP2} . ${BASE_DIR}/index/server/build.sh "${arch}"
 
 # Build the test devfile registry image
-docker build -t devfile-index:latest -f ${BASE_DIR}/.ci/Dockerfile ${BASE_DIR}
+docker build -t devfile-index:latest --platform "${arch}" -f ${BASE_DIR}/.ci/Dockerfile ${BASE_DIR}

index/server/README.md

@@ -39,12 +39,17 @@ See [swagger.io/docs](https://swagger.io/docs/specification/paths-and-operations
 ## Build
 To run the build scripts with `Podman` instead of `Docker` first run `export USE_PODMAN=true`.
 
-The registry index server is built into a container image, `devfile-index-base:latest`, by running the following script:
+The registry index server is built into a container image for the `linux/amd64` architecture by default named `devfile-index-base:latest`, by running the following script:
 
 ```sh
 bash build.sh
 ```
 
+to build for a different architecture like `linux/arm64`, pass it as an argument:
+```
+bash build.sh linux/arm64
+```
+
 You retag it with one of the two command:
 
 **Docker CLI**

index/server/build-multi-arch.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+#
+# Copyright Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the index container for the registry
+buildfolder="$(realpath $(dirname ${BASH_SOURCE[0]}))"
+# Due to command differences between podman and docker we need to separate the process
+# for creating and adding images to a multi-arch manifest
+podman=${USE_PODMAN:-false}
+# Base Repository
+BASE_REPO="quay.io/devfile/devfile-index-base"
+BASE_TAG="next"
+DEFAULT_IMG="$BASE_REPO:$BASE_TAG"
+# Platforms to build for
+PLATFORMS="linux/amd64,linux/arm64"
+
+# Generate OpenAPI endpoint and type definitions
+bash ${buildfolder}/codegen.sh
+
+if [ ${podman} == true ]; then
+  echo "Executing with podman"
+
+  podman manifest create "$DEFAULT_IMG"
+
+  podman build --platform="$PLATFORMS" --manifest "$DEFAULT_IMG" --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder
+
+  podman manifest push "$DEFAULT_IMG"
+
+  podman manifest rm "$DEFAULT_IMG"
+
+else
+  echo "Executing with docker"
+
+  docker buildx create --name index-base-builder
+
+  docker buildx use index-base-builder
+
+  docker buildx build --push --platform="$PLATFORMS" --tag "$DEFAULT_IMG" --provenance=false --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder
+
+  docker buildx rm index-base-builder
+
+fi

index/server/build.sh

@@ -21,11 +21,26 @@ if [ "$0" == "$BASH_SOURCE" ]; then
     . ../../setenv.sh
 fi
 
+DEFAULT_ARCH="linux/amd64"
+
+# Check if different architecture was passed for image build
+# Will default to $DEFAULT_ARCH if unset
+if [ ! -z "$1" ]
+  then
+    arch="$1"
+else
+    arch="$DEFAULT_ARCH"
+fi
+
+echo "BUILDING: devfile-index-base for ${arch}"
+
 # Build the index container for the registry
 buildfolder="$(realpath $(dirname ${BASH_SOURCE[0]}))"
 
+echo "RUNNING: bash ${buildfolders}/codegen.sh"
 # Generate OpenAPI endpoint and type definitions
 bash ${buildfolder}/codegen.sh
 
+echo "RUNNING: docker build -t devfile-index-base:latest --platform ${arch} --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder"
 # Build the index server
-docker build -t devfile-index-base:latest --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder
+docker build -t devfile-index-base:latest --platform "${arch}" --build-arg ENABLE_HTTP2=${ENABLE_HTTP2} $buildfolder