From 98f31efb71cf13551e9d3e3cfe0e930dd2c1bcde Mon Sep 17 00:00:00 2001
From: Javier Evans <j.evans@f5.com>
Date: Fri, 12 Jan 2024 13:57:18 -0800
Subject: [PATCH] Build multi-architecture images (#197)

# What
Allows us to push multi-architecture builds for `linux/amd64` and `linux/arm64` on merge to master. Fixes: https://github.com/nginxinc/nginx-s3-gateway/issues/196

Tests are performed by building the relevant image and tagging it with the image name the tests expect.  This follows the original implementation in `test.sh`.  This will likely change to an explicit image name specification as we move the test suite in to Javascript but it's being maintained for now in the interest of simplicity.

## Goals
* To decouple image build from the test run. This will help as we refactor the test suite for this issue https://github.com/nginxinc/nginx-s3-gateway/issues/191.
* Not have CI take a long time due to multi-architecture builds for interim pushes to pull requests.
* CI should fail fast if any of the image builds or tests fail
* Parallelize builds as much as possible.


## Notes
* Since the `unprivileged` and `latest-njs` images build off the base `oss` image, there were some strange moves that had to be done to make that work without rebuilding the base image every time. Specifically, we had to set `setup-buildx-action` to run in `docker` driver mode so we can simply `load` the base image. Otherwise we would have had to use a local repository.  However, in `docker` mode `upload-artifact` doesn't like the file produced so we have to save the file again.
* I wanted to build images once and then run tests against them and push at the end. However, there was not a clean way to get the full multi-architecture images all the way to the push step so I just build them again against all architectures and perform tests in the runner architecture.  This saves us from needing conditionals in the test portions.
---
 .github/workflows/main.yml | 337 ++++++++++++++++++++++++++-----------
 test.sh                    |  50 +++---
 2 files changed, 268 insertions(+), 119 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b1755cc7..6cd9263c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,16 +8,54 @@ on:
   pull_request:
     branches: [ master ]
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  build:
-    runs-on: ubuntu-latest
 
-    if: github.ref != 'refs/heads/master'
+env:
+  CI: true
+
+
+# Job progression.  We make sure that the base image [oss] builds and passes tests before kicking off the other builds
+
+    #                                ┌──────────────────┐        ┌────────────────┐      ┌────────────────┐
+    # ┌─────────┐     ┌─────────┬────► Build Latest NJS ├────────►Test Latest NJS ├─────►│Push Latest NJS │
+    # │Build OSS├────►│Test OSS │    └──────────────────┘        └────────────────┘      └────────────────┘
+    # └─────────┘     └──┬──────┤
+    #                    │      │    ┌──────────────────┐       ┌──────────────────┐     ┌─────────────────┐
+    #                    │      └────►Build Unprivileged├───────►Test Unprivileged ├────►│Push Unprivileged│
+    #                    │           └──────────────────┘       └──────────────────┘     ├────────┬────────┘
+    #                    │                                                               ├────────┤
+    #                    └──────────────────────────────────────────────────────────────►│Push OSS│
+    #                                                                                    └────────┘
+
+# As a last step, if we are on the main/master branch, multi-architecture images will be built and pushed to github packages
+# and docker hub
+
+jobs:
+  build-oss-for-test:
+    runs-on: ubuntu-22.04
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and export
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.oss
+          context: .
+          tags: nginx-s3-gateway , nginx-s3-gateway:oss
+          outputs: type=docker,dest=${{ runner.temp }}/oss.tar
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: oss
+          path: ${{ runner.temp }}/oss.tar
+          retention-days: 1
+          if-no-files-found: error
 
+  test-oss:
+    runs-on: ubuntu-22.04
+    needs: build-oss-for-test
+    steps:
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: sudo apt-get update -qq && sudo apt-get install -y curl wait-for-it
       - name: Restore cached binaries
@@ -34,35 +72,60 @@ jobs:
           curl --insecure --retry 6 --fail --silent --location "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2023-06-19T19-31-19Z.sha256sum" | sha256sum --check -
           mv mc.RELEASE.2023-06-19T19-31-19Z mc
           chmod +x mc
-      - name: Save cached binaries
-        id: cache-binaries-save
-        uses: actions/cache/save@v3
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
         with:
-          path: .bin
-          key: ${{ steps.cache-binaries-restore.outputs.cache-primary-key }}
-      - name: Run tests - latest njs version
-        run: ./test.sh --latest-njs --type oss
+          name: oss
+          path: ${{ runner.temp }}
+      - name: Load image
+        run: |
+          docker load --input ${{ runner.temp }}/oss.tar
       - name: Run tests - stable njs version
         run: ./test.sh --type oss
-      - name: Run tests - stable njs version - unprivileged process
-        run: ./test.sh --unprivileged --type oss
-      - name: Run tests - latest njs version - unprivileged process
-        run: ./test.sh --latest-njs --unprivileged --type oss
-
-  build_and_deploy:
-    runs-on: ubuntu-latest
 
-    if: github.ref == 'refs/heads/master'
+  build-latest-njs-for-test:
+    runs-on: ubuntu-22.04
+    needs: test-oss
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
-
-      - name: Get current date
-        id: date
-        run: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
-      - name: Configure Github Package Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u $GITHUB_ACTOR --password-stdin
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: oss
+          path: ${{ runner.temp }}
+      - name: Load image
+        run: |
+          docker load --input ${{ runner.temp }}/oss.tar
+      - name: Build and load oss image
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.latest-njs
+          context: .
+          tags: nginx-s3-gateway:latest-njs-oss
+          load: true
+      # Save manually here since we need to use the `docker` buildx `driver` but that can't output
+      # a file that upload-artifact likes.
+      - name: save image
+        run: |
+          docker save nginx-s3-gateway:latest-njs-oss > ${{ runner.temp }}/latest-njs.tar
+      - name: Upload artifact - latest-njs
+        uses: actions/upload-artifact@v3
+        with:
+          name: latest-njs
+          path: ${{ runner.temp }}/latest-njs.tar
+          retention-days: 1
+          if-no-files-found: error
 
+  test-latest-njs:
+    runs-on: ubuntu-22.04
+    needs: build-latest-njs-for-test
+    steps:
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: sudo apt-get update -qq && sudo apt-get install -y curl wait-for-it
       - name: Restore cached binaries
@@ -75,79 +138,161 @@ jobs:
         run: |
           mkdir .bin || exit 0
           cd .bin
-          curl --insecure --retry 6 --fail --silent --location --output mc.RELEASE.2023-06-19T19-31-19Z "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2023-06-19T19-31-19Z"
+          curl --insecure --retry 6 --fail --location --output mc.RELEASE.2023-06-19T19-31-19Z "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2023-06-19T19-31-19Z"
           curl --insecure --retry 6 --fail --silent --location "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2023-06-19T19-31-19Z.sha256sum" | sha256sum --check -
           mv mc.RELEASE.2023-06-19T19-31-19Z mc
           chmod +x mc
-
-      # Run tests and builds image
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: latest-njs
+          path: ${{ runner.temp }}
+      - name: Load image
+        run: |
+          docker load --input ${{ runner.temp }}/latest-njs.tar
+          docker tag nginx-s3-gateway:latest-njs-oss nginx-s3-gateway
       - name: Run tests - latest njs version
         run: ./test.sh --latest-njs --type oss
-      # latest-njs-oss image push [Github]
-      - name: Tag container image for Push to github [latest-njs-oss date]
-        run: docker tag nginx-s3-gateway:latest-njs-oss docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest-njs-oss-${{ steps.date.outputs.date }}
-      - name: Tag container image for Push to github [latest-njs-oss]
-        run: docker tag nginx-s3-gateway:latest-njs-oss docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest-njs-oss
-      - name: Push container image to github [latest-njs-oss date]
-        run: docker push docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest-njs-oss-${{ steps.date.outputs.date }}
-      - name: Push container image to github [latest-njs-oss]
-        run: docker push docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest-njs-oss
-
-      # Run tests and builds image
-      - name: Run tests - stable njs version - unprivileged process
+
+  build-unprivileged-for-test:
+    runs-on: ubuntu-22.04
+    needs: test-oss
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: oss
+          path: ${{ runner.temp }}
+      - name: Load image
+        run: |
+          docker load --input ${{ runner.temp }}/oss.tar
+      - name: Build and load oss image
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.unprivileged
+          context: .
+          tags: nginx-s3-gateway:unprivileged-oss
+          load: true
+      # Save manually here since we need to use the `docker` buildx `driver` but that can't output
+      # a file that upload-artifact likes.
+      - name: save image
+        run: |
+          docker save nginx-s3-gateway:unprivileged-oss > ${{ runner.temp }}/unprivileged.tar
+      - name: Upload artifact - unprivileged
+        uses: actions/upload-artifact@v3
+        with:
+          name: unprivileged
+          path: ${{ runner.temp }}/unprivileged.tar
+          retention-days: 1
+          if-no-files-found: error
+
+  test-unprivileged:
+    runs-on: ubuntu-22.04
+    needs: build-unprivileged-for-test
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y curl wait-for-it
+      - name: Restore cached binaries
+        id: cache-binaries-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: .bin
+          key: ${{ runner.os }}-binaries
+      - name: Install MinIO Client
+        run: |
+          mkdir .bin || exit 0
+          cd .bin
+          curl --insecure --retry 6 --fail --location --output mc.RELEASE.2023-06-19T19-31-19Z "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2023-06-19T19-31-19Z"
+          curl --insecure --retry 6 --fail --silent --location "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2023-06-19T19-31-19Z.sha256sum" | sha256sum --check -
+          mv mc.RELEASE.2023-06-19T19-31-19Z mc
+          chmod +x mc
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: unprivileged
+          path: ${{ runner.temp }}
+      - name: Load image
+        run: |
+          docker load --input ${{ runner.temp }}/unprivileged.tar
+          docker tag nginx-s3-gateway:unprivileged-oss nginx-s3-gateway
+      - name: Run tests - unprivileged
         run: ./test.sh --unprivileged --type oss
-      # unprivileged-oss image push [Github]
-      - name: Tag container image for Push to github [unprivileged-oss date]
-        run: docker tag nginx-s3-gateway:unprivileged-oss docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:unprivileged-oss-${{ steps.date.outputs.date }}
-      - name: Tag container image for Push to github [unprivileged-oss]
-        run: docker tag nginx-s3-gateway:unprivileged-oss docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:unprivileged-oss
-      - name: Push container image to github [unprivileged-oss date]
-        run: docker push docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:unprivileged-oss-${{ steps.date.outputs.date }}
-      - name: Push container image to github [unprivileged-oss]
-        run: docker push docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:unprivileged-oss
-
-      # Run tests and builds image
-      - name: Run tests - stable njs version
-        run: ./test.sh --type oss
-      # oss image push [Github]
-      - name: Tag container image for Push to github [oss date]
-        run: docker tag nginx-s3-gateway:oss docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest-${{ steps.date.outputs.date }}
-      - name: Tag container image for Push to github [oss]
-        run: docker tag nginx-s3-gateway:oss docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest
-      - name: Push container image to github [oss date]
-        run: docker push docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest-${{ steps.date.outputs.date }}
-      - name: Push container image to github [oss latest]
-        run: docker push docker.pkg.github.com/$GITHUB_REPOSITORY/nginx-oss-s3-gateway:latest
-      # Login to Docker Hub
+
+# After the tests are done, build multiarch and push to both github packages and dockerhub if we are on master/main
+  tag-and-push:
+    runs-on: ubuntu-22.04
+    needs: [test-oss, test-latest-njs, test-unprivileged]
+
+    if: |
+      github.ref == 'refs/heads/master' ||
+      github.ref == 'refs/heads/main'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-      # latest-njs-oss image push [Docker Hub]
-      - name: Tag container image for Push to Docker Hub [latest-njs-oss date]
-        run: docker tag nginx-s3-gateway:latest-njs-oss nginxinc/nginx-s3-gateway:latest-njs-oss-${{ steps.date.outputs.date }}
-      - name: Tag container image for Push to Docker Hub [latest-njs-oss]
-        run: docker tag nginx-s3-gateway:latest-njs-oss nginxinc/nginx-s3-gateway:latest-njs-oss
-      - name: Push container image to Docker Hub [latest-njs-oss date]
-        run: docker push nginxinc/nginx-s3-gateway:latest-njs-oss-${{ steps.date.outputs.date }}
-      - name: Push container image to Docker Hub [latest-njs-oss]
-        run: docker push nginxinc/nginx-s3-gateway:latest-njs-oss
-      # unprivileged-oss image push [Docker Hub]
-      - name: Tag container image for Push to Docker Hub [unprivileged-oss date]
-        run: docker tag nginx-s3-gateway:unprivileged-oss nginxinc/nginx-s3-gateway:unprivileged-oss-${{ steps.date.outputs.date }}
-      - name: Tag container image for Push to Docker Hub [unprivileged-oss]
-        run: docker tag nginx-s3-gateway:unprivileged-oss nginxinc/nginx-s3-gateway:unprivileged-oss
-      - name: Push container image to Docker Hub [unprivileged-oss date]
-        run: docker push nginxinc/nginx-s3-gateway:unprivileged-oss-${{ steps.date.outputs.date }}
-      - name: Push container image to Docker Hub [unprivileged-oss]
-        run: docker push nginxinc/nginx-s3-gateway:unprivileged-oss
-      # oss image push [Docker Hub]
-      - name: Tag container image for Push to Docker Hub [oss date]
-        run: docker tag nginx-s3-gateway:oss nginxinc/nginx-s3-gateway:latest-${{ steps.date.outputs.date }}
-      - name: Tag container image for Push to Docker Hub [oss]
-        run: docker tag nginx-s3-gateway:oss nginxinc/nginx-s3-gateway:latest
-      - name: Push container image to Docker Hub [oss date]
-        run: docker push nginxinc/nginx-s3-gateway:latest-${{ steps.date.outputs.date }}
-      - name: Push container image to Docker Hub [oss latest]
-        run: docker push nginxinc/nginx-s3-gateway:latest
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push image [oss]
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.oss
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}/nginx-oss-s3-gateway:latest-${{ steps.date.outputs.date }}
+            ghcr.io/${{ github.repository }}/nginx-oss-s3-gateway:latest
+            nginxinc/nginx-s3-gateway:latest-${{ steps.date.outputs.date }}
+            nginxinc/nginx-s3-gateway:latest
+
+      - name: Build and push image [latest-njs]
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.latest-njs
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}/nginx-oss-s3-gateway:latest-njs-oss-${{ steps.date.outputs.date }}
+            ghcr.io/${{ github.repository }}/nginx-oss-s3-gateway:latest-njs-oss
+            nginxinc/nginx-s3-gateway:latest-njs-oss-${{ steps.date.outputs.date }}
+            nginxinc/nginx-s3-gateway:latest-njs-oss
+
+      - name: Build and push image [unprivileged]
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile.unprivileged
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}/nginx-oss-s3-gateway:unprivileged-oss-${{ steps.date.outputs.date }}
+            ghcr.io/${{ github.repository }}/nginx-oss-s3-gateway:unprivileged-oss
+            nginxinc/nginx-s3-gateway:unprivileged-oss-${{ steps.date.outputs.date }}
+            nginxinc/nginx-s3-gateway:unprivileged-oss
diff --git a/test.sh b/test.sh
index 7ea8ad27..24fdf8e8 100755
--- a/test.sh
+++ b/test.sh
@@ -297,35 +297,39 @@ trap finish EXIT ERR SIGTERM SIGINT
 
 ### BUILD
 
-p "Building NGINX S3 gateway Docker image"
-if [ "${nginx_type}" = "plus" ]; then
-  if docker buildx > /dev/null 2>&1; then
-    p "Building using BuildKit"
-    export DOCKER_BUILDKIT=1
-    docker buildx build -f Dockerfile.buildkit.${nginx_type} \
-      --secret id=nginx-crt,src=plus/etc/ssl/nginx/nginx-repo.crt \
-      --secret id=nginx-key,src=plus/etc/ssl/nginx/nginx-repo.key \
-      --no-cache \
-      --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} .
+if [ "$CI" = "true" ]; then
+    echo "Skipping docker image build due to CI=true"
+else
+  p "Building NGINX S3 gateway Docker image"
+  if [ "${nginx_type}" = "plus" ]; then
+    if docker buildx > /dev/null 2>&1; then
+      p "Building using BuildKit"
+      export DOCKER_BUILDKIT=1
+      docker buildx build -f Dockerfile.buildkit.${nginx_type} \
+        --secret id=nginx-crt,src=plus/etc/ssl/nginx/nginx-repo.crt \
+        --secret id=nginx-key,src=plus/etc/ssl/nginx/nginx-repo.key \
+        --no-cache \
+        --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} .
+    else
+      docker build -f Dockerfile.${nginx_type} \
+        --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} .
+    fi
   else
     docker build -f Dockerfile.${nginx_type} \
       --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} .
   fi
-else
-  docker build -f Dockerfile.${nginx_type} \
-    --tag nginx-s3-gateway --tag nginx-s3-gateway:${nginx_type} .
-fi
 
-if [ ${njs_latest} -eq 1 ]; then
-  p "Layering in latest NJS build"
-  docker build -f Dockerfile.latest-njs \
-    --tag nginx-s3-gateway --tag nginx-s3-gateway:latest-njs-${nginx_type} .
-fi
+  if [ ${njs_latest} -eq 1 ]; then
+    p "Layering in latest NJS build"
+    docker build -f Dockerfile.latest-njs \
+      --tag nginx-s3-gateway --tag nginx-s3-gateway:latest-njs-${nginx_type} .
+  fi
 
-if [ ${unprivileged} -eq 1 ]; then
-  p "Layering in unprivileged build"
-  docker build -f Dockerfile.unprivileged \
-    --tag nginx-s3-gateway --tag nginx-s3-gateway:unprivileged-${nginx_type} .
+  if [ ${unprivileged} -eq 1 ]; then
+    p "Layering in unprivileged build"
+    docker build -f Dockerfile.unprivileged \
+      --tag nginx-s3-gateway --tag nginx-s3-gateway:unprivileged-${nginx_type} .
+  fi
 fi
 
 ### UNIT TESTS