Merge pull request #105 from ulucinar/matrix-service-artifacts

Switch to matrix strategy to parallize provider family builds
upbound · May 9, 2023 · e68d4a2 · e68d4a2
2 parents f93aaf4 + 7f0668f
commit e68d4a2
Showing 1 changed file with 39 additions and 14 deletions.
diff --git a/.github/workflows/provider-publish-service-artifacts.yml b/.github/workflows/provider-publish-service-artifacts.yml
@@ -12,6 +12,16 @@ on:
         description: 'Provider version (e.g. v0.1.0)'
         required: true
         type: string
+      size:
+        description: "Number of packages to build and push with each matrix build job"
+        required: true
+        default: 30
+        type: number
+      concurrency:
+        description: "Number of parallel package builds in each matrix job"
+        optional: true
+        default: 0
+        type: number
     secrets:
       UPBOUND_MARKETPLACE_PUSH_ROBOT_USR:
         required: true
@@ -29,8 +39,22 @@ env:
   # credentials have been provided before trying to run steps that need them.
   UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }}
 
-jobs:  
+jobs:
+  index:
+    runs-on: ubuntu-22.04
+    outputs:
+      indices: ${{ steps.calc.outputs.indices }}
+    steps:
+      - id: calc
+        run: |
+          python3 -c "import math; print(f'indices={list(range(0, math.ceil(len(\"${{ inputs.subpackages }}\".split()) / int(\"${{ inputs.size }}\"))))}')" >> "$GITHUB_OUTPUT"
+
   publish-service-artifacts:
+    strategy:
+      matrix:
+        index: ${{ fromJSON(needs.index.outputs.indices) }}
+
+    needs: index
     runs-on: ubuntu-22.04
     steps:
       - name: Setup QEMU
@@ -66,47 +90,48 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
 
       - name: Find the Go Build Cache
-        id: go
-        run: echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT
+        id: go_cache
+        run: |
+          echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \
+          echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT
 
       - name: Cache the Go Build Cache
         uses: actions/cache@v3
         with:
-          path: ${{ steps.go.outputs.cache }}
+          path: ${{ steps.go_cache.outputs.cache }}
           key: ${{ runner.os }}-build-publish-artifacts-${{ hashFiles('**/go.sum') }}
           restore-keys: ${{ runner.os }}-build-publish-artifacts-
 
       - name: Cache Go Dependencies
         uses: actions/cache@v3
         with:
-          path: .work/pkg
+          path: ${{ steps.go_cache.outputs.mod_cache }}
           key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }}
           restore-keys: ${{ runner.os }}-pkg-
 
       - name: Vendor Dependencies
         run: make vendor vendor.check
 
+      - name: Calculate packages to build & push
+        id: packages
+        run: |
+          echo target=$(python3 -c "print(' '.join(\"${{ inputs.subpackages }}\".split()[int(\"${{ matrix.index }}\") * int(\"${{ inputs.size }}\"): (int(\"${{ matrix.index }}\")+1) * int(\"${{ inputs.size }}\")]))") >> "$GITHUB_OUTPUT"
+
       - name: Build Artifacts
         id: build_artifacts
         run: |
-          packages=($(echo ${{ inputs.subpackages }} | tr ' ' '\n'))
+          packages=($(echo ${{ steps.packages.outputs.target }} | tr ' ' '\n'))
           num_packages=${#packages[@]}
           if [ $num_packages -gt 10 ]; then
             num_packages=10
           fi
-          make -j $num_packages SUBPACKAGES="${{ inputs.subpackages }}" XPKG_REG_ORGS=xpkg.upbound.io/upbound-release-candidates XPKG_REG_ORGS_NO_PROMOTE=xpkg.upbound.io/upbound-release-candidates VERSION=${{ inputs.version }} build.all
+          make -j $num_packages SUBPACKAGES="${{ steps.packages.outputs.target }}" XPKG_REG_ORGS=xpkg.upbound.io/upbound-release-candidates XPKG_REG_ORGS_NO_PROMOTE=xpkg.upbound.io/upbound-release-candidates VERSION=${{ inputs.version }} build.all
           echo "num_packages=$num_packages" >> $GITHUB_OUTPUT
         env:
           # We're using docker buildx, which doesn't actually load the images it
           # builds by default. Specifying --load does so.
           BUILD_ARGS: "--load"
 
-      - name: Upload Artifacts to GitHub
-        uses: actions/upload-artifact@v3
-        with:
-          name: output
-          path: _output/**
-
       - name: Publish Artifacts
         run: |
-          make -j ${{ steps.build_artifacts.outputs.num_packages }} SUBPACKAGES="${{ inputs.subpackages }}" XPKG_REG_ORGS=xpkg.upbound.io/upbound-release-candidates XPKG_REG_ORGS_NO_PROMOTE=xpkg.upbound.io/upbound-release-candidates VERSION=${{ inputs.version }} publish
+          make -j ${{ steps.build_artifacts.outputs.num_packages }} SUBPACKAGES="${{ steps.packages.outputs.target }}" XPKG_REG_ORGS=xpkg.upbound.io/upbound-release-candidates XPKG_REG_ORGS_NO_PROMOTE=xpkg.upbound.io/upbound-release-candidates VERSION=${{ inputs.version }} CONCURRENCY="${{ inputs.concurrency }}" publish