Remove parts of image building that cloud does not need (#94)

## Overview Flyte fork builds take 30+ minutes. This is mostly due to building images that cloud does not use (single binary, sandbox). This change: - removes parts of image building workflows that cloud does not use - removes couple of unrelated unused workflows. Context: https://unionai.slack.com/archives/C02CTFRHWDS/p1708664432669819 ## Test Plan *TODO: Summarize tests added, integration tests run, or other steps you took to validate this change. Include (or link to) relevant test output or screenshots.* ## Rollout Plan (if applicable) *TODO: Describe any deployment or compatibility considerations for rolling out this change.* ## Upstream Changes Should this change be upstreamed to OSS (flyteorg/flyte)? If so, please check this box for auditing. Note, this is the responsibility of each developer. See [this guide](https://unionai.atlassian.net/wiki/spaces/ENG/pages/447610883/Flyte+-+Union+Cloud+Development+Runbook/#When-are-versions-updated%3F). - [ ] To be upstreamed ## Jira Issue https://unionai.atlassian.net/browse/<project-number> ## Checklist * [ ] Added tests * [ ] Ran a deploy dry run and shared the terraform plan * [ ] Added logging and metrics * [ ] Updated [dashboards](https://unionai.grafana.net/dashboards) and [alerts](https://unionai.grafana.net/alerting/list) * [ ] Updated documentation
flyteorg · Feb 26, 2024 · e37d3c0 · e37d3c0
1 parent 462bf5f
commit e37d3c0
Show file tree

Hide file tree

Showing 9 changed files with 59 additions and 303 deletions.
diff --git a/.github/workflows/build-and-publish-all-images.yml b/.github/workflows/build-and-publish-all-images.yml
diff --git a/.github/workflows/publish-images.yml → ...lows/build-and-push-all-docker-images.yml b/.github/workflows/publish-images.yml → ...lows/build-and-push-all-docker-images.yml
@@ -1,16 +1,15 @@
-name: Build & Push All Components Images
+name: Build & Push All Component Docker Images
 
 on:
   workflow_call:
     inputs:
       push:
-        description: "Push to registry"
+        description: "Whether to push built images to registry"
         required: true
         type: boolean
 
 jobs:
   push_docker_image:
-    name: Build & Push Image
     strategy:
       fail-fast: false
       matrix:
@@ -20,7 +19,7 @@ jobs:
           - flytecopilot
           - flytepropeller
           - flytescheduler
-    uses: ./.github/workflows/publish.yml
+    uses: ./.github/workflows/build-and-push-docker-image.yml
     with:
       component: ${{ matrix.component }}
       dockerfile: Dockerfile.${{ matrix.component }}

diff --git a/.github/workflows/publish.yml → ...workflows/build-and-push-docker-image.yml b/.github/workflows/publish.yml → ...workflows/build-and-push-docker-image.yml
@@ -4,38 +4,32 @@ on:
   workflow_call:
     inputs:
       component:
-        description: "Component Name"
+        description: "Flyte component name"
         required: true
         type: string
       dockerfile:
-        description: "Dockerfile name"
+        description: "Dockerfile"
         required: true
         type: string
       push:
-        description: "Push to registry"
+        description: "Whether to push built image to registry"
         required: true
         type: boolean
-      before-build:
-        description: "Script to run before build"
-        required: false
-        type: string
 
 permissions:
   id-token: write
   contents: read
 
 jobs:
   push-github:
-    name: Push to Github Registry
+    name: Build and Push Image To ECR
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: '0'
+      - uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -48,16 +42,14 @@ jobs:
           registry-type: public
       - name: Prepare Image Tags
         id: tags
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/${{ inputs.component }}
           tags: |
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
-      - name: Before Build
-        run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ inputs.dockerfile }}

diff --git a/.github/workflows/build-and-push-images-for-cloud.yml b/.github/workflows/build-and-push-images-for-cloud.yml
@@ -0,0 +1,18 @@
+name: Build & Publish Images Used by Cloud
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build-and-push-docker-images:
+    uses: ./.github/workflows/build-and-push-all-docker-images.yml
+    with:
+      push: true
diff --git a/.github/workflows/single-binary.yml → ...ws/build-and-test-single-binary-image.yml b/.github/workflows/single-binary.yml → ...ws/build-and-test-single-binary-image.yml
@@ -12,12 +12,12 @@ jobs:
   test-bootstrap:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: "1.21"
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           working-directory: docker/sandbox-bundled/bootstrap
       - name: Check formatting
@@ -34,9 +34,9 @@ jobs:
     needs: [test-bootstrap]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Golang caches
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             /root/.cache/go-build
@@ -61,7 +61,7 @@ jobs:
           registry-type: public
       - name: Prepare Image Names
         id: image-names
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -70,14 +70,14 @@ jobs:
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Setup destination directories for image tarballs
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/arm64
@@ -88,7 +88,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64
@@ -99,12 +99,12 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar
       - name: Upload single binary image
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
       - name: Build and push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -129,18 +129,18 @@ jobs:
           FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: docker/sandbox-bundled
           load: true
@@ -151,7 +151,7 @@ jobs:
       - name: Prune Docker Buildx cache to reclaim storage
         run: docker buildx prune --all --force
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - uses: unionai/[email protected]
@@ -174,7 +174,7 @@ jobs:
           pip install flytekit flytekitplugins-deck-standard
           pip freeze
       - name: Checkout flytesnacks
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks
@@ -201,15 +201,15 @@ jobs:
     needs: [build-and-push-single-binary-image]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
@@ -229,7 +229,7 @@ jobs:
           registry-type: public
       - name: Prepare Image Names
         id: image-names
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -238,7 +238,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -119,6 +119,10 @@ jobs:
 
   build_docker_images:
     name: Build Images
-    uses: ./.github/workflows/publish-images.yml
+    uses: ./.github/workflows/build-and-push-all-docker-images.yml
     with:
       push: false
+
+  build_and_test_single_binary:
+    name: Build and Test Single Binary Image
+    uses: ./.github/workflows/build-and-test-single-binary-image.yml