From e37d3c03406c9c06df4b860499a9c8475df89d47 Mon Sep 17 00:00:00 2001
From: Kamal <54046807+kamaleybov@users.noreply.github.com>
Date: Mon, 26 Feb 2024 15:30:43 -0800
Subject: [PATCH] Remove parts of image building that cloud does not need (#94)

## Overview
Flyte fork builds take 30+ minutes. This is mostly due to building images that cloud does not use (single binary, sandbox).

This change:
- removes parts of image building workflows that cloud does not use
- removes couple of unrelated unused workflows.

Context: https://unionai.slack.com/archives/C02CTFRHWDS/p1708664432669819

## Test Plan
*TODO: Summarize tests added, integration tests run, or other steps you took to validate this change. Include (or link to) relevant test output or screenshots.*

## Rollout Plan (if applicable)
*TODO: Describe any deployment or compatibility considerations for rolling out this change.*

## Upstream Changes
Should this change be upstreamed to OSS (flyteorg/flyte)? If so, please check this box for auditing. Note, this is the responsibility of each developer. See [this guide](https://unionai.atlassian.net/wiki/spaces/ENG/pages/447610883/Flyte+-+Union+Cloud+Development+Runbook/#When-are-versions-updated%3F).
- [ ] To be upstreamed

## Jira Issue
https://unionai.atlassian.net/browse/<project-number>

## Checklist
* [ ] Added tests
* [ ] Ran a deploy dry run and shared the terraform plan
* [ ] Added logging and metrics
* [ ] Updated [dashboards](https://unionai.grafana.net/dashboards) and [alerts](https://unionai.grafana.net/alerting/list)
* [ ] Updated documentation
---
 .../build-and-publish-all-images.yml          |  23 ----
 ...l => build-and-push-all-docker-images.yml} |   7 +-
 ...sh.yml => build-and-push-docker-image.yml} |  26 ++--
 .../build-and-push-images-for-cloud.yml       |  18 +++
 ...=> build-and-test-single-binary-image.yml} |  48 ++++----
 .github/workflows/checks.yml                  |   6 +-
 .github/workflows/create_release.yml          |  95 ---------------
 .github/workflows/sandbox.yml                 | 115 ------------------
 .github/workflows/selfassign.yml              |  24 ----
 9 files changed, 59 insertions(+), 303 deletions(-)
 delete mode 100644 .github/workflows/build-and-publish-all-images.yml
 rename .github/workflows/{publish-images.yml => build-and-push-all-docker-images.yml} (72%)
 rename .github/workflows/{publish.yml => build-and-push-docker-image.yml} (71%)
 create mode 100644 .github/workflows/build-and-push-images-for-cloud.yml
 rename .github/workflows/{single-binary.yml => build-and-test-single-binary-image.yml} (89%)
 delete mode 100644 .github/workflows/create_release.yml
 delete mode 100644 .github/workflows/sandbox.yml
 delete mode 100644 .github/workflows/selfassign.yml

diff --git a/.github/workflows/build-and-publish-all-images.yml b/.github/workflows/build-and-publish-all-images.yml
deleted file mode 100644
index e88133ec31..0000000000
--- a/.github/workflows/build-and-publish-all-images.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: Build & Publish All Images
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-  workflow_dispatch:
-
-concurrency:
-  # for master branch we want every commit to have corresponding images
-  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.sha || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  single-binary:
-    uses: ./.github/workflows/single-binary.yml
-  components:
-    uses: ./.github/workflows/create_release.yml
-    needs: single-binary
-  sandbox:
-    uses: ./.github/workflows/sandbox.yml
-    needs: single-binary
diff --git a/.github/workflows/publish-images.yml b/.github/workflows/build-and-push-all-docker-images.yml
similarity index 72%
rename from .github/workflows/publish-images.yml
rename to .github/workflows/build-and-push-all-docker-images.yml
index 4243050a00..a9ca73094d 100644
--- a/.github/workflows/publish-images.yml
+++ b/.github/workflows/build-and-push-all-docker-images.yml
@@ -1,16 +1,15 @@
-name: Build & Push All Components Images
+name: Build & Push All Component Docker Images
 
 on:
   workflow_call:
     inputs:
       push:
-        description: "Push to registry"
+        description: "Whether to push built images to registry"
         required: true
         type: boolean
 
 jobs:
   push_docker_image:
-    name: Build & Push Image
     strategy:
       fail-fast: false
       matrix:
@@ -20,7 +19,7 @@ jobs:
           - flytecopilot
           - flytepropeller
           - flytescheduler
-    uses: ./.github/workflows/publish.yml
+    uses: ./.github/workflows/build-and-push-docker-image.yml
     with:
       component: ${{ matrix.component }}
       dockerfile: Dockerfile.${{ matrix.component }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/build-and-push-docker-image.yml
similarity index 71%
rename from .github/workflows/publish.yml
rename to .github/workflows/build-and-push-docker-image.yml
index 552739f994..528fb9fbc0 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/build-and-push-docker-image.yml
@@ -4,21 +4,17 @@ on:
   workflow_call:
     inputs:
       component:
-        description: "Component Name"
+        description: "Flyte component name"
         required: true
         type: string
       dockerfile:
-        description: "Dockerfile name"
+        description: "Dockerfile"
         required: true
         type: string
       push:
-        description: "Push to registry"
+        description: "Whether to push built image to registry"
         required: true
         type: boolean
-      before-build:
-        description: "Script to run before build"
-        required: false
-        type: string
 
 permissions:
   id-token: write
@@ -26,16 +22,14 @@ permissions:
 
 jobs:
   push-github:
-    name: Push to Github Registry
+    name: Build and Push Image To ECR
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: '0'
+      - uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -48,16 +42,14 @@ jobs:
           registry-type: public
       - name: Prepare Image Tags
         id: tags
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/${{ inputs.component }}
           tags: |
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
-      - name: Before Build
-        run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ inputs.dockerfile }}
diff --git a/.github/workflows/build-and-push-images-for-cloud.yml b/.github/workflows/build-and-push-images-for-cloud.yml
new file mode 100644
index 0000000000..b554ea0c8c
--- /dev/null
+++ b/.github/workflows/build-and-push-images-for-cloud.yml
@@ -0,0 +1,18 @@
+name: Build & Publish Images Used by Cloud
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build-and-push-docker-images:
+    uses: ./.github/workflows/build-and-push-all-docker-images.yml
+    with:
+      push: true
diff --git a/.github/workflows/single-binary.yml b/.github/workflows/build-and-test-single-binary-image.yml
similarity index 89%
rename from .github/workflows/single-binary.yml
rename to .github/workflows/build-and-test-single-binary-image.yml
index 07aee6b17d..7cd02f9cca 100644
--- a/.github/workflows/single-binary.yml
+++ b/.github/workflows/build-and-test-single-binary-image.yml
@@ -12,12 +12,12 @@ jobs:
   test-bootstrap:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: "1.21"
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           working-directory: docker/sandbox-bundled/bootstrap
       - name: Check formatting
@@ -34,9 +34,9 @@ jobs:
     needs: [test-bootstrap]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Golang caches
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             /root/.cache/go-build
@@ -61,7 +61,7 @@ jobs:
           registry-type: public
       - name: Prepare Image Names
         id: image-names
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -70,14 +70,14 @@ jobs:
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Setup destination directories for image tarballs
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/arm64
@@ -88,7 +88,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64
@@ -99,12 +99,12 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar
       - name: Upload single binary image
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
       - name: Build and push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -129,18 +129,18 @@ jobs:
           FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: docker/sandbox-bundled
           load: true
@@ -151,7 +151,7 @@ jobs:
       - name: Prune Docker Buildx cache to reclaim storage
         run: docker buildx prune --all --force
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - uses: unionai/flytectl-setup-action@v0.0.1
@@ -174,7 +174,7 @@ jobs:
           pip install flytekit flytekitplugins-deck-standard
           pip freeze
       - name: Checkout flytesnacks
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks
@@ -201,15 +201,15 @@ jobs:
     needs: [build-and-push-single-binary-image]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
@@ -229,7 +229,7 @@ jobs:
           registry-type: public
       - name: Prepare Image Names
         id: image-names
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -238,7 +238,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index a8521cd38a..9f692f0f4a 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -119,6 +119,10 @@ jobs:
 
   build_docker_images:
     name: Build Images
-    uses: ./.github/workflows/publish-images.yml
+    uses: ./.github/workflows/build-and-push-all-docker-images.yml
     with:
       push: false
+
+  build_and_test_single_binary:
+    name: Build and Test Single Binary Image
+    uses: ./.github/workflows/build-and-test-single-binary-image.yml
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
deleted file mode 100644
index a1eadd4ccb..0000000000
--- a/.github/workflows/create_release.yml
+++ /dev/null
@@ -1,95 +0,0 @@
-name: Create a flyte release
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-permissions:
-  id-token: write
-  contents: read
-
-jobs:
-  build-docker-images:
-    uses: ./.github/workflows/publish-images.yml
-    with:
-      push: true
-
-  publish-flyte-binary-image:
-    name: Publish flyte binary image for the release version
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::558384610515:role/github-actions-flyte-fork
-          aws-region: us-east-1 # public ECR is only available in us-east-1
-
-      - name: Login to Amazon ECR
-        id: ecr-login
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registry-type: public
-
-      - name: Tag image to release version (sha)
-        run: |
-          docker buildx imagetools create \
-            --tag "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/flyte-binary-release:${{ github.event.pull_request.head.sha || github.sha }}" \
-                  "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/flyte-binary:${{ github.event.pull_request.head.sha || github.sha }}"
-
-      - name: Tag image to release version (latest)
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        run: |
-          docker buildx imagetools create \
-            --tag "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/flyte-binary-release:latest" \
-                  "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/flyte-binary:${{ github.event.pull_request.head.sha || github.sha }}"
-
-  publish-flyte-component-image:
-    name: Publish flyte component image for the release version
-    runs-on: ubuntu-latest
-    needs:
-      - build-docker-images
-    strategy:
-      matrix:
-        component:
-          [
-              datacatalog,
-              flyteadmin,
-              flytecopilot,
-              flytepropeller,
-              flytescheduler,
-          ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::558384610515:role/github-actions-flyte-fork
-          aws-region: us-east-1 # public ECR is only available in us-east-1
-
-      - name: Login to Amazon ECR
-        id: ecr-login
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registry-type: public
-
-      - name: Tag image to release version (sha)
-        run: |
-          docker buildx imagetools create \
-            --tag "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/${{ matrix.component }}-release:${{ github.event.pull_request.head.sha || github.sha }}" \
-                  "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/${{ matrix.component }}:${{ github.event.pull_request.head.sha || github.sha }}"
-      
-      - name: Tag image to release version (latest)
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        run: |
-          docker buildx imagetools create \
-            --tag "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/${{ matrix.component }}-release:latest" \
-                  "${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/${{ matrix.component }}:${{ github.event.pull_request.head.sha || github.sha }}"
diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
deleted file mode 100644
index be67fd8498..0000000000
--- a/.github/workflows/sandbox.yml
+++ /dev/null
@@ -1,115 +0,0 @@
-name: Build & Push Sandbox Docker Image
-
-on:
-  workflow_call:
-  workflow_dispatch:
-
-permissions:
-  id-token: write
-  contents: read
-
-jobs:
-  trigger-sandbox-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - name: Set versions
-        run: |
-          echo "FLYTECONSOLE_VERSION=$(yq eval '.flyteconsole.image.tag' charts/flyte-core/values.yaml)" >> $GITHUB_ENV
-          echo "FLYTE_VERSION=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_ENV
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::558384610515:role/github-actions-flyte-fork
-          aws-region: us-east-1 # public ECR is only available in us-east-1
-      - name: Login to Amazon ECR
-        id: ecr-login
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registry-type: public
-      - name: Prepare DIND Image Names
-        id: dind-names
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/flyte-sandbox
-          tags: |
-            type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          platforms: linux/arm64, linux/amd64
-          push: true
-          target: dind
-          tags: ${{ steps.dind-names.outputs.tags }}
-          build-args: |
-            FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
-            FLYTE_VERSION=${{ env.FLYTE_VERSION }}
-          file: docker/sandbox/Dockerfile
-
-  trigger-sandbox-lite-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - name: Setup Golang caches
-        uses: actions/cache@v3
-        with:
-          path: |
-            /root/.cache/go-build
-            /root/go/pkg/mod
-          key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-golang-
-      - name: Set versions
-        run: |
-          echo "FLYTECONSOLE_VERSION=$(yq eval '.flyteconsole.image.tag' charts/flyte-core/values.yaml)" >> $GITHUB_ENV
-          echo "FLYTE_VERSION=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_ENV
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::558384610515:role/github-actions-flyte-fork
-          aws-region: us-east-1 # public ECR is only available in us-east-1
-      - name: Login to Amazon ECR
-        id: ecr-login
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registry-type: public
-      - name: Prepare DIND Image Names
-        id: dind-names
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ${{ steps.ecr-login.outputs.registry }}/p6w6o0n4/flyte-sandbox-lite
-          tags: |
-            type=raw,value=${{ github.event.pull_request.head.sha || github.sha }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          platforms: linux/arm64, linux/amd64
-          push: true
-          target: dind
-          tags: ${{ steps.dind-names.outputs.tags }}
-          build-args: |
-            FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
-            FLYTE_VERSION=${{ env.FLYTE_VERSION }}
-          file: Dockerfile.sandbox-lite
diff --git a/.github/workflows/selfassign.yml b/.github/workflows/selfassign.yml
deleted file mode 100644
index a60bcb4c70..0000000000
--- a/.github/workflows/selfassign.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Allow users to automatically tag themselves to issues
-#
-# Usage:
-#  - a github user (a member of the repo) needs to comment
-#    with "#self-assign" on an issue to be assigned to them.
-#------------------------------------------------------------
-
-name: Self-assign
-on:
-  issue_comment:
-    types: created
-jobs:
-  one:
-    runs-on: ubuntu-latest
-    if: >-
-      (github.event.comment.body == '#take' ||
-       github.event.comment.body == '#self-assign')
-    steps:
-      - run: |
-          echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
-          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' \
-                  https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
-          echo "Done 🔥 "