[Github workflows] Use latest boilerplate (flyteorg#201)

* [Github workflows] Use latest boilerplate * Update latest master workflow
eapolinario · Nov 2, 2020 · ac6c29d · ac6c29d
1 parent 39cf556
commit ac6c29d
Show file tree

Hide file tree

Showing 6 changed files with 301 additions and 90 deletions.
diff --git a/flytepropeller/.github/workflows/master.yml b/flytepropeller/.github/workflows/master.yml
@@ -6,8 +6,62 @@ on:
       - master
 
 jobs:
+  # Duplicated from pull request workflow because sharing is not yet supported
+  build-docker:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+          restore-keys: |
+            /tmp/docker-images-${{ github.event.before }}
+            /tmp/docker-images-${{ github.event.pull_request.base.sha }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        run: |
+          docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder .
+          docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder .
+
+      - name: Tag and cache docker image
+        run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar
+
+  # Duplicated from pull request workflow because sharing is not yet supported
+  endtoend:
+    name: End to End tests
+    runs-on: ubuntu-latest
+    needs: [build-docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+      - name: Prime docker cache
+        run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true
+      - uses: engineerd/[email protected]
+      - name: End2End
+        run: |
+          kubectl cluster-info
+          kubectl get pods -n kube-system
+          echo "current-context:" $(kubectl config current-context)
+          echo "environment-kubeconfig:" ${KUBECONFIG}
+          IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute
+
   bump-version:
+    name: Bump Version
+    if: github.actor != 'goreleaserbot'
     runs-on: ubuntu-latest
+    needs: build-docker # Only to ensure it can successfully build
     outputs:
       version: ${{ steps.bump-version.outputs.tag }}
     steps:
@@ -21,57 +75,48 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           WITH_V: true
           DEFAULT_BUMP: patch
+
   goreleaser:
+    name: Goreleaser
     runs-on: ubuntu-latest
     needs: [bump-version]
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: '0'
-      -
-        name: Set up Go
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.14
-      -
-        name: Run GoReleaser
+      - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:
           version: latest
           args: release --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }}
-  push-github-end2end:
+
+  push-github:
+    name: Push to Github Registry
     runs-on: ubuntu-latest
     needs: bump-version
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: '0'
-      - id: cache-docker
-        uses: actions/cache@v1
+      - name: Push Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
         with:
-          path: /tmp//tmp/docker-images
-          key: /tmp/docker-images-${{ hashFiles('Dockerfile') }}
-
-      - run: docker load -i /tmp//tmp/docker-images/snapshot-builder.tar || true
-        if: steps.cache-docker.outputs.cache-hit == 'true'      
-      - run: |
-          docker build -t lyft/flytepropeller:${{ github.sha }} . --target builder --cache-from=flytepropeller-cache
-          docker build -t lyft/flytepropeller:${{ github.sha }} .
-
-      - run: docker tag lyft/flytepropeller:${{ github.sha }} flytepropeller-cache && mkdir -p /tmp//tmp/docker-images && docker save flytepropeller-cache -o /tmp//tmp/docker-images/snapshot.tar && ls -lh /tmp//tmp/docker-images || true
-        if: always() && steps.cache-docker.outputs.cache-hit != 'true'
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+          image_name: ${{ github.repository }}/${{ github.event.repository.name }}
+          image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }}
+          push_git_tag: true
+          registry: docker.pkg.github.com
+          build_extra_args: "--compress=true"
 
-      - uses: engineerd/[email protected]
-      - name: End2End
-        run: |
-          kubectl cluster-info
-          kubectl get pods -n kube-system
-          echo "current-context:" $(kubectl config current-context)
-          echo "environment-kubeconfig:" ${KUBECONFIG}
-          PROPELLER=lyft/flytepropeller:${{ github.sha }} make end2end_execute
   push-dockerhub:
+    name: Push to Dockerhub
     runs-on: ubuntu-latest
     needs: bump-version
     steps:
@@ -87,7 +132,9 @@ jobs:
           image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }}
           push_git_tag: true
           build_extra_args: "--compress=true"
+
   tests-lint:
+    name: Run tests and lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

diff --git a/flytepropeller/.github/workflows/pull_request.yml b/flytepropeller/.github/workflows/pull_request.yml
@@ -4,38 +4,85 @@ on:
   pull_request
 
 jobs:
-  build-and-end2end:
+  build-docker:
+    name: Build Docker Image
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-        name: "Checkout"
-      - id: cache-docker
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
         uses: actions/cache@v1
         with:
-          path: /tmp//tmp/docker-images
-          key: /tmp/docker-images-${{ hashFiles('Dockerfile') }}
-      - run: docker load -i /tmp//tmp/docker-images/snapshot-builder.tar || true
-        if: steps.cache-docker.outputs.cache-hit == 'true'
-      - run: |
-          docker build -t lyft/flytepropeller:${{ github.sha }} . --target builder --cache-from=flytepropeller-cache
-          docker build -t lyft/flytepropeller:${{ github.sha }} .
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+          restore-keys: |
+            /tmp/docker-images-${{ github.event.before }}
+            /tmp/docker-images-${{ github.event.pull_request.base.sha }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        run: |
+          docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder .
+          docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder .
 
-      - run: docker tag lyft/flytepropeller:${{ github.sha }} flytepropeller-cache && mkdir -p /tmp//tmp/docker-images && docker save flytepropeller-cache -o /tmp//tmp/docker-images/snapshot.tar && ls -lh /tmp//tmp/docker-images || true
-        if: always() && steps.cache-docker.outputs.cache-hit != 'true'
+      - name: Tag and cache docker image
+        run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar
 
+  endtoend:
+    name: End to End tests
+    runs-on: ubuntu-latest
+    needs: [build-docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+      - name: Prime docker cache
+        run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true
       - uses: engineerd/[email protected]
       - name: End2End
         run: |
           kubectl cluster-info
           kubectl get pods -n kube-system
           echo "current-context:" $(kubectl config current-context)
           echo "environment-kubeconfig:" ${KUBECONFIG}
-          PROPELLER=lyft/flytepropeller:${{ github.sha }} make end2end_execute
+          IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute
+
+  integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    needs: [build-docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+      - name: Prime docker cache
+        run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true
+      - uses: engineerd/[email protected]
+      - name: Integration
+        run: |
+          kubectl cluster-info
+          kubectl get pods -n kube-system
+          echo "current-context:" $(kubectl config current-context)
+          echo "environment-kubeconfig:" ${KUBECONFIG}
+          IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:builder make k8s_integration_execute
 
   tests-lint:
+    name: Run tests and lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
       - name: Unit Tests
         uses: cedrickring/[email protected]
         env:

diff --git a/flytepropeller/boilerplate/lyft/end2end/Makefile b/flytepropeller/boilerplate/lyft/end2end/Makefile
@@ -6,3 +6,7 @@
 .PHONY: end2end_execute
 end2end_execute:
 	./boilerplate/lyft/end2end/end2end.sh
+
+.PHONY: k8s_integration_execute
+k8s_integration_execute::
+	echo "pass"
diff --git a/flytepropeller/boilerplate/lyft/end2end/end2end.sh b/flytepropeller/boilerplate/lyft/end2end/end2end.sh
@@ -13,19 +13,22 @@ OUT="${DIR}/tmp"
 rm -rf ${OUT}
 git clone https://github.com/lyft/flyte.git "${OUT}"
 
-# TODO: load all images
-kind  load docker-image ${PROPELLER}
+pushd ${OUT}
 
-echo "Setup Kubectl"
-kubectl cluster-info
-kubectl get pods -n kube-system
-echo "current-context:" $(kubectl config current-context)
-echo "environment-kubeconfig:" ${KUBECONFIG}
+if [ ! -z "$IMAGE" ]; 
+then
+  kind load docker-image ${IMAGE}
+  if [ ${IMAGE_NAME} -eq "flytepropeller" ]
+  then
+    sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
+  fi
+
+  if [ ${IMAGE} -eq "flyteadmin" ]
+  then
+    sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/admindeployment/deployment.yaml
+  fi
+fi
 
-pushd ${OUT}
-# TODO: Only replace propeller if it's passed in
-# TODO: Support replacing other images too
-sed -i.bak -e "s_docker.io/lyft/flytepropeller:.*_${PROPELLER}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
 make kustomize
 make end2end_execute
 popd