Remove all dependency on github secret for pull request (flyteorg#172)

* remove secret dependancy for contributors * fix lint * Update pull_request.yml * workflow lint fixes * workflow lint fixes * remove kind logic from e2e test * move logic from make to e2e shell script * remmove kind from e2e test * remove flytepropeller_repo from master * change repo name * Update .github/workflows/master.yml Co-authored-by: Haytham AbuelFutuh <[email protected]> * remove extra docker build * added docker build in pull workflow * removed unused variable in makefile * adding github caching * adding github caching * adding github caching * Update Makefile Co-authored-by: Haytham AbuelFutuh <[email protected]> * adding github caching * handle failed cases * revert changes * cache added * cache added * cache testing commit * Update master.yml * Update pull_request.yml * More changes * More changes * Update master.yml Co-authored-by: Haytham AbuelFutuh <[email protected]>
eapolinario · Sep 24, 2020 · 28690c7 · 28690c7
1 parent e49292e
commit 28690c7
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 43 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -28,27 +28,29 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: '0'
-      - name: Push Docker Image to Github Registry
-        uses: whoan/docker-build-with-cache-action@v5
+              - id: cache-docker
+        uses: actions/cache@v1
         with:
-          username: "${{ github.actor }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-          image_name: ${{ secrets.flytegithub_repo }}/flytepropeller
-          image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }}
-          push_git_tag: true
-          registry: docker.pkg.github.com
-          build_extra_args: "--compress=true"
+          path: /tmp//tmp/docker-images
+          key: /tmp/docker-images-${{ hashFiles('Dockerfile') }}
+
+      - run: docker load -i /tmp//tmp/docker-images/snapshot-builder.tar || true
+        if: steps.cache-docker.outputs.cache-hit == 'true'      
+      - run: |
+          docker build -t lyft/flytepropeller:${{ github.sha }} . --target builder --cache-from=flytepropeller-cache
+          docker build -t lyft/flytepropeller:${{ github.sha }} .
+
+      - run: docker tag lyft/flytepropeller:${{ github.sha }} flytepropeller-cache && mkdir -p /tmp//tmp/docker-images && docker save flytepropeller-cache -o /tmp//tmp/docker-images/snapshot.tar && ls -lh /tmp//tmp/docker-images || true
+        if: always() && steps.cache-docker.outputs.cache-hit != 'true'
+
       - uses: engineerd/[email protected]
       - name: End2End
-        env:
-          DOCKER_USERNAME: ${{ github.actor }}
-          DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           kubectl cluster-info
           kubectl get pods -n kube-system
           echo "current-context:" $(kubectl config current-context)
           echo "environment-kubeconfig:" ${KUBECONFIG}
-          PROPELLER=${{ secrets.flytegithub_repo }}/flytepropeller:${{ github.sha }} make end2end_execute
+          PROPELLER=lyft/flytepropeller:${{ github.sha }} make end2end_execute
   push-dockerhub:
     runs-on: ubuntu-latest
     needs: bump-version

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -8,39 +8,30 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Push Docker Image to Github Registry
-        uses: whoan/docker-build-with-cache-action@v5
+        name: "Checkout"
+      - id: cache-docker
+        uses: actions/cache@v1
         with:
-          username: "${{ github.actor }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-          image_name: ${{ secrets.flytegithub_repo }}/flytepropeller
-          image_tag: ${{ github.sha }}
-          push_git_tag: true
-          registry: docker.pkg.github.com
+          path: /tmp//tmp/docker-images
+          key: /tmp/docker-images-${{ hashFiles('Dockerfile') }}
+      - run: docker load -i /tmp//tmp/docker-images/snapshot-builder.tar || true
+        if: steps.cache-docker.outputs.cache-hit == 'true'
+      - run: |
+          docker build -t lyft/flytepropeller:${{ github.sha }} . --target builder --cache-from=flytepropeller-cache
+          docker build -t lyft/flytepropeller:${{ github.sha }} .
+
+      - run: docker tag lyft/flytepropeller:${{ github.sha }} flytepropeller-cache && mkdir -p /tmp//tmp/docker-images && docker save flytepropeller-cache -o /tmp//tmp/docker-images/snapshot.tar && ls -lh /tmp//tmp/docker-images || true
+        if: always() && steps.cache-docker.outputs.cache-hit != 'true'
+
       - uses: engineerd/[email protected]
       - name: End2End
-        env:
-          DOCKER_USERNAME: ${{ github.actor }}
-          DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           kubectl cluster-info
           kubectl get pods -n kube-system
           echo "current-context:" $(kubectl config current-context)
           echo "environment-kubeconfig:" ${KUBECONFIG}
-          PROPELLER=${{ secrets.flytegithub_repo }}/flytepropeller:${{ github.sha }} make end2end_execute
-  push-dockerhub:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Push Docker Image to Dockerhub
-        uses: whoan/docker-build-with-cache-action@v5
-        with:
-          username: "${{ secrets.DOCKERHUB_USERNAME }}"
-          password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          image_name: ${{ secrets.DOCKERHUB_IMAGE_NAME }}
-          image_tag: ${{ github.sha }}
-          push_git_tag: true
-          build_extra_args: "--compress=true"
+          PROPELLER=lyft/flytepropeller:${{ github.sha }} make end2end_execute
+
   tests-lint:
     runs-on: ubuntu-latest
     steps:

diff --git a/Makefile b/Makefile
@@ -3,6 +3,7 @@ include boilerplate/lyft/docker_build/Makefile
 include boilerplate/lyft/golang_test_targets/Makefile
 include boilerplate/lyft/end2end/Makefile
 
+
 .PHONY: update_boilerplate
 update_boilerplate:
 	@boilerplate/update.sh

diff --git a/boilerplate/lyft/end2end/end2end.sh b/boilerplate/lyft/end2end/end2end.sh
@@ -14,15 +14,18 @@ rm -rf ${OUT}
 git clone https://github.com/lyft/flyte.git "${OUT}"
 
 # TODO: load all images
-echo "Loading github docker images into 'kind' cluster to workaround this issue: https://github.com/containerd/containerd/issues/3291#issuecomment-631746985"
-docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} docker.pkg.github.com
-docker pull docker.pkg.github.com/${PROPELLER}
-kind load docker-image docker.pkg.github.com/${PROPELLER}
+kind  load docker-image ${PROPELLER}
+
+echo "Setup Kubectl"
+kubectl cluster-info
+kubectl get pods -n kube-system
+echo "current-context:" $(kubectl config current-context)
+echo "environment-kubeconfig:" ${KUBECONFIG}
 
 pushd ${OUT}
 # TODO: Only replace propeller if it's passed in
 # TODO: Support replacing other images too
-sed -i.bak -e "s_docker.io/lyft/flytepropeller:.*_docker.pkg.github.com/${PROPELLER}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
+sed -i.bak -e "s_docker.io/lyft/flytepropeller:.*_${PROPELLER}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
 make kustomize
 make end2end_execute
 popd