diff --git a/flytepropeller/.github/workflows/master.yml b/flytepropeller/.github/workflows/master.yml index b6b00ecc6..0c3db359d 100644 --- a/flytepropeller/.github/workflows/master.yml +++ b/flytepropeller/.github/workflows/master.yml @@ -6,8 +6,62 @@ on: - master jobs: + # Duplicated from pull request workflow because sharing is not yet supported + build-docker: + name: Build Docker Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + restore-keys: | + /tmp/docker-images-${{ github.event.before }} + /tmp/docker-images-${{ github.event.pull_request.base.sha }} + - name: Prime docker cache + run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) + - name: Build dockerfile + run: | + docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder . + docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder . + + - name: Tag and cache docker image + run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar + + # Duplicated from pull request workflow because sharing is not yet supported + endtoend: + name: End to End tests + runs-on: ubuntu-latest + needs: [build-docker] + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + - name: Prime docker cache + run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true + - uses: engineerd/setup-kind@v0.4.0 + - name: End2End + run: | + kubectl cluster-info + kubectl get pods -n kube-system + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute + bump-version: + name: Bump Version + if: github.actor != 'goreleaserbot' runs-on: ubuntu-latest + needs: build-docker # Only to ensure it can successfully build outputs: version: ${{ steps.bump-version.outputs.tag }} steps: @@ -21,57 +75,48 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true DEFAULT_BUMP: patch + goreleaser: + name: Goreleaser runs-on: ubuntu-latest needs: [bump-version] steps: - uses: actions/checkout@v2 with: fetch-depth: '0' - - - name: Set up Go + - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.14 - - - name: Run GoReleaser + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: version: latest args: release --rm-dist env: GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} - push-github-end2end: + + push-github: + name: Push to Github Registry runs-on: ubuntu-latest needs: bump-version steps: - uses: actions/checkout@v2 with: fetch-depth: '0' - - id: cache-docker - uses: actions/cache@v1 + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 with: - path: /tmp//tmp/docker-images - key: /tmp/docker-images-${{ hashFiles('Dockerfile') }} - - - run: docker load -i /tmp//tmp/docker-images/snapshot-builder.tar || true - if: steps.cache-docker.outputs.cache-hit == 'true' - - run: | - docker build -t lyft/flytepropeller:${{ github.sha }} . --target builder --cache-from=flytepropeller-cache - docker build -t lyft/flytepropeller:${{ github.sha }} . - - - run: docker tag lyft/flytepropeller:${{ github.sha }} flytepropeller-cache && mkdir -p /tmp//tmp/docker-images && docker save flytepropeller-cache -o /tmp//tmp/docker-images/snapshot.tar && ls -lh /tmp//tmp/docker-images || true - if: always() && steps.cache-docker.outputs.cache-hit != 'true' + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ github.repository }}/${{ github.event.repository.name }} + image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} + push_git_tag: true + registry: docker.pkg.github.com + build_extra_args: "--compress=true" - - uses: engineerd/setup-kind@v0.4.0 - - name: End2End - run: | - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - PROPELLER=lyft/flytepropeller:${{ github.sha }} make end2end_execute push-dockerhub: + name: Push to Dockerhub runs-on: ubuntu-latest needs: bump-version steps: @@ -87,7 +132,9 @@ jobs: image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true build_extra_args: "--compress=true" + tests-lint: + name: Run tests and lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/flytepropeller/.github/workflows/pull_request.yml b/flytepropeller/.github/workflows/pull_request.yml index 028477c3e..a51d93175 100644 --- a/flytepropeller/.github/workflows/pull_request.yml +++ b/flytepropeller/.github/workflows/pull_request.yml @@ -4,25 +4,46 @@ on: pull_request jobs: - build-and-end2end: + build-docker: + name: Build Docker Image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - name: "Checkout" - - id: cache-docker + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache uses: actions/cache@v1 with: - path: /tmp//tmp/docker-images - key: /tmp/docker-images-${{ hashFiles('Dockerfile') }} - - run: docker load -i /tmp//tmp/docker-images/snapshot-builder.tar || true - if: steps.cache-docker.outputs.cache-hit == 'true' - - run: | - docker build -t lyft/flytepropeller:${{ github.sha }} . --target builder --cache-from=flytepropeller-cache - docker build -t lyft/flytepropeller:${{ github.sha }} . + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + restore-keys: | + /tmp/docker-images-${{ github.event.before }} + /tmp/docker-images-${{ github.event.pull_request.base.sha }} + - name: Prime docker cache + run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) + - name: Build dockerfile + run: | + docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder . + docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder . - - run: docker tag lyft/flytepropeller:${{ github.sha }} flytepropeller-cache && mkdir -p /tmp//tmp/docker-images && docker save flytepropeller-cache -o /tmp//tmp/docker-images/snapshot.tar && ls -lh /tmp//tmp/docker-images || true - if: always() && steps.cache-docker.outputs.cache-hit != 'true' + - name: Tag and cache docker image + run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar + endtoend: + name: End to End tests + runs-on: ubuntu-latest + needs: [build-docker] + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + - name: Prime docker cache + run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - uses: engineerd/setup-kind@v0.4.0 - name: End2End run: | @@ -30,12 +51,38 @@ jobs: kubectl get pods -n kube-system echo "current-context:" $(kubectl config current-context) echo "environment-kubeconfig:" ${KUBECONFIG} - PROPELLER=lyft/flytepropeller:${{ github.sha }} make end2end_execute + IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute + + integration: + name: Integration tests + runs-on: ubuntu-latest + needs: [build-docker] + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + - name: Prime docker cache + run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true + - uses: engineerd/setup-kind@v0.4.0 + - name: Integration + run: | + kubectl cluster-info + kubectl get pods -n kube-system + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:builder make k8s_integration_execute tests-lint: + name: Run tests and lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Unit Tests uses: cedrickring/golang-action@1.5.2 env: diff --git a/flytepropeller/boilerplate/lyft/end2end/Makefile b/flytepropeller/boilerplate/lyft/end2end/Makefile index 1d8397f23..bfac15c15 100644 --- a/flytepropeller/boilerplate/lyft/end2end/Makefile +++ b/flytepropeller/boilerplate/lyft/end2end/Makefile @@ -6,3 +6,7 @@ .PHONY: end2end_execute end2end_execute: ./boilerplate/lyft/end2end/end2end.sh + +.PHONY: k8s_integration_execute +k8s_integration_execute:: + echo "pass" diff --git a/flytepropeller/boilerplate/lyft/end2end/end2end.sh b/flytepropeller/boilerplate/lyft/end2end/end2end.sh index 410b1e8ef..49c328cee 100755 --- a/flytepropeller/boilerplate/lyft/end2end/end2end.sh +++ b/flytepropeller/boilerplate/lyft/end2end/end2end.sh @@ -13,19 +13,22 @@ OUT="${DIR}/tmp" rm -rf ${OUT} git clone https://github.com/lyft/flyte.git "${OUT}" -# TODO: load all images -kind load docker-image ${PROPELLER} +pushd ${OUT} -echo "Setup Kubectl" -kubectl cluster-info -kubectl get pods -n kube-system -echo "current-context:" $(kubectl config current-context) -echo "environment-kubeconfig:" ${KUBECONFIG} +if [ ! -z "$IMAGE" ]; +then + kind load docker-image ${IMAGE} + if [ ${IMAGE_NAME} -eq "flytepropeller" ] + then + sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/propeller/deployment.yaml + fi + + if [ ${IMAGE} -eq "flyteadmin" ] + then + sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/admindeployment/deployment.yaml + fi +fi -pushd ${OUT} -# TODO: Only replace propeller if it's passed in -# TODO: Support replacing other images too -sed -i.bak -e "s_docker.io/lyft/flytepropeller:.*_${PROPELLER}_g" ${OUT}/kustomize/base/propeller/deployment.yaml make kustomize make end2end_execute popd diff --git a/flytepropeller/boilerplate/lyft/github_workflows/master.yml b/flytepropeller/boilerplate/lyft/github_workflows/master.yml index b240e8606..0c3db359d 100644 --- a/flytepropeller/boilerplate/lyft/github_workflows/master.yml +++ b/flytepropeller/boilerplate/lyft/github_workflows/master.yml @@ -6,11 +6,68 @@ on: - master jobs: + # Duplicated from pull request workflow because sharing is not yet supported + build-docker: + name: Build Docker Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + restore-keys: | + /tmp/docker-images-${{ github.event.before }} + /tmp/docker-images-${{ github.event.pull_request.base.sha }} + - name: Prime docker cache + run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) + - name: Build dockerfile + run: | + docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder . + docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder . + + - name: Tag and cache docker image + run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar + + # Duplicated from pull request workflow because sharing is not yet supported + endtoend: + name: End to End tests + runs-on: ubuntu-latest + needs: [build-docker] + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + - name: Prime docker cache + run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true + - uses: engineerd/setup-kind@v0.4.0 + - name: End2End + run: | + kubectl cluster-info + kubectl get pods -n kube-system + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute + bump-version: + name: Bump Version + if: github.actor != 'goreleaserbot' runs-on: ubuntu-latest + needs: build-docker # Only to ensure it can successfully build outputs: version: ${{ steps.bump-version.outputs.tag }} steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' - name: Bump version and push tag id: bump-version uses: anothrNick/github-tag-action@1.17.2 @@ -18,7 +75,29 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true DEFAULT_BUMP: patch - push-github-end2end: + + goreleaser: + name: Goreleaser + runs-on: ubuntu-latest + needs: [bump-version] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: '0' + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.14 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} + + push-github: + name: Push to Github Registry runs-on: ubuntu-latest needs: bump-version steps: @@ -30,23 +109,14 @@ jobs: with: username: "${{ github.actor }}" password: "${{ secrets.GITHUB_TOKEN }}" - image_name: ${{ secrets.flytegithub_repo }}/operator + image_name: ${{ github.repository }}/${{ github.event.repository.name }} image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true registry: docker.pkg.github.com build_extra_args: "--compress=true" - - uses: engineerd/setup-kind@v0.4.0 - - name: End2End - env: - DOCKER_USERNAME: ${{ github.actor }} - DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}" - run: | - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - PROPELLER=${{ secrets.flytegithub_repo }}/operator:${{ github.sha }} make end2end_execute + push-dockerhub: + name: Push to Dockerhub runs-on: ubuntu-latest needs: bump-version steps: @@ -62,7 +132,9 @@ jobs: image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} push_git_tag: true build_extra_args: "--compress=true" + tests-lint: + name: Run tests and lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/flytepropeller/boilerplate/lyft/github_workflows/pull_request.yml b/flytepropeller/boilerplate/lyft/github_workflows/pull_request.yml index cf7b2156a..a51d93175 100644 --- a/flytepropeller/boilerplate/lyft/github_workflows/pull_request.yml +++ b/flytepropeller/boilerplate/lyft/github_workflows/pull_request.yml @@ -4,47 +4,85 @@ on: pull_request jobs: - build-and-end2end: + build-docker: + name: Build Docker Image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 with: - username: "${{ github.actor }}" - password: "${{ secrets.GITHUB_TOKEN }}" - image_name: ${{ secrets.flytegithub_repo }}/flytepropeller - image_tag: ${{ github.sha }} - push_git_tag: true - registry: docker.pkg.github.com + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + restore-keys: | + /tmp/docker-images-${{ github.event.before }} + /tmp/docker-images-${{ github.event.pull_request.base.sha }} + - name: Prime docker cache + run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) + - name: Build dockerfile + run: | + docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder . + docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder . + + - name: Tag and cache docker image + run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar + + endtoend: + name: End to End tests + runs-on: ubuntu-latest + needs: [build-docker] + steps: + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 + with: + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + - name: Prime docker cache + run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - uses: engineerd/setup-kind@v0.4.0 - name: End2End - env: - DOCKER_USERNAME: ${{ github.actor }} - DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}" run: | kubectl cluster-info kubectl get pods -n kube-system echo "current-context:" $(kubectl config current-context) echo "environment-kubeconfig:" ${KUBECONFIG} - PROPELLER=${{ secrets.flytegithub_repo }}/flytepropeller:${{ github.sha }} make end2end_execute - push-dockerhub: + IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute + + integration: + name: Integration tests runs-on: ubuntu-latest + needs: [build-docker] steps: - - uses: actions/checkout@v2 - - name: Push Docker Image to Dockerhub - uses: whoan/docker-build-with-cache-action@v5 + - name: Checkout + uses: actions/checkout@v2 + - id: load-docker-cache + name: Load Docker Cache + uses: actions/cache@v1 with: - username: "${{ secrets.DOCKERHUB_USERNAME }}" - password: "${{ secrets.DOCKERHUB_PASSWORD }}" - image_name: ${{ secrets.DOCKERHUB_IMAGE_NAME }} - image_tag: ${{ github.sha }} - push_git_tag: true - build_extra_args: "--compress=true" + path: /tmp/tmp/docker-images + key: /tmp/docker-images-${{ github.event.after }} + - name: Prime docker cache + run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true + - uses: engineerd/setup-kind@v0.4.0 + - name: Integration + run: | + kubectl cluster-info + kubectl get pods -n kube-system + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:builder make k8s_integration_execute + tests-lint: + name: Run tests and lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Unit Tests uses: cedrickring/golang-action@1.5.2 env: