From 633cf55f1dd6eb874798280e1903e3ba35f7eb44 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 22 Jun 2021 13:40:46 -0700 Subject: [PATCH] Consolidate github workflows (#301) * Consolidate github workflows Signed-off-by: Haytham Abuelfutuh * try objects Signed-off-by: Haytham Abuelfutuh * wip Signed-off-by: Haytham Abuelfutuh * wip Signed-off-by: Haytham Abuelfutuh * wip Signed-off-by: Haytham Abuelfutuh * wip Signed-off-by: Haytham Abuelfutuh * Update contribution guide Signed-off-by: Haytham Abuelfutuh --- .github/workflows/ghcr_core_master.yml | 33 ---------- .../workflows/ghcr_house_price_prediction.yml | 33 ---------- .github/workflows/ghcr_k8s_spark.yml | 33 ---------- .github/workflows/ghcr_kfpytorch.yml | 33 ---------- .github/workflows/ghcr_pandera.yml | 33 ---------- .github/workflows/ghcr_papermilltasks.yml | 33 ---------- .github/workflows/ghcr_pima_diabetes.yml | 33 ---------- .github/workflows/ghcr_pod.yml | 33 ---------- .github/workflows/ghcr_push.yml | 58 ++++++++++++++++++ .github/workflows/ghcr_sagemaker_pytorch.yml | 33 ---------- .github/workflows/ghcr_sagemaker_training.yml | 33 ---------- .../workflows/ghcr_sqlite_datacleaning.yml | 33 ---------- .github/workflows/master.yml | 61 ++++--------------- .github/workflows/pull_request.yml | 54 ---------------- .../{build.yml => sandbox_register.yml} | 4 +- .github/workflows/serialize_examples.yml | 25 ++++++++ README.md | 52 ++++++++++++++-- 17 files changed, 143 insertions(+), 474 deletions(-) delete mode 100644 .github/workflows/ghcr_core_master.yml delete mode 100644 .github/workflows/ghcr_house_price_prediction.yml delete mode 100644 .github/workflows/ghcr_k8s_spark.yml delete mode 100644 .github/workflows/ghcr_kfpytorch.yml delete mode 100644 .github/workflows/ghcr_pandera.yml delete mode 100644 .github/workflows/ghcr_papermilltasks.yml delete mode 100644 .github/workflows/ghcr_pima_diabetes.yml delete mode 100644 .github/workflows/ghcr_pod.yml create mode 100644 .github/workflows/ghcr_push.yml delete mode 100644 .github/workflows/ghcr_sagemaker_pytorch.yml delete mode 100644 .github/workflows/ghcr_sagemaker_training.yml delete mode 100644 .github/workflows/ghcr_sqlite_datacleaning.yml delete mode 100644 .github/workflows/pull_request.yml rename .github/workflows/{build.yml => sandbox_register.yml} (93%) create mode 100644 .github/workflows/serialize_examples.yml diff --git a/.github/workflows/ghcr_core_master.yml b/.github/workflows/ghcr_core_master.yml deleted file mode 100644 index 8b2de615eb..0000000000 --- a/.github/workflows/ghcr_core_master.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Push Core Examples - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push Core examples to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: latest,core-latest,core-${{ github.sha }},core-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:core-${{ github.sha }}" - context: ./cookbook - dockerfile: core/Dockerfile diff --git a/.github/workflows/ghcr_house_price_prediction.yml b/.github/workflows/ghcr_house_price_prediction.yml deleted file mode 100644 index b809003b6f..0000000000 --- a/.github/workflows/ghcr_house_price_prediction.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build and Push House Price Prediction example Docker Images - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push House Price Prediction example to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: house_price_prediction-latest,house_price_prediction-${{ github.sha }},house_price_prediction-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:house_price_prediction-${{ github.sha }}" - context: ./cookbook/case_studies/ml_training/ - dockerfile: house_price_prediction/Dockerfile diff --git a/.github/workflows/ghcr_k8s_spark.yml b/.github/workflows/ghcr_k8s_spark.yml deleted file mode 100644 index 86f66681ef..0000000000 --- a/.github/workflows/ghcr_k8s_spark.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push Spark Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Spark Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: k8s_spark-latest,k8s_spark-${{ github.sha }},k8s_spark-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:k8s_spark-${{ github.sha }}" - context: ./cookbook/integrations/kubernetes/ - dockerfile: k8s_spark/Dockerfile diff --git a/.github/workflows/ghcr_kfpytorch.yml b/.github/workflows/ghcr_kfpytorch.yml deleted file mode 100644 index c9b78db765..0000000000 --- a/.github/workflows/ghcr_kfpytorch.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push Kubeflow Pytorch Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Kubeflow Pytorch Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: kfpytorch-latest,kfpytorch-${{ github.sha }},kfpytorch-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:kfpytorch-${{ github.sha }}" - context: ./cookbook/integrations/kubernetes/ - dockerfile: kfpytorch/Dockerfile diff --git a/.github/workflows/ghcr_pandera.yml b/.github/workflows/ghcr_pandera.yml deleted file mode 100644 index e0351d2af8..0000000000 --- a/.github/workflows/ghcr_pandera.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push Pandera Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Pandera Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: pandera-latest,pandera-${{ github.sha }},pandera-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:pandera-${{ github.sha }}" - context: ./cookbook/integrations/flytekit_plugins/ - dockerfile: pandera/Dockerfile diff --git a/.github/workflows/ghcr_papermilltasks.yml b/.github/workflows/ghcr_papermilltasks.yml deleted file mode 100644 index b5d7eb80a8..0000000000 --- a/.github/workflows/ghcr_papermilltasks.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push Papermill Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Papermill Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: papermilltasks-latest,papermilltasks-${{ github.sha }},papermilltasks-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:papermilltasks-${{ github.sha }}" - context: ./cookbook/integrations/flytekit_plugins/ - dockerfile: papermilltasks/Dockerfile diff --git a/.github/workflows/ghcr_pima_diabetes.yml b/.github/workflows/ghcr_pima_diabetes.yml deleted file mode 100644 index 3259dd3aa6..0000000000 --- a/.github/workflows/ghcr_pima_diabetes.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build and Push Pima Diabetes Docker Images - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push Pima Diabetes examples to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: pima_diabetes-latest,pima_diabetes-${{ github.sha }},pima_diabetes-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:pima_diabetes-${{ github.sha }}" - context: ./cookbook/case_studies/ml_training/ - dockerfile: pima_diabetes/Dockerfile diff --git a/.github/workflows/ghcr_pod.yml b/.github/workflows/ghcr_pod.yml deleted file mode 100644 index aa83caba4a..0000000000 --- a/.github/workflows/ghcr_pod.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push Pod Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Pod Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: pod-latest,pod-${{ github.sha }},pod-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:pod-${{ github.sha }}" - context: ./cookbook/integrations/kubernetes/ - dockerfile: pod/Dockerfile diff --git a/.github/workflows/ghcr_push.yml b/.github/workflows/ghcr_push.yml new file mode 100644 index 0000000000..3f456b6499 --- /dev/null +++ b/.github/workflows/ghcr_push.yml @@ -0,0 +1,58 @@ +name: Build & Push Docker Images + +on: + pull_request: + create: + tags: + - v* + +jobs: + push-github: + name: Push to GHCR + runs-on: ubuntu-latest + defaults: + run: + working-directory: cookbook + strategy: + matrix: + directory: + - name: core + path: . + - name: pod + path: integrations/kubernetes + - name: k8s_spark + path: integrations/kubernetes + - name: kfpytorch + path: integrations/kubernetes + - name: sqlite_datacleaning + path: case_studies/feature_engineering + - name: sagemaker_training + path: integrations/aws + - name: sagemaker_pytorch + path: integrations/aws + - name: pima_diabetes + path: case_studies/ml_training + - name: papermilltasks + path: integrations/flytekit_plugins + - name: pandera + path: integrations/flytekit_plugins + - name: house_price_prediction + path: case_studies/ml_training + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Push Spark Plugin Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" + image_name: ${{ github.repository_owner }}/flytecookbook + image_tag: ${{ matrix.directory.name }}-latest,${{ matrix.directory.name }}-${{ github.sha }},${{ matrix.directory.name }}-${{ github.event.ref }} + push_git_tag: ${{ github.event_name != 'pull_request' }} + push_image_and_stages: ${{ github.event_name != 'pull_request' }} + registry: ghcr.io + build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ matrix.directory.name }}-${{ github.sha }}" + context: ./cookbook/${{ matrix.directory.path }} + dockerfile: ${{ matrix.directory.name }}/Dockerfile diff --git a/.github/workflows/ghcr_sagemaker_pytorch.yml b/.github/workflows/ghcr_sagemaker_pytorch.yml deleted file mode 100644 index bae28523af..0000000000 --- a/.github/workflows/ghcr_sagemaker_pytorch.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push SageMaker Pytorch Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push SageMaker Pytorch Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: sagemaker_pytorch-latest,sagemaker_pytorch-${{ github.sha }},sagemaker_pytorch-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:sagemaker_pytorch-${{ github.sha }}" - context: ./cookbook/integrations/aws/ - dockerfile: sagemaker_pytorch/Dockerfile diff --git a/.github/workflows/ghcr_sagemaker_training.yml b/.github/workflows/ghcr_sagemaker_training.yml deleted file mode 100644 index 00bbdcae5f..0000000000 --- a/.github/workflows/ghcr_sagemaker_training.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build & Push SageMaker Training Plugin Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push SageMaker Training Plugin Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: sagemaker_training-latest,sagemaker_training-${{ github.sha }},sagemaker_training-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:sagemaker_training-${{ github.sha }}" - context: ./cookbook/integrations/aws/ - dockerfile: sagemaker_training/Dockerfile diff --git a/.github/workflows/ghcr_sqlite_datacleaning.yml b/.github/workflows/ghcr_sqlite_datacleaning.yml deleted file mode 100644 index 6d62452a93..0000000000 --- a/.github/workflows/ghcr_sqlite_datacleaning.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build and Push SQLite3 Data Cleaning Docker Image - -on: - pull_request: - create: - tags: - - v* - -jobs: - push-github: - name: Push SQLite3 Data Cleaning examples to GHCR - runs-on: ubuntu-latest - defaults: - run: - working-directory: cookbook - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: sqlite_datacleaning-latest,sqlite_datacleaning-${{ github.sha }},sqlite_datacleaning-${{ github.event.ref }} - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:sqlite_datacleaning-${{ github.sha }}" - context: ./cookbook/case_studies/feature_engineering/ - dockerfile: sqlite_datacleaning/Dockerfile diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c49ab5b6a7..22966bc8bc 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -23,70 +23,33 @@ jobs: GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} WITH_V: true DEFAULT_BUMP: patch - core-releaser: - name: Release Core example + examples-releaser: + name: Release Examples + strategy: + matrix: + directory: [ "core", "integrations", "case_studies" ] runs-on: ubuntu-latest - needs: [bump-version] + needs: [ bump-version ] steps: - uses: actions/checkout@v2 with: fetch-depth: "0" - uses: actions/cache@v1 with: - path: /tmp/release-snacks-core/ - key: release-snacks-core - - name: Generate Serialize proto - env: - REGISTRY: "ghcr.io/flyteorg" - run: | - make -C cookbook/core serialize - mkdir -p cookbook/release-snacks /tmp/release-snacks-core/ - make -C cookbook archive - cp -r cookbook/release-snacks/* /tmp/release-snacks-core/ - integrations-releaser: - name: Release Integrations example - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-integrations/ - key: release-snacks-plugins - - name: Generate Serialize proto - env: - REGISTRY: "ghcr.io/flyteorg" - run: | - make -C cookbook/integrations serialize - mkdir -p cookbook/release-snacks /tmp/release-snacks-integrations/ - make -C cookbook archive - cp -r cookbook/release-snacks/* /tmp/release-snacks-integrations/ - case_studies-releaser: - name: Release Case studies example - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-case-studies/ - key: release-snacks-case-studies + path: /tmp/release-snacks-${{ matrix.directory }}/ + key: release-snacks-${{ matrix.directory }} - name: Generate Serialize proto env: REGISTRY: "ghcr.io/flyteorg" run: | - make -C cookbook/case_studies serialize - mkdir -p cookbook/release-snacks /tmp/release-snacks-case-studies/ + make -C cookbook/${{ matrix.directory }} serialize + mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/ make -C cookbook archive - cp -r cookbook/release-snacks/* /tmp/release-snacks-case-studies/ + cp -r cookbook/release-snacks/* /tmp/release-snacks-${{ matrix.directory }}/ goreleaser: name: Release Case studies example runs-on: ubuntu-latest - needs: [core-releaser, case_studies-releaser, integrations-releaser] + needs: [ core-releaser, case_studies-releaser, integrations-releaser ] steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index b177d473f8..0000000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Serialize snacks - -on: - pull_request: - -jobs: - core-build: - name: Build Core example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-core/ - key: release-snacks-core - - name: Generate Serialize proto - env: - REGISTRY : "ghcr.io/flyteorg" - run: | - make -C cookbook/core serialize - integrations-build: - name: Build Integrations example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-integrations/ - key: release-snacks-plugins - - name: Generate Serialize proto - env: - REGISTRY: "ghcr.io/flyteorg" - run: | - make -C cookbook/integrations serialize - case_studies-build: - name: Build Case studies example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-case-studies/ - key: release-snacks-case-studies - - name: Generate Serialize proto - env: - REGISTRY: "ghcr.io/flyteorg" - run: | - make -C cookbook/case_studies serialize \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/sandbox_register.yml similarity index 93% rename from .github/workflows/build.yml rename to .github/workflows/sandbox_register.yml index d9ff58e2a6..364e83b573 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/sandbox_register.yml @@ -1,4 +1,4 @@ -name: Build +name: Sandbox Register on: pull_request: @@ -12,7 +12,7 @@ defaults: jobs: build: - name: Build + name: Sandbox Register runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/serialize_examples.yml b/.github/workflows/serialize_examples.yml new file mode 100644 index 0000000000..f63ffbfb0a --- /dev/null +++ b/.github/workflows/serialize_examples.yml @@ -0,0 +1,25 @@ +name: Serialize Examples + +on: + pull_request: + +jobs: + core-build: + name: Serialize Examples + strategy: + matrix: + directory: [ "core", "integrations", "case_studies" ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - uses: actions/cache@v1 + with: + path: /tmp/release-snacks-${{ matrix.directory }}/ + key: release-snacks-${{ matrix.directory }} + - name: Generate Serialize proto + env: + REGISTRY: "ghcr.io/flyteorg" + run: | + make -C cookbook/${{ matrix.directory }} serialize diff --git a/README.md b/README.md index 2a3ac69c87..3a0773a115 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,52 @@ Flytesnacks currently has all examples in Python (Flytekit Python SDK). In the future, Java examples employing Flytekit JAVA will be out. Here are the setup instructions to start contributing to `flytesnacks` repo: -- If the Python code has to be tested, run it locally -- If the Python code has to be tested in a cluster: - * Run the `make start` command in the root directory of the flytesnacks repo - * Visit https://localhost:30081 to view the Flyte console consisting of the examples present in flytesnacks/cookbook/core directory - * To fetch new dependencies and rebuild the image, run `make register` - * If examples from a different directory (other than `core`) have to registered, run the command `SANDBOX=1 make -C register` in the `flytesnacks` repo. + +### 💻 Contribute to examples + +1. Determine where to put your new code: + * [Core](https://github.com/flyteorg/blob/master/cookbook/core) + + Contains examples that demonstrates functionality available within core flytekit. These examples should be runnable + locally. + + * [Integrations](https://github.com/flyteorg/blob/master/cookbook/integrations) + + Contains examples that leverage one or more of the available plugins. + + * [Case Studies](https://github.com/flyteorg/blob/master/cookbook/case_studies) + + Contains examples that demonstrates usage of Flyte to solve real world problems. These are more generally more complex + examples that may require extra setup or that can only run on larger clusters.. + +2. Create a directory + + After determining where to put your example, create a directory under the appropriate parent directory. Each example + directory should contain: + + * Dockerfile + * Makefile + * README.rst + * __init__.py + * requirements.in + * sandbox.config + + It might be easier to copy one of the existing examples and modify it to your needs. + +3. Add it to CI + + Examples are references in [this github workflow](https://github.com/flyteorg/flytesnacks/blob/master/.github/workflows/ghcr_push.yml) + Add a new entry under ``strategy -> matrix -> directory`` with the name of your directory as well as its relative path. + +4. Test your code! + * If the Python code can be run locally, just use `python ` to run it. + * If the Python code has to be tested in a cluster: + * Run the `make start` command in the root directory of the flytesnacks repo + * Visit https://localhost:30081 to view the Flyte console consisting of the examples present in flytesnacks/cookbook/core directory + * To fetch new dependencies and rebuild the image, run `make register` + * If examples from a different directory (other than `core`) have to registered, run the command `SANDBOX=1 make -C register` in the `flytesnacks` repo. + +### 📝 Contribute to documentation `docs` folder in `flytesnacks` houses the user guide and tutorials present in the documentation. Refer to the [documentation contribution guide](https://docs.flyte.org/en/latest/community/contribute.html#documentation) to get acquainted with the guidelines.