diff --git a/.github/workflows/ghcr_push.yml b/.github/workflows/ghcr_push.yml index dfd733e478..0f52d4d11b 100644 --- a/.github/workflows/ghcr_push.yml +++ b/.github/workflows/ghcr_push.yml @@ -58,4 +58,4 @@ jobs: registry: ghcr.io build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ matrix.directory.name }}-${{ github.sha }}" context: ./cookbook/${{ matrix.directory.path }} - dockerfile: ${{ matrix.directory.name }}/Dockerfile + dockerfile: ${{ matrix.directory.name }}/Dockerfile \ No newline at end of file diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e2a602b195..ccc2907c09 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -22,64 +22,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} WITH_V: true - DEFAULT_BUMP: patch - examples-releaser: - name: Release Examples - strategy: - matrix: - directory: [ "core", "integrations", "case_studies" ] - runs-on: ubuntu-latest - needs: [ bump-version ] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-${{ matrix.directory }}/ - key: release-snacks-${{ matrix.directory }} - - name: Generate Serialize proto - env: - REGISTRY: "ghcr.io/flyteorg" - run: | - make -C cookbook/${{ matrix.directory }} serialize - mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/ - make -C cookbook archive - cp -r cookbook/release-snacks/* /tmp/release-snacks-${{ matrix.directory }}/ - goreleaser: - name: Release Case studies example - runs-on: ubuntu-latest - needs: [ examples-releaser ] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-core/ - key: release-snacks-core - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-case-studies/ - key: release-snacks-case-studies - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-integrations/ - key: release-snacks-plugins - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - ref: ${{ steps.bump-version.outputs.tag }} - - name: Copy artifacts - run: | - mkdir -p cookbook/release-snacks - cp -r /tmp/release-snacks-core/* cookbook/release-snacks/ - cp -r /tmp/release-snacks-case-studies/* cookbook/release-snacks/ - cp -r /tmp/release-snacks-integrations/* cookbook/release-snacks/ - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} \ No newline at end of file + DEFAULT_BUMP: patch \ No newline at end of file diff --git a/.github/workflows/release_examples.yml b/.github/workflows/release_examples.yml new file mode 100644 index 0000000000..46c7ddf692 --- /dev/null +++ b/.github/workflows/release_examples.yml @@ -0,0 +1,78 @@ +name: Serialize & Release Examples + +on: + pull_request: + paths-ignore: + - 'cookbook/docs/**' + - 'cookbook/deployment/**' + create: + tags: + - v* + +jobs: + serialize-example: + name: Serialize + strategy: + matrix: + directory: [ "core", "integrations", "case_studies" ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - uses: actions/cache@v1 + if: github.event_name != 'pull_request' + with: + path: /tmp/release-snacks-${{ matrix.directory }}/ + key: release-snacks-${{ matrix.directory }}-${{ github.sha }} + - name: Serialize Examples + env: + REGISTRY: "ghcr.io/flyteorg" + run: | + REGISTRY=ghcr.io/flyteorg VERSION=latest make -C cookbook/${{ matrix.directory }} serialize + - name: Cache Serialized protos + if: github.event_name != 'pull_request' + env: + REGISTRY: "ghcr.io/flyteorg" + run: | + mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/ + make -C cookbook archive + cp -r /tmp/release-snacks-${{ matrix.directory }}/* cookbook/release-snacks/ + releaser: + name: Release example + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + needs: [ serialize-example ] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - uses: actions/cache@v1 + with: + path: /tmp/release-snacks-core/ + key: release-snacks-core-${{ github.sha }} + - uses: actions/cache@v1 + with: + path: /tmp/release-snacks-case_studies/ + key: release-snacks-case_studies-${{ github.sha }} + - uses: actions/cache@v1 + with: + path: /tmp/release-snacks-integrations/ + key: release-snacks-integrations-${{ github.sha }} + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + ref: ${{ steps.bump-version.outputs.tag }} + - name: Copy artifacts + run: | + mkdir -p cookbook/release-snacks + cp -r /tmp/release-snacks-core/* cookbook/release-snacks/ + cp -r /tmp/release-snacks-case_studies/* cookbook/release-snacks/ + cp -r /tmp/release-snacks-integrations/* cookbook/release-snacks/ + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} \ No newline at end of file diff --git a/.github/workflows/sandbox_register.yml b/.github/workflows/sandbox_register.yml index 26beee48f3..984cf1801d 100644 --- a/.github/workflows/sandbox_register.yml +++ b/.github/workflows/sandbox_register.yml @@ -21,17 +21,8 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: "0" - - run: docker pull ghcr.io/flyteorg/flyte-sandbox:dind + - uses: unionai/flytectl-setup-action@v0.0.1 - name: setup sandbox - run: make setup - - name: wait for sandbox - run: | - n=0 - until [ "$n" -ge 5 ]; do - echo "try $n" - make wait && break - n=$((n+1)) - sleep 1 - done + run: flytectl sandbox start --sourcesPath=$GITHUB_WORKSPACE - name: fast register - run: REGISTRY=ghcr.io/flyteorg VERSION=latest make fast_register + run: REGISTRY=ghcr.io/flyteorg VERSION=latest make fast_register \ No newline at end of file diff --git a/.github/workflows/serialize_examples.yml b/.github/workflows/serialize_examples.yml deleted file mode 100644 index b27641c556..0000000000 --- a/.github/workflows/serialize_examples.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Serialize Examples - -on: - pull_request: - paths-ignore: - - 'cookbook/docs/**' - - 'cookbook/deployment/**' - -jobs: - core-build: - name: Serialize Examples - strategy: - matrix: - directory: [ "core", "integrations", "case_studies" ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/cache@v1 - with: - path: /tmp/release-snacks-${{ matrix.directory }}/ - key: release-snacks-${{ matrix.directory }} - - name: Generate Serialize proto - env: - REGISTRY: "ghcr.io/flyteorg" - run: | - make -C cookbook/${{ matrix.directory }} serialize diff --git a/Makefile b/Makefile index dd0b6a57d2..98e0d7c97e 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ FLYTE_PROXY_PORT := 30081 K8S_DASHBOARD_PROXY_PORT := 30082 MINIO_PROXY_PORT := 30084 FLYTE_SANDBOX_NAME := flyte-sandbox -KUBE_CONFIG := ~/kubeconfig +FLYTE_DIR := ~/.flyte # Module of cookbook examples to register EXAMPLES_MODULE := core @@ -33,6 +33,7 @@ update_boilerplate: @curl https://raw.githubusercontent.com/flyteorg/boilerplate/master/boilerplate/update.sh -o boilerplate/update.sh @boilerplate/update.sh + .PHONY: help help: ## Show help message @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -47,25 +48,10 @@ endif .PHONY: setup setup: $(call LOG,Starting Flyte sandbox) - docker run -d --rm --privileged --name $(FLYTE_SANDBOX_NAME) \ - -e SANDBOX=1 \ - -e KUBERNETES_API_PORT=$(KUBERNETES_API_PORT) \ - -e FLYTE_HOST=localhost:30081 \ - -e FLYTE_AWS_ENDPOINT=http://localhost:30084/ \ - -v $(CURDIR):/usr/src \ - -v $(KUBE_CONFIG):/etc/rancher/ \ - -p $(KUBERNETES_API_PORT):$(KUBERNETES_API_PORT) \ - -p $(FLYTE_PROXY_PORT):30081 \ - -p $(K8S_DASHBOARD_PROXY_PORT):30082 \ - -p $(MINIO_PROXY_PORT):30084 \ - cr.flyte.org/flyteorg/flyte-sandbox:dind > /dev/null - -.PHONY: wait -wait: - $(call RUN_IN_SANDBOX, wait-for-flyte.sh) + flytectl sandbox start --sourcesPath:=$(shell pwd) .PHONY: start -start: setup wait ## Start a local Flyte sandbox +start: setup $(call LOG,Registering examples from commit: latest) REGISTRY=cr.flyte.org/flyteorg VERSION=latest $(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) fast_register) @@ -74,7 +60,7 @@ start: setup wait ## Start a local Flyte sandbox .PHONY: teardown teardown: _requires-sandbox-up ## Teardown Flyte sandbox $(call LOG,Tearing down Flyte sandbox) - docker rm -f -v $(FLYTE_SANDBOX_NAME) > /dev/null ||: + flytectl sandbox teardown .PHONY: status status: _requires-sandbox-up ## Show status of Flyte deployment @@ -95,7 +81,7 @@ fast_register: _requires-sandbox-up ## Fast register Flyte cookbook workflows $(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) fast_register) .PHONY: setup-kubectl -kubectl-config: +kubectl-config: # In shell/bash, run: `eval $(make kubectl-config)` # Makefiles run recipes in sub-processes. A sub-process cannot modify the parent process's environment. # The best I (@EngHabu) can think of at the moment is to output this for the user to eval in the diff --git a/cookbook/Makefile b/cookbook/Makefile index d03dc71dd9..0a1dfc5d17 100644 --- a/cookbook/Makefile +++ b/cookbook/Makefile @@ -31,4 +31,8 @@ run-examples: ## Runs all examples .PHONY: archive archive: # archive directories (e.g. _pb_output/) - sh scripts/archive-all-example.sh + bash scripts/ci_helper.sh RELEASE + +.PHONY: register-workflow-ci +register-workflow-ci: + bash scripts/ci_helper.sh REGISTER diff --git a/cookbook/flyte_tests_manifest.json b/cookbook/flyte_tests_manifest.json index 8a29be17fd..b3001891a7 100644 --- a/cookbook/flyte_tests_manifest.json +++ b/cookbook/flyte_tests_manifest.json @@ -81,7 +81,7 @@ },{ "name": "case-studies-pima-diabetes", "priority": "P2", - "path": "case_studies/ml_training/pima-diabetes", + "path": "case_studies/ml_training/pima_diabetes", "exitCondition": { "exit_success": true, "exit_message": "" diff --git a/cookbook/scripts/archive-all-example.sh b/cookbook/scripts/archive-all-example.sh deleted file mode 100755 index 3b29daebfb..0000000000 --- a/cookbook/scripts/archive-all-example.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e - -for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do - if [ -d "./$(echo ${row} | jq -r '.path')/_pb_output/" ]; then - tar -cvzf "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz" "./$(echo ${row} | jq -r '.path')/_pb_output/" - fi -done \ No newline at end of file diff --git a/cookbook/scripts/ci_helper.sh b/cookbook/scripts/ci_helper.sh new file mode 100755 index 0000000000..23443ed085 --- /dev/null +++ b/cookbook/scripts/ci_helper.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +release_example() { + for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do + if [ -d "./$(echo ${row} | jq -r '.path')/_pb_output/" ]; then + tar -cvzf "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz" "./$(echo ${row} | jq -r '.path')/_pb_output/" + fi + done +} + +register_example() { + for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do + flytectl register file "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz" -p flytesnacks -d development --archive + done +} + +if [ "$1" == "RELEASE" ]; then + release_example +elif [ $1 == "REGISTER" ]; then + register_example +fi \ No newline at end of file