add ImageSpec in raw container as example #4006
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Master | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "docs/**" | |
| pull_request: | |
| paths-ignore: | |
| - "docs/**" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: [ "3.11" ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v2 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements files | |
| key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r dev-requirements.txt | |
| - name: Pip info | |
| run: python -m pip list | |
| - name: Lint | |
| run: pre-commit run --all --show-diff-on-failure | |
| # This is the build system for the new example directory structure | |
| list_examples: | |
| runs-on: ubuntu-latest | |
| name: "Create a list of example packages" | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: "Provide the list" | |
| id: create-example-list | |
| run: | | |
| # TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc) | |
| echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | |
| | grep -v -e 'testing' -e 'examples' \ | |
| | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'mnist_classifier' \ | |
| | sort \ | |
| | jq --raw-input . \ | |
| | jq --slurp . \ | |
| | jq -c .)" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| packages: "${{ steps.create-example-list.outputs.PACKAGES }}" | |
| serialize: | |
| needs: [list_examples] | |
| runs-on: "ubuntu-latest" | |
| env: | |
| FLYTE_SDK_RICH_TRACEBACKS: "0" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| example: "${{ fromJson(needs.list_examples.outputs.packages) }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: insightsengineering/disk-space-reclaimer@v1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Cache pip | |
| uses: actions/cache@v3 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements files | |
| key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} | |
| - name: Install dependencies | |
| working-directory: examples/${{ matrix.example }} | |
| run: | | |
| pip install uv | |
| uv venv $GITHUB_WORKSPACE/.venv | |
| source $GITHUB_WORKSPACE/.venv/bin/activate | |
| if [ -f requirements.in ]; then uv pip install -r requirements.in; fi | |
| uv pip install "flytekit>=1.12.2" "numpy<2.0.0" | |
| pip freeze | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| if: ${{ github.event_name != 'pull_request' }} | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.FLYTE_BOT_USERNAME }} | |
| password: ${{ secrets.FLYTE_BOT_PAT }} | |
| - name: Check if dockerfile exists | |
| working-directory: examples/${{ matrix.example }} | |
| id: dockerfile | |
| run: | | |
| if [ -f Dockerfile ] | |
| then | |
| echo "exist=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "exist=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Build and push default image | |
| if: ${{ steps.dockerfile.outputs.exist == 'true' }} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: examples/${{ matrix.example }} | |
| platforms: linux/amd64 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }},ghcr.io/flyteorg/flytecookbook:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Pyflyte package | |
| working-directory: examples/${{ matrix.example }} | |
| run: | | |
| export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }} | |
| default_image=ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }} | |
| source $GITHUB_WORKSPACE/.venv/bin/activate | |
| pyflyte \ | |
| --pkgs ${{ matrix.example }} package \ | |
| --image $default_image \ | |
| --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ | |
| --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ | |
| --output flyte-package.tgz \ | |
| --force | |
| tar -xvf flyte-package.tgz | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: snacks-examples-${{ matrix.example }} | |
| path: examples/${{ matrix.example }}/**/*.pb | |
| # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them. | |
| register: | |
| name: Register example to sandbox | |
| runs-on: ubuntu-latest | |
| needs: [ "serialize" ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: "0" | |
| - uses: unionai/[email protected] | |
| - name: setup download artifact dir | |
| run: | | |
| mkdir download-artifact | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./download-artifact/ | |
| - name: setup sandbox | |
| run: | | |
| flytectl demo start | |
| flytectl config init | |
| - name: Register examples | |
| run: | | |
| flytectl register files ./download-artifact/**/* -p flytesnacks -d development --version latest | |
| bump_version: | |
| name: Bump Version | |
| if: ${{ github.event_name != 'pull_request' }} | |
| needs: [ serialize ] # Only to ensure it can successfully build | |
| uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master | |
| secrets: | |
| FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | |
| prerelease: | |
| name: Create Prerelease | |
| needs: [ bump_version ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: "0" | |
| - name: Create Pre release | |
| id: prerelease | |
| run: | | |
| RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \ | |
| --header 'Accept: application/vnd.github.v3+json' \ | |
| --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ | |
| --data-raw '{ | |
| "tag_name": "'${{ needs.bump-version.outputs.version }}'", | |
| "prerelease": true | |
| }' | jq -r '.id') | |
| echo ::set-output name=release_id::$RELEASE_ID | |
| outputs: | |
| release_id: ${{ steps.prerelease.outputs.release_id }} | |
| # Download artifacts again and push them to the release only if this is not a pull request | |
| release_workflow: | |
| name: Publish artifacts to github release | |
| runs-on: ubuntu-latest | |
| needs: [ prerelease ] | |
| strategy: | |
| matrix: | |
| python-version: [ "3.11" ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: setup download artifact dir | |
| run: | | |
| mkdir download-artifact | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./download-artifact/ | |
| - name: Package Examples | |
| run: | | |
| mkdir -p release-snacks | |
| cd download-artifact | |
| for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait | |
| cd .. && sudo rm -rf download-artifact/ | |
| cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json | |
| cp flyte_tests.txt release-snacks/flyte_tests.txt | |
| - name: Release test manifest | |
| uses: goreleaser/goreleaser-action@v5 | |
| with: | |
| version: latest | |
| args: release --rm-dist | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} | |
| GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }} | |
| make_release: | |
| name: Mark github pre-release as Release | |
| needs: [ release_workflow ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: "0" | |
| - name: Update Release | |
| id: release | |
| run: | | |
| curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \ | |
| --header 'Accept: application/vnd.github.v3+json' \ | |
| --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ | |
| --data-raw '{ | |
| "tag_name": "'${{ needs.bump-version.outputs.version }}'", | |
| "prerelease": false | |
| }' | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| FLYTESNACKS_PRIORITIES: "P0" | |
| FLYTESNACKS_VERSION: "" | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: insightsengineering/disk-space-reclaimer@v1 | |
| - name: Set latest Flytesnacks release | |
| if: ${{ env.FLYTESNACKS_VERSION == '' }} | |
| run: | | |
| FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" | |
| echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - uses: unionai/[email protected] | |
| - name: Setup sandbox | |
| run: | | |
| mkdir -p ~/.flyte/sandbox | |
| cat << EOF > ~/.flyte/sandbox/config.yaml | |
| task_resources: | |
| defaults: | |
| cpu: "0" | |
| memory: "0" | |
| limits: | |
| cpu: "0" | |
| memory: "0" | |
| EOF | |
| flytectl demo start --imagePullPolicy Never | |
| - name: Install Python dependencies | |
| run: | | |
| pip install uv | |
| uv venv | |
| source .venv/bin/activate | |
| uv pip install "flytekit>=1.12.2" flytekitplugins-deck-standard torch tabulate pyarrow pandas | |
| pip freeze | |
| - name: Checkout flytesnacks | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: flyteorg/flytesnacks | |
| path: flytesnacks | |
| - name: Verify existence of the tests | |
| run: | | |
| source .venv/bin/activate | |
| python flyte_tests_validate.py | |
| - name: Register specific tests | |
| run: | | |
| source .venv/bin/activate | |
| export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }} | |
| while read -r line; | |
| do | |
| pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ | |
| register \ | |
| --project flytesnacks \ | |
| --domain development \ | |
| --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ | |
| --version ${{ env.FLYTESNACKS_VERSION }} \ | |
| flytesnacks/$line; | |
| done < flyte_tests.txt | |
| - name: Setup tmate session | |
| if: ${{ failure() }} | |
| uses: mxschmitt/action-tmate@v3 |