checks.yml

name: Master

on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
  pull_request:
    paths-ignore:
      - "docs/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ "3.11" ]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements files
          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r dev-requirements.txt
      - name: Pip info
        run: python -m pip list
      - name: Lint
        run: pre-commit run --all --show-diff-on-failure
  # This is the build system for the new example directory structure
  list_examples:
    runs-on: ubuntu-latest
    name: "Create a list of example packages"
    steps:
      - uses: actions/checkout@v2
      - name: "Provide the list"
        id: create-example-list
        run: |
          # TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc)
          echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \
          | grep -v -e 'testing' -e 'examples' \
          | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'mnist_classifier' \
          | sort \
          | jq --raw-input . \
          | jq --slurp . \
          | jq -c .)" >> "$GITHUB_OUTPUT"
    outputs:
      packages: "${{ steps.create-example-list.outputs.PACKAGES }}"
  serialize:
    needs: [list_examples]
    runs-on: "ubuntu-latest"
    env:
      FLYTE_SDK_RICH_TRACEBACKS: "0"
    strategy:
      fail-fast: false
      matrix:
        example: "${{ fromJson(needs.list_examples.outputs.packages) }}"
    steps:
      - uses: actions/checkout@v4
      - uses: insightsengineering/disk-space-reclaimer@v1
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: Cache pip
        uses: actions/cache@v3
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements files
          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
      - name: Install dependencies
        working-directory: examples/${{ matrix.example }}
        run: |
          pip install uv
          uv venv $GITHUB_WORKSPACE/.venv
          source $GITHUB_WORKSPACE/.venv/bin/activate
          if [ -f requirements.in ]; then uv pip install -r requirements.in; fi
          uv pip install "flytekit>=1.12.2" "numpy<2.0.0"
          pip freeze
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Container Registry
        if: ${{ github.event_name != 'pull_request' }}
        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ secrets.FLYTE_BOT_USERNAME }}
          password: ${{ secrets.FLYTE_BOT_PAT }}
      - name: Check if dockerfile exists
        working-directory: examples/${{ matrix.example }}
        id: dockerfile
        run: |
          if [ -f Dockerfile ]
          then
            echo "exist=true" >> "$GITHUB_OUTPUT"
          else
            echo "exist=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Build and push default image
        if: ${{ steps.dockerfile.outputs.exist == 'true' }}
        uses: docker/build-push-action@v5
        with:
          context: examples/${{ matrix.example }}
          platforms: linux/amd64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }},ghcr.io/flyteorg/flytecookbook:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max
      - name: Pyflyte package
        working-directory: examples/${{ matrix.example }}
        run: |
          export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }}
          default_image=ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }}
          source $GITHUB_WORKSPACE/.venv/bin/activate
          pyflyte \
          --pkgs ${{ matrix.example }} package \
          --image $default_image \
          --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
          --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
          --output flyte-package.tgz \
          --force
          tar -xvf flyte-package.tgz
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: snacks-examples-${{ matrix.example }}
          path: examples/${{ matrix.example }}/**/*.pb

  # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them.
  register:
    name: Register example to sandbox
    runs-on: ubuntu-latest
    needs: [ "serialize" ]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - uses: unionai/flytectl-setup-action@v0.0.3
      - name: setup download artifact dir
        run: |
          mkdir download-artifact
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./download-artifact/
      - name: setup sandbox
        run: |
          flytectl demo start
          flytectl config init
      - name: Register examples
        run: |
          flytectl register files ./download-artifact/**/* -p flytesnacks -d development --version latest

  bump_version:
    name: Bump Version
    if: ${{ github.event_name != 'pull_request' }}
    needs: [ serialize ] # Only to ensure it can successfully build
    uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
    secrets:
      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

  prerelease:
    name: Create Prerelease
    needs: [ bump_version ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: Create Pre release
        id: prerelease
        run: |
          RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \
          --header 'Accept: application/vnd.github.v3+json' \
          --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
          --data-raw '{
              "tag_name": "'${{ needs.bump-version.outputs.version }}'",
              "prerelease": true
          }' | jq -r '.id')
          echo ::set-output name=release_id::$RELEASE_ID
    outputs:
      release_id: ${{ steps.prerelease.outputs.release_id }}

  # Download artifacts again and push them to the release only if this is not a pull request
  release_workflow:
    name: Publish artifacts to github release
    runs-on: ubuntu-latest
    needs: [ prerelease ]
    strategy:
      matrix:
        python-version: [ "3.11" ]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: setup download artifact dir
        run: |
          mkdir download-artifact
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          path: ./download-artifact/
      - name: Package Examples
        run: |
          mkdir -p release-snacks
          cd download-artifact
          for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait
          cd .. && sudo rm -rf download-artifact/
          cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json
          cp flyte_tests.txt release-snacks/flyte_tests.txt
      - name: Release test manifest
        uses: goreleaser/goreleaser-action@v5
        with:
          version: latest
          args: release --rm-dist
        env:
          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
          GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }}

  make_release:
    name: Mark github pre-release as Release
    needs: [ release_workflow ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - name: Update Release
        id: release
        run: |
          curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \
          --header 'Accept: application/vnd.github.v3+json' \
          --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
          --data-raw '{
              "tag_name": "'${{ needs.bump-version.outputs.version }}'",
              "prerelease": false
          }'

  e2e-tests:
    runs-on: ubuntu-latest
    env:
      FLYTESNACKS_PRIORITIES: "P0"
      FLYTESNACKS_VERSION: ""
    timeout-minutes: 30
    steps:
      - uses: insightsengineering/disk-space-reclaimer@v1
      - name: Set latest Flytesnacks release
        if: ${{ env.FLYTESNACKS_VERSION == '' }}
        run: |
          FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
          echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - uses: unionai/flytectl-setup-action@v0.0.3
      - name: Setup sandbox
        run: |
          mkdir -p ~/.flyte/sandbox
          cat << EOF > ~/.flyte/sandbox/config.yaml
          task_resources:
            defaults:
              cpu: "0"
              memory: "0"
            limits:
              cpu: "0"
              memory: "0"
          EOF
          flytectl demo start --imagePullPolicy Never
      - name: Install Python dependencies
        run: |
          pip install uv
          uv venv
          source .venv/bin/activate
          uv pip install "flytekit>=1.12.2" flytekitplugins-deck-standard torch tabulate pyarrow
          pip freeze
      - name: Checkout flytesnacks
        uses: actions/checkout@v3
        with:
          repository: flyteorg/flytesnacks
          path: flytesnacks
      - name: Verify existence of the tests
        run: |
          source .venv/bin/activate
          python flyte_tests_validate.py
      - name: Register specific tests
        run: |
          source .venv/bin/activate
          export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }}
          while read -r line;
          do
              pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                  register \
                  --project flytesnacks \
                  --domain development \
                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                  --version ${{ env.FLYTESNACKS_VERSION }} \
                  flytesnacks/$line;
          done < flyte_tests.txt
      - name: Setup tmate session
        if: ${{ failure() }}
        uses: mxschmitt/action-tmate@v3