checks_new.yml

name: Master

on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
  pull_request:
    paths-ignore:
      - "docs/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  # This is the build system for the new example directory structure
  list_examples:
    runs-on: ubuntu-latest
    name: "Create a list of example packages"
    steps:
      - uses: actions/checkout@v2
      - name: "Provide the list"
        id: create-example-list
        run: |
          echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \
          | grep -v -e 'testing' -e 'examples' \
          | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' \
          | sort \
          | jq --raw-input . \
          | jq --slurp . \
          | jq -c .)" >> "$GITHUB_OUTPUT"
    outputs:
      packages: "${{ steps.create-example-list.outputs.PACKAGES }}"
  serialize:
    needs: [list_examples]
    runs-on: "ubuntu-latest"
    env:
      FLYTE_SDK_RICH_TRACEBACKS: "0"
      PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
    strategy:
      fail-fast: false
      matrix:
        example: "${{ fromJson(needs.list_examples.outputs.packages) }}"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: Cache pip
        uses: actions/cache@v3
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements files
          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
      - name: Install dependencies
        working-directory: examples/${{ matrix.example }}
        run: |
          pip install uv
          uv venv
          source .venv/bin/activate
          uv pip install flytekit flytekitplugins-envd
          if [ -f requirements.in ]; then uv pip install -r requirements.in; fi
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v1
        with:
          registry: ghcr.io
          username: ${{ secrets.FLYTE_BOT_USERNAME }}
          password: ${{ secrets.FLYTE_BOT_PAT }}
      - name: Pyflyte package
        working-directory: examples/${{ matrix.example }}
        run: |
          source .venv/bin/activate
          pyflyte \
          --pkgs ${{ matrix.example }} package \
          --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
          --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
          --output flyte-package.tgz \
          --force
          tar -xvf flyte-package.tgz
      - name: Upload artifacts
        uses: actions/upload-artifact@v2
        with:
          name: snacks-${{ matrix.example }}
          path: examples/${{ matrix.example }}/**/*.pb

  # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them.
  register:
    name: Register example to sandbox
    runs-on: ubuntu-latest
    needs: [ "serialize" ]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"
      - uses: unionai/flytectl-setup-action@v0.0.1
      - name: setup download artifact dir
        run: |
          mkdir download-artifact
      - name: Download artifacts
        uses: actions/download-artifact@v2
        with:
          path: ./download-artifact/
      - name: setup sandbox
        run: |
          flytectl demo start
          flytectl config init
      - name: Register examples
        uses: unionai/flyte-register-action@v0.0.2
        with:
          flytesnacks: false
          proto: ./download-artifact/**/*
          project: flytesnacks
          version: "latest"
          domain: development