nightly-ci.yml

name: nightly-ci

on:
  schedule:
    - cron: '00 08 * * *'  # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST.

# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532
#concurrency:
#  group: pr-integration-tests-${{ github.event.pull_request.number }}
#  cancel-in-progress: true

jobs:
  check_date:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    name: Check latest commit
    outputs:
      WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
      - id: check_date
        name: Check if there were commits in the last day
        if: ${{ github.event_name == 'schedule' }}
        run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
  cleanup_dynamo_tables:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    name: Cleanup Bigtable / Dynamo tables which can fail to cleanup
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: "3.11"
          architecture: x64
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Install Python dependencies
        run: |
          pip install boto3
          pip install google-cloud-bigtable
          pip install tqdm
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Run DynamoDB / Bigtable cleanup script
        run: python infra/scripts/cleanup_ci.py
  integration-test-python:
    if: github.repository == 'feast-dev/feast'
    needs: [check_date, cleanup_dynamo_tables]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.11" ]
        os: [ ubuntu-latest ]
    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
          submodules: recursive
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - name: Setup Go
        id: setup-go
        uses: actions/setup-go@v2
        with:
          go-version: 1.18.0
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Use AWS CLI
        run: aws sts get-caller-identity
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
      - name: Install apache-arrow on ubuntu
        if: matrix.os == 'ubuntu-latest'
        run: |
            sudo apt update
            sudo apt install -y -V ca-certificates lsb-release wget
            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
            sudo apt update
            sudo apt install -y -V libarrow-dev
      - name: Install apache-arrow on macos
        if: matrix.os == 'macos-13'
        run: brew install apache-arrow
      - name: Install dependencies
        run: make install-python-ci-dependencies-uv
      - name: Setup Redis Cluster
        run: |
          docker pull vishnunair/docker-redis-cluster:latest
          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
      - name: Test python
        if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
        env:
          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
        run: make test-python-integration