nightly-bnb.yml

name: BNB from source self-hosted runner with slow tests (scheduled)

on:
  workflow_dispatch:
  schedule:
    - cron: "0 2 * * *"

env:
  RUN_SLOW: "yes"
  IS_GITHUB_CI: "1"
  # To be able to run tests on CUDA 12.2
  NVIDIA_DISABLE_REQUIRE: "1"
  SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}


jobs:
  run_all_tests_single_gpu:
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
          docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"]
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
      CUDA_VISIBLE_DEVICES: "0"
      TEST_TYPE: "single_gpu_${{ matrix.docker-image-name }}"
    container:
      image: ${{ matrix.docker-image-name }}
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v3
      - name: Pip install
        run: |
          source activate peft
          pip install -e . --no-deps
          pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
          pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26
          mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone # rename to transformers clone to avoid modules conflict
          if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
            cd transformers-clone
            transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
            echo "Checking out tag for Transformers version: v$transformers_version"
            git fetch --tags
            git checkout tags/v$transformers_version
            cd ..
          fi

      - name: Test bnb import
        id: import
        if: always()
        run: |
          source activate peft
          python3 -m bitsandbytes
          python3 -c "import bitsandbytes as bnb"

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes import
          status: ${{ steps.import.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Run examples on single GPU
        id: examples_tests
        if: always()
        run: |
          source activate peft
          make tests_examples_single_gpu_bnb

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes examples tests - single GPU
          status: ${{ steps.examples_tests.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Run core tests on single GPU
        id: core_tests
        if: always()
        run: |
          source activate peft
          make tests_core_single_gpu_bnb

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes core tests - single GPU
          status: ${{ steps.core_tests.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      # TODO: this is a test to see if BNB multi-backend single-GPU tests succeed w/o regression tests
      # - name: Run BNB regression tests on single GPU
      #   id: regression_tests
      #   if: always()
      #   run: |
      #     source activate peft
      #     make tests_gpu_bnb_regression

      # - name: Post to Slack
      #   if: always()
      #   uses: huggingface/hf-workflows/.github/actions/post-slack@main
      #   with:
      #     slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
      #     title: 🤗 Results of bitsandbytes regression tests - single GPU
      #     status: ${{ steps.regression_tests.outcome }}
      #     slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Run transformers tests on single GPU
        id: transformers_tests
        if: always()
        run: |
          source activate peft
          make transformers_tests

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes transformers tests - single GPU
          status: ${{ steps.transformers_tests.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Generate Report
        if: always()
        run: |
          pip install slack_sdk tabulate
          python scripts/log_reports.py --slack_channel_name bnb-daily-ci-collab >> $GITHUB_STEP_SUMMARY

  run_all_tests_multi_gpu:
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"]
    runs-on:
      group: aws-g6-12xlarge-plus
    env:
      CUDA_VISIBLE_DEVICES: "0,1"
      TEST_TYPE: "multi_gpu_${{ matrix.docker-image-name }}"
    container:
      image: ${{ matrix.docker-image-name }}
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v3
      - name: Pip install
        run: |
          source activate peft
          pip install -e . --no-deps
          pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
          pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26
          mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone
          if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
            cd transformers-clone
            transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
            echo "Checking out tag for Transformers version: v$transformers_version"
            git fetch --tags
            git checkout tags/v$transformers_version
            cd ..
          fi

      - name: Test bnb import
        id: import
        if: always()
        run: |
          source activate peft
          python3 -m bitsandbytes
          python3 -c "import bitsandbytes as bnb"

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes import
          status: ${{ steps.import.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Run core GPU tests on multi-gpu
        if: always()
        run: |
          source activate peft

      - name: Run examples on multi GPU
        id: examples_tests
        if: always()
        run: |
          source activate peft
          make tests_examples_multi_gpu_bnb

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes examples tests - multi GPU
          status: ${{ steps.examples_tests.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Run core tests on multi GPU
        id: core_tests
        if: always()
        run: |
          source activate peft
          make tests_core_multi_gpu_bnb

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes core tests - multi GPU
          status: ${{ steps.core_tests.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Run transformers tests on multi GPU
        id: transformers_tests
        if: always()
        run: |
          source activate peft
          make transformers_tests

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
          title: 🤗 Results of bitsandbytes transformers tests - multi GPU
          status: ${{ steps.transformers_tests.outcome }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

      - name: Generate Report
        if: always()
        run: |
          pip install slack_sdk tabulate
          python scripts/log_reports.py --slack_channel_name bnb-daily-ci-collab >> $GITHUB_STEP_SUMMARY