benchmarks.yml

name: Continuous Benchmark
on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - "*"
  workflow_dispatch:

permissions:
  deployments: write
  contents: write

concurrency:
  # Documentation suggests ${{ github.head_ref }}, but that's only available on pull_request/pull_request_target triggers, so using ${{ github.ref }}.
  # On master, we want all builds to complete even if merging happens faster to make it easier to discover at which point something broke.
  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && format('ci-master-{0}', github.sha) || format('ci-{0}', github.ref) }}
  cancel-in-progress: true

jobs:

  benchmark_cpu:
    name: CPU Pytest benchmark
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Setup Environment
        run: |
          python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu 
          python -m pip install git+https://github.com/pytorch/tensordict 
          python setup.py develop
          python -m pip install pytest pytest-benchmark
          python -m pip install dm_control
      - name: Run benchmarks
        run: |
          cd benchmarks/
          python -m pytest --benchmark-json output.json
      - name: Store benchmark results
        if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: CPU Benchmark Results
          tool: 'pytest'
          output-file-path: benchmarks/output.json
          fail-on-alert: true
          alert-threshold: '200%'
          alert-comment-cc-users: '@vmoens'
          comment-on-alert: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          gh-pages-branch: gh-pages
          auto-push: true


  benchmark_gpu:
    name: GPU Pytest benchmark
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        include:
         - os: linux.4xlarge.nvidia.gpu
           python-version: 3.8
    defaults:
      run:
        shell: bash -l {0}
    container: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04
    steps:
    - name: Install deps
      run: |
        export TZ=Europe/London
        export DEBIAN_FRONTEND=noninteractive  # tzdata bug
        apt-get update -y
        apt-get install software-properties-common -y
        add-apt-repository ppa:git-core/candidate -y
        apt-get update -y
        apt-get upgrade -y
        apt-get -y install libglu1-mesa libgl1-mesa-glx libosmesa6 gcc curl g++ unzip wget libglfw3-dev libgles2-mesa-dev libglew-dev sudo git cmake libz-dev
    - name: Check ldd --version
      run: ldd --version
    - name: Checkout
      uses: actions/checkout@v3
    - name: Update pip
      run: |
        apt-get install python3.8 python3-pip -y
        pip3 install --upgrade pip
    - name: Setup git
      run: git config --global --add safe.directory /__w/rl/rl
    - name: setup Path
      run: |
        echo /usr/local/bin >> $GITHUB_PATH
    - name: Setup Environment
      run: |
        python3 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu118 
        python3 -m pip install git+https://github.com/pytorch/tensordict 
        python3 setup.py develop
        python3 -m pip install pytest pytest-benchmark
        python3 -m pip install dm_control
    - name: Run benchmarks
      run: |
        cd benchmarks/
        python3 -m pytest --benchmark-json output.json
    - name: Store benchmark results
      uses: benchmark-action/github-action-benchmark@v1
      if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
      with:
        name: GPU Benchmark Results
        tool: 'pytest'
        output-file-path: benchmarks/output.json
        fail-on-alert: true
        alert-threshold: '200%'
        alert-comment-cc-users: '@vmoens'
        comment-on-alert: true
        github-token: ${{ secrets.GITHUB_TOKEN }}
        gh-pages-branch: gh-pages
        auto-push: true