tests.yml

# this is the sibling workflow to tests-skip.yml, it is required to work around
# the skipped but required checks issue:
#   https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
name: Tests

on:
  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#push
  push:
    branches:
      - main
      - feature/**
      - '[0-9].*.x'  # e.g., 3.24.x

  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request
  pull_request:

  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch
  workflow_dispatch:

  # no payload
  schedule:
      # https://crontab.guru/#37_18_*_*_*
      - cron: 37 18 * * *

concurrency:
  # Concurrency group that uses the workflow name and PR number if available
  # or commit SHA as a fallback. If a new build is triggered under that
  # concurrency group while a previous build is running it will be canceled.
  # Repeated pushes to a PR will cancel all previous builds, while multiple
  # merges to main will not cancel.
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  # detect whether any code changes are included in this PR
  changes:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      code: ${{ steps.filter.outputs.code }}
    steps:
      - uses: actions/checkout@v3
        # dorny/paths-filter needs git clone for non-PR events
        # https://github.com/marketplace/actions/paths-changes-filter#supported-workflows
        if: github.event_name != 'pull_request'
      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
        id: filter
        with:
          filters: |
            code:
              - 'conda_build/**'
              - 'tests/**'
              - '*.py'
              - 'recipe/**'
              - '.github/workflows/tests.yml'

  # linux test suite
  linux:
    # only run test suite if there are code changes
    needs: changes
    if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true'

    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        # test all lower versions (w/ stable conda) and upper version (w/ canary conda)
        python-version: ['3.9', '3.10']
        conda-version: [release]
        test-type: [serial, parallel]
        include:
          # minimum Python/conda combo
          - python-version: '3.8'
            conda-version: 22.11.0
            test-type: serial
          - python-version: '3.8'
            conda-version: 22.11.0
            test-type: parallel
          # maximum Python/conda combo
          - python-version: '3.11'
            conda-version: canary
            test-type: serial
          - python-version: '3.11'
            conda-version: canary
            test-type: parallel
    env:
      CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
      CONDA_VERSION: ${{ contains('canary,release', matrix.conda-version) && 'conda' || format('conda={0}', matrix.conda-version) }}
      REPLAY_NAME: Linux-${{ matrix.conda-version }}-Py${{ matrix.python-version }}
      REPLAY_DIR: ${{ github.workspace }}/pytest-replay
      ALLURE_DIR: ${{ github.workspace }}/allure-results
      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}
      PYTEST_NUMPROCESSES: ${{ matrix.test-type == 'serial' && 0 || 'auto' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Timestamp
        run: echo "TIMESTAMP=$(date -u "+%Y%m")" >> $GITHUB_ENV
        shell: bash

      - name: Cache conda
        uses: actions/cache@v3
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-conda-${{ env.TIMESTAMP }}

      - name: Setup miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          condarc-file: ./ci/github/.condarc
          python-version: ${{ matrix.python-version }}
          run-post: false  # skip post cleanup

      - name: Setup environment
        run: |
          conda install -q -y -c defaults \
            --file ./tests/requirements.txt \
            --file ./tests/requirements-linux.txt \
            ${{ env.CONDA_CHANNEL_LABEL }}::${{ env.CONDA_VERSION }}
          pip install -e .

      - name: Show info
        run: |
          conda info -a
          conda list --show-channel-urls

      - name: Run tests
        run: |
          pytest \
            --color=yes \
            -v \
            -n "${{ env.PYTEST_NUMPROCESSES }}" \
            --basetemp "${{ runner.temp }}/${{ matrix.test-type }}" \
            --cov conda_build \
            --cov-append \
            --cov-branch \
            --cov-report xml \
            --replay-record-dir="${{ env.REPLAY_DIR }}" \
            --replay-base-name="${{ env.REPLAY_NAME }}" \
            --alluredir="${{ env.ALLURE_DIR }}" \
            -m "${{ env.PYTEST_MARKER }}" \
            ./tests

      - uses: codecov/codecov-action@v3
        with:
          flags: ${{ matrix.test-type }},${{ matrix.python-version }},linux-64

      - name: Tar Allure Results
        if: '!cancelled()'
        run: tar -zcf "${{ env.ALLURE_DIR }}.tar.gz" "${{ env.ALLURE_DIR }}"

      - name: Upload Allure Results
        if: '!cancelled()'
        uses: actions/upload-artifact@v3
        with:
          name: allure-Linux-${{ matrix.conda-version }}-Py${{ matrix.python-version }}-${{ matrix.test-type }}
          path: allure-results.tar.gz

      - name: Upload Pytest Replay
        if: '!cancelled()'
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.REPLAY_NAME }}-${{ matrix.test-type }}
          path: ${{ env.REPLAY_DIR }}

  # windows test suite
  windows:
    # only run test suite if there are code changes
    needs: changes
    if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true'

    runs-on: windows-2019
    strategy:
      fail-fast: false
      matrix:
        # test lower version (w/ stable conda) and upper version (w/ canary conda)
        python-version: ['3.8']
        conda-version: [release]
        test-type: [serial, parallel]
        include:
          - python-version: '3.11'
            conda-version: canary
            test-type: serial
          - python-version: '3.11'
            conda-version: canary
            test-type: parallel
    env:
      CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
      REPLAY_NAME: Win-${{ matrix.conda-version }}-Py${{ matrix.python-version }}
      REPLAY_DIR: ${{ github.workspace }}\pytest-replay
      ALLURE_DIR: ${{ github.workspace }}\allure-results
      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial and not slow' }}
      PYTEST_NUMPROCESSES: ${{ matrix.test-type == 'serial' && 0 || 'auto' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Timestamp
        run: echo "TIMESTAMP=$(date -u "+%Y%m")" >> $GITHUB_ENV
        shell: bash

      - name: Cache conda
        uses: actions/cache@v3
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-conda-${{ env.TIMESTAMP }}

      - name: Setup miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          condarc-file: .\ci\github\.condarc
          python-version: ${{ matrix.python-version }}
          run-post: false  # skip post cleanup

      - name: Setup environment
        run: |
          choco install visualstudio2017-workload-vctools
          conda install -q -y -c defaults `
            --file .\tests\requirements.txt `
            --file .\tests\requirements-windows.txt `
            ${{ env.CONDA_CHANNEL_LABEL }}::conda
          pip install -e .

      - name: Show info
        run: |
          conda info -a
          conda list --show-channel-urls

      - name: Run tests
        run: |
          pytest `
            --color=yes `
            -v `
            -n "${{ env.PYTEST_NUMPROCESSES }}" `
            --basetemp "${{ runner.temp }}\${{ matrix.test-type}}" `
            --cov conda_build `
            --cov-append `
            --cov-branch `
            --cov-report xml `
            --replay-record-dir="${{ env.REPLAY_DIR }}" `
            --replay-base-name="${{ env.REPLAY_NAME }}" `
            --alluredir="${{ env.ALLURE_DIR }}" `
            -m "${{ env.PYTEST_MARKER }}" `
            .\tests

      - uses: codecov/codecov-action@v3
        with:
          flags: ${{ matrix.test-type }},${{ matrix.python-version }},win-64

      - name: Tar Allure Results
        if: '!cancelled()'
        run: tar -zcf "${{ env.ALLURE_DIR }}.tar.gz" "${{ env.ALLURE_DIR }}"
        # windows-2019/powershell ships with GNU tar 1.28 which struggles with Windows paths
        # window-2019/cmd ships with bsdtar 3.5.2 which doesn't have this problem
        shell: cmd

      - name: Upload Allure Results
        if: '!cancelled()'
        uses: actions/upload-artifact@v3
        with:
          name: allure-Win-${{ matrix.conda-version }}-Py${{ matrix.python-version }}-${{ matrix.test-type  }}
          path: allure-results.tar.gz

      - name: Upload Pytest Replay
        if: '!cancelled()'
        uses: actions/upload-artifact@v3
        with:
          path: ${{ env.REPLAY_DIR }}
          name: ${{ env.REPLAY_NAME }}-${{ matrix.test-type }}

  # macos test suite
  macos:
    # only run test suite if there are code changes
    needs: changes
    if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true'

    runs-on: macos-11
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        # test lower version (w/ stable conda) and upper version (w/ canary conda)
        python-version: ['3.8']
        conda-version: [release]
        test-type: [serial, parallel]
        include:
          - python-version: '3.11'
            conda-version: canary
            test-type: serial
          - python-version: '3.11'
            conda-version: canary
            test-type: parallel
    env:
      CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
      REPLAY_NAME: macOS-${{ matrix.conda-version }}-Py${{ matrix.python-version }}
      REPLAY_DIR: ${{ github.workspace }}/pytest-replay
      ALLURE_DIR: ${{ github.workspace }}/allure-results
      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}
      PYTEST_NUMPROCESSES: ${{ matrix.test-type == 'serial' && 0 || 'auto' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Timestamp
        run: echo "TIMESTAMP=$(date -u "+%Y%m")" >> $GITHUB_ENV
        shell: bash

      - name: Cache conda
        uses: actions/cache@v3
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-conda-${{ env.TIMESTAMP }}

      - name: Setup miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          condarc-file: ./ci/github/.condarc
          python-version: ${{ matrix.python-version }}
          run-post: false  # skip post cleanup

      - name: Setup environment
        run: |
          sudo xcode-select --switch /Applications/Xcode_11.7.app
          conda install -q -y -c defaults \
            --file ./tests/requirements.txt \
            --file ./tests/requirements-macos.txt \
            ${{ env.CONDA_CHANNEL_LABEL }}::conda
          pip install -e .

      - name: Show info
        run: |
          conda info -a
          conda list --show-channel-urls

      - name: Run tests
        run: |
          pytest \
            --color=yes \
            -v \
            -n "${{ env.PYTEST_NUMPROCESSES }}" \
            --basetemp "${{ runner.temp }}/${{ matrix.test-type }}" \
            --cov conda_build \
            --cov-append \
            --cov-branch \
            --cov-report xml \
            --replay-record-dir="${{ env.REPLAY_DIR }}" \
            --replay-base-name="${{ env.REPLAY_NAME }}" \
            --alluredir="${{ env.ALLURE_DIR }}" \
            -m "${{ env.PYTEST_MARKER }}" \
            ./tests

      - uses: codecov/codecov-action@v3
        with:
          flags: ${{ matrix.test-type }},${{ matrix.python-version }},osx-64

      - name: Tar Allure Results
        if: '!cancelled()'
        run: tar -zcf "${{ env.ALLURE_DIR }}.tar.gz" "${{ env.ALLURE_DIR }}"

      - name: Upload Allure Results
        if: '!cancelled()'
        uses: actions/upload-artifact@v3
        with:
          name: allure-macOS-${{ matrix.conda-version }}-Py${{ matrix.python-version }}-${{ matrix.test-type }}
          path: allure-results.tar.gz

      - name: Upload Pytest Replay
        if: '!cancelled()'
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.REPLAY_NAME }}-${{ matrix.test-type  }}
          path: ${{ env.REPLAY_DIR }}

  # aggregate and upload
  aggregate:
    # only aggregate test suite if there are code changes
    needs: [changes, linux, windows, macos]
    if: >-
      !cancelled()
      && (
        github.event_name == 'schedule'
        || needs.changes.outputs.code == 'true'
      )

    runs-on: ubuntu-latest
    steps:
      - name: Download test results
        uses: actions/download-artifact@v3

      - name: Upload combined test results
        # provides one downloadable archive of all .coverage/test-report.xml files
        # of all matrix runs for further analysis.
        uses: actions/upload-artifact@v3
        with:
          name: test-results-${{ github.sha }}-all
          path: test-results-${{ github.sha }}-*
          retention-days: 90  # default: 90

      - name: Test Summary
        uses: test-summary/action@v2
        with:
          paths: ./test-results-${{ github.sha }}-**/test-report*.xml

  # required check
  analyze:
    name: Analyze results
    needs: [linux, windows, macos, aggregate]
    if: '!cancelled()'

    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
        with:
          allowed-skips: ${{ toJSON(needs) }}
          jobs: ${{ toJSON(needs) }}

  # canary builds
  build:
    name: Canary Build
    needs: [analyze]
    # only build canary build if
    # - prior steps succeeded,
    # - this is the main repo, and
    # - we are on the main, feature, or release branch
    if: >-
      !cancelled()
      && !github.event.repository.fork
      && (
        github.ref_name == 'main'
        || startsWith(github.ref_name, 'feature/')
        || endsWith(github.ref_name, '.x')
      )
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            subdir: linux-64
          - runner: macos-latest
            subdir: osx-64
          - runner: windows-latest
            subdir: win-64
    runs-on: ${{ matrix.runner }}
    steps:
      # Clean checkout of specific git ref needed for package metadata version
      # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.ref }}
          clean: true
          fetch-depth: 0

      # Explicitly use Python 3.11 since each of the OSes has a different default Python
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Detect label
        shell: python
        run: |
          from pathlib import Path
          from re import match
          from os import environ

          if "${{ github.ref_name }}" == "main":
              # main branch commits are uploaded to the dev label
              label = "dev"
          elif "${{ github.ref_name }}".startswith("feature/"):
              # feature branch commits are uploaded to a custom label
              label = "${{ github.ref_name }}"
          else:
              # release branch commits are added to the rc label
              # see https://github.com/conda/infrastructure/issues/760
              _, name = "${{ github.repository }}".split("/")
              label = f"rc-{name}-${{ github.ref_name }}"

          Path(environ["GITHUB_ENV"]).write_text(f"ANACONDA_ORG_LABEL={label}")

      - name: Create and upload canary build
        uses: conda/actions/canary-release@v23.7.0
        with:
          package-name: ${{ github.event.repository.name }}
          subdir: ${{ matrix.subdir }}
          anaconda-org-channel: conda-canary
          anaconda-org-label: ${{ env.ANACONDA_ORG_LABEL }}
          anaconda-org-token: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }}