dispatch.yml

# Trigger the execution of workflow in batches.
# This workflow is needed since GitHub Actions limits the matrix size to 256 jobs.
# We use one job per repository per batch.

name: Dispatch

on:
  workflow_dispatch:
    inputs:
      workflow:
        description: "Workflow to dispatch"
        required: true
      inputs:
        description: "Workflow inputs"
        required: false
        default: '{}'
      filter:
        description: "Filter to apply to the list of repositories"
        required: false
        default: 'true'
      dry-run:
        description: "Whether to run in dry run mode"
        required: false
        default: 'false'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  # Number of repositories in a batch.
  # Matrix jobs within a workflow run are run in parallel.
  # 256 is the upper limit on the number of matrix jobs.
  # Batching too many repositories together can result in
  #  could not create workflow dispatch event: HTTP 422: inputs are too large.
  # This value should be higher than max-parallel in workflow.
  MAX_REPOS_PER_WORKFLOW: 100
  # Number of seconds to wait before starting to watch workflow run.
  # Unfortunately, the interval on the watch is not configurable.
  # The delay helps us save on GH API requests.
  WORKFLOW_COMPLETION_CHECK_DELAY: 60

jobs:
  matrix:
    name: Batch targets
    runs-on: ubuntu-latest
    outputs:
      batches: ${{ steps.matrix.outputs.result }}
    steps:
      - id: matrix
        uses: actions/github-script@v6
        env:
          FILTER: ${{ github.event.inputs.filter }}
        with:
          github-token: ${{ secrets.UCI_GITHUB_TOKEN }}
          retries: 0
          script: |
            const request = async function(req, opts) {
              try {
                return await req(opts)
              } catch(err) {
                opts._attempt = (opts._attempt || 0) + 1
                if (err.status === 403) {
                  if (err.response.headers['x-ratelimit-remaining'] === '0') {
                    const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1
                    core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`)
                    await new Promise(resolve => setTimeout(resolve, retryAfter * 1000))
                    return request(req, opts)
                  }
                  if (err.message.toLowerCase().includes('secondary rate limit')) {
                    const retryAfter = Math.pow(2, opts._attempt)
                    core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`)
                    await new Promise(resolve => setTimeout(resolve, retryAfter * 1000))
                    return request(req, opts)
                  }
                }
                throw err
              }
            }
            github.hook.wrap('request', request)
            core.info(`Looking for repositories the user has direct access to`)
            const items = await github.paginate(github.rest.repos.listForAuthenticatedUser, {
              affiliation: 'collaborator'
            })
            const maxReposPerWorkflow = parseInt(process.env.MAX_REPOS_PER_WORKFLOW)
            const batches = []
            let batch = []
            for (const item of items) {
              if (item.archived) {
                core.info(`Skipping archived repository ${item.full_name}`)
                continue
              }
              try {
                await exec.exec('jq', ['-e', '-n', `${JSON.stringify(item)} | ${process.env.FILTER}`])
              } catch(e) {
                core.info(`Skipping repository ${item.full_name} due to filter`)
                continue
              }
              batch.push(item.full_name)
              if (batch.length === maxReposPerWorkflow) {
                batches.push({
                  key: batches.length,
                  value: batch
                })
                batch = []
              }
            }
            if (batch.length > 0) {
              batches.push({
                key: batches.length,
                value: batch
              })
            }
            return batches
  dispatch:
    needs: [ matrix ]
    name: Dispatch workflow(batch ${{ matrix.cfg.key }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # We end up with multiple "dispatch" jobs,
        # one per BATCHES "key" chunk above with a "value" array.
        # For each "dispatch" job, matrix.cfg.value is an array, like:
        #
        #   [
        #     "repo1",
        #     "repo2"
        #   ]
        #
        # The triggered workflow runs use that final array as their matrix.
        # Since max-parallel here is 1, we'll end up with at most max-parallel from workflow + 1 parallel jobs.
        # 20 is the upper limit on parallel jobs on a free plan.
        cfg: ${{ fromJSON(needs.matrix.outputs.batches) }}
      max-parallel: 1
    env:
      GITHUB_TOKEN: ${{ github.token }}
      WORKFLOW: ${{ github.event.inputs.workflow }}
      REPO: ${{ github.repository }}
    steps:
      - id: dispatch
        name: Dispatch workflow
        env:
          TARGETS: ${{ toJSON(matrix.cfg.value) }}
          INPUTS: ${{ github.event.inputs.inputs }}
          DRY_RUN: ${{ github.event.inputs.dry-run }}
        run: |
          start_date="$(date +%s)"
          args=()
          for key in $(jq -r 'keys[]' <<< "$INPUTS"); do
            args+=("--field" "$key=$(jq -rc '.["'"$key"'"]' <<< "$INPUTS")")
          done
          if [[ "$DRY_RUN" == "true" ]]; then
            echo "DRY RUN: gh workflow run $WORKFLOW --ref $GITHUB_REF --repo $REPO --field targets=$TARGETS ${args[@]}"
          else
            gh workflow run "$WORKFLOW" --ref "$GITHUB_REF" --repo "$REPO" --field "targets=$TARGETS" ${args[@]}
          fi
          echo "start_date=$start_date" | tee -a $GITHUB_OUTPUT
      - id: run
        name: Wait for workflow run to start
        if: github.event.inputs.dry-run == 'false'
        env:
          START_DATE: ${{ steps.dispatch.outputs.start_date }}
        run: |
          # checks every 3 seconds until the most recent workflow run's created_at is later than this job's start_date
          while sleep 3; do
            run="$(gh api "/repos/$REPO/actions/workflows/$WORKFLOW/runs?per_page=1" --jq '.workflow_runs[0]')"
            # nothing to check if no workflow run was returned
            if [[ ! -z "$run" ]]; then
              run_start_date="$(date --date="$(jq -r '.created_at' <<< "$run")" +%s)"
              if [[ "$run_start_date" > "$START_DATE" ]]; then
                echo "id=$(jq -r '.id' <<< "$run")" | tee -a $GITHUB_OUTPUT
                break
              fi
            fi
          done
      - name: Wait for workflow run to complete
        if: github.event.inputs.dry-run == 'false'
        env:
          RUN_ID: ${{ steps.run.outputs.id }}
        run: |
          # delays checking workflow's run status to save on GH API requests
          sleep $WORKFLOW_COMPLETION_CHECK_DELAY

          # checks every 3 seconds until the workflow run's status is completed
          # redirects the stdout to /dev/null because it is very chatty
          gh run watch "$RUN_ID" --repo "$REPO" > /dev/null