Dispatch #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Trigger the execution of workflow in batches. | |
| # This workflow is needed since GitHub Actions limits the matrix size to 256 jobs. | |
| # We use one job per repository per batch. | |
| name: Dispatch | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| workflow: | |
| description: "Workflow to dispatch" | |
| required: true | |
| inputs: | |
| description: "Workflow inputs" | |
| required: false | |
| default: '{}' | |
| filter: | |
| description: "Filter to apply to the list of repositories" | |
| required: false | |
| default: 'true' | |
| dry-run: | |
| description: "Whether to run in dry run mode" | |
| required: false | |
| default: 'false' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Number of repositories in a batch. | |
| # Matrix jobs within a workflow run are run in parallel. | |
| # 256 is the upper limit on the number of matrix jobs. | |
| # Batching too many repositories together can result in | |
| # could not create workflow dispatch event: HTTP 422: inputs are too large. | |
| # This value should be higher than max-parallel in workflow. | |
| MAX_REPOS_PER_WORKFLOW: 100 | |
| # Number of seconds to wait before starting to watch workflow run. | |
| # Unfortunately, the interval on the watch is not configurable. | |
| # The delay helps us save on GH API requests. | |
| WORKFLOW_COMPLETION_CHECK_DELAY: 60 | |
| jobs: | |
| matrix: | |
| name: Batch targets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| batches: ${{ steps.matrix.outputs.result }} | |
| steps: | |
| - id: matrix | |
| uses: actions/github-script@v6 | |
| env: | |
| FILTER: ${{ github.event.inputs.filter }} | |
| with: | |
| github-token: ${{ secrets.UCI_GITHUB_TOKEN }} | |
| retries: 0 | |
| script: | | |
| const request = async function(req, opts) { | |
| try { | |
| return await req(opts) | |
| } catch(err) { | |
| opts.request.retries = (opts.request.retries || 0) + 1 | |
| if (err.status === 403) { | |
| if (err.response.headers['x-ratelimit-remaining'] === '0') { | |
| const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 | |
| core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) | |
| await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) | |
| return request(req, opts) | |
| } | |
| if (err.message.toLowerCase().includes('secondary rate limit')) { | |
| const retryAfter = Math.pow(2, opts.request.retries) | |
| core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) | |
| await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) | |
| return request(req, opts) | |
| } | |
| } | |
| throw err | |
| } | |
| } | |
| github.hook.wrap('request', request) | |
| core.info(`Looking for repositories the user has direct access to`) | |
| const items = await github.paginate(github.rest.repos.listForAuthenticatedUser, { | |
| affiliation: 'collaborator' | |
| }) | |
| const maxReposPerWorkflow = parseInt(process.env.MAX_REPOS_PER_WORKFLOW) | |
| const batches = [] | |
| let batch = [] | |
| for (const item of items) { | |
| if (item.archived) { | |
| core.info(`Skipping archived repository ${item.full_name}`) | |
| continue | |
| } | |
| try { | |
| await exec.exec('jq', ['-e', '-n', `${JSON.stringify(item)} | ${process.env.FILTER}`]) | |
| } catch(e) { | |
| core.info(`Skipping repository ${item.full_name} due to filter`) | |
| continue | |
| } | |
| batch.push(item.full_name) | |
| if (batch.length === maxReposPerWorkflow) { | |
| batches.push({ | |
| key: batches.length, | |
| value: batch | |
| }) | |
| batch = [] | |
| } | |
| } | |
| if (batch.length > 0) { | |
| batches.push({ | |
| key: batches.length, | |
| value: batch | |
| }) | |
| } | |
| return batches | |
| dispatch: | |
| needs: [ matrix ] | |
| name: Dispatch workflow(batch ${{ matrix.cfg.key }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # We end up with multiple "dispatch" jobs, | |
| # one per BATCHES "key" chunk above with a "value" array. | |
| # For each "dispatch" job, matrix.cfg.value is an array, like: | |
| # | |
| # [ | |
| # "repo1", | |
| # "repo2" | |
| # ] | |
| # | |
| # The triggered workflow runs use that final array as their matrix. | |
| # Since max-parallel here is 1, we'll end up with at most max-parallel from workflow + 1 parallel jobs. | |
| # 20 is the upper limit on parallel jobs on a free plan. | |
| cfg: ${{ fromJSON(needs.matrix.outputs.batches) }} | |
| max-parallel: 1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} | |
| WORKFLOW: ${{ github.event.inputs.workflow }} | |
| steps: | |
| - id: dispatch | |
| name: Dispatch workflow | |
| env: | |
| TARGETS: ${{ toJSON(matrix.cfg.value) }} | |
| INPUTS: ${{ github.event.inputs.inputs }} | |
| REPO: ${{ github.repository }} | |
| DRY_RUN: ${{ github.event.inputs.dry-run }} | |
| run: | | |
| start_date="$(date +%s)" | |
| args=() | |
| for key in $(jq -r 'keys[]' <<< "$INPUTS"); do | |
| args+=("--field" "$key=$(jq -r ".$key" <<< "$INPUTS")") | |
| done | |
| if [[ "$DRY_RUN" == "true" ]]; then | |
| echo "DRY RUN: gh workflow run $WORKFLOW --ref $GITHUB_REF --repo $REPO --field targets=$TARGETS ${args[@]}" | |
| else | |
| gh workflow run "$WORKFLOW" --ref "$GITHUB_REF" --repo "$REPO" --field "targets=$TARGETS" ${args[@]} | |
| fi | |
| echo "start_date=$start_date" | tee -a $GITHUB_OUTPUT | |
| - id: run | |
| name: Wait for workflow run to start | |
| if: ${{ !github.event.inputs.dry-run }} | |
| env: | |
| START_DATE: ${{ steps.dispatch.outputs.start_date }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| # checks every 3 seconds until the most recent workflow run's created_at is later than this job's start_date | |
| while sleep 3; do | |
| run="$(gh api "/repos/$REPO/actions/workflows/$WORKFLOW/runs?per_page=1" --jq '.workflow_runs[0]')" | |
| # nothing to check if no workflow run was returned | |
| if [[ ! -z "$run" ]]; then | |
| run_start_date="$(date --date="$(jq -r '.created_at' <<< "$run")" +%s)" | |
| if [[ "$run_start_date" > "$START_DATE" ]]; then | |
| echo "id=$(jq -r '.id' <<< "$run")" | tee -a $GITHUB_OUTPUT | |
| break | |
| fi | |
| fi | |
| done | |
| - name: Wait for workflow run to complete | |
| if: ${{ !github.event.inputs.dry-run }} | |
| env: | |
| RUN_ID: ${{ steps.run.outputs.id }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| # delays checking workflow's run status to save on GH API requests | |
| sleep $WORKFLOW_COMPLETION_CHECK_DELAY | |
| # checks every 3 seconds until the workflow run's status is completed | |
| # redirects the stdout to /dev/null because it is very chatty | |
| gh run watch "$RUN_ID" --repo "$REPO" > /dev/null |