benchmark.yml

name: benchmark

env:
  CRANK_VERSION: '0.2.0-*'
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_NOLOGO: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
  DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1
  NUGET_XMLDOC_MODE: skip
  TERM: xterm

on:
  issue_comment:
    types: [ created ]

permissions:
  contents: read
  issues: write
  pull-requests: write
  statuses: write

jobs:
  benchmark:
    name: benchmark
    runs-on: ubuntu-latest
    if: ${{ github.event.repository.fork == false && github.event.issue.pull_request != '' && contains(github.event.comment.body, '/benchmark') }}

    steps:

    - name: Get pull request commit
      uses: actions/github-script@v6
      id: get-pr
      with:
        result-encoding: string
        script: |
          const result = await github.rest.pulls.get({
            owner: context.repo.owner,
            repo: context.repo.repo,
            pull_number: context.issue.number
          });
          return result.data.head.sha;

    - name: Extract benchmark argument
      uses: actions/github-script@v6
      id: benchmark-argument
      with:
        result-encoding: string
        script: |
          // Documents all available commands
          const availableBenchmarks = {
            "guid": "Runs the GUID benchmark",
            "root": "Runs the root benchmark",
            "time": "Runs the time benchmark"
          };

          // Verify the user is a collaborator
          const repoOwner = context.payload.repository.owner.login;
          const repoName = context.payload.repository.name;
          const commentUser = context.payload.comment.user.login;

          try {
            var membership = await github.rest.repos.checkCollaborator({
              owner: repoOwner,
              repo: repoName,
              username: commentUser
            });
          } catch (error) {
            var message = `Error: @${commentUser} is not a repo collaborator, benchmarking is not allowed.`;
            await github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: message
            });
            throw message;
          }

          core.info(`Verified ${commentUser} is a repo collaborator.`);

          if (context.eventName !== "issue_comment") {
            throw "Error: This action only works on issue_comment events.";
          }

          // Extract the benchmark arguments from the trigger phrase containing these characters: a-z, A-Z, digits, forward slash, dot, hyphen, underscore
          const regex = /\/benchmark ([a-zA-Z\d\/\.\-\_]+)/;
          const arguments = regex.exec(context.payload.comment.body);

          // Generate help text with all available commands
          if (arguments == null || arguments.length < 2 || !availableBenchmarks.hasOwnProperty(arguments[1])) {

              var body = 'The `/benchmark` command accepts these benchmarks:\n';
              for (var key in availableBenchmarks) {
                body += `- \`/benchmark ${key}\`: ${availableBenchmarks[key]}\n`;
              }

              await github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: body
              });

              throw "Error: Invalid arguments, workflow stopped.";
          }

          const benchmark = arguments[1];
          const workflowUrl = `${{ github.server_url }}/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`;

          core.info(`Benchmark: ${benchmark}`);

          const start_body = `Started [${benchmark} benchmark](${workflowUrl}). :hourglass:`;
          await github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: start_body
          });

          await github.rest.repos.createCommitStatus({
            owner: context.repo.owner,
            repo: context.repo.repo,
            sha: '${{ steps.get-pr.outputs.result }}',
            state: 'pending',
            target_url: workflowUrl,
            description: `Benchmark ${benchmark} started...`,
            context: `benchmarks / ${benchmark.toLowerCase()}`
          });

          return benchmark;

    - name: Checkout code
      uses: actions/checkout@v3

    - name: Setup .NET SDK
      uses: actions/setup-dotnet@v3

    - name: Install crank
      shell: pwsh
      run: |
        dotnet tool install --global Microsoft.Crank.Agent --version "${{ env.CRANK_VERSION }}"
        dotnet tool install --global Microsoft.Crank.Controller --version "${{ env.CRANK_VERSION }}"
        dotnet tool install --global Microsoft.Crank.PullRequestBot --version "${{ env.CRANK_VERSION }}"

    - name: Run crank
      shell: pwsh
      run: |
        ./benchmark-crank.ps1 `
          -Benchmark "${{ steps.benchmark-argument.outputs.result }}" `
          -Repository "${{ github.server_url }}/${{ github.repository }}" `
          -PullRequestId "${{ github.event.issue.number }}" `
          -AccessToken "${{ secrets.GITHUB_TOKEN }}" `
          -PublishResults

    - name: Report success
      uses: actions/github-script@v6
      with:
        script: |
          const benchmark = '${{ steps.benchmark-argument.outputs.result }}';
          const workflowUrl = `${{ github.server_url }}/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`;

          const success_body = `Benchmark [${benchmark}](${workflowUrl}) succeeded. :white_check_mark:`;
          await github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: success_body
          });

          await github.rest.repos.createCommitStatus({
            owner: context.repo.owner,
            repo: context.repo.repo,
            sha: '${{ steps.get-pr.outputs.result }}',
            state: 'success',
            target_url: workflowUrl,
            description: `Benchmark ${benchmark} succeeded.`,
            context: `benchmarks / ${benchmark.toLowerCase()}`
          });

    - name: Report failure
      uses: actions/github-script@v6
      if: ${{ failure() }}
      with:
        script: |
          const benchmark = '${{ steps.benchmark-argument.outputs.result }}';
          const workflowUrl = `${{ github.server_url }}/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`;

          const failure_body = `Benchmark [${benchmark}](${workflowUrl}) failed. :x:`;
          await github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: failure_body
          });

          await github.rest.repos.createCommitStatus({
            owner: context.repo.owner,
            repo: context.repo.repo,
            sha: '${{ steps.get-pr.outputs.result }}',
            state: 'failure',
            target_url: workflowUrl,
            description: `Benchmark ${benchmark} failed.`,
            context: `benchmarks / ${benchmark.toLowerCase()}`
          });