reusable-prober.yml

name: Reusable Sigstore Prober Workflow

on:
  workflow_call:
    secrets:
      PAGERDUTY_INTEGRATION_KEY:
        description: 'Integration key for PagerDuty'
        required: true
    inputs:
      rekor_url:
        required: false
        type: string
        default: 'https://rekor.sigstore.dev'
        description: 'Rekor URL'
      fulcio_url:
        required: false
        type: string
        default: 'https://fulcio.sigstore.dev'
        description: 'Fulcio URL'
      fulcio_grpc_url:
        required: false
        type: string
        default: 'fulcio.sigstore.dev'
        description: 'Fulcio GRPC URL'
      oidc_url:
        required: false
        type: string
        default: 'https://oauth2.sigstore.dev/auth'
        description: 'OIDC URL'
      enable_staging:
        required: false
        type: boolean
      tuf_repo:
        required: false
        type: string
        default: 'https://tuf-repo-cdn.sigstore.dev'
        description: 'TUF Repo'
      tuf_preprod_repo:
        required: false
        type: string
        default: 'https://sigstore.github.io/root-signing'
        description: 'Preprod TUF Repo'
      tuf_root_path:
        required: false
        type: string
        default: ".github/assets/sigstore.root.json"
        description: "path to the tuf root"
      triggerPagerDutyTest:
        description: 'Trigger PagerDuty test message'
        required: false
        type: string
      github_issues_repo:
        required: false
        type: string
        default: 'sigstore/sigstore-probers'
        description: 'GitHub repo to file alert issues under'

permissions:
  contents: read
  id-token: write
jobs:

  sigstore-probe:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    outputs:
      sigstore_probe: ${{ steps.msg.outputs.sigstore_probe }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Set up Go
        id: setup-go
        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
        with:
          go-version-file: 'prober/hack/toolz/go.mod'
          check-latest: true
          cache: false

      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-${{ hashFiles('prober/hack/toolz/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-

      - name: Install 'prober' from sigstore/scaffolding
        run: |
          make -C prober/ prober
          echo "PATH=$PATH:$PWD/prober/hack/toolz/bin" >> $GITHUB_ENV

      # Make sure rekor is up and we can get root info
      - name: Run prober
        env:
          DEBUG: 1
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 3
          max_attempts: 3
          retry_wait_seconds: 60
          retry_on: error
          command: prober --one-time --rekor-url ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --fulcio-grpc-url ${{ inputs.fulcio_grpc_url }}

      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          echo "sigstore_probe=good" >> $GITHUB_OUTPUT
          if [ "${{ job.status }}" == 'failure' ]; then echo "sigstore_probe=failure" >> $GITHUB_OUTPUT; fi


  root-probe:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      root_state: ${{ steps.msg.outputs.root_state }}
      dedup_key: ${{ steps.msg.outputs.dedup_key }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Set initial root.json
        run: cp "${{ inputs.tuf_root_path }}" ./root.json

      - name: Verify TUF repository state
        uses: theupdateframework/tuf-on-ci/actions/test-repository@fce107eeb8b1887239ca3a694dc250fcabf49c7f # v0.14.0
        with:
          metadata_url: ${{ inputs.tuf_repo }}
          valid_days: 2
          offline_valid_days: 15
          metadata_dir: tuf/prod/
          compare_source: false

      - name: Verify preprod TUF repository state (and the upgrade from prod)
        if: ${{ inputs.enable_staging == false }}
        uses: theupdateframework/tuf-on-ci/actions/test-repository@fce107eeb8b1887239ca3a694dc250fcabf49c7f # v0.14.0
        with:
          metadata_url: ${{ inputs.tuf_preprod_repo }}
          update_base_url:  ${{ inputs.tuf_repo }}
          valid_days: 2
          offline_valid_days: 15
          metadata_dir: tuf/preprod/
          compare_source: false

      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          # calculate a dedup key based on all TUF metadata the test client saw
          echo "dedup_key=$(sha256sum tuf/*/* | sha256sum | cut -d " " -f 1)" >> $GITHUB_OUTPUT

          echo "root_state=good" >> $GITHUB_OUTPUT
          if [ "${{ job.status }}" == 'failure' ]; then echo "root_state=failure" >> $GITHUB_OUTPUT; fi

  rekor-fulcio-e2e:
    timeout-minutes: 10
    permissions:
      id-token: write
      contents: read
    env:
      COSIGN_YES: "true"
      GIT_HASH: ${{ github.sha }}
      GIT_VERSION: unstable
      GITHUB_RUN_ID: ${{ github.run_id }}
      GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
      IMAGE: localhost:1338/image:${{ github.sha }}-${{ github.run_id }}
      REKOR_SERVER: ${{ inputs.rekor_url }}
      IDENTITY_REGEX: ${{ github.server_url }}/${{ github.repository }}/.github/workflows/reusable-prober.yml@refs/.*
    runs-on: ubuntu-latest
    outputs:
      rekor_fulcio_e2e: ${{ steps.msg.outputs.rekor_fulcio_e2e }}
      skip_pagerduty: ${{ steps.set-skip-pagerduty.outputs.skip_pagerduty }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      # This server is often down, resulting in a lot of flaky probers
      # If the server is down, and this step fails, we don't alert PagerDuty
      - name: Confirm Github OIDC Server is Available
        continue-on-error: true
        run: |
          curl -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" $ACTIONS_ID_TOKEN_REQUEST_URL&audience=sigstore

      # Since the server is down, we want to ignore the failure in this workflow
      # and skip paging PagerDuty
      - name: Set skip_pagerduty outputs
        id: set-skip-pagerduty
        if: failure()
        run: |
          echo "skip_pagerduty=true" >> $GITHUB_OUTPUT

      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
        id: setup-go
        with:
          go-version-file: 'prober/hack/toolz/go.mod'
          check-latest: true
          cache: false

      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-${{ hashFiles('prober/hack/toolz/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-

      # Install rekor-cli / cosign tools
      - name: Install (rekor-cli, cosign) tools
        run: |
          make -C prober/ rekor-cli cosign
          echo "PATH=$PATH:$PWD/prober/hack/toolz/bin" >> $GITHUB_ENV

      # Setup the registry on port 1338
      - uses: chainguard-dev/actions/setup-registry@main

      # this pulls a container from GHCR to avoid docker.io rate limiting and minimize network flake risk
      - name: Build and copy a container image
        run: |
          docker pull ghcr.io/linuxcontainers/alpine
          docker tag ghcr.io/linuxcontainers/alpine ${IMAGE}
          docker push ${IMAGE}

      # START: PREPRODUCTION VERIFICATION:
      #   * "Upgrade" cosign from prod to preprod TUF; sign and verify
      #   * Do not run this test for staging TUF
      # TODO: Create a matrix (https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs)
      #       to reduce duplication

      - name: Initialize prod TUF root
        if: ${{ inputs.enable_staging == false }}
        run: |
          for i in {1..5}
          do
            if cosign initialize --mirror=${{ inputs.tuf_repo }} --root=${{ inputs.tuf_root_path }}; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done

      - name: Initialize preprod TUF root
        if: ${{ inputs.enable_staging == false }}
        run: |
          for i in {1..5}
          do
            if cosign initialize --mirror=${{ inputs.tuf_preprod_repo }} --root=${{ inputs.tuf_root_path }}; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done

      # Test signing in preproduction
      - name: Sign and verify the image with preprod TUF
        if: ${{ inputs.enable_staging == false }}
        run: |
          cosign sign --yes ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --oidc-issuer ${{ inputs.oidc_url }} --oidc-provider github-actions
          cosign verify ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --certificate-oidc-issuer=https://token.actions.githubusercontent.com --certificate-identity-regexp="$IDENTITY_REGEX"

      - name: Remove preprod TUF
        run: |
          rm -rf ~/.sigstore
      # END: PREPRODUCTION VERIFICATION

      - name: Initialize TUF root
        run: |
          for i in {1..5}
          do
            if cosign initialize --mirror=${{ inputs.tuf_repo }} --root=${{ inputs.tuf_root_path }}; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done
          exit 1

      - name: Sign and verify the image
        run: |
          cosign sign --yes ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --oidc-issuer ${{ inputs.oidc_url }} --oidc-provider github-actions
          cosign verify ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --certificate-oidc-issuer=https://token.actions.githubusercontent.com --certificate-identity-regexp="$IDENTITY_REGEX"

      - name: Generate and upload attestation
        run: |
          cosign attest --predicate ./prober/attestation.json --type slsaprovenance --rekor-url  ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --oidc-issuer ${{ inputs.oidc_url }} ${IMAGE}
          cosign verify-attestation --rekor-url  ${{ inputs.rekor_url }} --type=slsaprovenance ${IMAGE} --certificate-oidc-issuer=https://token.actions.githubusercontent.com --certificate-identity-regexp="$IDENTITY_REGEX"

      - name: Read entries from all Rekor shards
        run: |
          set -e

          # get shard information from Rekor
          response=$(curl -fs ${{ inputs.rekor_url }}/api/v1/log) || {
            echo "Fetching loginfo from ${{ inputs.rekor_url }} failed, exiting..."; exit 1;
          }

          # shard_sizes[0] is the size of the active shard, shard_sizes [1...n] are the sizes of the inactive shards
          mapfile -t SHARD_SIZES < <(echo ${response} | jq '.. | objects | .treeSize')

          result=()
          offset=0
          # Check if there are any inactive shards returned (len >= 2)
          if (( ${#SHARD_SIZES[@]} >= 2 )); then
            for (( i = 1; i < ${#SHARD_SIZES[@]}; i++ )); do
              lower_bound=${offset}
              upper_bound=$((SHARD_SIZES[i]-1))
              random_number=$(( (RANDOM % (upper_bound - lower_bound + 1)) + lower_bound ))
              result+=($random_number)
              offset=$((upper_bound+offset+1))
            done
          fi

          # now handle the active shard
          if (( ${SHARD_SIZES[0]} > 0 )); then
            lower_bound=${offset}
            upper_bound=$((SHARD_SIZES[0]-1))
            random_number=$(( (RANDOM % (upper_bound - lower_bound + 1)) + lower_bound ))
            result+=($random_number)
          fi

          for index in "${result[@]}"; do
            echo -n "Fetching index $index from ${{ inputs.rekor_url }} ... "
            entry=$(curl -fs ${{ inputs.rekor_url }}/api/v1/log/entries?logIndex=$index) || {
              echo "failed!"; exit 1;
            }
            echo
            echo $entry | jq .
          done

      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          echo "rekor_fulcio_e2e=good" >> $GITHUB_OUTPUT
          if [ "${{ job.status }}" == 'failure' ]; then echo "rekor_fulcio_e2e=failure" >> $GITHUB_OUTPUT; fi

  compute-summary-msg:
    runs-on: ubuntu-latest
    outputs:
      summary: ${{ steps.msg.outputs.summary }}
      group: ${{ steps.msg.outputs.group }}
    steps:
      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          if [ "${{ inputs.triggerPagerDutyTest }}" == "true" ]; then
            echo "summary=Test Notification" >> $GITHUB_OUTPUT
          else
            echo "summary=Prober Failed" >> $GITHUB_OUTPUT
          fi
          echo "group=production" >> $GITHUB_OUTPUT
          if [ ${{ inputs.enable_staging }} == 'true' ]; then
            echo "group=staging" >> $GITHUB_OUTPUT;
          fi

  pagerduty-notification:
    if: github.event.inputs.triggerPagerDutyTest=='true' || (failure() && needs.rekor-fulcio-e2e.outputs.skip_pagerduty != 'true')
    needs: [sigstore-probe, root-probe, rekor-fulcio-e2e, compute-summary-msg]
    uses: ./.github/workflows/reusable-pager.yml
    secrets:
      PAGERDUTY_INTEGRATION_KEY: ${{ secrets.PAGERDUTY_INTEGRATION_KEY }}
    with:
      summary: ${{ needs.compute-summary-msg.outputs.summary }}
      component: "health prober"
      group: ${{ needs.compute-summary-msg.outputs.group }}
      dedup_key: "${{ needs.root-probe.outputs.dedup_key }}"
      details: >
        {
          "Environment": "${{ needs.compute-summary-msg.outputs.group }}",
          "Failure URL": "https://github.com/sigstore/sigstore-probers/actions/runs/${{ github.run_id }}",
          "Commit": "${{ github.sha }}",
          "Prober": "${{ needs.sigstore-probe.outputs.sigstore_probe }}",
          "GCS Root": "${{ needs.root-probe.outputs.root_state }}",
          "Rekor Fulcio E2E Test": "${{ needs.rekor-fulcio-e2e.outputs.rekor_fulcio_e2e }}"
        }
      links: >
        [
          {
            "href": "https://github.com/sigstore/public-good-instance/blob/main/playbooks/alerting/alerts/k8s-api-endpoint-prober.md",
            "text": "Prober Failure Playbook"
          },
          {
            "href": "https://github.com/sigstore/public-good-instance/blob/main/playbooks/alerting/tuf.md",
            "text": "Expiring TUF Metadata Playbook"
          }
        ]

  github-issue:
    if: always() && (needs.sigstore-probe.result == 'failure' || needs.root-probe.result == 'failure' || needs.rekor-fulcio-e2e.result == 'failure')
    runs-on: ubuntu-latest
    needs: [sigstore-probe, root-probe, rekor-fulcio-e2e, compute-summary-msg]
    steps:
      - env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          cat << 'EOF' | gh issue create --repo ${{ inputs.github_issues_repo }} --title "[ALERT][${{ needs.compute-summary-msg.outputs.group }}] Prober Failed" --label "alert,needs-assignee,oncall,${{ inputs.enable_staging && 'env:staging' || 'env:production' }}" -F -
          Environment: ${{ needs.compute-summary-msg.outputs.group }}
          Failure URL: https://github.com/sigstore/sigstore-probers/actions/runs/${{ github.run_id }}
          Commit: ${{ github.sha }}
          Prober: ${{ needs.sigstore-probe.outputs.sigstore_probe }}
          GCS Root: ${{ needs.root-probe.outputs.root_state }}
          Rekor Fulcio E2E Test: ${{ needs.rekor-fulcio-e2e.outputs.rekor_fulcio_e2e }}
          EOF