dapr-perf.yml

#
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Required secrets:
# - AZURE_CREDENTIALS: JSON object containing the Azure service principal credentials. Docs: https://github.com/Azure/login#configure-a-service-principal-with-a-secret
# - PERF_AZURE_STORAGE_ACCOUNT and PERF_AZURE_STORAGE_KEY: Credentials for the Storage Account where to store the result of perf tests
# - DAPR_BOT_TOKEN: Token for the Dapr bot
#
# Optional secrets:
# - AZURE_DIAG_LOG_ANALYTICS_WORKSPACE_ID: Resource ID of the Log Analytics Workspace where to store certain diagnostic logs (e.g. `/subscriptions/<subscription>/resourcegroups/<resource group>/providers/Microsoft.OperationalInsights/workspaces/<workspace name>`)
# - AZURE_DIAG_STORAGE_ID: Resource ID of the Azure Storage account where to store certain diagnostic logs (e.g. `/subscriptions/<subscription>/resourcegroups/<resource group>/providers/Microsoft.Storage/storageAccounts/<storage account name>`)

name: dapr-perf

on:
  # Run at minute 44 past hour 6, 14, and 22 on every day-of-week from Monday through Friday.
  # Run at 23:44 on Sunday and Saturday.
  schedule:
    - cron: "44 6,14,22 * * 1-5"
    - cron: "44 23 * * 0,6"
  # Manual trigger
  workflow_dispatch:
    inputs:
      tests:
        description: "Space-separated list of perf tests to run only"
        required: false
        type: string
  # Dispatch on external events
  repository_dispatch:
    types: [perf-test]

env:
  # Configure proxy for Go modules
  GOPROXY: https://proxy.golang.org
  # Version of kubectl
  KUBECTLVER: "v1.27.6"
  # Version of Helm
  HELMVER: "v3.10.0"
  # Kubernetes namespace to use
  DAPR_NAMESPACE: "dapr-tests"
  # Timeout for tests
  MAX_TEST_TIMEOUT: 5400
  # Space-separated of supported Azure regions: one will be picked randomly for each cluster
  AZURE_REGIONS: "eastus"
  # Container registry where to cache perf test images
  DAPR_CACHE_REGISTRY: "dapre2eacr.azurecr.io"

jobs:
  deploy-infrastructure:
    name: Deploy test infrastructure
    runs-on: ubuntu-latest
    steps:
      - name: Set up for manual runs
        if: github.event_name == 'workflow_dispatch'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
        shell: bash
      - name: Set up for scheduled test
        if: github.event_name == 'schedule'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/master" >> $GITHUB_ENV
        shell: bash
      - name: Set up for dispatched events
        if: github.event_name == 'repository_dispatch'
        uses: actions/github-script@v6.2.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            if (testPayload && testPayload.command == "ok-to-perf") {
              var fs = require('fs');
              // Set environment variables
              fs.appendFileSync(process.env.GITHUB_ENV,
                `CHECKOUT_REPO=${testPayload.pull_head_repo}\n`+
                `CHECKOUT_REF=${testPayload.pull_head_ref}\n`+
                `PR_NUMBER=${testPayload.issue.number}`
              );
            }
      - name: Create PR comment
        if: env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          hide: true
          hide_classify: OUTDATED
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            # Dapr perf test

            🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**

            Commit ref: ${{ env.CHECKOUT_REF }}
      - name: Check out code
        if: env.CHECKOUT_REPO != ''
        uses: actions/checkout@v4
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Login to Azure
        if: env.CHECKOUT_REPO != ''
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Build test prefix
        if: env.CHECKOUT_REPO != ''
        run: |
          BASE_STR="PERF|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          echo "Base string is ${BASE_STR}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          echo "Suffix is ${SUFFIX}"
          TEST_PREFIX="daprprf${SUFFIX}"
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "TEST_RESOURCE_GROUP=Dapr-Perf-${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Deploy the test cluster
        if: env.TEST_PREFIX != ''
        run: |
          # Select two random Azure regions
          REGIONS=(${{ env.AZURE_REGIONS }})
          REGIONS_SIZE=${#REGIONS[@]}
          REGIONS_IDX=$(($RANDOM % $REGIONS_SIZE))
          REGION=${REGIONS[$REGIONS_IDX]}
          echo "AZURE_REGION=${REGION}" >> $GITHUB_ENV
          echo "Deploying to Azure region: ${REGION}"

           # Tags
          TAGS="date=$(date --iso-8601=seconds)"
          echo "Tags: ${TAGS}"

          # Create a resource group
          az group create \
            --resource-group "${{ env.TEST_RESOURCE_GROUP }}" \
            --location ${REGION} \
            --tags "${TAGS}"

          # Deploy the test cluster, deploying AKS only
          # Retry the deployment twice in case of transient failures (such as capacity constraints)
          success=false
          for i in 1 2 3; do 
            az deployment group create \
              --resource-group "${{ env.TEST_RESOURCE_GROUP }}" \
              --template-file ./tests/test-infra/azure-aks.bicep \
              --parameters \
                namePrefix="${{ env.TEST_PREFIX }}" \
                location=${REGION} \
                linuxVMSize=Standard_D8s_v4 \
                diagLogAnalyticsWorkspaceResourceId="${{ secrets.AZURE_DIAG_LOG_ANALYTICS_WORKSPACE_ID_EAST }}" \
                diagStorageResourceId="${{ secrets.AZURE_DIAG_STORAGE_ID_EAST }}" \
              && success=true \
              && break \
              || sleep 120
          done
          # Exit with error if failed
          $success || exit 1
        shell: bash
      - name: Update PR comment for success
        if: success() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ✅ Infrastructure deployed

            - Resource group name: `Dapr-Perf-${{ env.TEST_PREFIX }}`
            - Azure region: ${{ env.AZURE_REGION }}

      - name: Update PR comment for failure
        if: failure() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ❌ Infrastructure deployment failed

            - Resource group name: `Dapr-Perf-${{ env.TEST_PREFIX }}`
            - Azure region: ${{ env.AZURE_REGION }}

            Please check the logs for details on the failure.

  build:
    name: Build
    runs-on: ubuntu-latest
    env:
      GOOS: linux
      GOARCH: amd64
      TARGET_OS: linux
      TARGET_ARCH: amd64
    steps:
      - name: Set up for manual runs
        if: github.event_name == 'workflow_dispatch'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
        shell: bash
      - name: Set up for scheduled test
        if: github.event_name == 'schedule'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/master" >> $GITHUB_ENV
        shell: bash
      - name: Set up for dispatched events
        if: github.event_name == 'repository_dispatch'
        uses: actions/github-script@v6.2.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            if (testPayload && testPayload.command == "ok-to-perf") {
              var fs = require('fs');
              // Set environment variables
              fs.appendFileSync(process.env.GITHUB_ENV,
                `CHECKOUT_REPO=${testPayload.pull_head_repo}\n`+
                `CHECKOUT_REF=${testPayload.pull_head_ref}\n`+
                `PR_NUMBER=${testPayload.issue.number}`
              );
            }
      - name: Check out code
        if: env.CHECKOUT_REPO != ''
        uses: actions/checkout@v4
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Set up Go
        id: setup-go
        uses: actions/setup-go@v5
        with:
          go-version-file: 'go.mod'
      - name: Login to Azure
        if: env.CHECKOUT_REPO != ''
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Login to cache registry
        if: env.CHECKOUT_REPO != '' && env.DAPR_CACHE_REGISTRY != ''
        run: |
          az acr login --name ${{ env.DAPR_CACHE_REGISTRY }}
        shell: bash
      - name: Build test prefix and set env vars
        if: env.CHECKOUT_REPO != ''
        run: |
          BASE_STR="PERF|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          echo "Base string is ${BASE_STR}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          echo "Suffix is ${SUFFIX}"
          TEST_PREFIX="daprprf${SUFFIX}"
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_REGISTRY=${TEST_PREFIX}acr.azurecr.io" >> $GITHUB_ENV
          echo "TEST_CLUSTER=${TEST_PREFIX}-aks" >> $GITHUB_ENV
          echo "DAPR_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_TEST_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "TEST_RESOURCE_GROUP=Dapr-Perf-${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_GO_BUILD_TAGS=wfbackendsqlite" >> $GITHUB_ENV
        shell: bash
      - name: Build dapr and its docker image
        if: env.TEST_PREFIX != ''
        run: |
          make build
          make docker-build
        shell: bash
      - name: Wait for Azure Container Registry deployment
        timeout-minutes: 30
        if: env.TEST_PREFIX != ''
        run: |
          until az acr show --name ${{ env.TEST_PREFIX }}acr --query "id"
          do
              echo "Azure Container Registry not ready yet: sleeping for 20 seconds"
              sleep 20
          done
        shell: bash
      - name: Login to Azure Container Registry
        if: env.TEST_PREFIX != ''
        run: |
          az acr login --name ${{ env.TEST_PREFIX }}acr
        shell: bash
      - name: Push Dapr container images
        if: env.TEST_PREFIX != ''
        run: |
          make docker-push
        shell: bash
      - name: Build and push perf test apps
        if: env.TEST_PREFIX != ''
        run: |
          make build-push-perf-app-all
        shell: bash
      - name: Update PR comment for success
        if: success() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ✅ Build succeeded

            - Image tag: `${{ env.DAPR_TAG }}`
            - Test image tag: `${{ env.DAPR_TEST_TAG }}`
      - name: Update PR comment for failure
        if: failure() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ❌ Build failed

            Please check the logs for details on the error.

  test-perf:
    name: Perf tests
    needs:
      - build
      - deploy-infrastructure
    runs-on: ubuntu-latest
    env:
      GOOS: linux
      GOARCH: amd64
      AZURE_STORAGE_ACCOUNT: ${{ secrets.PERF_AZURE_STORAGE_ACCOUNT }}
      AZURE_STORAGE_ACCESS_KEY: ${{ secrets.PERF_AZURE_STORAGE_KEY }}
      DAPR_PERF_METRICS_PROMETHEUS_PUSHGATEWAY_URL: ${{ secrets.DAPR_PERF_METRICS_PROMETHEUS_PUSHGATEWAY_URL }}
      DAPR_PERF_METRICS_PROMETHEUS_PUSHGATEWAY_USERNAME: ${{ secrets.DAPR_PERF_METRICS_PROMETHEUS_PUSHGATEWAY_USERNAME }}
      DAPR_PERF_METRICS_PROMETHEUS_PUSHGATEWAY_PASSWORD: ${{ secrets.DAPR_PERF_METRICS_PROMETHEUS_PUSHGATEWAY_PASSWORD }}
      DAPR_SCHEDULER_IN_MEMORY_STORAGE: true
      PULL_POLICY: IfNotPresent
    steps:
      - name: Setup test output
        run: |
          export TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report
          echo "TEST_OUTPUT_FILE_PREFIX=$TEST_OUTPUT_FILE_PREFIX" >> $GITHUB_ENV
        shell: bash
      - name: Set up log paths
        run: |
          echo "DAPR_CONTAINER_LOG_PATH=$GITHUB_WORKSPACE/container_logs/perf_tests" >> $GITHUB_ENV
          echo "DAPR_TEST_LOG_PATH=$GITHUB_WORKSPACE/test_logs/perf_tests" >> $GITHUB_ENV
        shell: bash
      - name: Set up for manual runs
        if: github.event_name == 'workflow_dispatch'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
          if [ "${{ inputs.tests }}" != "" ] ; then
            echo "DAPR_PERF_TEST=${{ inputs.tests }}" >> $GITHUB_ENV
          fi
        shell: bash
      - name: Set up for scheduled test
        if: github.event_name == 'schedule'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/master" >> $GITHUB_ENV
        shell: bash
      - name: Set up for dispatched events
        if: github.event_name == 'repository_dispatch'
        uses: actions/github-script@v6.2.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            if (testPayload && testPayload.command == "ok-to-perf") {
              let selectedTestsEnvVar = ""
              if (testPayload?.args) {
                selectedTestsEnvVar = `DAPR_PERF_TEST=${testPayload.args}\n`
              }
              var fs = require('fs');
              // Set environment variables
              fs.appendFileSync(process.env.GITHUB_ENV,
                `${selectedTestsEnvVar}`+
                `CHECKOUT_REPO=${testPayload.pull_head_repo}\n`+
                `CHECKOUT_REF=${testPayload.pull_head_ref}\n`+
                `PR_NUMBER=${testPayload.issue.number}`
              );
            }
      - name: Check out code
        if: env.CHECKOUT_REPO != ''
        uses: actions/checkout@v4
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Set up Go
        id: setup-go
        uses: actions/setup-go@v5
        with:
          go-version-file: 'go.mod'
      - uses: azure/setup-kubectl@v3
        with:
          version: ${{ env.KUBECTLVER }}
        id: install
      - name: Set up Helm ${{ env.HELMVER }}
        uses: azure/setup-helm@v3
        with:
          version: ${{ env.HELMVER }}
      - name: Login to Azure
        if: env.CHECKOUT_REPO != ''
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Build test prefix and set env vars
        if: env.CHECKOUT_REPO != ''
        run: |
          BASE_STR="PERF|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          echo "Base string is ${BASE_STR}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          echo "Suffix is ${SUFFIX}"
          TEST_PREFIX="daprprf${SUFFIX}"
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_REGISTRY=${TEST_PREFIX}acr.azurecr.io" >> $GITHUB_ENV
          echo "TEST_CLUSTER=${TEST_PREFIX}-aks" >> $GITHUB_ENV
          echo "DAPR_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_TEST_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "TEST_RESOURCE_GROUP=Dapr-Perf-${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Connect to Kubernetes
        if: env.TEST_PREFIX != ''
        run: |
          az aks get-credentials -n "${{ env.TEST_CLUSTER }}" -g "${{ env.TEST_RESOURCE_GROUP }}"
          kubectl create namespace ${{ env.DAPR_NAMESPACE }}
        shell: bash
      - name: Preparing AKS cluster for test
        if: env.TEST_PREFIX != ''
        run: |
          make setup-helm-init
          make setup-test-env
          kubectl get pods -n ${{ env.DAPR_NAMESPACE }}
      - name: Deploy dapr to AKS cluster
        if: env.TEST_PREFIX != ''
        env:
          ADDITIONAL_HELM_SET: "dapr_operator.logLevel=debug,dapr_operator.watchInterval=20s,dapr_dashboard.enabled=false"
        run: make docker-deploy-k8s
      - name: Deploy test components
        if: env.TEST_PREFIX != ''
        run: make setup-test-components
      - name: Run all performance tests
        if: env.TEST_PREFIX != ''
        run: make test-perf-all
      - name: Save control plane logs
        if: always() && env.TEST_PREFIX != ''
        run: |
          make save-dapr-control-plane-k8s-logs
      - name: Upload container logs
        if: always() && env.TEST_PREFIX != ''
        uses: actions/upload-artifact@v4
        with:
          name: perf_container_logs
          path: ${{ env.DAPR_CONTAINER_LOG_PATH }}
      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: perf_test_logs
          path: ${{ env.DAPR_TEST_LOG_PATH }}
      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          #TODO: .json suffix can be removed from artifact name after test analytics scripts are updated
          name: test_perf.json
          path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_perf*.*
      - name: Add job test summary
        if: always()
        uses: test-summary/action@v2
        with:
          paths: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_perf*.xml
      - name: Add job test outputs
        if: always()
        uses: actions/github-script@v6
        with:
          script: |
            const script = require('./.github/scripts/dapr_tests_summary.js')
            await script({core, glob})
      - name: Update PR comment for success
        if: success() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ✅ Perf tests succeeded

            - Image tag: `${{ env.DAPR_TAG }}`
            - Test image tag: `${{ env.DAPR_TEST_TAG }}`
      - name: Update PR comment for failure
        if: failure() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ❌ Perf tests failed

            Please check the logs for details on the error.
      - name: Update PR comment for cancellation
        if: cancelled() && env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          message: |
            ## ⚠️ Perf tests cancelled

            The Action has been canceled

  cleanup:
    name: Clean up Azure resources
    runs-on: ubuntu-latest
    needs:
      - test-perf
    if: always()
    steps:
      - name: Login to Azure
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Build test prefix
        run: |
          BASE_STR="PERF|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          echo "Base string is ${BASE_STR}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          echo "Suffix is ${SUFFIX}"
          TEST_PREFIX="daprprf${SUFFIX}"
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Delete cluster
        run: |
          # We are not waiting for these commands to complete, and we're ignoring errors
          echo "Starting removal of resource group Dapr-Perf-${TEST_PREFIX}"
          az group delete --no-wait --yes --name "Dapr-Perf-${TEST_PREFIX}" || true
        shell: bash