e2e-profiling.yml

#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Run E2E profiling"
on:
  push:
    tags:
      - "*.*.*"
      - "v*.*.*"
      - "*.*.*-*"
      - "v*.*.*-*"
  pull_request:
    types:
      - "labeled"
env:
  DATASET: fashion-mnist-784-euclidean.hdf5
jobs:
  dump-contexts-to-log:
    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/dump-context
  e2e-profiling:
    name: "E2E profiling"
    needs: [dump-contexts-to-log]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - name: Set Git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Setup E2E environment
        id: setup_e2e
        uses: ./.github/actions/setup-e2e
      - name: Deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
          values: .github/helm/values/values-profile.yaml
          wait_for_selector: app=vald-lb-gateway
      - name: Deploy profefe
        run: |
          make k8s/metrics/profefe/deploy
          kubectl patch cronjob kprofefe -p '{"spec": {"schedule": "*/1 * * * *"}}'
          kubectl wait --for=condition=ready pod -l app=profefe --timeout=300s
      - name: Run E2E CRUD
        continue-on-error: true
        run: |
          make hack/benchmark/assets/dataset/${{ env.DATASET }}
          make E2E_BIND_PORT=8081 \
            E2E_DATASET_NAME=${{ env.DATASET }} \
            E2E_INSERT_COUNT=10000 \
            E2E_SEARCH_COUNT=100 \
            E2E_SEARCH_BY_ID_COUNT=100 \
            E2E_GET_OBJECT_COUNT=100 \
            E2E_UPDATE_COUNT=100 \
            E2E_UPSERT_COUNT=100 \
            E2E_REMOVE_COUNT=100 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
            E2E_TARGET_POD_NAME=${POD_NAME} \
            E2E_TARGET_NAMESPACE=default \
            e2e
        env:
          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
      - name: Get profiles
        run: |
          mkdir -p profiles
          kubectl port-forward deployment/profefe 10100:10100 &
          sleep 3
          for svc in vald-agent-ngt vald-lb-gateway vald-discoverer vald-manager-index
          do
            for t in heap cpu goroutine threadcreate
            do
              curl "http://localhost:10100/api/0/profiles/merge?service=${svc}&type=${t}&from=2021-07-01T00:00:00&to=2030-07-01T00:00:00" \
                --output profiles/${svc}-${t}.pb
            done
          done
      - uses: actions/upload-artifact@v4
        with:
          name: profiles
          path: profiles/*
      - uses: actions/cache@v4
        id: cache
        with:
          path: ./profiles-main
          key: ${{ runner.os }}-profiles-main-${{ github.sha }}
          restore-keys: ${{ runner.os }}-profiles-main-
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y graphviz
      - name: Generate graphs
        run: |
          tag=$(cat profiles-main/VALD_VERSION || echo "unknown")
          mkdir -p graphs
          for svc in vald-agent-ngt vald-lb-gateway vald-discoverer vald-manager-index
          do
            for t in heap cpu goroutine threadcreate
            do
              echo "generating: graphs/${svc}-${t}-${GITHUB_SHA::6}.svg"
              go tool pprof --svg profiles/${svc}-${t}.pb > graphs/${svc}-${t}-${GITHUB_SHA::6}.svg
              echo "generating: graphs/${svc}-${t}-${GITHUB_SHA::6}.png"
              go tool pprof --png profiles/${svc}-${t}.pb > graphs/${svc}-${t}-${GITHUB_SHA::6}.png
              if [ -f profiles-main/${svc}-${t}.pb ]; then
                echo "generating: graphs/${svc}-${t}-main-${GITHUB_SHA::6}.svg"
                go tool pprof --svg -base=profiles-main/${svc}-${t}.pb profiles/${svc}-${t}.pb > graphs/${svc}-${t}-diff-${tag}-${GITHUB_SHA::6}.svg
                echo "generating: graphs/${svc}-${t}-main-${GITHUB_SHA::6}.png"
                go tool pprof --png -base=profiles-main/${svc}-${t}.pb profiles/${svc}-${t}.pb > graphs/${svc}-${t}-diff-${tag}-${GITHUB_SHA::6}.png
              else
                echo "skipping: profiles-main/${svc}-${t}.pb does not exist."
              fi
            done
          done
      - uses: actions/upload-artifact@v4
        with:
          name: graphs
          path: graphs/*
      - name: Update cache
        if: startsWith( github.ref, 'refs/tags/')
        run: |
          mkdir -p profiles-main
          cp -f profiles/* profiles-main/
          cp -f versions/VALD_VERSION profiles-main/
      - name: Upload to vald-ci-images repository
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
        run: |
          CLONE_DIR=$(mktemp -d)
          git clone --depth 1 https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/vdaas-ci/vald-ci-images.git $CLONE_DIR
          cp -r graphs $CLONE_DIR/${GITHUB_SHA::6}
          cd $CLONE_DIR
          git config user.email "vald@vdaas.org"
          git config user.name "vdaas-ci"
          git add .
          git commit -m ":robot: Add ${GITHUB_SHA::6}"
          git push https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/vdaas-ci/vald-ci-images.git --set-upstream main
        env:
          GITHUB_USER: ${{ secrets.DISPATCH_USER }}
          GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
      - name: Comment
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
        run: |
          base="https://raw.githubusercontent.com/vdaas-ci/vald-ci-images/main/${GITHUB_SHA::6}"
          body="<table><tr><th>type</th>"
          for svc in vald-agent-ngt vald-lb-gateway vald-discoverer vald-manager-index
          do
            body="$body<th>$svc</th>"
          done
          body="$body</tr>"
          for t in cpu heap
          do
            body="$body<tr><td>${t}</td>"
            for svc in vald-agent-ngt vald-lb-gateway vald-discoverer vald-manager-index
            do
              body="$body<td><img src=\\\"$base/${svc}-${t}-${GITHUB_SHA::6}.png\\\" width=\\\"100%\\\"></td>"
            done
            body="$body</tr>"
          done
          body="$body</table>"
          curl --include --verbose --fail \
          -H "Accept: application/json" \
          -H "Content-Type:application/json" \
          -H "Authorization: token ${GITHUB_TOKEN}" \
          --request POST \
          --data "{\"body\": \"# Profile Report\n$body\n<a href=\\\"https://github.com/vdaas-ci/vald-ci-images/tree/main/${GITHUB_SHA::6}\\\">other images</a>\"}" \
          $API_URL
        env:
          GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
          API_URL: ${{ github.event.pull_request.comments_url }}