NIGHTLY:EVM:PERFORMANCE:TESTING #249
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "NIGHTLY:EVM:PERFORMANCE:TESTING" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| endpoint: | |
| description: 'The EVM endpoint you want to test.' | |
| required: false | |
| default: 'http://localhost:9545' | |
| schedule: | |
| # run at 6AM UTC Daily | |
| # 6AM UTC -> 11PM PT | |
| - cron: "0 6 * * *" | |
| jobs: | |
| nightly_evm_api_performance_test: | |
| name: "NIGHTLY:EVM:API:PERFORMANCE:TEST" | |
| runs-on: "buildjet-16vcpu-ubuntu-2204" | |
| env: | |
| endpoint_to_test: "http://localhost:9545" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "INSTALL:NODEJS" | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: "START:LOCAL:NET:WITH:STATE" | |
| run: | | |
| make start-e2e-import-mainnet-test | |
| - name: "INSTALL:ARTILLERY" | |
| run: | | |
| npm install -g artillery@latest | |
| - name: "CONFIGURE:PERFORMANCE:TEMPLATE" | |
| env: | |
| CLOUDFLARE_UUID: ${{ secrets.CLOUDFLARE_UUID }} | |
| ENDPOINT: ${{ inputs.endpoint || 'http://localhost:9545' }} | |
| GITHUB_ACTION: ${{ github.action }} | |
| run: | | |
| # Replace -=endpoint_to_test=- placeholder in the art.yaml file with the endpoint_to_test value | |
| sed -i "s|-=endpoint_to_test=-|${ENDPOINT}|g" .github/actions/performance-tests/art.yaml | |
| # Replace -=CLOUDFLARE_UUID=- placeholder in the art.yaml file with the correct value | |
| sed -i "s|-=CLOUDFLARE_UUID=-|${CLOUDFLARE_UUID}|g" .github/actions/performance-tests/art.yaml | |
| # Replace -=GITHUB_ACTION=- placeholder in the art.yaml file with the correct value | |
| sed -i "s|-=GITHUB_ACTION=-|${GITHUB_ACTION}|g" .github/actions/performance-tests/art.yaml | |
| cat .github/actions/performance-tests/art.yaml | |
| - name: "EXECUTE:PERFORMANCE:TESTS" | |
| run: | | |
| # Execute Artillery to run performance tests using the specified configuration file | |
| artillery run .github/actions/performance-tests/art.yaml \ | |
| --record \ | |
| --key ${{ secrets.ARTILLERY_KEY }} \ | |
| --output ./report.json | |
| - name: "CHECK:TEST:RESULTS" | |
| run: | | |
| # Check Artillery exit status | |
| if [ $? -ne 0 ]; then | |
| echo "Artillery command failed to execute." | |
| exit 1 | |
| fi | |
| # Parse the results.json file to check for failed vusers and HTTP response codes | |
| failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json) | |
| http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json) | |
| http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json) | |
| if [ "$failed_vusers" -gt 1 ] || [ "$http_codes_200" -ne "$http_responses" ]; then | |
| echo "Performance Testing had Failed Tests." | |
| exit 1 | |
| else | |
| echo "EVM Performance Testing Successful" | |
| fi | |
| # Read the JSON file and extract the required metrics | |
| p99_values=$(jq -r '.aggregate.summaries["http.response_time"].p99, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p99, .aggregate.summaries["vusers.session_length"].p99' ./report.json) | |
| p50_values=$(jq -r '.aggregate.summaries["http.response_time"].p50, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p50, .aggregate.summaries["vusers.session_length"].p50' ./report.json) | |
| # Check P99 values | |
| for p99 in $p99_values; do | |
| if (( $(echo "$p99 > 2000" | bc -l) )); then | |
| echo "P99 value $p99 exceeds 2000ms" | |
| exit 1 | |
| fi | |
| done | |
| # Check P50 values | |
| for p50 in $p50_values; do | |
| if (( $(echo "$p50 > 40" | bc -l) )); then | |
| echo "P50 value $p50 exceeds 40ms" | |
| exit 1 | |
| fi | |
| done | |
| - name: "GENERATE:REPORT" | |
| if: always() | |
| run: | | |
| artillery report report.json --output artillery_report.html | |
| - name: "UPLOAD:REPORT" | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: artillery-report | |
| path: ./artillery_report.html | |
| - name: "NOTIFY:SLACK:FAILURE" | |
| if: failure() && github.event_name == 'schedule' | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: ${{ job.status }} | |
| fields: repo,message,commit,author,action,eventName,ref,workflow,job,took | |
| job_name: "NIGHTLY:EVM:API:PERFORMANCE:TEST" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }} |