From d10cc11a36074ec1497a86988f4ffdcc180c62df Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 14 Dec 2023 10:17:36 +0100 Subject: [PATCH 01/12] fix: Docker parameter --- .github/workflows/performance-tests.yml | 2 +- k6/performance-tests/generate-github-action.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 81ae748ad..d2d6138cf 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -13,5 +13,5 @@ jobs: uses: addnab/docker-run-action@v3 with: image: grafana/k6:latest - options: -rm -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} + options: -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} run: k6 run -o experimental-prometheus-rw --tag testid=StateAccounts_Office/4/ --env ENV=test --env CUBE=StateAccounts_Office/4/ - { return queries.map((query) => { From 92748958f1853412c63bf447e721bfc527ae3951 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 14 Dec 2023 11:11:09 +0100 Subject: [PATCH 02/12] fix: Use correct path to read the files inside Docker container --- .github/workflows/performance-tests.yml | 4 ++-- k6/performance-tests/generate-github-action.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index d2d6138cf..42f15979f 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -13,5 +13,5 @@ jobs: uses: addnab/docker-run-action@v3 with: image: grafana/k6:latest - options: -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=StateAccounts_Office/4/ --env ENV=test --env CUBE=StateAccounts_Office/4/ - { return queries.map((query) => { return cubes.map((cube) => { - return `k6 run -o experimental-prometheus-rw --tag testid=${cube} --env ENV=${env} --env CUBE=${cube} - Date: Thu, 14 Dec 2023 11:20:26 +0100 Subject: [PATCH 03/12] fix: Testid --- .github/workflows/performance-tests.yml | 2 +- k6/performance-tests/generate-github-action.js | 4 ++-- .../graphql/{metadata.js => DataCubeMetadata.js} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename k6/performance-tests/graphql/{metadata.js => DataCubeMetadata.js} (100%) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 42f15979f..bf5cf8fcf 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -14,4 +14,4 @@ jobs: with: image: grafana/k6:latest options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=StateAccounts_Office/4/ --env ENV=test --env CUBE=StateAccounts_Office/4/ - { @@ -25,7 +25,7 @@ jobs: .map((env) => { return queries.map((query) => { return cubes.map((cube) => { - return `k6 run -o experimental-prometheus-rw --tag testid=${cube} --env ENV=${env} --env CUBE=${cube} - Date: Thu, 14 Dec 2023 11:26:01 +0100 Subject: [PATCH 04/12] feat: Separate cube iri and label --- .github/workflows/performance-tests.yml | 2 +- .../generate-github-action.js | 9 +++++-- .../graphql/DataCubeMetadata.js | 25 ++++++++++--------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index bf5cf8fcf..cdc21abb4 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -14,4 +14,4 @@ jobs: with: image: grafana/k6:latest options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=DataCubeMetadata --env ENV=test --env CUBE=StateAccounts_Office/4/ - { const file = `name: Performance tests @@ -25,7 +30,7 @@ jobs: .map((env) => { return queries.map((query) => { return cubes.map((cube) => { - return `k6 run -o experimental-prometheus-rw --tag testid=${query} --env ENV=${env} --env CUBE=${cube} - Date: Thu, 14 Dec 2023 11:39:40 +0100 Subject: [PATCH 05/12] feat: Add new cube to performance tests & optimize script --- .github/workflows/performance-tests.yml | 12 +++---- .../generate-github-action.js | 33 +++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index cdc21abb4..96294ec83 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -1,17 +1,17 @@ -name: Performance tests +name: GraphQL performance tests on: workflow_dispatch jobs: - compile: - name: Compile site assets + run_tests: + name: Run tests runs-on: ubuntu-latest steps: - - name: Check out the repo + - name: Check out uses: actions/checkout@v2 - - name: Run the build process with Docker + - name: Run k6 and upload results to Prometheus uses: addnab/docker-run-action@v3 with: image: grafana/k6:latest options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=DataCubeMetadata --env ENV=test --env CUBE_IRI=https://culture.ld.admin.ch/sfa/StateAccounts_Office/4/ --env CUBE_LABEL=StateAccounts_Office/4/ - + queries.flatMap((q) => cubes.map((c) => getRunCommand(e, q, c))) + ) + .join(" && "); const generate = () => { - const file = `name: Performance tests + const file = `name: GraphQL performance tests on: workflow_dispatch jobs: - compile: - name: Compile site assets + run_tests: + name: Run tests runs-on: ubuntu-latest steps: - - name: Check out the repo + - name: Check out uses: actions/checkout@v2 - - name: Run the build process with Docker + - name: Run k6 and upload results to Prometheus uses: addnab/docker-run-action@v3 with: image: grafana/k6:latest options: -v \${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=\${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=\${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=\${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: ${envs - .map((env) => { - return queries.map((query) => { - return cubes.map((cube) => { - return `k6 run -o experimental-prometheus-rw --tag testid=${query} --env ENV=${env} --env CUBE_IRI=${cube.iri} --env CUBE_LABEL=${cube.label} - Date: Thu, 14 Dec 2023 12:33:00 +0100 Subject: [PATCH 06/12] feat: Update performance test cubes --- .github/workflows/performance-tests.yml | 2 +- k6/performance-tests/generate-github-action.js | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 96294ec83..5ac6df667 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -14,4 +14,4 @@ jobs: with: image: grafana/k6:latest options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=DataCubeMetadata --env ENV=test --env CUBE_IRI=https://culture.ld.admin.ch/sfa/StateAccounts_Office/4/ --env CUBE_LABEL=StateAccounts_Office/4/ - Date: Thu, 14 Dec 2023 12:35:15 +0100 Subject: [PATCH 07/12] feat: Add DataCubeComponents performance test --- .github/workflows/performance-tests.yml | 2 +- .../generate-github-action.js | 2 +- .../graphql/DataCubeComponents.js | 50 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 k6/performance-tests/graphql/DataCubeComponents.js diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 5ac6df667..375dd6ada 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -14,4 +14,4 @@ jobs: with: image: grafana/k6:latest options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=DataCubeMetadata --env ENV=test --env CUBE_IRI=https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9 --env CUBE_LABEL=Photovoltaikanlagen/9 - Date: Thu, 14 Dec 2023 12:39:04 +0100 Subject: [PATCH 08/12] refactor: Make the run commands more readable --- .github/workflows/performance-tests.yml | 20 ++++++++++++++++++- .../generate-github-action.js | 5 +++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 375dd6ada..f840733be 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -14,4 +14,22 @@ jobs: with: image: grafana/k6:latest options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: k6 run -o experimental-prometheus-rw --tag testid=DataCubeComponents --env ENV=test --env CUBE_IRI=https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9 --env CUBE_LABEL=Photovoltaikanlagen/9 - queries.flatMap((q) => cubes.map((c) => getRunCommand(e, q, c))) ) - .join(" && "); + .join(" &&\n "); const generate = () => { const file = `name: GraphQL performance tests @@ -39,7 +39,8 @@ jobs: with: image: grafana/k6:latest options: -v \${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=\${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=\${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=\${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - run: ${commands}`; + run: | + ${commands}`; fs.writeFileSync("./.github/workflows/performance-tests.yml", file); }; From 96eb27379e174b8e4fbb681bc667df59d7240aff Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 14 Dec 2023 13:15:12 +0100 Subject: [PATCH 09/12] feat: Add DataCubePreview performance tests --- .github/workflows/performance-tests.yml | 17 ++++- .../generate-github-action.js | 8 ++- .../graphql/DataCubePreview.js | 67 +++++++++++++++++++ 3 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 k6/performance-tests/graphql/DataCubePreview.js diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index f840733be..53faf3d29 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -13,7 +13,11 @@ jobs: uses: addnab/docker-run-action@v3 with: image: grafana/k6:latest - options: -v ${{ github.workspace }}:/root -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} + options: | + -v ${{ github.workspace }}:/root + -e K6_PROMETHEUS_RW_USERNAME=${{ secrets.K6_PROMETHEUS_RW_USERNAME }} + -e K6_PROMETHEUS_RW_PASSWORD=${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} + -e K6_PROMETHEUS_RW_SERVER_URL=${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} run: | k6 run -o experimental-prometheus-rw --tag testid=DataCubeComponents --env ENV=test --env CUBE_IRI=https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9 --env CUBE_LABEL=Photovoltaikanlagen/9 - Date: Thu, 14 Dec 2023 14:21:12 +0100 Subject: [PATCH 10/12] feat: Only point to LINDAS prod --- k6/performance-tests/graphql/DataCubeComponents.js | 2 +- k6/performance-tests/graphql/DataCubeMetadata.js | 2 +- k6/performance-tests/graphql/DataCubePreview.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k6/performance-tests/graphql/DataCubeComponents.js b/k6/performance-tests/graphql/DataCubeComponents.js index 5396d989a..2809d1fc9 100644 --- a/k6/performance-tests/graphql/DataCubeComponents.js +++ b/k6/performance-tests/graphql/DataCubeComponents.js @@ -22,7 +22,7 @@ const cubeLabel = __ENV.CUBE_LABEL; const variables = { locale: "en", sourceType: "sparql", - sourceUrl: `https://${env === "prod" ? "" : `${env}.`}lindas.admin.ch/query`, + sourceUrl: "https://lindas.admin.ch/query", cubeFilter: { iri: cubeIri, }, diff --git a/k6/performance-tests/graphql/DataCubeMetadata.js b/k6/performance-tests/graphql/DataCubeMetadata.js index 1c7391ba1..1552b7bf3 100644 --- a/k6/performance-tests/graphql/DataCubeMetadata.js +++ b/k6/performance-tests/graphql/DataCubeMetadata.js @@ -22,7 +22,7 @@ const cubeLabel = __ENV.CUBE_LABEL; const variables = { locale: "en", sourceType: "sparql", - sourceUrl: `https://${env === "prod" ? "" : `${env}.`}lindas.admin.ch/query`, + sourceUrl: "https://lindas.admin.ch/query", cubeFilter: { iri: cubeIri, }, diff --git a/k6/performance-tests/graphql/DataCubePreview.js b/k6/performance-tests/graphql/DataCubePreview.js index 994c65d7c..8f8c64e44 100644 --- a/k6/performance-tests/graphql/DataCubePreview.js +++ b/k6/performance-tests/graphql/DataCubePreview.js @@ -40,7 +40,7 @@ const cubeLabel = __ENV.CUBE_LABEL; const variables = { iri: cubeIri, sourceType: "sparql", - sourceUrl: `https://${env === "prod" ? "" : `${env}.`}lindas.admin.ch/query`, + sourceUrl: "https://lindas.admin.ch/query", locale: "en", latest: false, }; From e64eb365b5003c4889756d6faf9e26e5155b3fe9 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 14 Dec 2023 14:24:44 +0100 Subject: [PATCH 11/12] feat: Schedule performance tests --- k6/performance-tests/generate-github-action.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/k6/performance-tests/generate-github-action.js b/k6/performance-tests/generate-github-action.js index 7bbbfd181..6cba27f88 100644 --- a/k6/performance-tests/generate-github-action.js +++ b/k6/performance-tests/generate-github-action.js @@ -25,7 +25,10 @@ const commands = envs const generate = () => { const file = `name: GraphQL performance tests -on: workflow_dispatch +on: + workflow_dispatch: + schedule: + - cron: 37 * * * *" jobs: run_tests: From edad59ab437d180b938f80ae84f81f719342687f Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 14 Dec 2023 14:36:05 +0100 Subject: [PATCH 12/12] docs: Update README --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 24105e010..46cf611d0 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,18 @@ Playwright is run on every successful deployment of a branch. Screenshots are ma A special [test page](http://localhost:3000/en/__test) shows all the charts that are screenshotted. Those charts configurations are kept in the repository. +## GraphQL performance tests + +The project uses a combination of [k6](https://k6.io) and [Grafana](https://grafana.com) with [Prometheus](https://k6.io/docs/results-output/real-time/prometheus-remote-write/) for GraphQL performance testing. + +### Automation + +To ensure constant monitoring of the performance of selected GraphQL queries, the performance tests are run [once an hour](https://github.com/visualize-admin/visualization-tool/blob/main/.github/workflows/performance-tests.yml) against each environment of the application. The results are then sent to the governmental Grafana dashboards. + +### How to add or modify the tests + +To add or modify the performance tests, go to the [k6/performance-tests](https://github.com/visualize-admin/visualization-tool/tree/main/k6/performance-tests) folder. The GitHub Action is generated by running the `yarn run github:codegen` command; be sure to run it after modifying the [generate-github-action.js](`https://github.com/visualize-admin/visualization-tool/blob/main/k6/performance-tests/generate-github-action.js`) file. + ## Load tests The project uses [k6](https://k6.io) for load testing.