diff --git a/.github/workflows/performance-tests-pr.yml b/.github/workflows/performance-tests-pr.yml new file mode 100644 index 000000000..f047caf66 --- /dev/null +++ b/.github/workflows/performance-tests-pr.yml @@ -0,0 +1,33 @@ +name: GraphQL performance tests (PR) + +on: [deployment_status] + +jobs: + run_tests: + if: github.event.deployment_status.state == 'success' + name: Run tests + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v2 + - name: Run k6 + uses: addnab/docker-run-action@v3 + with: + image: grafana/k6:latest + options: -v ${{ github.workspace }}:/root + run: | + k6 run --tag testid=DataCubeComponents --env ENV=PR --env ENDPOINT=${{ github.event.deployment_status.target_url }}/api/graphql --env CUBE_IRI=https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9 --env CUBE_LABEL=Photovoltaikanlagen/9 --env ROOT_PATH=/root/ - - queries.flatMap((query) => - cubes.map((cube) => getRunCommand(env, query, cube)) - ) - ) - .join(" &&\n "); - -const generate = () => { - const file = `name: GraphQL performance tests - -on: - workflow_dispatch: - schedule: - - cron: "37 * * * *" - -jobs: - run_tests: - name: Run tests - runs-on: ubuntu-latest - steps: - - name: Check out - uses: actions/checkout@v2 - - name: Run k6 and upload results to Prometheus - uses: addnab/docker-run-action@v3 - with: - image: grafana/k6:latest - options: | - -v \${{ github.workspace }}:/root - -e K6_PROMETHEUS_RW_USERNAME=\${{ secrets.K6_PROMETHEUS_RW_USERNAME }} - -e K6_PROMETHEUS_RW_PASSWORD=\${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} - -e K6_PROMETHEUS_RW_SERVER_URL=\${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} - -e K6_PROMETHEUS_RW_TREND_STATS=avg - run: | - ${commands}`; - - fs.writeFileSync("./.github/workflows/performance-tests.yml", file); -}; - -generate(); - -function getRunCommand(env, query, cube) { - return `k6 run -o experimental-prometheus-rw --tag testid=${query} --env ENV=${env} --env CUBE_IRI=${cube.iri} --env CUBE_LABEL=${cube.label} - { + const commands = envs + .flatMap((env) => + queries.flatMap((query) => + cubes.map((cube) => + getRunCommand( + env, + query, + cube, + `https://${ + env === "prod" ? "" : `${env}.` + }visualize.admin.ch/api/graphql` + ) + ) + ) + ) + .join(" &&\n "); + const file = `name: GraphQL performance tests (auto) + +on: + workflow_dispatch: + schedule: + - cron: "37 * * * *" + +jobs: + run_tests: + name: Run tests + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v2 + - name: Run k6 and upload results to Prometheus + uses: addnab/docker-run-action@v3 + with: + image: grafana/k6:latest + options: | + -v \${{ github.workspace }}:/root + -e K6_PROMETHEUS_RW_USERNAME=\${{ secrets.K6_PROMETHEUS_RW_USERNAME }} + -e K6_PROMETHEUS_RW_PASSWORD=\${{ secrets.K6_PROMETHEUS_RW_PASSWORD }} + -e K6_PROMETHEUS_RW_SERVER_URL=\${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }} + -e K6_PROMETHEUS_RW_TREND_STATS=avg + run: | + ${commands} +`; + + fs.writeFileSync("./.github/workflows/performance-tests.yml", file); +}; + +generateAutoTests(); + +const generatePRTests = () => { + const commands = queries + .flatMap((query) => + cubes.map((cube) => + getRunCommand( + "PR", + query, + cube, + "${{ github.event.deployment_status.target_url }}/api/graphql", + false + ) + ) + ) + + .join(" &&\n "); + const file = `name: GraphQL performance tests (PR) + +on: [deployment_status] + +jobs: + run_tests: + if: github.event.deployment_status.state == 'success' + name: Run tests + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v2 + - name: Send an HTTP request to get the start up the server + run: curl 'https://test.visualize.admin.ch/api/graphql' -X 'POST' -H 'Content-Type: application/json' -d '{"operationName":"DataCubeObservations","variables":{"locale":"en","sourceType":"sparql","sourceUrl":"https://lindas.admin.ch/query","cubeFilter":{"iri":"https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9","filters":{"https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/Kanton":{"type":"single","value":"https://ld.admin.ch/canton/1"}}}},"query":"query DataCubeObservations($sourceType: String!, $sourceUrl: String!, $locale: String!, $cubeFilter: DataCubeObservationFilter!) {\n dataCubeObservations(\n sourceType: $sourceType\n sourceUrl: $sourceUrl\n locale: $locale\n cubeFilter: $cubeFilter\n )\n}\n"}' + - name: Run k6 + uses: addnab/docker-run-action@v3 + with: + image: grafana/k6:latest + options: -v \${{ github.workspace }}:/root + run: | + ${commands} +`; + + fs.writeFileSync("./.github/workflows/performance-tests-pr.yml", file); +}; + +generatePRTests(); + +function getRunCommand(env, query, cube, endpoint, sendToPrometheus = true) { + return `k6 run${ + sendToPrometheus ? " -o experimental-prometheus-rw" : "" + } --tag testid=${query} --env ENV=${env} --env ENDPOINT=${endpoint} --env CUBE_IRI=${ + cube.iri + } --env CUBE_LABEL=${ + cube.label + } --env ROOT_PATH=/root/ - cube.iri === cubeIri); const variables = { locale: "en", @@ -40,6 +37,11 @@ const variables = { /** @type {import("k6/options").Options} */ export const options = { iterations: 2, + thresholds: { + http_req_duration: [ + `avg<${2 * metadata.queries.DataCubeComponents.expectedDuration}`, + ], + }, }; const headers = { @@ -51,11 +53,9 @@ export default function Components() { exec.vu.metrics.tags.env = env; exec.vu.metrics.tags.cube = cubeLabel; - const res = http.post( - `https://${env === "prod" ? "" : `${env}.`}visualize.admin.ch/api/graphql`, - JSON.stringify({ query, variables }), - { headers } - ); + const res = http.post(endpoint, JSON.stringify({ query, variables }), { + headers, + }); check(res, { "Response must have data": (res) => { diff --git a/k6/performance-tests/graphql/DataCubeMetadata.js b/k6/performance-tests/graphql/DataCubeMetadata.js index 0c92d5742..ddeb17534 100644 --- a/k6/performance-tests/graphql/DataCubeMetadata.js +++ b/k6/performance-tests/graphql/DataCubeMetadata.js @@ -1,7 +1,10 @@ -import { check, fail } from "k6"; +import { check } from "k6"; import exec from "k6/execution"; import http from "k6/http"; +const rootPath = __ENV.ROOT_PATH || "../../../"; +const cubes = require(`${rootPath}k6/performance-tests/data.js`); + const query = `query DataCubeMetadata( $sourceType: String! $sourceUrl: String! @@ -16,17 +19,11 @@ const query = `query DataCubeMetadata( ) }`; -const metadataByCubeIri = { - "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9": - {}, - "https://environment.ld.admin.ch/foen/nfi/nfi_C-20/cube/2023-3": {}, - "https://energy.ld.admin.ch/elcom/electricityprice": {}, -}; - const env = __ENV.ENV; const cubeIri = __ENV.CUBE_IRI; const cubeLabel = __ENV.CUBE_LABEL; -const metadata = metadataByCubeIri[cubeIri]; +const endpoint = __ENV.ENDPOINT; +const metadata = cubes.find((cube) => cube.iri === cubeIri); const variables = { locale: "en", @@ -40,6 +37,11 @@ const variables = { /** @type {import("k6/options").Options} */ export const options = { iterations: 2, + thresholds: { + http_req_duration: [ + `avg<${2 * metadata.queries.DataCubeMetadata.expectedDuration}`, + ], + }, }; const headers = { @@ -51,11 +53,9 @@ export default function Components() { exec.vu.metrics.tags.env = env; exec.vu.metrics.tags.cube = cubeLabel; - const res = http.post( - `https://${env === "prod" ? "" : `${env}.`}visualize.admin.ch/api/graphql`, - JSON.stringify({ query, variables }), - { headers } - ); + const res = http.post(endpoint, JSON.stringify({ query, variables }), { + headers, + }); check(res, { "Response must have data": (res) => { diff --git a/k6/performance-tests/graphql/DataCubeObservations.js b/k6/performance-tests/graphql/DataCubeObservations.js index 968b2c657..e76726568 100644 --- a/k6/performance-tests/graphql/DataCubeObservations.js +++ b/k6/performance-tests/graphql/DataCubeObservations.js @@ -1,7 +1,10 @@ -import { check, fail } from "k6"; +import { check } from "k6"; import exec from "k6/execution"; import http from "k6/http"; +const rootPath = __ENV.ROOT_PATH || "../../../"; +const cubes = require(`${rootPath}k6/performance-tests/data.js`); + const query = `query DataCubeObservations( $sourceType: String! $sourceUrl: String! @@ -16,95 +19,30 @@ const query = `query DataCubeObservations( ) }`; -const metadataByCubeIri = { - "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9": - { - cubeFilter: { - iri: "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9", - filters: { - "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/Kanton": - { - type: "single", - value: "https://ld.admin.ch/canton/1", - }, - }, - }, - }, - "https://environment.ld.admin.ch/foen/nfi/nfi_C-20/cube/2023-3": { - cubeFilter: { - iri: "https://environment.ld.admin.ch/foen/nfi/nfi_C-20/cube/2023-3", - filters: { - "https://environment.ld.admin.ch/foen/nfi/unitOfReference": { - type: "single", - value: "https://ld.admin.ch/country/CHE", - }, - "https://environment.ld.admin.ch/foen/nfi/classificationUnit": { - type: "single", - value: - "https://environment.ld.admin.ch/foen/nfi/ClassificationUnit/Total", - }, - "https://environment.ld.admin.ch/foen/nfi/inventory": { - type: "single", - value: "https://environment.ld.admin.ch/foen/nfi/Inventory/150", - }, - "https://environment.ld.admin.ch/foen/nfi/unitOfEvaluation": { - type: "single", - value: - "https://environment.ld.admin.ch/foen/nfi/UnitOfEvaluation/2382", - }, - "https://environment.ld.admin.ch/foen/nfi/evaluationType": { - type: "single", - value: "https://environment.ld.admin.ch/foen/nfi/EvaluationType/1", - }, - }, - }, - }, - "https://energy.ld.admin.ch/elcom/electricityprice": { - cubeFilter: { - iri: "https://energy.ld.admin.ch/elcom/electricityprice", - filters: { - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/municipality": - { - type: "single", - value: "https://ld.admin.ch/municipality/1", - }, - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/category": - { - type: "single", - value: - "https://energy.ld.admin.ch/elcom/electricityprice/category/C1", - }, - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/operator": - { - type: "single", - value: - "https://energy.ld.admin.ch/elcom/electricityprice/operator/486", - }, - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/product": { - type: "single", - value: - "https://energy.ld.admin.ch/elcom/electricityprice/product/standard", - }, - }, - }, - }, -}; - const env = __ENV.ENV; const cubeIri = __ENV.CUBE_IRI; const cubeLabel = __ENV.CUBE_LABEL; -const metadata = metadataByCubeIri[cubeIri]; +const endpoint = __ENV.ENDPOINT; +const metadata = cubes.find((cube) => cube.iri === cubeIri); const variables = { locale: "en", sourceType: "sparql", sourceUrl: "https://lindas.admin.ch/query", - cubeFilter: metadata.cubeFilter, + cubeFilter: { + iri: cubeIri, + filters: metadata.filters, + }, }; /** @type {import("k6/options").Options} */ export const options = { iterations: 2, + thresholds: { + http_req_duration: [ + `avg<${2 * metadata.queries.DataCubeObservations.expectedDuration}`, + ], + }, }; const headers = { @@ -116,11 +54,9 @@ export default function Observations() { exec.vu.metrics.tags.env = env; exec.vu.metrics.tags.cube = cubeLabel; - const res = http.post( - `https://${env === "prod" ? "" : `${env}.`}visualize.admin.ch/api/graphql`, - JSON.stringify({ query, variables }), - { headers } - ); + const res = http.post(endpoint, JSON.stringify({ query, variables }), { + headers, + }); check(res, { "Response must have data": (res) => { diff --git a/k6/performance-tests/graphql/DataCubePreview.js b/k6/performance-tests/graphql/DataCubePreview.js index 8cbbc8d79..396fefe87 100644 --- a/k6/performance-tests/graphql/DataCubePreview.js +++ b/k6/performance-tests/graphql/DataCubePreview.js @@ -1,7 +1,10 @@ -import { check, fail } from "k6"; +import { check } from "k6"; import exec from "k6/execution"; import http from "k6/http"; +const rootPath = __ENV.ROOT_PATH || "../../../"; +const cubes = require(`${rootPath}k6/performance-tests/data.js`); + const query = `query DataCubePreview( $sourceType: String! $sourceUrl: String! @@ -16,17 +19,11 @@ const query = `query DataCubePreview( ) }`; -const metadataByCubeIri = { - "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9": - {}, - "https://environment.ld.admin.ch/foen/nfi/nfi_C-20/cube/2023-3": {}, - "https://energy.ld.admin.ch/elcom/electricityprice": {}, -}; - const env = __ENV.ENV; const cubeIri = __ENV.CUBE_IRI; const cubeLabel = __ENV.CUBE_LABEL; -const metadata = metadataByCubeIri[cubeIri]; +const endpoint = __ENV.ENDPOINT; +const metadata = cubes.find((cube) => cube.iri === cubeIri); const variables = { locale: "en", @@ -41,6 +38,11 @@ const variables = { /** @type {import("k6/options").Options} */ export const options = { iterations: 2, + thresholds: { + http_req_duration: [ + `avg<${2 * metadata.queries.DataCubePreview.expectedDuration}`, + ], + }, }; const headers = { @@ -52,11 +54,9 @@ export default function Components() { exec.vu.metrics.tags.env = env; exec.vu.metrics.tags.cube = cubeLabel; - const res = http.post( - `https://${env === "prod" ? "" : `${env}.`}visualize.admin.ch/api/graphql`, - JSON.stringify({ query, variables }), - { headers } - ); + const res = http.post(endpoint, JSON.stringify({ query, variables }), { + headers, + }); check(res, { "Response must have data": (res) => { diff --git a/k6/performance-tests/graphql/PossibleFilters.js b/k6/performance-tests/graphql/PossibleFilters.js index eb7b7f61a..58bc5241a 100644 --- a/k6/performance-tests/graphql/PossibleFilters.js +++ b/k6/performance-tests/graphql/PossibleFilters.js @@ -1,7 +1,10 @@ -import { check, fail } from "k6"; +import { check } from "k6"; import exec from "k6/execution"; import http from "k6/http"; +const rootPath = __ENV.ROOT_PATH || "../../../"; +const cubes = require(`${rootPath}k6/performance-tests/data.js`); + const query = `query PossibleFilters( $iri: String! $sourceType: String! @@ -20,73 +23,11 @@ const query = `query PossibleFilters( } }`; -const metadataByCubeIri = { - "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9": - { - iri: "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/9", - filters: { - "https://energy.ld.admin.ch/sfoe/bfe_ogd84_einmalverguetung_fuer_photovoltaikanlagen/Kanton": - { - type: "single", - value: "https://ld.admin.ch/canton/1", - }, - }, - }, - "https://environment.ld.admin.ch/foen/nfi/nfi_C-20/cube/2023-3": { - iri: "https://environment.ld.admin.ch/foen/nfi/nfi_C-20/cube/2023-3", - filters: { - "https://environment.ld.admin.ch/foen/nfi/unitOfReference": { - type: "single", - value: "https://ld.admin.ch/country/CHE", - }, - "https://environment.ld.admin.ch/foen/nfi/classificationUnit": { - type: "single", - value: - "https://environment.ld.admin.ch/foen/nfi/ClassificationUnit/Total", - }, - "https://environment.ld.admin.ch/foen/nfi/inventory": { - type: "single", - value: "https://environment.ld.admin.ch/foen/nfi/Inventory/150", - }, - "https://environment.ld.admin.ch/foen/nfi/unitOfEvaluation": { - type: "single", - value: "https://environment.ld.admin.ch/foen/nfi/UnitOfEvaluation/2382", - }, - "https://environment.ld.admin.ch/foen/nfi/evaluationType": { - type: "single", - value: "https://environment.ld.admin.ch/foen/nfi/EvaluationType/1", - }, - }, - }, - "https://energy.ld.admin.ch/elcom/electricityprice": { - iri: "https://energy.ld.admin.ch/elcom/electricityprice", - filters: { - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/municipality": - { - type: "single", - value: "https://ld.admin.ch/municipality/1", - }, - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/category": { - type: "single", - value: "https://energy.ld.admin.ch/elcom/electricityprice/category/C1", - }, - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/operator": { - type: "single", - value: "https://energy.ld.admin.ch/elcom/electricityprice/operator/486", - }, - "https://energy.ld.admin.ch/elcom/electricityprice/dimension/product": { - type: "single", - value: - "https://energy.ld.admin.ch/elcom/electricityprice/product/standard", - }, - }, - }, -}; - const env = __ENV.ENV; const cubeIri = __ENV.CUBE_IRI; const cubeLabel = __ENV.CUBE_LABEL; -const metadata = metadataByCubeIri[cubeIri]; +const endpoint = __ENV.ENDPOINT; +const metadata = cubes.find((cube) => cube.iri === cubeIri); const variables = { iri: cubeIri, @@ -99,6 +40,11 @@ const variables = { /** @type {import("k6/options").Options} */ export const options = { iterations: 2, + thresholds: { + http_req_duration: [ + `avg<${2 * metadata.queries.PossibleFilters.expectedDuration}`, + ], + }, }; const headers = { @@ -110,11 +56,9 @@ export default function Observations() { exec.vu.metrics.tags.env = env; exec.vu.metrics.tags.cube = cubeLabel; - const res = http.post( - `https://${env === "prod" ? "" : `${env}.`}visualize.admin.ch/api/graphql`, - JSON.stringify({ query, variables }), - { headers } - ); + const res = http.post(endpoint, JSON.stringify({ query, variables }), { + headers, + }); check(res, { "Response must have data": (res) => { diff --git a/package.json b/package.json index 280e236fc..000fbeadb 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "e2e": "percy exec -- playwright test", "cube": "NODE_ENV=development ts-node app/scripts/cube.ts", "dev:circular-deps": "madge --warning --extensions js,jsx,ts,tsx -b ./app -c ./app/pages/ --ts-config ./app/tsconfig.json | bun scripts/circular-deps-analysis.ts -", - "github:codegen": "node k6/performance-tests/generate-github-action.js" + "github:codegen": "node k6/performance-tests/generate-github-actions.mjs" }, "dependencies": { "@babel/runtime": "^7.11.2",