From 3b4c5bf98c73d0cf4115de8272def560c0da754e Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Tue, 10 Dec 2024 13:02:11 +0100 Subject: [PATCH 01/27] created tests with thresholds --- .github/workflows/ci-cd-yt01.yml | 28 +++++++++++++++++++ .../workflows/workflow-run-k6-performance.yml | 10 ++++++- .../enduser/enduserSearchWithThresholds.js | 26 +++++++++++++++++ tests/k6/tests/scripts/generate_tokens.sh | 23 +++++++++++++-- .../createDialogWithThresholds.js | 12 ++++++++ 5 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 tests/k6/tests/enduser/enduserSearchWithThresholds.js create mode 100644 tests/k6/tests/serviceowner/createDialogWithThresholds.js diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 734ff03ed..49cf714a5 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -140,6 +140,34 @@ jobs: checks: write pull-requests: write + run-performance: + name: "Run K6 performance tests" + # we want the performance tests to be dependent on deployment of infrastructure and apps, but if infrastructure is skipped, we still want to run the tests + if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true' || needs.check-for-changes.outputs.hasInfraChanges == 'true') }} + needs: [deploy-apps, deploy-infra, check-for-changes] + #needs: [deploy-apps, check-for-changes] + uses: ./.github/workflows/workflow-run-k6-performance.yml + secrets: + TOKEN_GENERATOR_USERNAME: ${{ secrets.TOKEN_GENERATOR_USERNAME }} + TOKEN_GENERATOR_PASSWORD: ${{ secrets.TOKEN_GENERATOR_PASSWORD }} + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} + K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} + with: + environment: yt01 + apiVersion: v1 + vus: 1 + duration: 1m + tokens: both + numberOfTokens: 100 + # testSuitePath: | + # tests/k6/tests/serviceowner/createDialogWithThresholds.js + # tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js + # tests/k6/tests/serviceowner/yetAnotherTestSuite.js + testSuitePath: tests/k6/tests/serviceowner/createDialogWithThresholds.js + permissions: + checks: write + pull-requests: write + send-slack-message-on-failure: name: Send Slack message on failure needs: [deploy-infra, deploy-apps, deploy-slack-notifier, run-e2e-tests, publish] diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 17ddcadf5..8b5f9fbc1 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -21,6 +21,14 @@ on: tokens: required: true type: string + numberOfTokens: + required: false + type: number + default: 0 + ttl: + required: false + type: number + default: 3600 secrets: TOKEN_GENERATOR_USERNAME: required: true @@ -45,7 +53,7 @@ jobs: uses: grafana/setup-k6-action@v1 - name: Run K6 tests (${{ inputs.testSuitePath }}) run: | - ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} + ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out=cloud --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 diff --git a/tests/k6/tests/enduser/enduserSearchWithThresholds.js b/tests/k6/tests/enduser/enduserSearchWithThresholds.js new file mode 100644 index 000000000..ca11389ce --- /dev/null +++ b/tests/k6/tests/enduser/enduserSearchWithThresholds.js @@ -0,0 +1,26 @@ +import { default as run } from "./performance/enduser-search.js"; + +export let options = { + summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], + vus: 1, + duration: "30m", + thresholds: { + "http_req_duration{name:get dialog}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get dialog activities}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get dialog activity}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get seenlogs}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get transmissions}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get transmission}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get labellog}": ["p(95)<100", "p(99)<300"], + "http_reqs{name:get dialog activities}": [], + "http_reqs{name:get dialog activity}": [], + "http_reqs{name:get seenlogs}": [], + "http_reqs{name:get transmissions}": [], + "http_reqs{name:get transmission}": [], + "http_reqs{name:get dialog}": [], + "http_reqs{name:get labellog}": [], + } +} + +export default function (data) { run(data); } + diff --git a/tests/k6/tests/scripts/generate_tokens.sh b/tests/k6/tests/scripts/generate_tokens.sh index 4373de459..f9d39ab9a 100755 --- a/tests/k6/tests/scripts/generate_tokens.sh +++ b/tests/k6/tests/scripts/generate_tokens.sh @@ -11,11 +11,14 @@ usage() { echo "Usage: $0 " echo " : Path to the test data files" echo " : Type of tokens to generate (both, enterprise, or personal)" + echo " : limit number of tokens to generate. 0 means generate all" + echo " : Time to live in seconds for the generated tokens" + echo "Example: $0 /path/to/testdata both 10 3600" exit 1 } # Validate arguments -if [ $# -ne 2 ]; then +if [ $# -ne 4 ]; then usage fi @@ -37,6 +40,8 @@ esac testdatafilepath=$1 tokens=$2 +limit=$3 +ttl=$4 # Validate tokens argument if [[ ! "$tokens" =~ ^(both|enterprise|personal)$ ]]; then @@ -55,9 +60,13 @@ if [ "$tokens" = "both" ] || [ "$tokens" = "enterprise" ]; then exit 1 fi echo "org,orgno,scopes,resource,token" > $serviceowner_tokenfile + generated=0 while IFS=, read -r org orgno scopes resource do - url="https://altinn-testtools-token-generator.azurewebsites.net/api/GetEnterpriseToken?org=$org&env=$env&orgno=$orgno&ttl=3600" + if [ $limit -gt 0 ] && [ $generated -ge $limit ]; then + break + fi + url="https://altinn-testtools-token-generator.azurewebsites.net/api/GetEnterpriseToken?org=$org&env=$env&orgno=$orgno&ttl=$ttl" token=$(curl -s -f --get --data-urlencode "scopes=$scopes" $url -u "$tokengenuser:$tokengenpasswd" ) if [ $? -ne 0 ]; then echo "Error: Failed to generate enterprise token for: $env, $org, $orgno, $scopes " @@ -67,6 +76,8 @@ if [ "$tokens" = "both" ] || [ "$tokens" = "enterprise" ]; then status=$? if [ $status -ne 0 ]; then echo "Error: Failed to write enterprise token to file for: $env, $org, $orgno, $scopes" + else + ((generated++)) fi done < <(tail -n +2 $serviceowner_datafile) fi @@ -77,9 +88,13 @@ if [ "$tokens" = "both" ] || [ "$tokens" = "personal" ]; then exit 1 fi echo "ssn,resource,scopes,token" > $enduser_tokenfile + generated=0 while IFS=, read -r ssn resource scopes do - url="https://altinn-testtools-token-generator.azurewebsites.net/api/GetPersonalToken?env=$env&scopes=$scopes&pid=$ssn&ttl=3600" + if [ $limit -gt 0 ] && [ $generated -ge $limit ]; then + break + fi + url="https://altinn-testtools-token-generator.azurewebsites.net/api/GetPersonalToken?env=$env&scopes=$scopes&pid=$ssn&ttl=$ttl" token=$(curl -s -f $url -u "$tokengenuser:$tokengenpasswd" ) if [ $? -ne 0 ]; then echo "Error: Failed to generate personal token for: $ssn, $scopes " @@ -89,6 +104,8 @@ if [ "$tokens" = "both" ] || [ "$tokens" = "personal" ]; then status=$? if [ $status -ne 0 ]; then echo "Error: Failed to write personal token to file for: $ssn, $scopes" + else + ((generated++)) fi done < <(tail -n +2 $enduser_datafile) fi diff --git a/tests/k6/tests/serviceowner/createDialogWithThresholds.js b/tests/k6/tests/serviceowner/createDialogWithThresholds.js new file mode 100644 index 000000000..31db16e45 --- /dev/null +++ b/tests/k6/tests/serviceowner/createDialogWithThresholds.js @@ -0,0 +1,12 @@ +import { default as run } from "./performance/create-dialog.js"; + +export let options = { + summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], + vus: 1, + duration: "30s", + thresholds: { + "http_req_duration": ["p(95)<100", "p(99)<300"], + } +} + +export default function (data) { run(data); } \ No newline at end of file From 46fa464fdfc00220320a75cb317444d8140414e6 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Tue, 10 Dec 2024 14:58:24 +0100 Subject: [PATCH 02/27] a testversion for running multiple k6 tests --- .github/workflows/ci-cd-yt01.yml | 167 ++---------------- .../workflows/workflow-run-k6-performance.yml | 2 +- .../enduser/enduserSearchWithThresholds.js | 2 +- 3 files changed, 15 insertions(+), 156 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 49cf714a5..3697d8f07 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -10,176 +10,35 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} jobs: - get-versions-from-github: - name: Get Latest Deployed Version Info from GitHub - uses: ./.github/workflows/workflow-get-latest-deployed-version-info-from-github.yml - with: - environment: yt01 - secrets: - GH_TOKEN: ${{ secrets.RELEASE_VERSION_STORAGE_PAT }} - - check-for-changes: - name: Check for changes - needs: [get-versions-from-github] - uses: ./.github/workflows/workflow-check-for-changes.yml - with: - infra_base_sha: ${{ needs.get-versions-from-github.outputs.infra_version_sha }} - apps_base_sha: ${{ needs.get-versions-from-github.outputs.apps_version_sha }} - - get-current-version: - name: Get current version - uses: ./.github/workflows/workflow-get-current-version.yml - - publish: - name: Build and publish docker images - uses: ./.github/workflows/workflow-publish.yml - if: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true' }} - needs: [get-current-version, check-for-changes] - secrets: - GCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - with: - dockerImageBaseName: ghcr.io/digdir/dialogporten- - version: ${{ needs.get-current-version.outputs.version }} - - deploy-infra: - name: Deploy infra to yt01 - if: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasInfraChanges == 'true' }} - needs: [get-current-version, check-for-changes] - uses: ./.github/workflows/workflow-deploy-infra.yml - secrets: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - AZURE_SOURCE_KEY_VAULT_NAME: ${{ secrets.AZURE_SOURCE_KEY_VAULT_NAME }} - AZURE_SOURCE_KEY_VAULT_SUBSCRIPTION_ID: ${{ secrets.AZURE_SOURCE_KEY_VAULT_SUBSCRIPTION_ID }} - AZURE_SOURCE_KEY_VAULT_RESOURCE_GROUP: ${{ secrets.AZURE_SOURCE_KEY_VAULT_RESOURCE_GROUP }} - AZURE_SOURCE_KEY_VAULT_SSH_JUMPER_SSH_PUBLIC_KEY: ${{ secrets.AZURE_SOURCE_KEY_VAULT_SSH_JUMPER_SSH_PUBLIC_KEY }} - with: - environment: yt01 - region: norwayeast - version: ${{ needs.get-current-version.outputs.version }} - - store-infra-version: - name: Store Latest Deployed Infra Version as GitHub Variable - needs: [deploy-infra, get-current-version] - if: ${{ needs.deploy-infra.result == 'success' }} - uses: ./.github/workflows/workflow-store-github-env-variable.yml - with: - variable_name: LATEST_DEPLOYED_INFRA_VERSION - variable_value: ${{ needs.get-current-version.outputs.version }} - environment: yt01 - secrets: - GH_TOKEN: ${{ secrets.RELEASE_VERSION_STORAGE_PAT }} + - deploy-apps: - name: Deploy apps to yt01 - needs: - [get-current-version, check-for-changes, deploy-infra, publish] - # we want deployment of apps to be dependent on deployment of infrastructure, but if infrastructure is skipped, we still want to deploy the apps - if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true') }} - uses: ./.github/workflows/workflow-deploy-apps.yml - secrets: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - # todo: consider resolving these in another way since they are created in the infra-step - AZURE_RESOURCE_GROUP_NAME: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - AZURE_ENVIRONMENT_KEY_VAULT_NAME: ${{ secrets.AZURE_ENVIRONMENT_KEY_VAULT_NAME }} - AZURE_CONTAINER_APP_ENVIRONMENT_NAME: ${{ secrets.AZURE_CONTAINER_APP_ENVIRONMENT_NAME }} - AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} - AZURE_APP_CONFIGURATION_NAME: ${{ secrets.AZURE_APP_CONFIGURATION_NAME }} - AZURE_SERVICE_BUS_NAMESPACE_NAME: ${{ secrets.AZURE_SERVICE_BUS_NAMESPACE_NAME }} - with: - environment: yt01 - region: norwayeast - version: ${{ needs.get-current-version.outputs.version }} - runMigration: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasMigrationChanges == 'true' }} - - store-apps-version: - name: Store Latest Deployed Apps Version as GitHub Variable - needs: [deploy-apps, get-current-version] - if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.deploy-apps.outputs.deployment_executed == 'true') }} - uses: ./.github/workflows/workflow-store-github-env-variable.yml - with: - variable_name: LATEST_DEPLOYED_APPS_VERSION - variable_value: ${{ needs.get-current-version.outputs.version }} - environment: yt01 - secrets: - GH_TOKEN: ${{ secrets.RELEASE_VERSION_STORAGE_PAT }} - - deploy-slack-notifier: - name: Deploy slack notifier (yt01) - needs: [check-for-changes] - if: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasSlackNotifierChanges == 'true' }} - uses: ./.github/workflows/workflow-deploy-function.yml - secrets: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - # todo: resolve this automatically, or use tags - AZURE_FUNCTION_APP_NAME: ${{ secrets.AZURE_SLACK_NOTIFIER_FUNCTION_APP_NAME }} - with: - function-app-name: "slack-notifier" - function-project-path: "./src/Digdir.Tool.Dialogporten.SlackNotifier" - environment: yt01 - - run-e2e-tests: - name: "Run K6 functional end-to-end tests" - # we want the end-to-end tests to be dependent on deployment of infrastructure and apps, but if infrastructure is skipped, we still want to run the tests - if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true') }} - needs: [deploy-apps, check-for-changes] - uses: ./.github/workflows/workflow-run-k6-tests.yml - secrets: - TOKEN_GENERATOR_USERNAME: ${{ secrets.TOKEN_GENERATOR_USERNAME }} - TOKEN_GENERATOR_PASSWORD: ${{ secrets.TOKEN_GENERATOR_PASSWORD }} - with: - environment: yt01 - apiVersion: v1 - testSuitePath: tests/k6/suites/all-single-pass.js - permissions: - checks: write - pull-requests: write - - run-performance: + run-performance-ci-cd: name: "Run K6 performance tests" # we want the performance tests to be dependent on deployment of infrastructure and apps, but if infrastructure is skipped, we still want to run the tests - if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true' || needs.check-for-changes.outputs.hasInfraChanges == 'true') }} - needs: [deploy-apps, deploy-infra, check-for-changes] + #needs: [deploy-apps, check-for-changes] + uses: ./.github/workflows/workflow-run-k6-performance.yml secrets: TOKEN_GENERATOR_USERNAME: ${{ secrets.TOKEN_GENERATOR_USERNAME }} TOKEN_GENERATOR_PASSWORD: ${{ secrets.TOKEN_GENERATOR_PASSWORD }} K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} + + strategy: + matrix: + files: + - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js + - tests/k6/tests/serviceowner/createDialogWithThresholds.js + - tests/k6/tests/enduser/enduserSearchWithThresholds.js with: environment: yt01 apiVersion: v1 - vus: 1 + vus: 2 duration: 1m tokens: both numberOfTokens: 100 - # testSuitePath: | - # tests/k6/tests/serviceowner/createDialogWithThresholds.js - # tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - # tests/k6/tests/serviceowner/yetAnotherTestSuite.js - testSuitePath: tests/k6/tests/serviceowner/createDialogWithThresholds.js + testSuitePath: ${{ matrix.files }} permissions: checks: write pull-requests: write - - send-slack-message-on-failure: - name: Send Slack message on failure - needs: [deploy-infra, deploy-apps, deploy-slack-notifier, run-e2e-tests, publish] - if: ${{ always() && failure() && !cancelled() }} - uses: ./.github/workflows/workflow-send-ci-cd-status-slack-message.yml - with: - environment: yt01 - infra_status: ${{ needs.deploy-infra.result }} - apps_status: ${{ needs.deploy-apps.result }} - slack_notifier_status: ${{ needs.deploy-slack-notifier.result }} - e2e_tests_status: ${{ needs.run-e2e-tests.result }} - publish_status: ${{ needs.publish.result }} - secrets: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_FOR_CI_CD_STATUS }} diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 8b5f9fbc1..3003af626 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -55,7 +55,7 @@ jobs: run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ - --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out=cloud --out csv=./results.csv + --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 env: API_ENVIRONMENT: ${{ inputs.environment }} diff --git a/tests/k6/tests/enduser/enduserSearchWithThresholds.js b/tests/k6/tests/enduser/enduserSearchWithThresholds.js index ca11389ce..b041fdc36 100644 --- a/tests/k6/tests/enduser/enduserSearchWithThresholds.js +++ b/tests/k6/tests/enduser/enduserSearchWithThresholds.js @@ -3,7 +3,7 @@ import { default as run } from "./performance/enduser-search.js"; export let options = { summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], vus: 1, - duration: "30m", + duration: "30s", thresholds: { "http_req_duration{name:get dialog}": ["p(95)<100", "p(99)<300"], "http_req_duration{name:get dialog activities}": ["p(95)<100", "p(99)<300"], From 8dbab9b071d166dc126b2077ebc78682e89b864e Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 10:41:26 +0100 Subject: [PATCH 03/27] run performance in sequence --- .github/workflows/ci-cd-yt01.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 3697d8f07..ea9705848 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -26,6 +26,7 @@ jobs: K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} strategy: + max-parallel: 1 matrix: files: - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js From aeb1a46b9559dd1c58df82a695f718a9720fc6eb Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 10:51:56 +0100 Subject: [PATCH 04/27] encapsulate filepaths --- .github/workflows/ci-cd-yt01.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index ea9705848..5bbd75e6b 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -29,9 +29,9 @@ jobs: max-parallel: 1 matrix: files: - - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - - tests/k6/tests/serviceowner/createDialogWithThresholds.js - - tests/k6/tests/enduser/enduserSearchWithThresholds.js + - "tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js" + - "tests/k6/tests/serviceowner/createDialogWithThresholds.js" + - "tests/k6/tests/enduser/enduserSearchWithThresholds.js" with: environment: yt01 apiVersion: v1 From d27e95d5f3736e8916fcd3d9a7ea484dd6caf9de Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 11:00:54 +0100 Subject: [PATCH 05/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 5bbd75e6b..ea9705848 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -29,9 +29,9 @@ jobs: max-parallel: 1 matrix: files: - - "tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js" - - "tests/k6/tests/serviceowner/createDialogWithThresholds.js" - - "tests/k6/tests/enduser/enduserSearchWithThresholds.js" + - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js + - tests/k6/tests/serviceowner/createDialogWithThresholds.js + - tests/k6/tests/enduser/enduserSearchWithThresholds.js with: environment: yt01 apiVersion: v1 From 19792f24eb3cdf0d7e453fed12cc7d8de0db5841 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 11:06:52 +0100 Subject: [PATCH 06/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index ea9705848..bb23f5266 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -29,9 +29,9 @@ jobs: max-parallel: 1 matrix: files: - - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - - tests/k6/tests/serviceowner/createDialogWithThresholds.js - - tests/k6/tests/enduser/enduserSearchWithThresholds.js + - ./tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js + - ./tests/k6/tests/serviceowner/createDialogWithThresholds.js + - ./tests/k6/tests/enduser/enduserSearchWithThresholds.js with: environment: yt01 apiVersion: v1 From 79c00594ef1f81501be0a43aa5d9aabd5c816f15 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 12:57:43 +0100 Subject: [PATCH 07/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index bb23f5266..da205855c 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -29,9 +29,9 @@ jobs: max-parallel: 1 matrix: files: - - ./tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - - ./tests/k6/tests/serviceowner/createDialogWithThresholds.js - - ./tests/k6/tests/enduser/enduserSearchWithThresholds.js + - "\"tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js\"" + - "\"tests/k6/tests/serviceowner/createDialogWithThresholds.js\"" + - "\"tests/k6/tests/enduser/enduserSearchWithThresholds.js\"" with: environment: yt01 apiVersion: v1 @@ -39,7 +39,7 @@ jobs: duration: 1m tokens: both numberOfTokens: 100 - testSuitePath: ${{ matrix.files }} + testSuitePath: ${{ fromJson(matrix.files) }} permissions: checks: write pull-requests: write From 7dc98517fc5f60991df5609665be429a9d78ca15 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:01:05 +0100 Subject: [PATCH 08/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index da205855c..991307188 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -39,7 +39,7 @@ jobs: duration: 1m tokens: both numberOfTokens: 100 - testSuitePath: ${{ fromJson(matrix.files) }} + testSuitePath: ${{ matrix.files }} permissions: checks: write pull-requests: write From 5339c26a2d0251a24bc540581bb19e95e24c800c Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:05:35 +0100 Subject: [PATCH 09/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 3003af626..918a56fb1 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -54,7 +54,7 @@ jobs: - name: Run K6 tests (${{ inputs.testSuitePath }}) run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} - k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ + k6 run ${{ fromJson(inputs.testSuitePath) }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 env: From 8b04a8fa6190c90a551ae7098595c2a5350e67e6 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:27:16 +0100 Subject: [PATCH 10/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 918a56fb1..bce8fa05e 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -54,6 +54,7 @@ jobs: - name: Run K6 tests (${{ inputs.testSuitePath }}) run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} + echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" k6 run ${{ fromJson(inputs.testSuitePath) }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 From edc298b123636857e505832c60b31f9411fcff34 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:29:17 +0100 Subject: [PATCH 11/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index bce8fa05e..3b1427a7f 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -54,7 +54,7 @@ jobs: - name: Run K6 tests (${{ inputs.testSuitePath }}) run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} - echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" + echo "Running k6 test suite ${{ fromJson(inputs.testSuitePath) }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" k6 run ${{ fromJson(inputs.testSuitePath) }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 From 6c7eef6076834a6fa1524ebf1c5f7766d8cd9b94 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:34:16 +0100 Subject: [PATCH 12/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 991307188..96f93d4a7 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -39,7 +39,7 @@ jobs: duration: 1m tokens: both numberOfTokens: 100 - testSuitePath: ${{ matrix.files }} + testSuitePath: "\"tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js\"" # ${{ matrix.files }} permissions: checks: write pull-requests: write From 85c8cb79d01132a949814860d780316b052af9b6 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:37:28 +0100 Subject: [PATCH 13/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 96f93d4a7..65416bfa3 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -25,13 +25,13 @@ jobs: K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} - strategy: - max-parallel: 1 - matrix: - files: - - "\"tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js\"" - - "\"tests/k6/tests/serviceowner/createDialogWithThresholds.js\"" - - "\"tests/k6/tests/enduser/enduserSearchWithThresholds.js\"" + # strategy: + # max-parallel: 1 + # matrix: + # files: + # - "\"tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js\"" + # - "\"tests/k6/tests/serviceowner/createDialogWithThresholds.js\"" + # - "\"tests/k6/tests/enduser/enduserSearchWithThresholds.js\"" with: environment: yt01 apiVersion: v1 @@ -39,7 +39,7 @@ jobs: duration: 1m tokens: both numberOfTokens: 100 - testSuitePath: "\"tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js\"" # ${{ matrix.files }} + testSuitePath: "tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js" # ${{ matrix.files }} permissions: checks: write pull-requests: write From ce3e44ab1920011bbffa227a066d40afa817ad98 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:42:35 +0100 Subject: [PATCH 14/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 65416bfa3..bd8bc68bd 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -10,7 +10,7 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} jobs: - + run-performance-ci-cd: name: "Run K6 performance tests" @@ -39,7 +39,7 @@ jobs: duration: 1m tokens: both numberOfTokens: 100 - testSuitePath: "tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js" # ${{ matrix.files }} + testSuitePath: tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js # ${{ matrix.files }} permissions: checks: write pull-requests: write From ebb6e44929ff8a469f239ba97f22cb1c11369ef5 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 13:52:11 +0100 Subject: [PATCH 15/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 3b1427a7f..88a839e6c 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -54,8 +54,8 @@ jobs: - name: Run K6 tests (${{ inputs.testSuitePath }}) run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} - echo "Running k6 test suite ${{ fromJson(inputs.testSuitePath) }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" - k6 run ${{ fromJson(inputs.testSuitePath) }} --quiet --log-output=stdout --include-system-env-vars \ + echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" + k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 env: From 6becb2b6c5dc691b92e8a368c7182ff35947fb6f Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 14:00:42 +0100 Subject: [PATCH 16/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 88a839e6c..ec7647746 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -55,6 +55,7 @@ jobs: run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" + cat README.md k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 From 281128a7e204d1a2b38a0c7d59bc4f0af2106181 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 14:03:23 +0100 Subject: [PATCH 17/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index ec7647746..07fbcddf8 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -55,7 +55,7 @@ jobs: run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" - cat README.md + cat ${{ inputs.testSuitePath }} k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 From 5170472150213f3be119ad652c3df881119ba067 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 14:08:39 +0100 Subject: [PATCH 18/27] trying filepaths again --- .github/workflows/workflow-run-k6-performance.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 07fbcddf8..11d58f1e7 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -55,7 +55,8 @@ jobs: run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" - cat ${{ inputs.testSuitePath }} + ls -l ./tests/k6/tests/serviceowner + git status k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 From 4f9ae20a50ba518ae258c5a9dadedb10dc177a85 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 14:14:28 +0100 Subject: [PATCH 19/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 16 ++++++------- .../serviceOwnerSearchWithThresholds.js | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index bd8bc68bd..47bab264b 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -25,13 +25,13 @@ jobs: K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} - # strategy: - # max-parallel: 1 - # matrix: - # files: - # - "\"tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js\"" - # - "\"tests/k6/tests/serviceowner/createDialogWithThresholds.js\"" - # - "\"tests/k6/tests/enduser/enduserSearchWithThresholds.js\"" + strategy: + max-parallel: 1 + matrix: + files: + - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js + - tests/k6/tests/serviceowner/createDialogWithThresholds.js + - tests/k6/tests/enduser/enduserSearchWithThresholds.js with: environment: yt01 apiVersion: v1 @@ -39,7 +39,7 @@ jobs: duration: 1m tokens: both numberOfTokens: 100 - testSuitePath: tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js # ${{ matrix.files }} + testSuitePath: ${{ matrix.files }} permissions: checks: write pull-requests: write diff --git a/tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js b/tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js new file mode 100644 index 000000000..ed0fac469 --- /dev/null +++ b/tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js @@ -0,0 +1,24 @@ +import { default as run } from "./performance/serviceowner-search.js"; + +export let options = { + summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], + vus: 1, + duration: "30s", + thresholds: { + "http_req_duration{name:serviceowner search}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get dialog activities}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get dialog activity}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get seenlogs}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get transmissions}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:get transmission}": ["p(95)<100", "p(99)<300"], + "http_reqs{name:get dialog activities}": [], + "http_reqs{name:get dialog activity}": [], + "http_reqs{name:get seenlogs}": [], + "http_reqs{name:get transmissions}": [], + "http_reqs{name:get transmission}": [], + "http_reqs{name:serviceowner search}": [], + } +} + +export default function (data) { run(data); } + From 0e687777d08342b1b89b775c75e52a14ba573827 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 15:19:26 +0100 Subject: [PATCH 20/27] trying filepaths again --- .github/workflows/ci-cd-yt01.yml | 5 +++-- .github/workflows/workflow-run-k6-performance.yml | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 47bab264b..f87bb4826 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -32,11 +32,12 @@ jobs: - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - tests/k6/tests/serviceowner/createDialogWithThresholds.js - tests/k6/tests/enduser/enduserSearchWithThresholds.js + fail-fast: false with: environment: yt01 apiVersion: v1 - vus: 2 - duration: 1m + vus: 1 + duration: 30s tokens: both numberOfTokens: 100 testSuitePath: ${{ matrix.files }} diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 11d58f1e7..88a839e6c 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -55,8 +55,6 @@ jobs: run: | ./tests/k6/tests/scripts/generate_tokens.sh ./tests/k6/tests/performancetest_data ${{ inputs.tokens }} ${{ inputs.numberOfTokens }} ${{ inputs.ttl }} echo "Running k6 test suite ${{ inputs.testSuitePath }} with ${{ inputs.vus }} VUs for ${{ inputs.duration }}" - ls -l ./tests/k6/tests/serviceowner - git status k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 From 52c5ee43123dcefeb38002bdf789435a10e8d4c3 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 19:26:02 +0100 Subject: [PATCH 21/27] changed thresholds --- .../tests/enduser/enduserSearchWithThresholds.js | 16 +++++++++------- .../serviceowner/createDialogWithThresholds.js | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/k6/tests/enduser/enduserSearchWithThresholds.js b/tests/k6/tests/enduser/enduserSearchWithThresholds.js index b041fdc36..8855cb07a 100644 --- a/tests/k6/tests/enduser/enduserSearchWithThresholds.js +++ b/tests/k6/tests/enduser/enduserSearchWithThresholds.js @@ -5,13 +5,15 @@ export let options = { vus: 1, duration: "30s", thresholds: { - "http_req_duration{name:get dialog}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get dialog activities}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get dialog activity}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get seenlogs}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get transmissions}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get transmission}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get labellog}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:enduser search}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get dialog}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get dialog activities}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get dialog activity}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get seenlogs}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get transmissions}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get transmission}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:get labellog}": ["p(95)<300", "p(99)<500"], + "http_reqs{name:enduser search}": [], "http_reqs{name:get dialog activities}": [], "http_reqs{name:get dialog activity}": [], "http_reqs{name:get seenlogs}": [], diff --git a/tests/k6/tests/serviceowner/createDialogWithThresholds.js b/tests/k6/tests/serviceowner/createDialogWithThresholds.js index 31db16e45..56b473244 100644 --- a/tests/k6/tests/serviceowner/createDialogWithThresholds.js +++ b/tests/k6/tests/serviceowner/createDialogWithThresholds.js @@ -5,7 +5,7 @@ export let options = { vus: 1, duration: "30s", thresholds: { - "http_req_duration": ["p(95)<100", "p(99)<300"], + "http_req_duration": ["p(95)<300", "p(99)<500"], } } From 94579566e934715a315673da07a91096637473e4 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 19:40:06 +0100 Subject: [PATCH 22/27] continue on error --- .github/workflows/workflow-run-k6-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 88a839e6c..0075006c1 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -46,6 +46,7 @@ jobs: permissions: checks: write pull-requests: write + continue-on-error: true steps: - name: Checkout code uses: actions/checkout@v4 From 52207f0ddad76e9046f835061b990c849c4ed48a Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 19:47:00 +0100 Subject: [PATCH 23/27] continue on error --- .github/workflows/workflow-run-k6-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 0075006c1..874a0ab1e 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -46,7 +46,6 @@ jobs: permissions: checks: write pull-requests: write - continue-on-error: true steps: - name: Checkout code uses: actions/checkout@v4 @@ -59,6 +58,7 @@ jobs: k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 + continue-on-error: true env: API_ENVIRONMENT: ${{ inputs.environment }} API_VERSION: ${{ inputs.apiVersion }} From 49550ee091eee502cb4a4e2bb0f11a8cf00905fc Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 19:56:37 +0100 Subject: [PATCH 24/27] continue on error --- .github/workflows/workflow-run-k6-performance.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 874a0ab1e..88a839e6c 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -58,7 +58,6 @@ jobs: k6 run ${{ inputs.testSuitePath }} --quiet --log-output=stdout --include-system-env-vars \ --vus=${{ inputs.vus }} --duration=${{ inputs.duration }} --out csv=./results.csv grep http_req_duration ./results.csv | sort --field-separator=',' --key=3 -nr | head -10 - continue-on-error: true env: API_ENVIRONMENT: ${{ inputs.environment }} API_VERSION: ${{ inputs.apiVersion }} From f65907774da1f32dc00fe37c240e31c0509f4d87 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Wed, 11 Dec 2024 20:07:37 +0100 Subject: [PATCH 25/27] back to correct workflow --- .github/workflows/ci-cd-yt01.yml | 151 ++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index f87bb4826..f5e00287c 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -10,14 +10,142 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} jobs: + get-versions-from-github: + name: Get Latest Deployed Version Info from GitHub + uses: ./.github/workflows/workflow-get-latest-deployed-version-info-from-github.yml + with: + environment: yt01 + secrets: + GH_TOKEN: ${{ secrets.RELEASE_VERSION_STORAGE_PAT }} + + check-for-changes: + name: Check for changes + needs: [get-versions-from-github] + uses: ./.github/workflows/workflow-check-for-changes.yml + with: + infra_base_sha: ${{ needs.get-versions-from-github.outputs.infra_version_sha }} + apps_base_sha: ${{ needs.get-versions-from-github.outputs.apps_version_sha }} + + get-current-version: + name: Get current version + uses: ./.github/workflows/workflow-get-current-version.yml + + publish: + name: Build and publish docker images + uses: ./.github/workflows/workflow-publish.yml + if: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true' }} + needs: [get-current-version, check-for-changes] + secrets: + GCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + with: + dockerImageBaseName: ghcr.io/digdir/dialogporten- + version: ${{ needs.get-current-version.outputs.version }} + + deploy-infra: + name: Deploy infra to yt01 + if: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasInfraChanges == 'true' }} + needs: [get-current-version, check-for-changes] + uses: ./.github/workflows/workflow-deploy-infra.yml + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_SOURCE_KEY_VAULT_NAME: ${{ secrets.AZURE_SOURCE_KEY_VAULT_NAME }} + AZURE_SOURCE_KEY_VAULT_SUBSCRIPTION_ID: ${{ secrets.AZURE_SOURCE_KEY_VAULT_SUBSCRIPTION_ID }} + AZURE_SOURCE_KEY_VAULT_RESOURCE_GROUP: ${{ secrets.AZURE_SOURCE_KEY_VAULT_RESOURCE_GROUP }} + AZURE_SOURCE_KEY_VAULT_SSH_JUMPER_SSH_PUBLIC_KEY: ${{ secrets.AZURE_SOURCE_KEY_VAULT_SSH_JUMPER_SSH_PUBLIC_KEY }} + with: + environment: yt01 + region: norwayeast + version: ${{ needs.get-current-version.outputs.version }} + + store-infra-version: + name: Store Latest Deployed Infra Version as GitHub Variable + needs: [deploy-infra, get-current-version] + if: ${{ needs.deploy-infra.result == 'success' }} + uses: ./.github/workflows/workflow-store-github-env-variable.yml + with: + variable_name: LATEST_DEPLOYED_INFRA_VERSION + variable_value: ${{ needs.get-current-version.outputs.version }} + environment: yt01 + secrets: + GH_TOKEN: ${{ secrets.RELEASE_VERSION_STORAGE_PAT }} + + deploy-apps: + name: Deploy apps to yt01 + needs: + [get-current-version, check-for-changes, deploy-infra, publish] + # we want deployment of apps to be dependent on deployment of infrastructure, but if infrastructure is skipped, we still want to deploy the apps + if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true') }} + uses: ./.github/workflows/workflow-deploy-apps.yml + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # todo: consider resolving these in another way since they are created in the infra-step + AZURE_RESOURCE_GROUP_NAME: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} + AZURE_ENVIRONMENT_KEY_VAULT_NAME: ${{ secrets.AZURE_ENVIRONMENT_KEY_VAULT_NAME }} + AZURE_CONTAINER_APP_ENVIRONMENT_NAME: ${{ secrets.AZURE_CONTAINER_APP_ENVIRONMENT_NAME }} + AZURE_APP_INSIGHTS_CONNECTION_STRING: ${{ secrets.AZURE_APP_INSIGHTS_CONNECTION_STRING }} + AZURE_APP_CONFIGURATION_NAME: ${{ secrets.AZURE_APP_CONFIGURATION_NAME }} + AZURE_SERVICE_BUS_NAMESPACE_NAME: ${{ secrets.AZURE_SERVICE_BUS_NAMESPACE_NAME }} + with: + environment: yt01 + region: norwayeast + version: ${{ needs.get-current-version.outputs.version }} + runMigration: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasMigrationChanges == 'true' }} + + store-apps-version: + name: Store Latest Deployed Apps Version as GitHub Variable + needs: [deploy-apps, get-current-version] + if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.deploy-apps.outputs.deployment_executed == 'true') }} + uses: ./.github/workflows/workflow-store-github-env-variable.yml + with: + variable_name: LATEST_DEPLOYED_APPS_VERSION + variable_value: ${{ needs.get-current-version.outputs.version }} + environment: yt01 + secrets: + GH_TOKEN: ${{ secrets.RELEASE_VERSION_STORAGE_PAT }} + deploy-slack-notifier: + name: Deploy slack notifier (yt01) + needs: [check-for-changes] + if: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasSlackNotifierChanges == 'true' }} + uses: ./.github/workflows/workflow-deploy-function.yml + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # todo: resolve this automatically, or use tags + AZURE_FUNCTION_APP_NAME: ${{ secrets.AZURE_SLACK_NOTIFIER_FUNCTION_APP_NAME }} + with: + function-app-name: "slack-notifier" + function-project-path: "./src/Digdir.Tool.Dialogporten.SlackNotifier" + environment: yt01 + + run-e2e-tests: + name: "Run K6 functional end-to-end tests" + # we want the end-to-end tests to be dependent on deployment of infrastructure and apps, but if infrastructure is skipped, we still want to run the tests + if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true') }} + needs: [deploy-apps, check-for-changes] + uses: ./.github/workflows/workflow-run-k6-tests.yml + secrets: + TOKEN_GENERATOR_USERNAME: ${{ secrets.TOKEN_GENERATOR_USERNAME }} + TOKEN_GENERATOR_PASSWORD: ${{ secrets.TOKEN_GENERATOR_PASSWORD }} + with: + environment: yt01 + apiVersion: v1 + testSuitePath: tests/k6/suites/all-single-pass.js + permissions: + checks: write + pull-requests: write - run-performance-ci-cd: + run-performance: name: "Run K6 performance tests" # we want the performance tests to be dependent on deployment of infrastructure and apps, but if infrastructure is skipped, we still want to run the tests - + if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasBackendChanges == 'true' || needs.check-for-changes.outputs.hasInfraChanges == 'true') }} + needs: [deploy-apps, deploy-infra, check-for-changes] #needs: [deploy-apps, check-for-changes] - uses: ./.github/workflows/workflow-run-k6-performance.yml secrets: TOKEN_GENERATOR_USERNAME: ${{ secrets.TOKEN_GENERATOR_USERNAME }} @@ -32,7 +160,6 @@ jobs: - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - tests/k6/tests/serviceowner/createDialogWithThresholds.js - tests/k6/tests/enduser/enduserSearchWithThresholds.js - fail-fast: false with: environment: yt01 apiVersion: v1 @@ -44,3 +171,19 @@ jobs: permissions: checks: write pull-requests: write + + send-slack-message-on-failure: + name: Send Slack message on failure + needs: [deploy-infra, deploy-apps, deploy-slack-notifier, run-e2e-tests, publish] + if: ${{ always() && failure() && !cancelled() }} + uses: ./.github/workflows/workflow-send-ci-cd-status-slack-message.yml + with: + environment: yt01 + infra_status: ${{ needs.deploy-infra.result }} + apps_status: ${{ needs.deploy-apps.result }} + slack_notifier_status: ${{ needs.deploy-slack-notifier.result }} + e2e_tests_status: ${{ needs.run-e2e-tests.result }} + publish_status: ${{ needs.publish.result }} + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_FOR_CI_CD_STATUS }} From 4344776f07070c86d7fd977cd43205d7e4741c51 Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Thu, 12 Dec 2024 11:34:35 +0100 Subject: [PATCH 26/27] trying to avoid duplicated code --- .../enduser/enduserSearchWithThresholds.js | 28 +++++++++---------- .../enduser/performance/enduser-search.js | 16 +++++------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/k6/tests/enduser/enduserSearchWithThresholds.js b/tests/k6/tests/enduser/enduserSearchWithThresholds.js index 8855cb07a..eb2d3b6c1 100644 --- a/tests/k6/tests/enduser/enduserSearchWithThresholds.js +++ b/tests/k6/tests/enduser/enduserSearchWithThresholds.js @@ -6,21 +6,21 @@ export let options = { duration: "30s", thresholds: { "http_req_duration{name:enduser search}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get dialog}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get dialog activities}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get dialog activity}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get seenlogs}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get transmissions}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get transmission}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get labellog}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get dialog}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get dialog activities}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get dialog activity}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get seenlogs}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get transmissions}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get transmission}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:enduser get labellog}": ["p(95)<300", "p(99)<500"], "http_reqs{name:enduser search}": [], - "http_reqs{name:get dialog activities}": [], - "http_reqs{name:get dialog activity}": [], - "http_reqs{name:get seenlogs}": [], - "http_reqs{name:get transmissions}": [], - "http_reqs{name:get transmission}": [], - "http_reqs{name:get dialog}": [], - "http_reqs{name:get labellog}": [], + "http_reqs{name:enduser get dialog activities}": [], + "http_reqs{name:enduser get dialog activity}": [], + "http_reqs{name:enduser get seenlogs}": [], + "http_reqs{name:enduser get transmissions}": [], + "http_reqs{name:enduser get transmission}": [], + "http_reqs{name:enduser get dialog}": [], + "http_reqs{name:enduser get labellog}": [], } } diff --git a/tests/k6/tests/enduser/performance/enduser-search.js b/tests/k6/tests/enduser/performance/enduser-search.js index 41f1b0d7c..90fd03737 100644 --- a/tests/k6/tests/enduser/performance/enduser-search.js +++ b/tests/k6/tests/enduser/performance/enduser-search.js @@ -8,14 +8,14 @@ const traceCalls = (__ENV.traceCalls ?? 'false') === 'true'; export let options = { summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], thresholds: getDefaultThresholds(['http_req_duration', 'http_reqs'],['enduser search', - 'get dialog', - 'get dialog activities', - 'get dialog activity', - 'get seenlogs', - 'get seenlog', - 'get transmissions', - 'get transmission', - 'get labellog' + 'enduser get dialog', + 'enduser get dialog activities', + 'enduser get dialog activity', + 'enduser get seenlogs', + 'enduser get seenlog', + 'enduser get transmissions', + 'enduser get transmission', + 'enduser get labellog' ]) }; From 2176ea3e51b0c28e9fa07e9e184c3d71f7c658fc Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Thu, 12 Dec 2024 13:52:52 +0100 Subject: [PATCH 27/27] add fail-fast: false --- .github/workflows/ci-cd-yt01.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index f5e00287c..af94c7f8f 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -160,6 +160,7 @@ jobs: - tests/k6/tests/serviceowner/serviceOwnerSearchWithThresholds.js - tests/k6/tests/serviceowner/createDialogWithThresholds.js - tests/k6/tests/enduser/enduserSearchWithThresholds.js + fail-fast: false with: environment: yt01 apiVersion: v1