From 65f6c2aa3e680f7ca6cc5d9fd85a3afb76d8eb0c Mon Sep 17 00:00:00 2001 From: Tomasz Konieczny Date: Tue, 1 Oct 2024 00:36:46 +0200 Subject: [PATCH] feat: Workflow tests - junit reports (#5881) * status page examples, curl workflows updated * cypress test updated - junit * playwright and postman - junit examples * workflow tests - pytest project * workflow tests - pytest project - test files renamed * workflow tests - pytest - pytest.ini * workflow tests - pytest - pytest.ini updated * workflow tests - junit5 * workflow tests - junit5 tests - pom updated * workflow tests - junit5 tests - filenames updated * workflow tests - junit5 * workflow tests - junit5 tests added * workflow tests - junit5 pom updated * workflow tests - junit5 pom updated * workflow tests - junit5 pom updated * workflow tests - junit5 pom updated * workflow tests - junit5 * workflow tests - junit5 * workflow tests - junit5 * workflow tests - junit5 * workflow tests - junit5 * workflow tests - junit5 * junit 5 project - pom fixed * junit 5 - pom updated (junit report) * junit 5 - pom updated (junit report) * junit 5 - tests updated * junit 5 - tests extended * workflow tests - suites updated * workflow tests - pytest * workflow tests - suites extended, run scirpt * junit target removed --- .../executor-tests/crd-workflow/smoke.yaml | 6 +- .../executor-tests/crd-workflow/smoke.yaml | 32 +------ test/examples/status-page/crd.yaml | 53 ++++++++++++ test/junit/crd-workflow/smoke.yaml | 80 +++++++++++++++++ test/junit/junit5-project/pom.xml | 36 ++++++++ .../src/test/java/TestSmoke1.java | 61 +++++++++++++ .../src/test/java/TestSmoke2.java | 41 +++++++++ .../src/test/java/TestSmokeFail.java | 58 +++++++++++++ .../executor-tests/crd-workflow/smoke.yaml | 65 ++++++++++++++ .../executor-tests/crd-workflow/smoke.yaml | 27 ++++++ test/pytest/crd-workflow/smoke.yaml | 85 +++++++++++++++++++ test/pytest/pytest-project/pytest.ini | 3 + test/pytest/pytest-project/requirements.txt | 1 + .../tests/fail/smoke-3-fail_test.py | 15 ++++ .../tests/success/smoke-2_test.py | 20 +++++ .../tests/success/smoke_test.py | 15 ++++ test/scripts/executor-tests/run.sh | 38 ++++++++- .../combined-workflow-test.yaml | 13 +++ test/suites/test-workflows/curl-workflow.yaml | 14 +++ .../test-workflows/cypress-workflow.yaml | 3 +- test/suites/test-workflows/full-smoke.yaml | 3 + .../suites/test-workflows/junit-workflow.yaml | 14 +++ .../test-workflows/playwright-workflow.yaml | 2 + .../test-workflows/postman-workflow.yaml | 1 + .../test-workflows/pytest-workflow.yaml | 13 +++ test/suites/test-workflows/small-smoke.yaml | 1 + 26 files changed, 663 insertions(+), 37 deletions(-) create mode 100644 test/examples/status-page/crd.yaml create mode 100644 test/junit/crd-workflow/smoke.yaml create mode 100644 test/junit/junit5-project/pom.xml create mode 100644 test/junit/junit5-project/src/test/java/TestSmoke1.java create mode 100644 test/junit/junit5-project/src/test/java/TestSmoke2.java create mode 100644 test/junit/junit5-project/src/test/java/TestSmokeFail.java create mode 100644 test/pytest/crd-workflow/smoke.yaml create mode 100644 test/pytest/pytest-project/pytest.ini create mode 100644 test/pytest/pytest-project/requirements.txt create mode 100644 test/pytest/pytest-project/tests/fail/smoke-3-fail_test.py create mode 100644 test/pytest/pytest-project/tests/success/smoke-2_test.py create mode 100644 test/pytest/pytest-project/tests/success/smoke_test.py create mode 100644 test/suites/test-workflows/combined-workflow-test.yaml create mode 100644 test/suites/test-workflows/curl-workflow.yaml create mode 100644 test/suites/test-workflows/junit-workflow.yaml create mode 100644 test/suites/test-workflows/pytest-workflow.yaml diff --git a/test/curl/executor-tests/crd-workflow/smoke.yaml b/test/curl/executor-tests/crd-workflow/smoke.yaml index 7c81cbed835..d4d13869eeb 100644 --- a/test/curl/executor-tests/crd-workflow/smoke.yaml +++ b/test/curl/executor-tests/crd-workflow/smoke.yaml @@ -12,7 +12,7 @@ spec: memory: 32Mi steps: - name: Run tests - shell: curl https://testkube.io + shell: curl -f -LI https://testkube.io container: image: curlimages/curl:8.7.1 --- @@ -33,7 +33,7 @@ spec: memory: 32Mi steps: - name: Run tests - shell: curl https://testkube.io && sleep 10 + shell: curl -f -LI https://testkube.io && sleep 10 container: image: tkoniecznykubeshop/example-private-repo:curl-8.7.1 --- @@ -54,6 +54,6 @@ spec: parallel: matrix: url: ['https://testkube.io', 'https://docs.testkube.io'] - shell: curl '{{ matrix.url }}' + shell: curl -f -LI '{{ matrix.url }}' container: image: curlimages/curl:8.7.1 diff --git a/test/cypress/executor-tests/crd-workflow/smoke.yaml b/test/cypress/executor-tests/crd-workflow/smoke.yaml index 3fdae48f0aa..0f069b80a46 100644 --- a/test/cypress/executor-tests/crd-workflow/smoke.yaml +++ b/test/cypress/executor-tests/crd-workflow/smoke.yaml @@ -35,9 +35,6 @@ spec: artifacts: paths: - '**/*' - events: - - cronjob: - cron: "1 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -76,9 +73,6 @@ spec: artifacts: paths: - '**/*' - events: - - cronjob: - cron: "2 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -114,9 +108,6 @@ spec: artifacts: paths: - '**/*' - events: - - cronjob: - cron: "3 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -148,9 +139,6 @@ spec: config: version: 13.5.0 run: "npx cypress run --env NON_CYPRESS_ENV=NON_CYPRESS_ENV_value --config '{\"screenshotsFolder\":\"/data/artifacts/screenshots\",\"videosFolder\":\"/data/artifacts/videos\"}'" - events: - - cronjob: - cron: "4 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -182,9 +170,6 @@ spec: config: version: 13.5.0 run: "npx cypress run --env NON_CYPRESS_ENV=NON_CYPRESS_ENV_value --config '{\"screenshotsFolder\":\"/data/artifacts/screenshots\",\"videosFolder\":\"/data/artifacts/videos\"}'" - events: - - cronjob: - cron: "5 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -218,9 +203,6 @@ spec: config: version: 13.5.0 run: "npx cypress run --env NON_CYPRESS_ENV=NON_CYPRESS_ENV_value --config '{\"screenshotsFolder\":\"/data/artifacts/screenshots\",\"videosFolder\":\"/data/artifacts/videos\"}'" - events: - - cronjob: - cron: "6 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -259,9 +241,6 @@ spec: artifacts: paths: - '**/*' - events: - - cronjob: - cron: "7 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -304,9 +283,6 @@ spec: artifacts: paths: - '**/*' - events: - - cronjob: - cron: "8 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow @@ -355,14 +331,11 @@ spec: workingDir: /data/artifacts paths: - '**/*' - events: - - cronjob: - cron: "10 */4 * * *" --- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow metadata: - name: cypress-workflow-smoke-matrix + name: cypress-workflow-smoke-matrix-junit labels: core-tests: workflows spec: @@ -407,6 +380,3 @@ spec: workingDir: /data/artifacts paths: - '**/*' - events: - - cronjob: - cron: "12 */4 * * *" diff --git a/test/examples/status-page/crd.yaml b/test/examples/status-page/crd.yaml new file mode 100644 index 00000000000..3b4500c939e --- /dev/null +++ b/test/examples/status-page/crd.yaml @@ -0,0 +1,53 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: tw-testkube-website-5m +spec: + events: + - cronjob: + cron: '*/5 * * * *' + container: + image: curlimages/curl:8.7.1 + resources: + requests: + cpu: 32m + memory: 32Mi + steps: + - name: Run tests + shell: curl -f -LI https://testkube.io +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: tw-testkube-docs-5m +spec: + events: + - cronjob: + cron: '*/5 * * * *' + container: + image: curlimages/curl:8.7.1 + resources: + requests: + cpu: 32m + memory: 32Mi + steps: + - name: Run tests + shell: curl -f -LI https://docs.testkube.io +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: tw-testkube-blog-5m +spec: + events: + - cronjob: + cron: '*/5 * * * *' + container: + image: curlimages/curl:8.7.1 + resources: + requests: + cpu: 32m + memory: 32Mi + steps: + - name: Run tests + shell: curl -f -LI https://testkube.io/blog diff --git a/test/junit/crd-workflow/smoke.yaml b/test/junit/crd-workflow/smoke.yaml new file mode 100644 index 00000000000..0736e109b28 --- /dev/null +++ b/test/junit/crd-workflow/smoke.yaml @@ -0,0 +1,80 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: junit5-workflow-smoke + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: workflow-tests-junit # TODO: update + paths: + - test/junit/junit5-project + container: + image: maven:3.9.9-eclipse-temurin-11-alpine + resources: + requests: + cpu: 256m + memory: 256Mi + workingDir: /data/repo/test/junit/junit5-project + steps: + - name: Run tests + shell: mvn test -Dtest=TestSmoke1,TestSmoke2 + artifacts: + paths: + - 'target/surefire-reports/**/*' +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: junit5-workflow-smoke-expected-fail + labels: + core-tests: expected-fail +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: workflow-tests-junit # TODO: update + paths: + - test/junit/junit5-project + container: + image: maven:3.9.9-eclipse-temurin-11-alpine + resources: + requests: + cpu: 256m + memory: 256Mi + workingDir: /data/repo/test/junit/junit5-project + steps: + - name: Run tests + shell: mvn test -Dtest=TestSmokeFail + artifacts: + paths: + - 'target/surefire-reports/**/*' +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: junit5-workflow-smoke-mixed-results + labels: + core-tests: expected-fail +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: workflow-tests-junit # TODO: update + paths: + - test/junit/junit5-project + container: + image: maven:3.9.9-eclipse-temurin-11-alpine + resources: + requests: + cpu: 256m + memory: 256Mi + workingDir: /data/repo/test/junit/junit5-project + steps: + - name: Run tests + shell: mvn test + artifacts: + paths: + - 'target/surefire-reports/**/*' diff --git a/test/junit/junit5-project/pom.xml b/test/junit/junit5-project/pom.xml new file mode 100644 index 00000000000..f22ecbe7341 --- /dev/null +++ b/test/junit/junit5-project/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + + example.testsuite + example-testsuite + 1.0 + + + + org.junit.jupiter + junit-jupiter-engine + 5.11.1 + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + diff --git a/test/junit/junit5-project/src/test/java/TestSmoke1.java b/test/junit/junit5-project/src/test/java/TestSmoke1.java new file mode 100644 index 00000000000..03fc71292d6 --- /dev/null +++ b/test/junit/junit5-project/src/test/java/TestSmoke1.java @@ -0,0 +1,61 @@ +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.*; + +public class TestSmoke1 { + @BeforeEach + public void beforeEach() { + try { + TimeUnit.SECONDS.sleep(1); + } + catch(Exception e) { + System.out.println(e); + } + } + + @Test + public void test1() { + assertEquals(1, 1); + } + + @Test + public void test2() { + assertEquals(1, 1); + } + + @Test + public void test3() { + assertEquals(1, 1); + } + + @Test + public void test4() { + try { + TimeUnit.SECONDS.sleep(3); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 1); + } + + @Test + public void test5() { + assertEquals(1, 1); + } + + @Test + public void test6() { + try { + TimeUnit.SECONDS.sleep(6); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 1); + } +} diff --git a/test/junit/junit5-project/src/test/java/TestSmoke2.java b/test/junit/junit5-project/src/test/java/TestSmoke2.java new file mode 100644 index 00000000000..68f95c21719 --- /dev/null +++ b/test/junit/junit5-project/src/test/java/TestSmoke2.java @@ -0,0 +1,41 @@ +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.*; + +public class TestSmoke2 { + @Test + public void testAaa() { + try { + TimeUnit.SECONDS.sleep(1); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 1); + } + + @Test + public void testBbb() { + assertEquals(1, 1); + } + + @Test + public void testCcc() { + try { + TimeUnit.SECONDS.sleep(2); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 1); + } + + @Test + public void testDdd() { + assertEquals(1, 1); + } +} diff --git a/test/junit/junit5-project/src/test/java/TestSmokeFail.java b/test/junit/junit5-project/src/test/java/TestSmokeFail.java new file mode 100644 index 00000000000..aaf4baf3401 --- /dev/null +++ b/test/junit/junit5-project/src/test/java/TestSmokeFail.java @@ -0,0 +1,58 @@ +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.*; + +public class TestSmokeFail { + @Test + public void test1() { + assertEquals(1, 2); + } + + @Test + public void test2() { + try { + TimeUnit.SECONDS.sleep(2); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 1); + } + + @Test + public void test3() { + try { + TimeUnit.SECONDS.sleep(5); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 2); + } + + @Test + public void test4() { + assertEquals(1, 2); + } + + @Test + public void test5() { + try { + TimeUnit.SECONDS.sleep(2); + } + catch(Exception e) { + System.out.println(e); + } + + assertEquals(1, 1); + } + + @Test + public void test6() { + assertEquals(1, 1); + } +} diff --git a/test/playwright/executor-tests/crd-workflow/smoke.yaml b/test/playwright/executor-tests/crd-workflow/smoke.yaml index 6ffff58f0cc..43e5a5e3299 100644 --- a/test/playwright/executor-tests/crd-workflow/smoke.yaml +++ b/test/playwright/executor-tests/crd-workflow/smoke.yaml @@ -129,6 +129,39 @@ spec: paths: - '**/*' --- +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: playwright-workflow-smoke-junit + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/playwright/executor-tests/playwright-project + container: + image: mcr.microsoft.com/playwright:v1.32.3-focal + resources: + requests: + cpu: 2 + memory: 2Gi + workingDir: /data/repo/test/playwright/executor-tests/playwright-project + steps: + - name: Install dependencies + shell: "npm ci" + - name: Run tests + shell: "mkdir /data/artifacts && npx playwright test --reporter=junit > /data/artifacts/junit-report.xml" # workaround - PLAYWRIGHT_JUNIT_OUTPUT_DIR and PLAYWRIGHT_JUNIT_OUTPUT_NAME are not working + steps: + - name: Save artifacts + workingDir: /data/artifacts + artifacts: + paths: + - '**/*' +--- apiVersion: testworkflows.testkube.io/v1 kind: TestWorkflow metadata: @@ -234,3 +267,35 @@ spec: artifacts: paths: - "playwright-report/**" +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: distributed-playwright-workflow-smoke-junit + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/playwright/executor-tests/playwright-project + container: + image: mcr.microsoft.com/playwright:v1.38.0-focal + workingDir: /data/repo/test/playwright/executor-tests/playwright-project + steps: + - name: Install dependencies + # TODO: Upgrade our Playwright tests to use newer version; Blob reporter was not available in 1.32.3 + shell: 'npm install --save-dev @playwright/test@1.38.0 && npm ci' + - name: Run tests + parallel: + count: 2 + transfer: + - from: /data/repo + run: + shell: 'mkdir /data/artifacts && npx playwright test --reporter=junit --shard {{ index + 1 }}/{{ count }} > /data/artifacts/junit-report{{ index + 1 }}.xml' + artifacts: + workingDir: /data/artifacts/ + paths: + - '**/*' diff --git a/test/postman/executor-tests/crd-workflow/smoke.yaml b/test/postman/executor-tests/crd-workflow/smoke.yaml index f644a772429..c238d0464d1 100644 --- a/test/postman/executor-tests/crd-workflow/smoke.yaml +++ b/test/postman/executor-tests/crd-workflow/smoke.yaml @@ -138,3 +138,30 @@ spec: events: - cronjob: cron: "*/5 * * * *" +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: postman-workflow-smoke-junit + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/postman/executor-tests/postman-executor-smoke.postman_collection.json + container: + image: postman/newman:6-alpine + resources: + requests: + cpu: 256m + memory: 128Mi + workingDir: /data/repo/test/postman/executor-tests + steps: + - name: Run test + shell: newman run postman-executor-smoke.postman_collection.json --env-var TESTKUBE_POSTMAN_PARAM=TESTKUBE_POSTMAN_PARAM_value -r cli,junit --reporter-junit-export /data/artifacts/junit-report.xml + artifacts: + paths: + - "/data/artifacts/**" diff --git a/test/pytest/crd-workflow/smoke.yaml b/test/pytest/crd-workflow/smoke.yaml new file mode 100644 index 00000000000..acf7afdd2a4 --- /dev/null +++ b/test/pytest/crd-workflow/smoke.yaml @@ -0,0 +1,85 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: pytest-workflow-smoke + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: workflow-tests-junit # TODO: update + paths: + - test/pytest/pytest-project + container: + image: python:3.12.6-alpine3.20 + resources: + requests: + cpu: 256m + memory: 256Mi + workingDir: /data/repo/test/pytest/pytest-project + steps: + - name: Run test + shell: | + pip install -r requirements.txt + pytest tests/success +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: pytest-workflow-smoke-junit-success-only + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: workflow-tests-junit # TODO: update + paths: + - test/pytest/pytest-project + container: + image: python:3.12.6-alpine3.20 + resources: + requests: + cpu: 256m + memory: 256Mi + workingDir: /data/repo/test/pytest/pytest-project + steps: + - name: Run test + shell: | + pip install -r requirements.txt + mkdir /data/artifacts + pytest tests/success --junit-xml=/data/artifacts/pytest-report.xml + artifacts: + paths: + - "/data/artifacts/**" +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: pytest-workflow-smoke-junit-mixed-failures + labels: + core-tests: expected-fail +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: workflow-tests-junit # TODO: update + paths: + - test/pytest/pytest-project + container: + image: python:3.12.6-alpine3.20 + resources: + requests: + cpu: 256m + memory: 256Mi + workingDir: /data/repo/test/pytest/pytest-project + steps: + - name: Run test + shell: | + pip install -r requirements.txt + mkdir /data/artifacts + pytest tests --junit-xml=/data/artifacts/pytest-report.xml + artifacts: + paths: + - "/data/artifacts/**" diff --git a/test/pytest/pytest-project/pytest.ini b/test/pytest/pytest-project/pytest.ini new file mode 100644 index 00000000000..d3e7bd17a4a --- /dev/null +++ b/test/pytest/pytest-project/pytest.ini @@ -0,0 +1,3 @@ +# pytest.ini +[pytest] +junit_suite_name = Smoke Test Suite diff --git a/test/pytest/pytest-project/requirements.txt b/test/pytest/pytest-project/requirements.txt new file mode 100644 index 00000000000..40543aababf --- /dev/null +++ b/test/pytest/pytest-project/requirements.txt @@ -0,0 +1 @@ +pytest==8.3.3 diff --git a/test/pytest/pytest-project/tests/fail/smoke-3-fail_test.py b/test/pytest/pytest-project/tests/fail/smoke-3-fail_test.py new file mode 100644 index 00000000000..c870fb3f0af --- /dev/null +++ b/test/pytest/pytest-project/tests/fail/smoke-3-fail_test.py @@ -0,0 +1,15 @@ +import time + +def test_AAA(): + assert 1 == 2 + +def test_BBB(): + time.sleep(3) + assert 1 == 1 + +def test_CCC(): + assert 1 == 2 + +def test_DDD(): + time.sleep(2) + assert 1 == 1 diff --git a/test/pytest/pytest-project/tests/success/smoke-2_test.py b/test/pytest/pytest-project/tests/success/smoke-2_test.py new file mode 100644 index 00000000000..e2143fcbc76 --- /dev/null +++ b/test/pytest/pytest-project/tests/success/smoke-2_test.py @@ -0,0 +1,20 @@ +import time + +def test_lorem(): + time.sleep(1) + assert 1 == 1 + +def test_ipsum(): + time.sleep(3) + assert 1 == 1 + +def test_dolor(): + assert 1 == 1 + +def test_sit(): + time.sleep(5) + assert 1 == 1 + +def test_amet(): + time.sleep(2) + assert 1 == 1 diff --git a/test/pytest/pytest-project/tests/success/smoke_test.py b/test/pytest/pytest-project/tests/success/smoke_test.py new file mode 100644 index 00000000000..3b03d76394d --- /dev/null +++ b/test/pytest/pytest-project/tests/success/smoke_test.py @@ -0,0 +1,15 @@ +import time + +def test_1(): + assert 1 == 1 + +def test_2(): + time.sleep(3) + assert 1 == 1 + +def test_3(): + assert 1 == 1 + +def test_4(): + time.sleep(2) + assert 1 == 1 diff --git a/test/scripts/executor-tests/run.sh b/test/scripts/executor-tests/run.sh index a76987463e7..1e4e53cef1c 100755 --- a/test/scripts/executor-tests/run.sh +++ b/test/scripts/executor-tests/run.sh @@ -432,6 +432,15 @@ workflow-artillery-smoke() { common_workflow_run "$name" "$workflow_crd_file" "$workflow_suite_name" "$workflow_suite_file" } +workflow-curl-smoke() { + name="Test Workflow - Curl" + workflow_crd_file="test/curl/executor-tests/crd-workflow/smoke.yaml" + workflow_suite_name="curl-workflow-suite" + workflow_suite_file="test/suites/test-workflows/curl-workflow.yaml" + + common_workflow_run "$name" "$workflow_crd_file" "$workflow_suite_name" "$workflow_suite_file" +} + workflow-cypress-smoke() { name="Test Workflow - Cypress" workflow_crd_file="test/cypress/executor-tests/crd-workflow/smoke.yaml" @@ -459,6 +468,15 @@ workflow-jmeter-smoke() { common_workflow_run "$name" "$workflow_crd_file" "$workflow_suite_name" "$workflow_suite_file" } +workflow-junit-smoke() { + name="Test Workflow - JUnit 5" + workflow_crd_file="test/junit/crd-workflow/smoke.yaml" + workflow_suite_name="junit-workflow-suite" + workflow_suite_file="test/suites/test-workflows/junit-workflow.yaml" + + common_workflow_run "$name" "$workflow_crd_file" "$workflow_suite_name" "$workflow_suite_file" +} + workflow-k6-smoke() { name="Test Workflow - k6" workflow_crd_file="test/k6/executor-tests/crd-workflow/smoke.yaml" @@ -495,6 +513,15 @@ workflow-postman-smoke() { common_workflow_run "$name" "$workflow_crd_file" "$workflow_suite_name" "$workflow_suite_file" } +workflow-pytest-smoke() { + name="Test Workflow - Pytest" + workflow_crd_file="test/pytest/crd-workflow/smoke.yaml" + workflow_suite_name="pytest-workflow-suite" + workflow_suite_file="test/suites/test-workflows/pytest-workflow.yaml" + + common_workflow_run "$name" "$workflow_crd_file" "$workflow_suite_name" "$workflow_suite_file" +} + workflow-soapui-smoke() { name="Test Workflow - SoapUI" workflow_crd_file="test/soapui/executor-smoke/crd-workflow/smoke.yaml" @@ -526,7 +553,6 @@ main() { container-postman-smoke container-playwright-smoke container-soapui-smoke - curl-smoke cypress-smoke ginkgo-smoke gradle-smoke @@ -540,13 +566,16 @@ main() { playwright-smoke soapui-smoke workflow-artillery-smoke + workflow-curl-smoke workflow-cypress-smoke workflow-gradle-smoke workflow-jmeter-smoke + workflow-junit-smoke workflow-k6-smoke workflow-maven-smoke workflow-playwright-smoke workflow-postman-smoke + workflow-pytest-smoke workflow-soapui-smoke workflow-special-cases-failures ;; @@ -561,7 +590,6 @@ main() { container-postman-smoke container-playwright-smoke container-soapui-smoke - curl-smoke cypress-smoke ginkgo-smoke gradle-smoke @@ -573,13 +601,16 @@ main() { postman-smoke soapui-smoke workflow-artillery-smoke + workflow-curl-smoke workflow-cypress-smoke workflow-gradle-smoke workflow-jmeter-smoke + workflow-junit-smoke workflow-k6-smoke workflow-maven-smoke workflow-playwright-smoke workflow-postman-smoke + workflow-pytest-smoke workflow-soapui-smoke ;; special) @@ -590,13 +621,16 @@ main() { ;; workflow) workflow-artillery-smoke + workflow-curl-smoke workflow-cypress-smoke workflow-gradle-smoke workflow-jmeter-smoke + workflow-junit-smoke workflow-k6-smoke workflow-maven-smoke workflow-playwright-smoke workflow-postman-smoke + workflow-pytest-smoke workflow-soapui-smoke workflow-special-cases-failures ;; diff --git a/test/suites/test-workflows/combined-workflow-test.yaml b/test/suites/test-workflows/combined-workflow-test.yaml new file mode 100644 index 00000000000..bec114f8a72 --- /dev/null +++ b/test/suites/test-workflows/combined-workflow-test.yaml @@ -0,0 +1,13 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: artillery-workflow-suite-combined + labels: + core-tests: workflows-suite +spec: + steps: + - execute: + workflows: + - name: artillery-workflow-smoke + tests: + - name: artillery-executor-smoke diff --git a/test/suites/test-workflows/curl-workflow.yaml b/test/suites/test-workflows/curl-workflow.yaml new file mode 100644 index 00000000000..2dc0d0ce964 --- /dev/null +++ b/test/suites/test-workflows/curl-workflow.yaml @@ -0,0 +1,14 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: curl-workflow-suite + labels: + core-tests: workflows-suite +spec: + steps: + - execute: + parallelism: 2 + workflows: + - name: curl-workflow-smoke + - name: curl-workflow-smoke-matrix + diff --git a/test/suites/test-workflows/cypress-workflow.yaml b/test/suites/test-workflows/cypress-workflow.yaml index 9cdc2ea7c3e..c6eba169e93 100644 --- a/test/suites/test-workflows/cypress-workflow.yaml +++ b/test/suites/test-workflows/cypress-workflow.yaml @@ -18,4 +18,5 @@ spec: - name: cypress-workflow-smoke-12.7.0 - name: cypress-workflow-smoke-13-junit-report - name: distributed-cypress-workflow-smoke-13 - - name: cypress-workflow-smoke-matrix + - name: cypress-workflow-smoke-matrix-junit + diff --git a/test/suites/test-workflows/full-smoke.yaml b/test/suites/test-workflows/full-smoke.yaml index ff4e1f6867d..d4cfb3be247 100644 --- a/test/suites/test-workflows/full-smoke.yaml +++ b/test/suites/test-workflows/full-smoke.yaml @@ -11,10 +11,13 @@ spec: workflows: - name: artillery-workflow-suite - name: cypress-workflow-suite + - name: curl-workflow-suite - name: gradle-workflow-suite - name: jmeter-workflow-suite + - name: junit-workflow-suite - name: k6-workflow-suite - name: maven-workflow-suite - name: playwright-workflow-suite - name: postman-workflow-suite + - name: pytest-workflow-suite - name: soapui-workflow-suite diff --git a/test/suites/test-workflows/junit-workflow.yaml b/test/suites/test-workflows/junit-workflow.yaml new file mode 100644 index 00000000000..2dc0d0ce964 --- /dev/null +++ b/test/suites/test-workflows/junit-workflow.yaml @@ -0,0 +1,14 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: curl-workflow-suite + labels: + core-tests: workflows-suite +spec: + steps: + - execute: + parallelism: 2 + workflows: + - name: curl-workflow-smoke + - name: curl-workflow-smoke-matrix + diff --git a/test/suites/test-workflows/playwright-workflow.yaml b/test/suites/test-workflows/playwright-workflow.yaml index 4374c3a7a84..e36fc63d4ba 100644 --- a/test/suites/test-workflows/playwright-workflow.yaml +++ b/test/suites/test-workflows/playwright-workflow.yaml @@ -12,6 +12,8 @@ spec: - name: playwright-workflow-smoke-v1.32.3 - name: playwright-workflow-smoke-v1.32.3-custom-report-dir - name: playwright-workflow-smoke-v1.32.3-shell + - name: playwright-workflow-smoke-junit - name: playwright-workflow-smoke-artifacts-double-asterisk - name: playwright-workflow-smoke-template - name: distributed-playwright-workflow-smoke-artifacts + - name: distributed-playwright-workflow-smoke-junit diff --git a/test/suites/test-workflows/postman-workflow.yaml b/test/suites/test-workflows/postman-workflow.yaml index 051ebaa2d0d..226e131cf50 100644 --- a/test/suites/test-workflows/postman-workflow.yaml +++ b/test/suites/test-workflows/postman-workflow.yaml @@ -14,3 +14,4 @@ spec: - name: postman-workflow-smoke-template - name: postman-workflow-smoke-template-without-envs - name: postman-workflow-smoke-cron-trigger-test + - name: postman-workflow-smoke-junit diff --git a/test/suites/test-workflows/pytest-workflow.yaml b/test/suites/test-workflows/pytest-workflow.yaml new file mode 100644 index 00000000000..1eb516f0aa9 --- /dev/null +++ b/test/suites/test-workflows/pytest-workflow.yaml @@ -0,0 +1,13 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: pytest-workflow-suite + labels: + core-tests: workflows-suite +spec: + steps: + - execute: + parallelism: 2 + workflows: + - name: pytest-workflow-smoke + - name: pytest-workflow-smoke-junit-success-only diff --git a/test/suites/test-workflows/small-smoke.yaml b/test/suites/test-workflows/small-smoke.yaml index 93307ec51e9..81312891e6e 100644 --- a/test/suites/test-workflows/small-smoke.yaml +++ b/test/suites/test-workflows/small-smoke.yaml @@ -15,3 +15,4 @@ spec: - name: k6-workflow-smoke-artifacts - name: postman-workflow-smoke-template - name: soapui-workflow-smoke + - name: pytest-workflow-smoke-junit-success-only