Smoke test daily #77
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke test daily | |
| on: | |
| schedule: | |
| - cron: '25 3 * * *' | |
| workflow_dispatch: | |
| env: | |
| API_ARTIFACT: api-daily--run-${{ github.run_number }} | |
| E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }} | |
| FORCE_COLOR: 1 | |
| PLUGIN_SLACK_BLOCKS_ARTIFACT: plugin-blocks | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: {} | |
| jobs: | |
| api-tests: | |
| name: API tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| outputs: | |
| test-result: ${{ steps.run-api-composite-action.outputs.result }} | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report | |
| BASE_URL: ${{ secrets.SMOKE_TEST_URL }} | |
| API_BASE_URL: ${{ secrets.SMOKE_TEST_API_URL }} | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| UPDATE_WC: 'nightly' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install-filters: woocommerce | |
| build: false | |
| - name: Update site to nightly version | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: ${{ env.API_ARTIFACT }} | |
| tests: update-woocommerce.spec.js | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| UPDATE_WC: nightly | |
| - name: Run API tests | |
| id: run-api-composite-action | |
| uses: ./.github/actions/tests/run-api-tests | |
| with: | |
| report-name: ${{ env.API_ARTIFACT }} | |
| env: | |
| USER_KEY: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| USER_SECRET: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| e2e-tests: | |
| name: E2E tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| outputs: | |
| test-result: ${{ steps.run-e2e-composite-action.outputs.result }} | |
| # needs: [api-tests] | |
| env: | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| BASE_URL: ${{ secrets.SMOKE_TEST_URL }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install-filters: woocommerce | |
| build: false | |
| - name: Run E2E tests | |
| id: run-e2e-composite-action | |
| timeout-minutes: 60 | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: ${{ env.E2E_ARTIFACT }} | |
| env: | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| BASE_URL: ${{ secrets.SMOKE_TEST_URL }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| E2E_MAX_FAILURES: 25 | |
| RESET_SITE: true | |
| k6-tests: | |
| name: k6 tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| if: success() || failure() | |
| outputs: | |
| test-result: ${{ steps.run-k6-tests.conclusion }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install-filters: woocommerce | |
| build: false | |
| - name: Update performance test site with E2E test | |
| id: update-perf-site | |
| continue-on-error: true | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: k6-daily-update-site--run-${{ github.run_number }} | |
| tests: update-woocommerce.spec.js | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/ | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| UPDATE_WC: nightly | |
| - name: Install k6 | |
| run: | | |
| curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 | |
| - name: Run k6 smoke tests | |
| id: run-k6-tests | |
| env: | |
| URL: ${{ secrets.SMOKE_TEST_PERF_URL }} | |
| HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }} | |
| A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| P_ID: 22733 | |
| run: | | |
| ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js | |
| test-plugins: | |
| name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooCommerce Payments' | |
| repo: 'automattic/woocommerce-payments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| repo: 'woocommerce/woocommerce-paypal-payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| repo: 'automattic/woocommerce-services' | |
| slug: woocommerce-services | |
| - plugin: 'WooCommerce Subscriptions' | |
| repo: WC_SUBSCRIPTIONS_REPO | |
| private: true | |
| slug: woocommerce-subscriptions | |
| - plugin: 'Gutenberg' | |
| repo: 'WordPress/gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| repo: 'bph/gutenberg' | |
| slug: gutenberg-nightly | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| build-filters: woocommerce | |
| - name: Setup local test environment | |
| uses: ./.github/actions/tests/setup-local-test-environment | |
| with: | |
| test-type: e2e | |
| - name: Run 'Upload plugin' test | |
| id: run-upload-plugin-test | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| tests: upload-plugin.spec.js | |
| env: | |
| PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| - name: Run the rest of E2E tests | |
| id: run-e2e-composite-action | |
| timeout-minutes: 60 | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| playwright-config: ignore-plugin-tests.playwright.config.js | |
| report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| env: | |
| E2E_MAX_FAILURES: 15 | |
| - name: Create context block and save as JSON file | |
| if: success() || failure() | |
| id: create-block-json | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const script = require( './.github/actions/tests/slack-summary-daily/scripts/create-blocks-plugin-tests.js' ) | |
| script( { core } ); | |
| env: | |
| UPLOAD_RESULT: ${{ steps.run-upload-plugin-test.outputs.result }} | |
| E2E_RESULT: ${{ steps.run-e2e-composite-action.outputs.result }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| PLUGIN_SLUG: ${{ matrix.slug }} | |
| - name: Upload JSON file as artifact | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} | |
| path: ${{ steps.create-block-json.outputs.path }} | |
| trunk-results: | |
| name: Publish report on smoke tests on nightly build | |
| if: | | |
| ( success() || failure() ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| steps: | |
| - name: Create dirs | |
| run: | | |
| mkdir -p repo | |
| mkdir -p artifacts/api | |
| mkdir -p artifacts/e2e | |
| mkdir -p output | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| path: repo | |
| - name: Download API test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: artifacts/api | |
| - name: Download E2E test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: artifacts/e2e | |
| - name: Post test summary | |
| uses: actions/github-script@v6 | |
| env: | |
| API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json | |
| E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json | |
| with: | |
| result-encoding: string | |
| script: | | |
| const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' ) | |
| return await script( { core } ) | |
| - name: Publish report | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| gh workflow run publish-test-reports-daily.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f api_artifact="$API_ARTIFACT" \ | |
| -f e2e_artifact="$E2E_ARTIFACT" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| plugins-results: | |
| name: Publish report on Smoke tests on trunk with plugins | |
| if: | | |
| ( success() || failure() ) && | |
| ( needs.test-plugins.result != 'skipped' ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooCommerce Payments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| slug: woocommerce-services | |
| - plugin: 'WooCommerce Subscriptions' | |
| slug: woocommerce-subscriptions | |
| - plugin: 'Gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| slug: gutenberg-nightly | |
| steps: | |
| - name: Download test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.ARTIFACT }} | |
| - name: Publish reports | |
| run: | | |
| gh workflow run publish-test-reports-daily-plugins.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f artifact="${{ env.ARTIFACT }}" \ | |
| -f plugin="${{ matrix.plugin }}" \ | |
| -f slug="${{ matrix.slug }}" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| post-slack-summary: | |
| name: Post Slack summary | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| if: | | |
| success() || ( | |
| failure() && contains( needs.*.result, 'failure' ) | |
| ) | |
| needs: | |
| - api-tests | |
| - e2e-tests | |
| - k6-tests | |
| - test-plugins | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download Slack blocks from plugin tests | |
| if: needs.test-plugins.result != 'skipped' | |
| id: download-plugin-blocks | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} | |
| path: /tmp/plugin-blocks | |
| - name: Construct Slack payload | |
| id: construct-slack-payload | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const script = require('./.github/actions/tests/slack-summary-daily/scripts/construct-slack-payload.js'); | |
| await script( { context, core, github } ); | |
| env: | |
| API_RESULT: ${{ needs.api-tests.outputs.test-result }} | |
| E2E_RESULT: ${{ needs.e2e-tests.outputs.test-result || needs.e2e-tests.result }} | |
| k6_RESULT: ${{ needs.k6-tests.outputs.test-result || needs.k6-tests.result }} | |
| PLUGINS_BLOCKS_PATH: ${{ steps.download-plugin-blocks.outputs.download-path }} | |
| PLUGIN_TESTS_RESULT: ${{ needs.test-plugins.result }} | |
| - name: Send Slack message | |
| id: send-slack-message | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.DAILY_TEST_SLACK_CHANNEL }} | |
| payload: ${{ steps.construct-slack-payload.outputs.payload }} | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }} |