Smoke test daily #400
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke test daily | |
| on: | |
| schedule: | |
| - cron: '25 3 * * *' | |
| workflow_dispatch: | |
| env: | |
| API_ARTIFACT: api-daily--run-${{ github.run_number }} | |
| E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }} | |
| FORCE_COLOR: 1 | |
| PLUGIN_SLACK_BLOCKS_ARTIFACT: plugin-blocks | |
| concurrency: | |
| group: '${{ github.workflow }}-${{ github.ref }}' | |
| cancel-in-progress: true | |
| permissions: {} | |
| jobs: | |
| api-tests: | |
| name: API tests on nightly build | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| test-result: ${{ steps.run-api-composite-action.outputs.result }} | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Load docker images and start containers. | |
| working-directory: plugins/woocommerce | |
| env: | |
| ENABLE_HPOS: 1 | |
| run: pnpm --filter=@woocommerce/plugin-woocommerce env:test | |
| - name: Run API tests | |
| id: run-api-composite-action | |
| uses: ./.github/actions/tests/run-api-tests | |
| with: | |
| report-name: ${{ env.API_ARTIFACT }} | |
| env: | |
| BASE_URL: http://localhost:8086 | |
| USER_KEY: admin | |
| USER_SECRET: password | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| - name: Generate Playwright API Test report. | |
| id: generate_api_report | |
| if: | | |
| always() && | |
| ( | |
| steps.run-api-composite-action.conclusion != 'cancelled' || | |
| steps.run-api-composite-action.conclusion != 'skipped' | |
| ) | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive Playwright API test report | |
| if: | | |
| always() && | |
| steps.generate_api_report.conclusion == 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 20 | |
| e2e-tests: | |
| name: E2E tests on nightly build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: | |
| [ | |
| { number: 1, name: 1/5 }, | |
| { number: 2, name: 2/5 }, | |
| { number: 3, name: 3/5 }, | |
| { number: 4, name: 4/5 }, | |
| { number: 5, name: 5/5 }, | |
| ] | |
| permissions: | |
| contents: read | |
| outputs: | |
| test-result: ${{ steps.run_playwright_e2e_tests.outputs.result }} | |
| env: | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Load docker images and start containers. | |
| working-directory: plugins/woocommerce | |
| env: | |
| WP_ENV_PHP_VERSION: 7.4 | |
| ENABLE_HPOS: 1 | |
| run: pnpm env:test | |
| - name: Download and install Chromium browser. | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright install chromium | |
| - name: Run E2E tests | |
| timeout-minutes: 60 | |
| id: run_playwright_e2e_tests | |
| env: | |
| USE_WP_ENV: 1 | |
| E2E_MAX_FAILURES: 15 | |
| FORCE_COLOR: 1 | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js --shard ${{ matrix.shard.name }} | |
| - name: Upload reports to GitHub Actions Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: all-blob-reports-${{ matrix.shard.number }} | |
| path: ${{ env.ALLURE_RESULTS_DIR }} | |
| retention-days: 1 | |
| compression-level: 9 | |
| merge-reports: | |
| name: Merge e2e test reports | |
| if: always() | |
| needs: e2e-tests | |
| runs-on: ubuntu-latest | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Download blob reports from GitHub Actions Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ${{ env.ALLURE_RESULTS_DIR }} | |
| pattern: all-blob-reports-* | |
| run-id: e2e-tests-run | |
| merge-multiple: true | |
| - name: Generate Test report. | |
| id: generate_e2e_report | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive Playwright E2E test report | |
| if: | | |
| always() && | |
| steps.generate_e2e_report.conclusion == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| k6-tests: | |
| name: k6 tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| if: success() || failure() | |
| outputs: | |
| test-result: ${{ steps.run-k6-tests.conclusion }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| - name: Update performance test site with E2E test | |
| id: update-perf-site | |
| continue-on-error: true | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: k6-daily-update-site--run-${{ github.run_number }} | |
| tests: update-woocommerce.spec.js | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/ | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| UPDATE_WC: nightly | |
| - name: Install k6 | |
| run: | | |
| curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 | |
| - name: Run k6 smoke tests | |
| id: run-k6-tests | |
| env: | |
| URL: ${{ secrets.SMOKE_TEST_PERF_URL }} | |
| HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }} | |
| A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| P_ID: 22733 | |
| run: | | |
| ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js | |
| test-plugins: | |
| name: E2E tests with ${{ matrix.plugin }} plugin installed | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| PLUGIN_SLUG: ${{ matrix.slug }} | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooPayments' | |
| repo: 'automattic/woocommerce-payments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| repo: 'woocommerce/woocommerce-paypal-payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| repo: 'automattic/woocommerce-services' | |
| slug: woocommerce-services | |
| - plugin: 'Gutenberg' | |
| repo: 'WordPress/gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| repo: 'bph/gutenberg' | |
| slug: gutenberg | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Setup local test environment | |
| uses: ./.github/actions/tests/setup-local-test-environment | |
| with: | |
| test-type: e2e | |
| - name: Setup plugin for the main e2e suite | |
| working-directory: ./plugins/woocommerce | |
| run: ./tests/e2e-pw/bin/install-plugin.sh | |
| - name: Run 'Upload plugin' test | |
| id: run-upload-plugin-test | |
| if: ${{ failure() }} # only if the plugin setup failed to check if the plugin really cannot be installed | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| tests: upload-plugin.spec.js | |
| - name: Run the rest of E2E tests | |
| id: run-e2e-composite-action | |
| timeout-minutes: 60 | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| playwright-config: ignore-plugin-tests.playwright.config.js | |
| report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| env: | |
| E2E_MAX_FAILURES: 15 | |
| - name: Create context block and save as JSON file | |
| if: success() || failure() | |
| id: create-block-json | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const script = require( './.github/actions/tests/slack-summary-daily/scripts/create-blocks-plugin-tests.js' ) | |
| script( { core } ); | |
| env: | |
| UPLOAD_RESULT: ${{ steps.run-upload-plugin-test.outputs.result }} | |
| E2E_RESULT: ${{ steps.run-e2e-composite-action.outputs.result }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| PLUGIN_SLUG: ${{ matrix.slug }} | |
| - name: Upload JSON file as artifact | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} | |
| path: ${{ steps.create-block-json.outputs.path }} | |
| trunk-results: | |
| name: Publish report on smoke tests on nightly build | |
| if: | | |
| ( success() || failure() ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| steps: | |
| - name: Create dirs | |
| run: | | |
| mkdir -p repo | |
| mkdir -p artifacts/api | |
| mkdir -p artifacts/e2e | |
| mkdir -p output | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| path: repo | |
| - name: Download API test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: artifacts/api | |
| - name: Download E2E test report artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: artifacts/e2e | |
| - name: Post test summary | |
| uses: actions/github-script@v6 | |
| env: | |
| API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json | |
| E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json | |
| with: | |
| result-encoding: string | |
| script: | | |
| const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' ) | |
| return await script( { core } ) | |
| - name: Publish report | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| gh workflow run publish-test-reports-daily.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f api_artifact="$API_ARTIFACT" \ | |
| -f e2e_artifact="$E2E_ARTIFACT" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| plugins-results: | |
| name: Publish report on Smoke tests on trunk with plugins | |
| if: | | |
| ( success() || failure() ) && | |
| ( needs.test-plugins.result != 'skipped' ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooPayments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| slug: woocommerce-services | |
| - plugin: 'Gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| slug: gutenberg-nightly | |
| steps: | |
| - name: Download test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.ARTIFACT }} | |
| - name: Publish reports | |
| run: | | |
| gh workflow run publish-test-reports-daily-plugins.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f artifact="${{ env.ARTIFACT }}" \ | |
| -f plugin="${{ matrix.plugin }}" \ | |
| -f slug="${{ matrix.slug }}" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| post-slack-summary: | |
| name: Post Slack summary | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| if: | | |
| github.ref_name == 'trunk' && | |
| success() || ( | |
| failure() && contains( needs.*.result, 'failure' ) | |
| ) | |
| needs: | |
| - api-tests | |
| - e2e-tests | |
| - k6-tests | |
| - test-plugins | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download Slack blocks from plugin tests | |
| if: needs.test-plugins.result != 'skipped' | |
| id: download-plugin-blocks | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} | |
| path: /tmp/plugin-blocks | |
| - name: Construct Slack payload | |
| id: construct-slack-payload | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const script = require('./.github/actions/tests/slack-summary-daily/scripts/construct-slack-payload.js'); | |
| await script( { context, core, github } ); | |
| env: | |
| API_RESULT: ${{ needs.api-tests.outputs.test-result }} | |
| E2E_RESULT: ${{ needs.e2e-tests.outputs.test-result || needs.e2e-tests.result }} | |
| k6_RESULT: ${{ needs.k6-tests.outputs.test-result || needs.k6-tests.result }} | |
| PLUGINS_BLOCKS_PATH: ${{ steps.download-plugin-blocks.outputs.download-path }} | |
| PLUGIN_TESTS_RESULT: ${{ needs.test-plugins.result }} | |
| - name: Send Slack message | |
| id: send-slack-message | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.DAILY_TEST_SLACK_CHANNEL }} | |
| payload: ${{ steps.construct-slack-payload.outputs.payload }} | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }} |