From 90e1a233e613e71a99b6429a0a2d67579f1c06d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 27 Sep 2023 21:59:38 +0200
Subject: [PATCH 01/19] Add actions and workflows for QIT tests

---
 .github/workflows/run-qit-all.yml             |  78 +++++++++
 .github/workflows/run-qit-extension.yml       | 153 ++++++++++++++++++
 packages/github-actions/README.md             |   1 +
 .../actions/run-qit-annotate/README.md        |  56 +++++++
 .../actions/run-qit-annotate/action.yml       |  94 +++++++++++
 5 files changed, 382 insertions(+)
 create mode 100644 .github/workflows/run-qit-all.yml
 create mode 100644 .github/workflows/run-qit-extension.yml
 create mode 100644 packages/github-actions/actions/run-qit-annotate/README.md
 create mode 100644 packages/github-actions/actions/run-qit-annotate/action.yml

diff --git a/.github/workflows/run-qit-all.yml b/.github/workflows/run-qit-all.yml
new file mode 100644
index 00000000..7f9ad171
--- /dev/null
+++ b/.github/workflows/run-qit-all.yml
@@ -0,0 +1,78 @@
+name: Run QIT for all extensions
+
+# **What it does**: Runs a suite of tests for all Grow extensions.
+# **Why we have it**: To be able to run tests for all extensions at once. For example when we want to compatibility test a new version of the platform (WP/WC).
+
+on:
+  workflow_dispatch:
+    inputs:
+      # Basic params.
+      version:
+        description: 'Version to be tested'
+        required: true
+        type: choice
+        options:
+        - latest
+        - dev
+
+      # Configure which tests to run.
+      test-activation:
+        description: 'Should activation be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-security:
+        description: 'Should security be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-phpstan:
+        description: 'Should phpstan be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-api:
+        description: 'Should API be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-e2e:
+        description: 'Should E2E be tested? (takes a lot of time)'
+        required: true
+        default: false
+        type: boolean
+
+      # Advanced customization.
+      options:
+        description: 'Additional options for `qit` command, like `--optional_features=hpos`.'
+        required: false
+
+  # Used to test this PR, to be removed after before merging.
+  push:
+    branches:
+      - add/qit-workflows
+
+jobs:
+    qit-tests:
+        name: Run QIT Tests
+        uses: ./.github/workflows/run-qit-extension.yml
+        secrets: inherit
+        strategy:
+          # Allow to test extensions even if one of them fails.
+          fail-fast: false
+          matrix:
+            # List of extensions to be tested.
+            extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration]
+            # Run both options for PR review purposes.
+            version: [latest, dev]
+        with:
+          # We're running tests for all versions, just for PR review purposes to be replaced before merging.
+          # version: ${{ inputs.version }}
+          version: ${{ matrix.version }}
+          # Conditional statements are here to allow testing on push triggers, without manual input. To be removed before merging.
+          test-activation: ${{ inputs.test-activation || true }}
+          test-security: ${{ inputs.test-security || true }}
+          test-phpstan: ${{ inputs.test-phpstan || true }}
+          test-api: ${{ inputs.test-api || false }}
+          test-e2e: ${{ inputs.test-e2e || false }}
+          extension: ${{ matrix.extension }}
diff --git a/.github/workflows/run-qit-extension.yml b/.github/workflows/run-qit-extension.yml
new file mode 100644
index 00000000..c2bab28c
--- /dev/null
+++ b/.github/workflows/run-qit-extension.yml
@@ -0,0 +1,153 @@
+name: Run QIT for a given extensions
+
+# **What it does**: Runs a set of QIT tests for a given extension.
+# **Why we have it**: To reuse across other repos, to make a full test of a single extension.
+
+on:
+  workflow_call:
+    inputs:
+      # Basic params.
+      extension:
+        description: Extension to test
+        required: true
+        type: string
+      version:
+        description: |
+          Version to be tested: `latest` or `dev`.
+          The `dev` option will make the action look up for an `{extension}.zip` file in the root
+          of the repository under the `gha-dev-build` tag.
+        required: true
+        default: 'latest'
+        type: string
+
+      # Customize which types to run.
+      test-activation:
+        description: 'Should activation be tested?'
+        default: true
+        type: boolean
+      test-security:
+        description: 'Should security be tested?'
+        default: true
+        type: boolean
+      test-phpstan:
+        description: 'Should PHPStan be tested?'
+        default: true
+        type: boolean
+      test-api:
+        description: 'Should API be tested?'
+        default: true
+        type: boolean
+      test-e2e:
+        description: 'Should E2E be tested?'
+        default: true
+        type: boolean
+
+      # Advanced customization.
+      options:
+        description: 'Additional options for `qit` command, like `--optional_features=hpos`'
+        type: string
+
+    outputs:
+        statuses:
+            description: "Statuses of all tests. Array of integers."
+            value: ${{ jobs.qit-tests.outputs.statuses }}
+
+
+jobs:
+    qit-tests:
+        name: Run QIT Tests
+        runs-on: ubuntu-20.04
+        env:
+            QIT_DISABLE_ONBOARDING: yes
+            dev_build: ${{ inputs.version == 'dev' && format('{0}.zip', inputs.extension) || '' }}
+        outputs:
+            statuses: ${{ toJSON( steps.*.outputs.status ) }}
+        steps:
+
+            # Checkout repository to reuse local action. To be removed once we switch to the published action.
+            - uses: actions/checkout@v3
+              with:
+                fetch-depth: 2
+                path: 'this-repo'
+
+            - name: Download plugins
+              if: ${{ env.dev_build != '' }}
+              uses: robinraju/release-downloader@v1.8
+              with:
+                repository: "woocommerce/${{ inputs.extension }}"
+                tag: 'gha-dev-build'
+                fileName: ${{ env.dev_build }}
+                token: ${{ secrets.BOT_GH_TOKEN }}
+
+            - name: Install QIT via composer
+              run: composer require woocommerce/qit-cli
+            - name: Add Partner
+              run: |
+                ./vendor/bin/qit partner:add \
+                  --user='${{ secrets.QIT_PARTNER_USER }}' \
+                  --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
+
+            - name: Create results dir
+              run: mkdir -p ./qit-results/${{ inputs.extension }}
+
+            - name: Activation test
+              id: activation-test
+              if: ${{ inputs.test-activation == true }}
+              # Switch to published action once released.
+              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              timeout-minutes: 5
+              with:
+                type: activation
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: Security test
+              id: security-test
+              if: ${{ inputs.test-security == true }}
+              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              timeout-minutes: 5
+              with:
+                type: security
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: PHPStan test
+              id: phpstan-test
+              if: ${{ inputs.test-phpstan == true }}
+              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              timeout-minutes: 5
+              with:
+                type: phpstan
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: API test
+              id: api-test
+              if: ${{ inputs.test-api == true }}
+              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              timeout-minutes: 5
+              with:
+                type: api
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: E2E test
+              id: e2e-test
+              if: ${{ inputs.test-e2e == true }}
+              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              timeout-minutes: 30
+              with:
+                type: e2e
+                extension: ${{ inputs.extension }}
+                extension-file: ${{ env.dev_build }}
+                options: ${{ inputs.options }}
+
+            - name: Upload results
+              uses: actions/upload-artifact@v1
+              with:
+                  name: ${{ inputs.extension }}
+                  path: ./qit-results/${{ inputs.extension }}
diff --git a/packages/github-actions/README.md b/packages/github-actions/README.md
index 91823530..d9065c37 100644
--- a/packages/github-actions/README.md
+++ b/packages/github-actions/README.md
@@ -20,6 +20,7 @@ Custom GitHub actions that help to composite GitHub workflows across the repos m
 - [`prepare-node`](actions/prepare-node) - Set up Node.js with a specific version, load npm cache, install Node dependencies
 - [`prepare-php`](actions/prepare-php) - Set up PHP with a specific version and tools, load Composer cache, install Composer dependencies
 - [`publish-extension-dev-build`](actions/publish-extension-dev-build) - Publish extension development build
+- [`run-qit-annotate`](actions/run-qit-annotate) - Runs QIT test and annotates the results
 - [`stylelint-annotation`](actions/stylelint-annotation) - Annotate stylelint results via stylelint formatter
 - [`update-version-tags`](actions/update-version-tags) - Update version tags
 
diff --git a/packages/github-actions/actions/run-qit-annotate/README.md b/packages/github-actions/actions/run-qit-annotate/README.md
new file mode 100644
index 00000000..4a007c1e
--- /dev/null
+++ b/packages/github-actions/actions/run-qit-annotate/README.md
@@ -0,0 +1,56 @@
+# Run QIT test
+
+This action provides the following functionality for GitHub Actions users:
+
+- Run `qit` test of given type for a given extension
+- Annotate the results
+- Forward status code, so the consumer can decide how to conclude the results
+
+
+## Usage
+
+See [action.yml](action.yml)
+
+### Prerequisites
+
+- QIT needs to be [installed](https://woocommerce.github.io/qit-documentation/#/cli/getting-started?id=installing-qit) and [authenticated](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli)
+   - To setup QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`.
+- The action assumes there is `qit-results` directory. You can change it's name using `results-folder` input.
+
+
+### Basic:
+
+```yaml
+jobs:
+  qit-test:
+    name: Run QIT Tests
+    runs-on: ubuntu-20.04
+    env:
+      QIT_DISABLE_ONBOARDING: yes
+    steps:
+      - name: Install QIT via composer
+        run: composer require woocommerce/qit-cli
+
+      - name: Add Partner
+        run: |
+          ./vendor/bin/qit partner:add \
+            --user='${{ secrets.QIT_PARTNER_USER }}' \
+            --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
+
+      - name: Create results dir
+        run: mkdir -p ./qit-results/automatewoo
+
+      - name: Security test
+        id: security-test
+        uses: woocommerce/grow/run-qit-annotate@actions-v1
+        timeout-minutes: 5
+        with:
+          type: security
+          extension: automatewoo
+          options: '--optional_features=hpos'
+
+      - name: Echo status
+        shell: bash
+        run: echo ${{ jobs.security-test.outputs.status }}
+	
+```
diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
new file mode 100644
index 00000000..a0f88d88
--- /dev/null
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -0,0 +1,94 @@
+name: Run QIT test
+description: Runs QIT test and annotates the results.
+
+# **What it does**: Runs a single QIT test and annotates the results.
+# **Why we have it**: To reuse across other workflows to test extensions.
+
+inputs:
+  # Basic params.
+  extension:
+    description: Extension to test
+    required: true
+  extension-file:
+    description: Custom build of the extension to test. If not given, the latest release will be used.
+    required: false
+  type:
+    description: Type of test to run
+    required: false
+    default: activation
+  # Advanced customization.
+  options:
+    description: Options to pass to the qit command
+    required: false
+    default: ''
+  results-filename:
+    description: Custom file name for results
+    required: false
+  results-folder:
+    description: Folder to store results
+    required: false
+    default: qit-results
+
+outputs:
+  status:
+    description: "Exit code of the test. May be used, for example, to fail the workflow if the test fails."
+    value: ${{ steps.run-test.outputs.status }}
+  summary:
+    description: "Short summary of the test"
+    value: ${{ steps.read-summary.outputs.summary }}
+  resultsURL:
+    description: "URL to the results of the test"
+    value: ${{ steps.read-summary.outputs.resultURL }}
+
+branding:
+  icon: 'award'
+  color: 'purple'
+
+runs:
+  using: composite
+  steps:
+    - name: Run test
+      id: run-test
+      # Do not fail when the `qit` fails, so we can annotate the results.
+      shell: bash --noprofile --norc {0}
+      continue-on-error: true
+      env:
+        # If the custom build is provided, pass it to the `zip` param.
+        zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}
+        report_file: ${{ inputs.results-filename || format( '{0}/{1}/{2}/{3}.txt', github.workspace, inputs.results-folder, inputs.extension, inputs.type ) }}
+        status: 0
+      run: |
+        ./vendor/bin/qit run:${{ inputs.type }} ${{ inputs.extension }} \
+          $zip \
+          ${{ inputs.options }} \
+          --wait \
+          > $report_file || status=$?
+        echo "status=$status" >> "$GITHUB_OUTPUT"
+        echo "report_file=$report_file" >> "$GITHUB_OUTPUT"
+        cat $report_file
+
+    # Parse the report file, to fetch the essential information.
+    - name: Read summary
+      id: read-summary
+      shell: bash
+      if: '!cancelled()'
+      run: |
+        summary=`grep -Po "(?<=Test Summary)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never`
+        resultURL=`grep -Po "(?<=Result Url)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never`
+        echo "summary=$summary" >> $GITHUB_OUTPUT
+        echo "resultURL=$resultURL" >> $GITHUB_OUTPUT
+
+    # Annotate the results according to the exit code.
+    - name: Annotate
+      if: '!cancelled()'
+      shell: bash
+      run: |
+        summary="${{ inputs.type }}: ${{ steps.read-summary.outputs.summary }} - ${{ steps.read-summary.outputs.resultURL }}";
+        case ${{ steps.run-test.outputs.status }} in
+          0) echo "::notice ::$summary"
+          ;;
+          2) echo "::warning ::$summary"
+          ;;
+          *) echo "::error ::$summary"
+          ;;
+        esac

From 764e2d762ffc2f64c08ace9521848a5cb3620224 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 27 Sep 2023 22:24:32 +0200
Subject: [PATCH 02/19] Use the pre-released `run-qit-annotate` action.

---
 .github/workflows/run-qit-extension.yml | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/run-qit-extension.yml b/.github/workflows/run-qit-extension.yml
index c2bab28c..5dc6af6a 100644
--- a/.github/workflows/run-qit-extension.yml
+++ b/.github/workflows/run-qit-extension.yml
@@ -63,13 +63,6 @@ jobs:
         outputs:
             statuses: ${{ toJSON( steps.*.outputs.status ) }}
         steps:
-
-            # Checkout repository to reuse local action. To be removed once we switch to the published action.
-            - uses: actions/checkout@v3
-              with:
-                fetch-depth: 2
-                path: 'this-repo'
-
             - name: Download plugins
               if: ${{ env.dev_build != '' }}
               uses: robinraju/release-downloader@v1.8
@@ -93,8 +86,7 @@ jobs:
             - name: Activation test
               id: activation-test
               if: ${{ inputs.test-activation == true }}
-              # Switch to published action once released.
-              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
               timeout-minutes: 5
               with:
                 type: activation
@@ -105,7 +97,7 @@ jobs:
             - name: Security test
               id: security-test
               if: ${{ inputs.test-security == true }}
-              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
               timeout-minutes: 5
               with:
                 type: security
@@ -116,7 +108,7 @@ jobs:
             - name: PHPStan test
               id: phpstan-test
               if: ${{ inputs.test-phpstan == true }}
-              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
               timeout-minutes: 5
               with:
                 type: phpstan
@@ -127,7 +119,7 @@ jobs:
             - name: API test
               id: api-test
               if: ${{ inputs.test-api == true }}
-              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
               timeout-minutes: 5
               with:
                 type: api
@@ -138,7 +130,7 @@ jobs:
             - name: E2E test
               id: e2e-test
               if: ${{ inputs.test-e2e == true }}
-              uses: ./this-repo/packages/js/github-actions/actions/run-qit-annotate
+              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
               timeout-minutes: 30
               with:
                 type: e2e

From ad7a5facf43b4084908875ecf06a07499e2595fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 27 Sep 2023 22:54:58 +0200
Subject: [PATCH 03/19] Prefix results artifact to avoid conflict with build

---
 .github/workflows/run-qit-extension.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-qit-extension.yml b/.github/workflows/run-qit-extension.yml
index 5dc6af6a..b7c4b328 100644
--- a/.github/workflows/run-qit-extension.yml
+++ b/.github/workflows/run-qit-extension.yml
@@ -141,5 +141,5 @@ jobs:
             - name: Upload results
               uses: actions/upload-artifact@v1
               with:
-                  name: ${{ inputs.extension }}
+                  name: "qit-results-${{ inputs.extension }}"
                   path: ./qit-results/${{ inputs.extension }}

From d8f0673f9880347a115445e6749d61788a7fb42b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 27 Sep 2023 22:55:09 +0200
Subject: [PATCH 04/19] Forward options input

---
 .github/workflows/run-qit-all.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-qit-all.yml b/.github/workflows/run-qit-all.yml
index 7f9ad171..25d1f307 100644
--- a/.github/workflows/run-qit-all.yml
+++ b/.github/workflows/run-qit-all.yml
@@ -76,3 +76,4 @@ jobs:
           test-api: ${{ inputs.test-api || false }}
           test-e2e: ${{ inputs.test-e2e || false }}
           extension: ${{ matrix.extension }}
+          options: ${{ inputs.options }}

From 15d3e405d279efb517d3dfbd0195dec628d94cc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 27 Sep 2023 22:56:44 +0200
Subject: [PATCH 05/19] Support local builds

---
 .github/workflows/run-qit-extension.yml | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-qit-extension.yml b/.github/workflows/run-qit-extension.yml
index b7c4b328..231033f6 100644
--- a/.github/workflows/run-qit-extension.yml
+++ b/.github/workflows/run-qit-extension.yml
@@ -13,9 +13,11 @@ on:
         type: string
       version:
         description: |
-          Version to be tested: `latest` or `dev`.
-          The `dev` option will make the action look up for an `{extension}.zip` file in the root
-          of the repository under the `gha-dev-build` tag.
+          Version to be tested: `latest`, `dev`, or `local`.
+          `latest` will test latest released version,
+          `dev` will make the action look up for an `{extension}.zip` file in the root
+          of the repository under the `gha-dev-build` tag,
+          `local` will look up for an `{extension}.zip` artifact in the current workflow.
         required: true
         default: 'latest'
         type: string
@@ -59,12 +61,13 @@ jobs:
         runs-on: ubuntu-20.04
         env:
             QIT_DISABLE_ONBOARDING: yes
-            dev_build: ${{ inputs.version == 'dev' && format('{0}.zip', inputs.extension) || '' }}
+            dev_build: ${{ inputs.version != 'latest' && format('{0}.zip', inputs.extension) || '' }}
         outputs:
             statuses: ${{ toJSON( steps.*.outputs.status ) }}
         steps:
-            - name: Download plugins
-              if: ${{ env.dev_build != '' }}
+
+            - name: Download `gha-dev-build`
+              if: ${{ inputs.version == 'dev' }}
               uses: robinraju/release-downloader@v1.8
               with:
                 repository: "woocommerce/${{ inputs.extension }}"
@@ -72,6 +75,12 @@ jobs:
                 fileName: ${{ env.dev_build }}
                 token: ${{ secrets.BOT_GH_TOKEN }}
 
+            - name: Download artifact
+              if: ${{ inputs.version == 'local' }}
+              uses: actions/download-artifact@v3
+              with:
+                name: ${{ env.dev_build }}
+
             - name: Install QIT via composer
               run: composer require woocommerce/qit-cli
             - name: Add Partner

From cfb8348e07c6ed5173b265c8585032a766aa1dfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 27 Sep 2023 23:23:44 +0200
Subject: [PATCH 06/19] Run all QIT tests only on demand

remove  `on push` used for testing.
---
 .github/workflows/run-qit-all.yml | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-qit-all.yml b/.github/workflows/run-qit-all.yml
index 25d1f307..d6021905 100644
--- a/.github/workflows/run-qit-all.yml
+++ b/.github/workflows/run-qit-all.yml
@@ -47,11 +47,6 @@ on:
         description: 'Additional options for `qit` command, like `--optional_features=hpos`.'
         required: false
 
-  # Used to test this PR, to be removed after before merging.
-  push:
-    branches:
-      - add/qit-workflows
-
 jobs:
     qit-tests:
         name: Run QIT Tests
@@ -63,17 +58,13 @@ jobs:
           matrix:
             # List of extensions to be tested.
             extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration]
-            # Run both options for PR review purposes.
-            version: [latest, dev]
         with:
-          # We're running tests for all versions, just for PR review purposes to be replaced before merging.
-          # version: ${{ inputs.version }}
-          version: ${{ matrix.version }}
           # Conditional statements are here to allow testing on push triggers, without manual input. To be removed before merging.
-          test-activation: ${{ inputs.test-activation || true }}
-          test-security: ${{ inputs.test-security || true }}
-          test-phpstan: ${{ inputs.test-phpstan || true }}
-          test-api: ${{ inputs.test-api || false }}
-          test-e2e: ${{ inputs.test-e2e || false }}
+          version: ${{ inputs.version }}
+          test-activation: ${{ inputs.test-activation }}
+          test-security: ${{ inputs.test-security }}
+          test-phpstan: ${{ inputs.test-phpstan }}
+          test-api: ${{ inputs.test-api }}
+          test-e2e: ${{ inputs.test-e2e }}
           extension: ${{ matrix.extension }}
           options: ${{ inputs.options }}

From a8f23bae23cbe0e896189ec561099897622e4ac1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Mon, 2 Oct 2023 20:20:22 +0200
Subject: [PATCH 07/19] Fix indentation in `run-qit-annotate/README`

Co-authored-by: Eason <eason.su.tw@gmail.com>
---
 packages/github-actions/actions/run-qit-annotate/README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/README.md b/packages/github-actions/actions/run-qit-annotate/README.md
index 4a007c1e..f7330de3 100644
--- a/packages/github-actions/actions/run-qit-annotate/README.md
+++ b/packages/github-actions/actions/run-qit-annotate/README.md
@@ -52,5 +52,4 @@ jobs:
       - name: Echo status
         shell: bash
         run: echo ${{ jobs.security-test.outputs.status }}
-	
 ```

From d7e552ea0e420901c3b5baff6629a5af04be16f9 Mon Sep 17 00:00:00 2001
From: Eason Su <eason.su.tw@gmail.com>
Date: Tue, 3 Oct 2023 15:43:27 +0800
Subject: [PATCH 08/19] Transform the `run-qit-extension` workflow into an
 action.

---
 .../actions/run-qit-extension/action.yml      | 143 ++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100644 packages/github-actions/actions/run-qit-extension/action.yml

diff --git a/packages/github-actions/actions/run-qit-extension/action.yml b/packages/github-actions/actions/run-qit-extension/action.yml
new file mode 100644
index 00000000..7b63a656
--- /dev/null
+++ b/packages/github-actions/actions/run-qit-extension/action.yml
@@ -0,0 +1,143 @@
+name: Run QIT for a given extension
+description: Runs QIT tests and annotates the results for a given extension.
+
+# **What it does**: Runs a set of QIT tests for a given extension.
+# **Why we have it**: To reuse across other repos, to make a full test of a single extension.
+
+inputs:
+  # Basic params.
+  qit-partner-user:
+    description: 'QIT partner username'
+    required: true
+  qit-partner-secret:
+    description: 'QIT partner password'
+    required: true
+  github-token:
+    description: 'Github token to access private repos'
+    required: false
+    default: ${{ github.token }}
+  extension:
+    description: Extension to test
+    required: true
+  version:
+    description: |
+      Version to be tested: `latest`, `dev`, or `local`.
+      `latest` will test latest released version,
+      `dev` will make the action look up for an `{extension}.zip` file in the root
+      of the repository under the `gha-dev-build` tag,
+      `local` will look up for an `{extension}.zip` artifact in the current workflow.
+    required: true
+    default: 'latest'
+
+  # Customize which types to run.
+  test-activation:
+    description: 'Should activation be tested?'
+    default: 'true'
+  test-security:
+    description: 'Should security be tested?'
+    default: 'true'
+  test-phpstan:
+    description: 'Should PHPStan be tested?'
+    default: 'true'
+  test-api:
+    description: 'Should API be tested?'
+    default: 'true'
+  test-e2e:
+    description: 'Should E2E be tested?'
+    default: 'true'
+
+  # Advanced customization.
+  options:
+    description: 'Additional options for `qit` command, like `--optional_features=hpos`'
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: Prepare env variables
+      shell: bash
+      run: |
+        dev_build="${{ inputs.version != 'latest' && format('{0}.zip', inputs.extension) || '' }}"
+
+        echo "dev_build=$dev_build" >> $GITHUB_ENV
+        echo "QIT_DISABLE_ONBOARDING=yes" >> $GITHUB_ENV
+
+    - name: Download `gha-dev-build`
+      if: ${{ inputs.version == 'dev' }}
+      uses: robinraju/release-downloader@v1.8
+      with:
+        repository: "woocommerce/${{ inputs.extension }}"
+        tag: 'gha-dev-build'
+        fileName: ${{ env.dev_build }}
+        token: ${{ inputs.github-token }}
+
+    - name: Install QIT via composer
+      shell: bash
+      run: composer require woocommerce/qit-cli
+
+    - name: Add Partner
+      shell: bash
+      run: |
+        ./vendor/bin/qit partner:add \
+          --user='${{ inputs.qit-partner-user }}' \
+          --application_password='${{ inputs.qit-partner-secret }}'
+
+    - name: Create results dir
+      shell: bash
+      run: mkdir -p ./qit-results/${{ inputs.extension }}
+
+    - name: Activation test
+      id: activation-test
+      if: ${{ inputs.test-activation == 'true' }}
+      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      with:
+        type: activation
+        extension: ${{ inputs.extension }}
+        extension-file: ${{ env.dev_build }}
+        options: ${{ inputs.options }}
+
+    - name: Security test
+      id: security-test
+      if: ${{ inputs.test-security == 'true' }}
+      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      with:
+        type: security
+        extension: ${{ inputs.extension }}
+        extension-file: ${{ env.dev_build }}
+        options: ${{ inputs.options }}
+
+    - name: PHPStan test
+      id: phpstan-test
+      if: ${{ inputs.test-phpstan == 'true' }}
+      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      with:
+        type: phpstan
+        extension: ${{ inputs.extension }}
+        extension-file: ${{ env.dev_build }}
+        options: ${{ inputs.options }}
+
+    - name: API test
+      id: api-test
+      if: ${{ inputs.test-api == 'true' }}
+      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      with:
+        type: api
+        extension: ${{ inputs.extension }}
+        extension-file: ${{ env.dev_build }}
+        options: ${{ inputs.options }}
+
+    - name: E2E test
+      id: e2e-test
+      if: ${{ inputs.test-e2e == 'true' }}
+      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      with:
+        type: e2e
+        extension: ${{ inputs.extension }}
+        extension-file: ${{ env.dev_build }}
+        options: ${{ inputs.options }}
+
+    - name: Upload results
+      uses: actions/upload-artifact@v1
+      with:
+        name: "qit-results-${{ inputs.extension }}"
+        path: ./qit-results/${{ inputs.extension }}

From 38a0364ae1f8f59e1cac5a15c91f9f24689760d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Mon, 29 Jan 2024 20:48:42 +0100
Subject: [PATCH 09/19] Use JSON format of qit command

Use changes added in https://github.com/woocommerce/qit-cli/pull/109

Add `ignore_fail` parameter, to simplify continue-on-error behavior.

Drop the report file, use annotations only.

Hide result URLs, expose `test_run_id` for further checks.
---
 .../actions/run-qit-annotate/action.yml       | 66 +++++++++----------
 1 file changed, 31 insertions(+), 35 deletions(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
index a0f88d88..1601f249 100644
--- a/packages/github-actions/actions/run-qit-annotate/action.yml
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -12,6 +12,10 @@ inputs:
   extension-file:
     description: Custom build of the extension to test. If not given, the latest release will be used.
     required: false
+  ignore-fail:
+    description: Should pass even if awaited tests fails.
+    required: false
+    default: false
   type:
     description: Type of test to run
     required: false
@@ -21,24 +25,17 @@ inputs:
     description: Options to pass to the qit command
     required: false
     default: ''
-  results-filename:
-    description: Custom file name for results
-    required: false
-  results-folder:
-    description: Folder to store results
-    required: false
-    default: qit-results
 
 outputs:
   status:
-    description: "Exit code of the test. May be used, for example, to fail the workflow if the test fails."
-    value: ${{ steps.run-test.outputs.status }}
+    description: "Status of the test run. May be used, for example, to fail the workflow if the test fails."
+    value: ${{ steps.read-summary.outputs.status }}
   summary:
     description: "Short summary of the test"
     value: ${{ steps.read-summary.outputs.summary }}
-  resultsURL:
-    description: "URL to the results of the test"
-    value: ${{ steps.read-summary.outputs.resultURL }}
+  test_run_id:
+    description: "ID of the QIT test run"
+    value: ${{ steps.read-summary.outputs.test_run_id }}
 
 branding:
   icon: 'award'
@@ -47,47 +44,46 @@ branding:
 runs:
   using: composite
   steps:
+    # Run the qit command.
     - name: Run test
       id: run-test
       # Do not fail when the `qit` fails, so we can annotate the results.
       shell: bash --noprofile --norc {0}
-      continue-on-error: true
       env:
         # If the custom build is provided, pass it to the `zip` param.
         zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}
-        report_file: ${{ inputs.results-filename || format( '{0}/{1}/{2}/{3}.txt', github.workspace, inputs.results-folder, inputs.extension, inputs.type ) }}
-        status: 0
+        ignore_fail: ${{ inputs.ignore-fail == 'true' && '--ignore-fail' || '' }}
       run: |
-        ./vendor/bin/qit run:${{ inputs.type }} ${{ inputs.extension }} \
-          $zip \
-          ${{ inputs.options }} \
-          --wait \
-          > $report_file || status=$?
-        echo "status=$status" >> "$GITHUB_OUTPUT"
-        echo "report_file=$report_file" >> "$GITHUB_OUTPUT"
-        cat $report_file
+        json=`./vendor/bin/qit run:${{ inputs.type }} \
+         ${{ inputs.extension }} \
+         $zip \
+         ${{ inputs.options }} \
+         $ignore_fail \
+         -n \
+         --json | jq '. | {test_run_id, status, test_summary}'`
+        echo $json | jq
+        echo "result="$json"" >> "$GITHUB_OUTPUT"
 
-    # Parse the report file, to fetch the essential information.
+    # Parse the result json to populate GHA outputs.
     - name: Read summary
       id: read-summary
       shell: bash
-      if: '!cancelled()'
       run: |
-        summary=`grep -Po "(?<=Test Summary)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never`
-        resultURL=`grep -Po "(?<=Result Url)\s+(.*)" ${{ steps.run-test.outputs.report_file }} --color=never`
-        echo "summary=$summary" >> $GITHUB_OUTPUT
-        echo "resultURL=$resultURL" >> $GITHUB_OUTPUT
+        echo "test_run_id=${{fromJson(steps.run-test.outputs.result).test_run_id}}" >> $GITHUB_OUTPUT
+        echo "status=${{fromJson(steps.run-test.outputs.result).status}}" >> $GITHUB_OUTPUT
+        echo "summary=${{fromJson(steps.run-test.outputs.result).test_summary}}" >> $GITHUB_OUTPUT
 
-    # Annotate the results according to the exit code.
+    # Annotate the results according to the status.
     - name: Annotate
-      if: '!cancelled()'
       shell: bash
       run: |
-        summary="${{ inputs.type }}: ${{ steps.read-summary.outputs.summary }} - ${{ steps.read-summary.outputs.resultURL }}";
-        case ${{ steps.run-test.outputs.status }} in
-          0) echo "::notice ::$summary"
+        summary="${{ inputs.type }} (${{ steps.read-summary.outputs.test_run_id }}): ${{ steps.read-summary.outputs.status }} - ${{ steps.read-summary.outputs.summary }} \`qit get ${{ steps.read-summary.outputs.test_run_id }}\`";
+        case ${{ steps.read-summary.outputs.status }} in
+          "success") echo "::notice ::$summary"
+          ;;
+          "pending") echo "::notice ::$summary"
           ;;
-          2) echo "::warning ::$summary"
+          "warning") echo "::warning ::$summary"
           ;;
           *) echo "::error ::$summary"
           ;;

From 265e00f8a5f36377cc557b62472fabe02dff87dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Mon, 29 Jan 2024 20:49:08 +0100
Subject: [PATCH 10/19] Add wait input to qit action

Addresses https://github.com/woocommerce/grow/pull/81#pullrequestreview-1655013455
---
 packages/github-actions/actions/run-qit-annotate/action.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
index 1601f249..a4e44b27 100644
--- a/packages/github-actions/actions/run-qit-annotate/action.yml
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -20,6 +20,10 @@ inputs:
     description: Type of test to run
     required: false
     default: activation
+  wait:
+    description: Wait for the test to finish
+    required: false
+    default: false
   # Advanced customization.
   options:
     description: Options to pass to the qit command
@@ -52,12 +56,14 @@ runs:
       env:
         # If the custom build is provided, pass it to the `zip` param.
         zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}
+        wait: ${{ inputs.wait == 'true' && '--wait' || '' }}
         ignore_fail: ${{ inputs.ignore-fail == 'true' && '--ignore-fail' || '' }}
       run: |
         json=`./vendor/bin/qit run:${{ inputs.type }} \
          ${{ inputs.extension }} \
          $zip \
          ${{ inputs.options }} \
+         $wait \
          $ignore_fail \
          -n \
          --json | jq '. | {test_run_id, status, test_summary}'`

From e93d3c6970c863b47f3d942b99e4c40b077039f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Mon, 29 Jan 2024 21:43:37 +0100
Subject: [PATCH 11/19] Fail GHA when QIT fails (use ignore-fail param if
 needed)

---
 packages/github-actions/actions/run-qit-annotate/action.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
index a4e44b27..a31acb0a 100644
--- a/packages/github-actions/actions/run-qit-annotate/action.yml
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -13,7 +13,7 @@ inputs:
     description: Custom build of the extension to test. If not given, the latest release will be used.
     required: false
   ignore-fail:
-    description: Should pass even if awaited tests fails.
+    description: Should pass even if the awaited test fails.
     required: false
     default: false
   type:
@@ -51,8 +51,7 @@ runs:
     # Run the qit command.
     - name: Run test
       id: run-test
-      # Do not fail when the `qit` fails, so we can annotate the results.
-      shell: bash --noprofile --norc {0}
+      shell: bash
       env:
         # If the custom build is provided, pass it to the `zip` param.
         zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}

From 44409f55413240ad1fd4de5ce40c5a5b7722bf68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Mon, 29 Jan 2024 23:56:24 +0100
Subject: [PATCH 12/19] Remove results dir from QIT-extensions, add wait param,

Update docs.
---
 .github/workflows/run-qit-extension.yml       |  3 --
 .../actions/run-qit-annotate/README.md        |  7 +--
 .../actions/run-qit-extension/README.md       | 34 ++++++++++++++
 .../actions/run-qit-extension/action.yml      | 45 ++++++++++---------
 4 files changed, 59 insertions(+), 30 deletions(-)
 create mode 100644 packages/github-actions/actions/run-qit-extension/README.md

diff --git a/.github/workflows/run-qit-extension.yml b/.github/workflows/run-qit-extension.yml
index 231033f6..462981e1 100644
--- a/.github/workflows/run-qit-extension.yml
+++ b/.github/workflows/run-qit-extension.yml
@@ -89,9 +89,6 @@ jobs:
                   --user='${{ secrets.QIT_PARTNER_USER }}' \
                   --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
 
-            - name: Create results dir
-              run: mkdir -p ./qit-results/${{ inputs.extension }}
-
             - name: Activation test
               id: activation-test
               if: ${{ inputs.test-activation == true }}
diff --git a/packages/github-actions/actions/run-qit-annotate/README.md b/packages/github-actions/actions/run-qit-annotate/README.md
index f7330de3..6b1d6de9 100644
--- a/packages/github-actions/actions/run-qit-annotate/README.md
+++ b/packages/github-actions/actions/run-qit-annotate/README.md
@@ -4,7 +4,6 @@ This action provides the following functionality for GitHub Actions users:
 
 - Run `qit` test of given type for a given extension
 - Annotate the results
-- Forward status code, so the consumer can decide how to conclude the results
 
 
 ## Usage
@@ -14,8 +13,7 @@ See [action.yml](action.yml)
 ### Prerequisites
 
 - QIT needs to be [installed](https://woocommerce.github.io/qit-documentation/#/cli/getting-started?id=installing-qit) and [authenticated](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli)
-   - To setup QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`.
-- The action assumes there is `qit-results` directory. You can change it's name using `results-folder` input.
+   - To set up QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`.
 
 
 ### Basic:
@@ -37,9 +35,6 @@ jobs:
             --user='${{ secrets.QIT_PARTNER_USER }}' \
             --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
 
-      - name: Create results dir
-        run: mkdir -p ./qit-results/automatewoo
-
       - name: Security test
         id: security-test
         uses: woocommerce/grow/run-qit-annotate@actions-v1
diff --git a/packages/github-actions/actions/run-qit-extension/README.md b/packages/github-actions/actions/run-qit-extension/README.md
new file mode 100644
index 00000000..db2aa99e
--- /dev/null
+++ b/packages/github-actions/actions/run-qit-extension/README.md
@@ -0,0 +1,34 @@
+# Run QIT tests
+
+This action provides the following functionality for GitHub Actions users:
+
+- Install QIT-CLI
+- Run a set of `qit` tests for a given extension
+- Annotate the results
+
+
+## Usage
+
+See [action.yml](action.yml)
+
+### Prerequisites
+
+- You need to obtain valid [authentication keys](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli).
+
+
+### Basic:
+
+```yaml
+jobs:
+  qit-test:
+    name: Run QIT Tests
+    runs-on: ubuntu-20.04
+    steps:    
+      - name: Delegate QIT Tests
+        uses: woocommerce/grow/run-qit-extension@actions-v1
+        with:
+          qit-partner-user: ${{ secrets.QIT_PARTNER_USER }}
+          qit-partner-secret: ${{ secrets.QIT_PARTNER_SECRET }}
+          extension: 'my-extension'
+          test-activation: true
+```
diff --git a/packages/github-actions/actions/run-qit-extension/action.yml b/packages/github-actions/actions/run-qit-extension/action.yml
index 7b63a656..a898a0f0 100644
--- a/packages/github-actions/actions/run-qit-extension/action.yml
+++ b/packages/github-actions/actions/run-qit-extension/action.yml
@@ -1,5 +1,5 @@
 name: Run QIT for a given extension
-description: Runs QIT tests and annotates the results for a given extension.
+description: Runs many types of QIT tests and annotates the results for a given extension.
 
 # **What it does**: Runs a set of QIT tests for a given extension.
 # **Why we have it**: To reuse across other repos, to make a full test of a single extension.
@@ -19,6 +19,10 @@ inputs:
   extension:
     description: Extension to test
     required: true
+  ignore-fail:
+    description: Should pass even if any awaited test fails.
+    required: false
+    default: false
   version:
     description: |
       Version to be tested: `latest`, `dev`, or `local`.
@@ -28,6 +32,10 @@ inputs:
       `local` will look up for an `{extension}.zip` artifact in the current workflow.
     required: true
     default: 'latest'
+  wait:
+    description: Wait for the test to finish
+    required: false
+    default: false
 
   # Customize which types to run.
   test-activation:
@@ -82,62 +90,57 @@ runs:
           --user='${{ inputs.qit-partner-user }}' \
           --application_password='${{ inputs.qit-partner-secret }}'
 
-    - name: Create results dir
-      shell: bash
-      run: mkdir -p ./qit-results/${{ inputs.extension }}
-
     - name: Activation test
-      id: activation-test
+      id: activation
       if: ${{ inputs.test-activation == 'true' }}
-      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build
       with:
         type: activation
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
+        wait: ${{ inputs.wait }}
         options: ${{ inputs.options }}
 
     - name: Security test
-      id: security-test
+      id: security
       if: ${{ inputs.test-security == 'true' }}
-      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build
       with:
         type: security
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
+        wait: ${{ inputs.wait }}
         options: ${{ inputs.options }}
 
     - name: PHPStan test
-      id: phpstan-test
+      id: phpstan
       if: ${{ inputs.test-phpstan == 'true' }}
-      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build
       with:
         type: phpstan
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
+        wait: ${{ inputs.wait }}
         options: ${{ inputs.options }}
 
     - name: API test
-      id: api-test
+      id: api
       if: ${{ inputs.test-api == 'true' }}
-      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build
       with:
         type: api
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
+        wait: ${{ inputs.wait }}
         options: ${{ inputs.options }}
 
     - name: E2E test
-      id: e2e-test
+      id: e2e
       if: ${{ inputs.test-e2e == 'true' }}
-      uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
+      uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build
       with:
         type: e2e
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
+        wait: ${{ inputs.wait }}
         options: ${{ inputs.options }}
-
-    - name: Upload results
-      uses: actions/upload-artifact@v1
-      with:
-        name: "qit-results-${{ inputs.extension }}"
-        path: ./qit-results/${{ inputs.extension }}

From 5bdc33d570cbe0986cf09691729d5495f6c0c633 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Tue, 30 Jan 2024 00:06:16 +0100
Subject: [PATCH 13/19] Update QIT workflow to use `qit-extension` action

---
 .github/workflows/run-qit-all.yml       |  39 ++++--
 .github/workflows/run-qit-extension.yml | 151 ------------------------
 2 files changed, 27 insertions(+), 163 deletions(-)
 delete mode 100644 .github/workflows/run-qit-extension.yml

diff --git a/.github/workflows/run-qit-all.yml b/.github/workflows/run-qit-all.yml
index d6021905..88f4724c 100644
--- a/.github/workflows/run-qit-all.yml
+++ b/.github/workflows/run-qit-all.yml
@@ -14,6 +14,10 @@ on:
         options:
         - latest
         - dev
+      wait:
+        description: 'Should wait for results'
+        default: false
+        type: boolean
 
       # Configure which tests to run.
       test-activation:
@@ -43,28 +47,39 @@ on:
         type: boolean
 
       # Advanced customization.
+      ignore-fail:
+        description: Should pass even if any awaited test fails.
+        required: false
+        default: false
+        type: boolean
       options:
         description: 'Additional options for `qit` command, like `--optional_features=hpos`.'
         required: false
 
+
 jobs:
     qit-tests:
         name: Run QIT Tests
-        uses: ./.github/workflows/run-qit-extension.yml
-        secrets: inherit
+        runs-on: ubuntu-latest
         strategy:
           # Allow to test extensions even if one of them fails.
           fail-fast: false
           matrix:
             # List of extensions to be tested.
             extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration]
-        with:
-          # Conditional statements are here to allow testing on push triggers, without manual input. To be removed before merging.
-          version: ${{ inputs.version }}
-          test-activation: ${{ inputs.test-activation }}
-          test-security: ${{ inputs.test-security }}
-          test-phpstan: ${{ inputs.test-phpstan }}
-          test-api: ${{ inputs.test-api }}
-          test-e2e: ${{ inputs.test-e2e }}
-          extension: ${{ matrix.extension }}
-          options: ${{ inputs.options }}
+        steps:
+          - name: Run QIT
+            uses: woocommerce/grow/run-qit-extension@add/qit-workflows-test-build
+            with:
+              qit-partner-user: ${{ secrets.QIT_PARTNER_USER }}
+              qit-partner-secret: ${{ secrets.QIT_PARTNER_SECRET }}
+              version: ${{ inputs.version }}
+              wait: ${{ inputs.wait }}
+              test-activation: ${{ inputs.test-activation }}
+              test-security: ${{ inputs.test-security }}
+              test-phpstan: ${{ inputs.test-phpstan }}
+              test-api: ${{ inputs.test-api }}
+              test-e2e: ${{ inputs.test-e2e }}
+              extension: ${{ matrix.extension }}
+              ignore-fail: ${{ inputs.ignore-fail }}
+              options: ${{ inputs.options }}
diff --git a/.github/workflows/run-qit-extension.yml b/.github/workflows/run-qit-extension.yml
deleted file mode 100644
index 462981e1..00000000
--- a/.github/workflows/run-qit-extension.yml
+++ /dev/null
@@ -1,151 +0,0 @@
-name: Run QIT for a given extensions
-
-# **What it does**: Runs a set of QIT tests for a given extension.
-# **Why we have it**: To reuse across other repos, to make a full test of a single extension.
-
-on:
-  workflow_call:
-    inputs:
-      # Basic params.
-      extension:
-        description: Extension to test
-        required: true
-        type: string
-      version:
-        description: |
-          Version to be tested: `latest`, `dev`, or `local`.
-          `latest` will test latest released version,
-          `dev` will make the action look up for an `{extension}.zip` file in the root
-          of the repository under the `gha-dev-build` tag,
-          `local` will look up for an `{extension}.zip` artifact in the current workflow.
-        required: true
-        default: 'latest'
-        type: string
-
-      # Customize which types to run.
-      test-activation:
-        description: 'Should activation be tested?'
-        default: true
-        type: boolean
-      test-security:
-        description: 'Should security be tested?'
-        default: true
-        type: boolean
-      test-phpstan:
-        description: 'Should PHPStan be tested?'
-        default: true
-        type: boolean
-      test-api:
-        description: 'Should API be tested?'
-        default: true
-        type: boolean
-      test-e2e:
-        description: 'Should E2E be tested?'
-        default: true
-        type: boolean
-
-      # Advanced customization.
-      options:
-        description: 'Additional options for `qit` command, like `--optional_features=hpos`'
-        type: string
-
-    outputs:
-        statuses:
-            description: "Statuses of all tests. Array of integers."
-            value: ${{ jobs.qit-tests.outputs.statuses }}
-
-
-jobs:
-    qit-tests:
-        name: Run QIT Tests
-        runs-on: ubuntu-20.04
-        env:
-            QIT_DISABLE_ONBOARDING: yes
-            dev_build: ${{ inputs.version != 'latest' && format('{0}.zip', inputs.extension) || '' }}
-        outputs:
-            statuses: ${{ toJSON( steps.*.outputs.status ) }}
-        steps:
-
-            - name: Download `gha-dev-build`
-              if: ${{ inputs.version == 'dev' }}
-              uses: robinraju/release-downloader@v1.8
-              with:
-                repository: "woocommerce/${{ inputs.extension }}"
-                tag: 'gha-dev-build'
-                fileName: ${{ env.dev_build }}
-                token: ${{ secrets.BOT_GH_TOKEN }}
-
-            - name: Download artifact
-              if: ${{ inputs.version == 'local' }}
-              uses: actions/download-artifact@v3
-              with:
-                name: ${{ env.dev_build }}
-
-            - name: Install QIT via composer
-              run: composer require woocommerce/qit-cli
-            - name: Add Partner
-              run: |
-                ./vendor/bin/qit partner:add \
-                  --user='${{ secrets.QIT_PARTNER_USER }}' \
-                  --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
-
-            - name: Activation test
-              id: activation-test
-              if: ${{ inputs.test-activation == true }}
-              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
-              timeout-minutes: 5
-              with:
-                type: activation
-                extension: ${{ inputs.extension }}
-                extension-file: ${{ env.dev_build }}
-                options: ${{ inputs.options }}
-
-            - name: Security test
-              id: security-test
-              if: ${{ inputs.test-security == true }}
-              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
-              timeout-minutes: 5
-              with:
-                type: security
-                extension: ${{ inputs.extension }}
-                extension-file: ${{ env.dev_build }}
-                options: ${{ inputs.options }}
-
-            - name: PHPStan test
-              id: phpstan-test
-              if: ${{ inputs.test-phpstan == true }}
-              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
-              timeout-minutes: 5
-              with:
-                type: phpstan
-                extension: ${{ inputs.extension }}
-                extension-file: ${{ env.dev_build }}
-                options: ${{ inputs.options }}
-
-            - name: API test
-              id: api-test
-              if: ${{ inputs.test-api == true }}
-              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
-              timeout-minutes: 5
-              with:
-                type: api
-                extension: ${{ inputs.extension }}
-                extension-file: ${{ env.dev_build }}
-                options: ${{ inputs.options }}
-
-            - name: E2E test
-              id: e2e-test
-              if: ${{ inputs.test-e2e == true }}
-              uses: woocommerce/grow/run-qit-annotate@actions-v1.10.2-pre
-              timeout-minutes: 30
-              with:
-                type: e2e
-                extension: ${{ inputs.extension }}
-                extension-file: ${{ env.dev_build }}
-                options: ${{ inputs.options }}
-
-            - name: Upload results
-              uses: actions/upload-artifact@v1
-              with:
-                  name: "qit-results-${{ inputs.extension }}"
-                  path: ./qit-results/${{ inputs.extension }}

From 5cf8761044f76a411bbc2c14d4243c630b26807e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Tue, 30 Jan 2024 23:59:36 +0100
Subject: [PATCH 14/19] Make qit action bash permisive to failures, so we can
 annotate them

---
 packages/github-actions/actions/run-qit-annotate/action.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
index a31acb0a..4b74f7ef 100644
--- a/packages/github-actions/actions/run-qit-annotate/action.yml
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -51,7 +51,8 @@ runs:
     # Run the qit command.
     - name: Run test
       id: run-test
-      shell: bash
+      # Do not fail when the `qit` fails, so we can annotate the results.
+      shell: bash --noprofile --norc {0}
       env:
         # If the custom build is provided, pass it to the `zip` param.
         zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}

From f09dda3cb986d857eb435d764b281767db97aa72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 31 Jan 2024 00:31:52 +0100
Subject: [PATCH 15/19] Forward qit exitcode to github action

---
 .../github-actions/actions/run-qit-annotate/action.yml     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
index 4b74f7ef..c8d10776 100644
--- a/packages/github-actions/actions/run-qit-annotate/action.yml
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -67,6 +67,7 @@ runs:
          $ignore_fail \
          -n \
          --json | jq '. | {test_run_id, status, test_summary}'`
+        echo "exitcode=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
         echo $json | jq
         echo "result="$json"" >> "$GITHUB_OUTPUT"
 
@@ -79,8 +80,9 @@ runs:
         echo "status=${{fromJson(steps.run-test.outputs.result).status}}" >> $GITHUB_OUTPUT
         echo "summary=${{fromJson(steps.run-test.outputs.result).test_summary}}" >> $GITHUB_OUTPUT
 
-    # Annotate the results according to the status.
-    - name: Annotate
+
+    # Annotate the results according to the status, forward qit exit code.
+    - name: Annotate and exit
       shell: bash
       run: |
         summary="${{ inputs.type }} (${{ steps.read-summary.outputs.test_run_id }}): ${{ steps.read-summary.outputs.status }} - ${{ steps.read-summary.outputs.summary }} \`qit get ${{ steps.read-summary.outputs.test_run_id }}\`";
@@ -94,3 +96,4 @@ runs:
           *) echo "::error ::$summary"
           ;;
         esac
+        exit ${{ steps.run-test.outputs.exitcode }}

From c5922891e3fba1516c756b5e9a43d9feeb0fb86a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 31 Jan 2024 00:59:19 +0100
Subject: [PATCH 16/19] Forward qit exitcode to github action

---
 packages/github-actions/actions/run-qit-annotate/action.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml
index c8d10776..2d3da5dc 100644
--- a/packages/github-actions/actions/run-qit-annotate/action.yml
+++ b/packages/github-actions/actions/run-qit-annotate/action.yml
@@ -66,8 +66,9 @@ runs:
          $wait \
          $ignore_fail \
          -n \
-         --json | jq '. | {test_run_id, status, test_summary}'`
+         --json`
         echo "exitcode=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+        json=$(echo $json | jq '. | {test_run_id, status, test_summary}')
         echo $json | jq
         echo "result="$json"" >> "$GITHUB_OUTPUT"
 

From c0a8ee15405c353d16253b04a7110250f385bc11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Wed, 31 Jan 2024 02:21:03 +0100
Subject: [PATCH 17/19] Forward ignore fail to qit-extension

---
 packages/github-actions/actions/run-qit-extension/action.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/github-actions/actions/run-qit-extension/action.yml b/packages/github-actions/actions/run-qit-extension/action.yml
index a898a0f0..24a03601 100644
--- a/packages/github-actions/actions/run-qit-extension/action.yml
+++ b/packages/github-actions/actions/run-qit-extension/action.yml
@@ -99,6 +99,7 @@ runs:
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
         wait: ${{ inputs.wait }}
+        ignore-fail: ${{ inputs.ignore-fail }}
         options: ${{ inputs.options }}
 
     - name: Security test
@@ -110,6 +111,7 @@ runs:
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
         wait: ${{ inputs.wait }}
+        ignore-fail: ${{ inputs.ignore-fail }}
         options: ${{ inputs.options }}
 
     - name: PHPStan test
@@ -121,6 +123,7 @@ runs:
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
         wait: ${{ inputs.wait }}
+        ignore-fail: ${{ inputs.ignore-fail }}
         options: ${{ inputs.options }}
 
     - name: API test
@@ -132,6 +135,7 @@ runs:
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
         wait: ${{ inputs.wait }}
+        ignore-fail: ${{ inputs.ignore-fail }}
         options: ${{ inputs.options }}
 
     - name: E2E test
@@ -143,4 +147,5 @@ runs:
         extension: ${{ inputs.extension }}
         extension-file: ${{ env.dev_build }}
         wait: ${{ inputs.wait }}
+        ignore-fail: ${{ inputs.ignore-fail }}
         options: ${{ inputs.options }}

From 5f318fbcafa9b3f12289b72ddeec32d2bfd05470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Tue, 6 Feb 2024 22:53:12 +0100
Subject: [PATCH 18/19] Add `run-qit-extension` to the actions README

Addresses https://github.com/woocommerce/grow/pull/81#discussion_r1473727724
---
 packages/github-actions/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/github-actions/README.md b/packages/github-actions/README.md
index c9d7033c..5fd0a791 100644
--- a/packages/github-actions/README.md
+++ b/packages/github-actions/README.md
@@ -21,6 +21,7 @@ Custom GitHub actions that help to composite GitHub workflows across the repos m
 - [`prepare-php`](actions/prepare-php) - Set up PHP with a specific version and tools, load Composer cache, install Composer dependencies
 - [`publish-extension-dev-build`](actions/publish-extension-dev-build) - Publish extension development build
 - [`run-qit-annotate`](actions/run-qit-annotate) - Runs QIT test and annotates the results
+- [`run-qit-extension`](actions/run-qit-extension) - Run QIT tests for a given extension
 - [`stylelint-annotation`](actions/stylelint-annotation) - Annotate stylelint results via stylelint formatter
 - [`update-version-tags`](actions/update-version-tags) - Update version tags
 

From d252f0e6be5f3cfe030373c4260ddb9d13a9999f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomek=20Wytr=C4=99bowicz?= <tomalecpub@gmail.com>
Date: Tue, 6 Feb 2024 22:55:49 +0100
Subject: [PATCH 19/19] Use `qit_token` instead of `application_password`

Addresses https://github.com/woocommerce/grow/pull/81#discussion_r1473729942
---
 packages/github-actions/actions/run-qit-annotate/README.md   | 2 +-
 packages/github-actions/actions/run-qit-extension/action.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/github-actions/actions/run-qit-annotate/README.md b/packages/github-actions/actions/run-qit-annotate/README.md
index 6b1d6de9..f0b1147b 100644
--- a/packages/github-actions/actions/run-qit-annotate/README.md
+++ b/packages/github-actions/actions/run-qit-annotate/README.md
@@ -33,7 +33,7 @@ jobs:
         run: |
           ./vendor/bin/qit partner:add \
             --user='${{ secrets.QIT_PARTNER_USER }}' \
-            --application_password='${{ secrets.QIT_PARTNER_SECRET }}'
+            --qit_token='${{ secrets.QIT_PARTNER_SECRET }}'
 
       - name: Security test
         id: security-test
diff --git a/packages/github-actions/actions/run-qit-extension/action.yml b/packages/github-actions/actions/run-qit-extension/action.yml
index 24a03601..246f7aee 100644
--- a/packages/github-actions/actions/run-qit-extension/action.yml
+++ b/packages/github-actions/actions/run-qit-extension/action.yml
@@ -88,7 +88,7 @@ runs:
       run: |
         ./vendor/bin/qit partner:add \
           --user='${{ inputs.qit-partner-user }}' \
-          --application_password='${{ inputs.qit-partner-secret }}'
+          --qit_token='${{ inputs.qit-partner-secret }}'
 
     - name: Activation test
       id: activation