Add smoke E2E test workflow that ensures typical user-experience work (…

…#1837)



Co-authored-by: Igor Lema <[email protected]>

.github/workflows/e2e.yaml

@@ -0,0 +1,224 @@
+name: E2E test
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: macos-latest
+    if: github.repository == 'mobile-dev-inc/maestro'
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 8
+
+      - name: Set up Gradle
+        uses: gradle/actions/setup-gradle@v3
+
+      - name: Build xctest-runner
+        run: ./maestro-ios-xctest-runner/build-maestro-ios-runner.sh
+
+      - name: Build Maestro CLI
+        run: ./gradlew :maestro-cli:distZip
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: maestro-cli
+          path: maestro-cli/build/distributions/maestro.zip
+          retention-days: 1
+
+  test-local:
+    runs-on: ubuntu-latest
+    if: github.repository == 'mobile-dev-inc/maestro'
+    needs: build
+
+    env:
+      EMULATOR_API_LEVEL: 33
+      EMULATOR_TARGET: google_apis
+      EMULATOR_ARCH: x86_64
+
+    steps:
+      - name: Enable KVM group perms
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Clone repository (only needed for the e2e directory)
+        uses: actions/checkout@v4
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 17
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: maestro-cli
+
+      - name: Add Maestro CLI executable to PATH
+        run: |
+          unzip maestro.zip -d maestro_extracted
+          echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH
+
+      - name: Check if Maestro CLI executable starts up
+        run: |
+          maestro --help
+          maestro --version
+
+      - name: Start emulator and run a Flow
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: ${{ env.EMULATOR_API_LEVEL }}
+          target: ${{ env.EMULATOR_TARGET }}
+          arch: ${{ env.EMULATOR_ARCH }}
+          force-avd-creation: false
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          disable-animations: false
+          working-directory: ${{ github.workspace }}/e2e
+          script: |
+            ./download_apps
+            ./install_apps
+            ./run_tests
+
+      - name: Save logcat output
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: logcat
+          path: artifacts/logcat.log
+
+  test-cloud:
+    runs-on: ubuntu-latest
+    if: github.repository == 'mobile-dev-inc/maestro'
+    needs: build
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: maestro-cli
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 8
+
+      - name: Add Maestro CLI executable to PATH
+        run: |
+          unzip maestro.zip -d maestro_extracted
+          echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH
+
+      - name: Check if Maestro CLI executable starts up
+        run: |
+          maestro --help
+          maestro --version
+
+      - name: maestro download-samples
+        run: maestro download-samples
+
+      - name: Run iOS test
+        run: |
+          maestro cloud \
+            --apiKey=${{secrets.E2E_MOBILE_DEV_API_KEY}} \
+            --timeout 180 \
+            --fail-on-cancellation \
+            --include-tags advanced \
+            samples/sample.zip \
+            samples
+
+      - name: Run Android test
+        run: |
+          maestro cloud \
+            --apiKey ${{secrets.E2E_MOBILE_DEV_API_KEY}} \
+            --fail-on-cancellation \
+            --include-tags advanced \
+            samples/sample.apk \
+            samples
+
+      - name: Trigger alert on failure
+        if: ${{ false }}
+        # if: failure()
+        run: |
+          curl --request POST \
+              --url "https://events.pagerduty.com/v2/enqueue" \
+              --header 'Content-Type: application/json' \
+              --data '{
+            "payload": {
+              "summary": "E2E test failed",
+              "source": "E2E test",
+              "severity": "critical"
+            },
+            "routing_key": "${{ secrets.E2E_PAGER_DUTY_INTEGRATION_KEY }}",
+            "event_action": "trigger",
+            "links": [
+                {
+                  "href": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                  "text": "Failed E2E test - Github Action"
+                }
+            ]
+          }'
+
+  test-cloud-production:
+    # this job is the same as e2e-production in mobile-dev-inc/monorepo
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 8
+
+      - name: Install Maestro
+        run: |
+          curl -Ls --retry 3 --retry-all-errors "https://get.maestro.mobile.dev" | bash
+          echo "${HOME}/.maestro/bin" >> $GITHUB_PATH
+
+      - name: Print Maestro version
+        run: maestro --version
+
+      - name: Download samples
+        run: maestro download-samples
+
+      - name: Run iOS test
+        run: |
+          maestro cloud --apiKey=${{secrets.E2E_MOBILE_DEV_API_KEY}} --timeout=180 --fail-on-cancellation --include-tags=advanced samples/sample.zip samples
+      - name: Run Android test
+        run: |
+          maestro cloud --apiKey=${{secrets.E2E_MOBILE_DEV_API_KEY}} --fail-on-cancellation --include-tags=advanced samples/sample.apk samples
+      - name: Trigger alert on failure
+        if: ${{ false }}
+        # if: failure()
+        run: |
+          curl --request POST \
+              --url "https://events.pagerduty.com/v2/enqueue" \
+              --header 'Content-Type: application/json' \
+              --data '{
+            "payload": {
+              "summary": "E2E test failed",
+              "source": "E2E test",
+              "severity": "critical"
+            },
+            "routing_key": "${{ secrets.E2E_PAGER_DUTY_INTEGRATION_KEY }}",
+            "event_action": "trigger",
+            "links": [
+                {
+                  "href": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                  "text": "Failed E2E test - Github Action"
+                }
+            ]
+          }'

.github/workflows/test.yml

@@ -1,7 +1,6 @@
 name: Test
 
 on:
-  push:
   pull_request:
 
 jobs:

.github/workflows/warn_build_xctestrunner.yml

@@ -1,9 +1,6 @@
 name: Warn Build XCTest runner
 
 on:
-  push:
-    paths:
-      - 'maestro-ios-xctest-runner/**'
   pull_request:
     paths:
       - 'maestro-ios-xctest-runner/**'

e2e/.gitignore

@@ -0,0 +1 @@
+apps/

e2e/README.md

@@ -0,0 +1,12 @@
+# e2e
+
+This directory contains glue code for testing Maestro itself.
+
+Typical workflow is:
+
+1. Start Android emulator and iOS simulator
+2. `download_apps`
+3. `install_apps`
+4. `run_tests`
+
+We try to keep scripts in files, so we don't get too tightly coupled to GitHub Action.

e2e/download_apps

@@ -0,0 +1,23 @@
+#!/usr/bin/env sh
+
+set -eu
+
+# Download apps from URLs listed in manifest.txt.
+#
+# We assume that if the downloaded file is a zip file, it's an iOS app and must
+# be unzipped.
+
+[ "$(basename "$PWD")" = "e2e" ] || { echo "must be run from e2e directory" && exit 1; }
+
+command -v curl >/dev/null 2>&1 || { echo "curl is required" && exit 1; }
+
+mkdir -p ./apps
+while read -r url; do
+	echo "download $url"
+	app_file="$(curl -fsSL --output-dir ./apps --write-out "%{filename_effective}" -OJ "$url")"
+	extension="${app_file##*.}"
+	if [ "$extension" = "zip" ]; then
+		unzip -qq -o -d ./apps "$app_file" -x "__MACOSX/*"
+		rm "$app_file"
+	fi
+done <manifest.txt

e2e/flows/nowinandroid/bookmarks.yaml

@@ -0,0 +1,10 @@
+appId: com.google.samples.apps.nowinandroid.demo.debug
+---
+- launchApp:
+    clearState: true
+- tapOn: Headlines
+- tapOn: Done
+- tapOn: Bookmark
+- tapOn: Saved
+- tapOn: Unbookmark
+- assertVisible: No saved updates

e2e/install_apps

@@ -0,0 +1,21 @@
+#!/usr/bin/env sh
+
+set -eu
+
+[ "$(basename "$PWD")" = "e2e" ] || { echo "must be run from e2e directory" && exit 1; }
+
+command -v adb >/dev/null 2>&1 || { echo "adb is required" && exit 1; }
+
+for file in ./apps/*; do
+	filename="$(basename "$file")"
+	echo "install $filename"
+
+	extension="${file##*.}"
+	if [ "$extension" = "apk" ]; then
+		adb install -r "$file" >/dev/null
+	elif [ "$extension" = "app" ] && [ "$(uname)" = "Darwin" ]; then
+		xcrun simctl install booted "$file"
+	else
+		echo "unknown file extension $extension"
+	fi
+done

e2e/manifest.txt

@@ -0,0 +1,3 @@
+https://storage.googleapis.com/mobile.dev/samples/e2e_apps/wikipedia.apk
+https://storage.googleapis.com/mobile.dev/samples/e2e_apps/Wikipedia.zip
+https://storage.googleapis.com/mobile.dev/samples/e2e_apps/nowinandroid.apk

e2e/run_tests

@@ -0,0 +1,25 @@
+#!/usr/bin/env sh
+
+set -eu
+
+command -v maestro >/dev/null 2>&1 || { echo "maestro is required" && exit 1; }
+
+[ "$(basename "$PWD")" = "e2e" ] || { echo "must be run from e2e directory" && exit 1; }
+
+selected_app="${1:-}"
+
+FAILED=false
+for file in ./flows/*; do
+	filename="$(basename "$file")"
+
+	if [ -z "$selected_app" ] || [ "$selected_app" = "$filename" ]; then
+		cd "$file" || exit 1
+		maestro --device emulator-5554 test . || FAILED=true
+		cd - >/dev/null || exit 1
+	fi
+done
+
+if [ "$FAILED" = true ]; then
+  echo "Some tests failed"
+  exit 1
+fi

recipes/googleplay/install_twitter.yaml

@@ -10,4 +10,4 @@ name: Install Twitter
 - tapOn:
     text: twitter
     index: 1
-- tapOn: Install
+- tapOn: Install

recipes/nowinandroid/pick_interests.yaml

@@ -5,4 +5,4 @@ name: Pick Interests
     clearState: true
 - tapOn: Headlines
 - tapOn: Testing
-- tapOn: Done
+- tapOn: Done

recipes/square/pos/android/signup.yaml

@@ -14,4 +14,4 @@ name: Sign Up
     rightOf: I agree to.*
     retryTapIfNoChange: false
 - tapOn: Continue
-- tapOn: Confirm
+- tapOn: Confirm

recipes/twitter/android/create_account.yaml

@@ -16,4 +16,4 @@ name: Create Account
 - longPressOn: 2021
 - tapOn: Next
 - tapOn: Next
-- tapOn: Sign up
+- tapOn: Sign up

recipes/twitter/android/follow.yaml

@@ -6,4 +6,4 @@ appId: com.twitter.android
 - inputText: "@mobile__dev"
 - tapOn: mobile.dev
 - tapOn: Follow
-- assertVisible: Following
+- assertVisible: Following