More e2e tests (mobile-dev-inc#2124)

* Add additional failure e2e tests for different assert commands

* Add additional failure e2e tests for launching apps

* Add e2e test to tour commands

* Fix escaping of structured logging in e2e tests

* Allow e2e-tests longer to run

* Relax timings on extendedWaitUntil e2e test

* Relax tap retries when waiting for repaint

.github/workflows/test-e2e.yaml

@@ -165,7 +165,7 @@ jobs:
 
       - name: Run tests
         working-directory: ${{ github.workspace }}/e2e
-        timeout-minutes: 10
+        timeout-minutes: 20
         run: ./run_tests android
 
       - name: Stop screen recording of AVD

e2e/run_tests

@@ -10,15 +10,15 @@ command -v maestro >/dev/null 2>&1 || { echo "maestro is required" && exit 1; }
 ALL_PASS=true
 
 _h1() {
-	printf "=>\n=> $1\n=>\n"
+	printf "=>\n=> %s\n=>\n" "$1"
 }
 
 _h2() {
-	printf "==> [$1] $2\n"
+	printf "==> [%s] %s\n" "$1" "$2"
 }
 
 _h3() {
-	printf "==> [$1] [$2] => $3\n"
+	printf "==> [%s] [%s] => %s\n" "$1" "$2" "$3"
 }
 
 platform="${1:-}"

e2e/workspaces/demo_app/commands/assertNotVisible.yaml

@@ -0,0 +1,13 @@
+appId: com.example.example
+---
+
+- launchApp # For idempotence of sections
+
+- assertNotVisible: 'kwyjibo'
+
+- assertNotVisible:
+    text: 'kwyjibo'
+
+- assertNotVisible:
+    text: 'Form Test'
+    enabled: false

e2e/workspaces/demo_app/commands/assertTrue.yaml

@@ -0,0 +1,12 @@
+appId: com.example.example
+---
+
+- launchApp # For idempotence of sections
+
+- assertTrue: ${"test" == "test"}
+
+- assertTrue:
+    condition: ${12 < 20}
+
+- assertTrue:
+    condition: ${THING == "five"} # Using the env at the top of the file

e2e/workspaces/demo_app/commands/assertVisible.yaml

@@ -0,0 +1,12 @@
+appId: com.example.example
+---
+
+- launchApp # For idempotence of sections
+
+- assertVisible: 'Form Test'
+
+- assertVisible:
+    text: 'Form Test'
+
+- assertVisible:
+    id: 'fabAddIcon'

e2e/workspaces/demo_app/commands/back.yaml

@@ -0,0 +1,8 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Form Test'
+- assertVisible: 'Login'
+- back
+- assertVisible: 'Form Test'

e2e/workspaces/demo_app/commands/copyTextFrom.yaml

@@ -0,0 +1,10 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn:
+    id: 'fabAddIcon'
+    retryTapIfNoChange: false
+- copyTextFrom:
+    text: '\d+'
+- assertTrue: ${maestro.copiedText == '1'}

e2e/workspaces/demo_app/commands/eraseText.yaml

@@ -0,0 +1,18 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Form Test'
+- tapOn: 'Email'
+- inputText: 'foo'
+- assertVisible: 'foo'
+- eraseText
+- assertNotVisible: 'foo'
+
+- inputText: 'testing'
+- assertVisible: 'testing'
+- eraseText: 3
+- assertNotVisible: 'testing'
+- assertVisible:
+    text: 'test'
+    optional: true # FIXME: This still takes an extra character sometimes, even after #2123

e2e/workspaces/demo_app/commands/evalScript.yaml

@@ -0,0 +1,7 @@
+appId: com.example.example
+---
+
+- launchApp # For idempotence of sections
+
+- evalScript: ${output.test = 'foo'}
+- assertTrue: ${output.test == 'foo'}

e2e/workspaces/demo_app/commands/extendedWaitUntil.yaml

@@ -0,0 +1,13 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- extendedWaitUntil:
+    timeout: 10000
+    visible:
+      text: 'Swipe Test'
+
+- extendedWaitUntil:
+    timeout: 100
+    notVisible:
+      text: 'Non Existent Text'

e2e/workspaces/demo_app/commands/hideKeyboard.yaml

@@ -0,0 +1,11 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Form Test'
+- tapOn: 'Email'
+- assertVisible:
+    id: com.google.android.inputmethod.latin:id/key_pos_shift # The shift key on the Android keyboard
+- hideKeyboard
+- assertNotVisible:
+    id: com.google.android.inputmethod.latin:id/key_pos_shift

e2e/workspaces/demo_app/commands/inputRandomEmail.yaml

@@ -0,0 +1,9 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Input Test'
+- tapOn:
+    id: 'textInput'
+- inputRandomEmail
+- assertVisible: '.+@.+'

e2e/workspaces/demo_app/commands/inputRandomNumber.yaml

@@ -0,0 +1,18 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Input Test'
+- tapOn:
+    id: 'textInput'
+- inputRandomNumber
+- assertVisible:
+    text: '\d{8}' # The default length is 8
+    id: 'textInput'
+
+- eraseText
+- inputRandomNumber:
+    length: 4
+- assertVisible:
+    text: '\d{4}'
+    id: 'textInput'

e2e/workspaces/demo_app/commands/inputRandomPersonName.yaml

@@ -0,0 +1,11 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Input Test'
+- tapOn:
+    id: 'textInput'
+- inputRandomPersonName
+- assertVisible: 
+    text: '[A-Z][a-z]+ [A-Z][a-z]+'
+    id: 'textInput'

e2e/workspaces/demo_app/commands/inputRandomText.yaml

@@ -0,0 +1,18 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Input Test'
+- tapOn:
+    id: 'textInput'
+- inputRandomText
+- assertVisible:
+    text: '[a-z0-9]{8}' # The default length is 8
+    id: 'textInput'
+
+- eraseText
+- inputRandomText:
+    length: 4
+- assertVisible:
+    text: '[a-z0-9]{4}'
+    id: 'textInput'

e2e/workspaces/demo_app/commands/inputText.yaml

@@ -0,0 +1,9 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- tapOn: 'Input Test'
+- tapOn:
+    id: 'textInput'
+- inputText: 'foo'
+- assertVisible: 'foo'

e2e/workspaces/demo_app/commands/killApp.yaml

@@ -0,0 +1,7 @@
+appId: com.example.example
+---
+
+- launchApp
+- assertVisible: 'Form Test'
+- killApp
+- assertNotVisible: 'Form Test'

e2e/workspaces/demo_app/commands/launchApp.yaml

@@ -0,0 +1,14 @@
+appId: com.example.example
+---
+- launchApp:
+    appId: com.example.example
+    clearState: true
+    clearKeychain: true
+    stopApp: true
+    permissions:
+      all: allow
+- assertVisible: 'Form Test'
+
+- stopApp
+- launchApp
+- assertVisible: 'Form Test'

e2e/workspaces/demo_app/commands/pasteText.yaml

@@ -0,0 +1,13 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- evalScript: ${maestro.copiedText = 'foo'}
+- tapOn: 'Input Test'
+- tapOn:
+    id: 'textInput'
+- inputText: 'foo'
+- copyTextFrom:
+    id: 'textInput'
+- pasteText
+- assertVisible: 'foofoo'

e2e/workspaces/demo_app/commands/pressKey.yaml

@@ -0,0 +1,15 @@
+appId: com.example.example
+---
+- launchApp
+
+- assertVisible: 'Form Test'
+- pressKey: 'Home'
+- assertNotVisible: 'Form Test'
+
+- launchApp
+
+- assertVisible: 'Form Test'
+- tapOn: 'Form Test'
+- assertNotVisible: 'Form Test'
+- pressKey: 'Back'
+- assertVisible: 'Form Test'

e2e/workspaces/demo_app/commands/repeat.yaml

@@ -0,0 +1,19 @@
+appId: com.example.example
+---
+- launchApp # For idempotence of sections
+
+- assertVisible: '0'
+- repeat:
+    times: 3
+    commands:
+      - tapOn:
+          id: 'fabAddIcon'
+- assertVisible: '3'
+- assertNotVisible: '0'
+
+- evalScript: ${output.counter = 0}
+- repeat:
+    while:
+      true: ${output.counter < 3}
+    commands:
+      - evalScript: ${output.counter = output.counter + 1}

e2e/workspaces/demo_app/commands/runFlow.yaml

@@ -0,0 +1,20 @@
+appId: com.example.example
+---
+
+# runFlow with file: isn't included since it's in the root flow
+
+- launchApp # For idempotence of sections
+
+- runFlow:
+    commands:
+      - evalScript: ${output.test = 'bar'}
+      - assertTrue: ${output.test == 'bar'}
+      - tapOn:
+          id: 'fabAddIcon'
+- assertVisible: '1'
+
+- runFlow:
+    env:
+      THIS_THING: "six"
+    commands:
+      - assertTrue: ${THIS_THING == "six"}

e2e/workspaces/demo_app/commands/runScript.js

@@ -0,0 +1,5 @@
+if (THIS_THING == "six"){
+    output.something = "foo"
+} else {
+    output.something = "bar"
+}

e2e/workspaces/demo_app/commands/runScript.yaml

@@ -0,0 +1,23 @@
+appId: com.example.example
+---
+
+- launchApp # For idempotence of sections
+
+- evalScript: ${output.something = 'baz'}
+
+- runScript: runScript.js
+- assertTrue: ${output.something == 'bar'}
+
+- evalScript: ${output.something = 'baz'}
+
+- runScript:
+    file: runScript.js
+- assertTrue: ${output.something == 'bar'}
+
+- evalScript: ${output.something = 'baz'}
+
+- runScript:
+    env:
+      THIS_THING: "six"
+    file: runScript.js
+- assertTrue: ${output.something == 'foo'}