From a50f5cccd9c7188748bfec7247d7ded3b3ed268b Mon Sep 17 00:00:00 2001 From: Aaron Train Date: Tue, 20 Sep 2022 12:57:03 -0400 Subject: [PATCH] For #7698 - Change active Firebase Test Lab emulator from x86 to ARM in UI jobs --- .../androidTest/flank-arm-beta.yml | 36 ++++++++++ .../androidTest/flank-arm-start-test.yml | 36 ++++++++++ .../androidTest/flank-arm64-v8a.yml | 36 ++++++++++ .../androidTest/flank-x86-beta.yml | 65 ------------------- .../androidTest/flank-x86-start-test.yml | 65 ------------------- .../taskcluster/androidTest/flank-x86.yml | 43 ++---------- automation/taskcluster/androidTest/ui-test.sh | 10 +-- taskcluster/ci/ui-test/kind.yml | 20 +++--- 8 files changed, 128 insertions(+), 183 deletions(-) create mode 100644 automation/taskcluster/androidTest/flank-arm-beta.yml create mode 100644 automation/taskcluster/androidTest/flank-arm-start-test.yml create mode 100644 automation/taskcluster/androidTest/flank-arm64-v8a.yml delete mode 100644 automation/taskcluster/androidTest/flank-x86-beta.yml delete mode 100644 automation/taskcluster/androidTest/flank-x86-start-test.yml diff --git a/automation/taskcluster/androidTest/flank-arm-beta.yml b/automation/taskcluster/androidTest/flank-arm-beta.yml new file mode 100644 index 00000000000..83abaf78ce1 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-beta.yml @@ -0,0 +1,36 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: focus_android_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.focus.activity.SearchTest#testBlankSearchDoesNothing + - class org.mozilla.focus.activity.EraseBrowsingDataTest#deleteHistoryOnRestartTest + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true \ No newline at end of file diff --git a/automation/taskcluster/androidTest/flank-arm-start-test.yml b/automation/taskcluster/androidTest/flank-arm-start-test.yml new file mode 100644 index 00000000000..aea78598e2f --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-start-test.yml @@ -0,0 +1,36 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: focus_android_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.focus.activity.SearchTest#testBlankSearchDoesNothing + - class org.mozilla.focus.activity.EraseBrowsingDataTest#deleteHistoryOnRestartTest + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true \ No newline at end of file diff --git a/automation/taskcluster/androidTest/flank-arm64-v8a.yml b/automation/taskcluster/androidTest/flank-arm64-v8a.yml new file mode 100644 index 00000000000..84aa044a24b --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml @@ -0,0 +1,36 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: focus_android_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - package org.mozilla.focus.activity + - package org.mozilla.focus.privacy + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: -1 + num-test-runs: 1 + output-style: compact + full-junit-result: true \ No newline at end of file diff --git a/automation/taskcluster/androidTest/flank-x86-beta.yml b/automation/taskcluster/androidTest/flank-x86-beta.yml deleted file mode 100644 index 622983ecc15..00000000000 --- a/automation/taskcluster/androidTest/flank-x86-beta.yml +++ /dev/null @@ -1,65 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: focus_android_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: false - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.focus.activity.SearchTest#testBlankSearchDoesNothing - - class org.mozilla.focus.activity.EraseBrowsingDataTest#deleteHistoryOnRestartTest - - device: - - model: Pixel3 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 - max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. - output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. - full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-x86-start-test.yml b/automation/taskcluster/androidTest/flank-x86-start-test.yml deleted file mode 100644 index 622983ecc15..00000000000 --- a/automation/taskcluster/androidTest/flank-x86-start-test.yml +++ /dev/null @@ -1,65 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: focus_android_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: false - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.focus.activity.SearchTest#testBlankSearchDoesNothing - - class org.mozilla.focus.activity.EraseBrowsingDataTest#deleteHistoryOnRestartTest - - device: - - model: Pixel3 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 - max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. - output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. - full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml index 13ed3c85d35..abf065577e1 100644 --- a/automation/taskcluster/androidTest/flank-x86.yml +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -1,32 +1,14 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ gcloud: results-bucket: focus_android_test_artifacts record-video: true - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. num-flaky-test-attempts: 1 - # test and app are the only required args - app: /app/path + app: /app/path test: /test/path auto-google-login: false @@ -42,24 +24,13 @@ gcloud: - package org.mozilla.focus.privacy device: - - model: Pixel3 - version: 30 + - model: Pixel2 + version: 30 + locale: en_US flank: project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. - full-junit-result: true + full-junit-result: true \ No newline at end of file diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 8e02e4be05b..ea64f9eda9e 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -76,14 +76,10 @@ set +e if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then flank_template="${PATH_TEST}/flank-${device_type}.yml" -elif [[ "${device_type}" == "x86-start-test" ]]; then - flank_template="${PATH_TEST}/flank-x86-start-test.yml" elif [[ "${device_type}" == "arm-start-test" ]]; then - flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" -elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" -elif [[ "${device_type}" == "x86-beta-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-beta.yml" + flank_template="${PATH_TEST}/flank-arm-start-test.yml" +elif [[ "${device_type}" == "arm-beta-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-beta.yml" else echo "FAILURE: flank config file not found!" exitcode=1 diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index 6e10e519557..4236b0c0965 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -48,19 +48,19 @@ task-defaults: path: .firebase_token.json json: true commands: - - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', app.apk] - [wget, {artifact-reference: ''}, '-O', android-test.apk] tasks: - x86-debug: + arm-debug: description: 'UI tests with firebase' run-on-tasks-for: [github-pull-request, github-push] run: commands: - - ['automation/taskcluster/androidTest/ui-test.sh', 'x86', 'app.apk', 'android-test.apk', '-1'] + - ['automation/taskcluster/androidTest/ui-test.sh', 'arm64-v8a', 'app.apk', 'android-test.apk', '50'] treeherder: - symbol: debug(ui-test-x86) - x86-nightly: + symbol: debug(ui-test-arm) + arm-nightly: attributes: build-type: nightly-firebase description: 'UI tests on Nightly with firebase' @@ -70,10 +70,10 @@ tasks: signed-android-test: signing-android-test-nightly run: commands: - - ['automation/taskcluster/androidTest/ui-test.sh', 'x86-start-test', 'app.apk', 'android-test.apk', '-1'] + - ['automation/taskcluster/androidTest/ui-test.sh', 'arm-start-test', 'app.apk', 'android-test.apk', '1'] treeherder: - symbol: nightly(ui-test-x86-nightly) - x86-beta: + symbol: nightly(ui-test-arm-nightly) + arm-beta: attributes: build-type: nightly-firebase description: 'UI tests on Nightly with firebase' @@ -83,6 +83,6 @@ tasks: signed-android-test: signing-android-test-beta run: commands: - - ['automation/taskcluster/androidTest/ui-test.sh', 'x86-beta-tests', 'app.apk', 'android-test.apk', '-1'] + - ['automation/taskcluster/androidTest/ui-test.sh', 'arm-beta-tests', 'app.apk', 'android-test.apk', '1'] treeherder: - symbol: beta(ui-test-x86-beta) + symbol: beta(ui-test-arm-beta)