From a1da5da162f93d35c61a98392d6dad695397a560 Mon Sep 17 00:00:00 2001 From: Aaron Train Date: Fri, 2 Sep 2022 11:55:09 -0400 Subject: [PATCH] For #26792 - Change active Firebase Test Lab emulator from x86 to ARM Update startup-test kind.yml --- .../androidTest/flank-arm-beta.yml | 42 ++++++++++ .../flank-arm-legacy-api-tests.yml | 47 +++++++++++ .../flank-arm-screenshots-tests.yml | 35 +++++++++ .../androidTest/flank-arm-start-test.yml | 41 ++++++++++ .../androidTest/flank-arm64-v8a.yml | 46 ++++------- .../flank-armeabi-v7a-start-test.yml | 48 ------------ .../androidTest/flank-armeabi-v7a.yml | 47 ----------- .../androidTest/flank-x86-beta.yml | 61 --------------- .../flank-x86-legacy-api-tests.yml | 77 ------------------- .../flank-x86-screenshots-tests.yml | 55 ------------- .../androidTest/flank-x86-start-test.yml | 61 --------------- .../taskcluster/androidTest/flank-x86.yml | 36 +-------- automation/taskcluster/androidTest/ui-test.sh | 16 ++-- taskcluster/ci/startup-test/kind.yml | 12 +-- taskcluster/ci/ui-test/kind.yml | 32 ++++---- 15 files changed, 210 insertions(+), 446 deletions(-) create mode 100644 automation/taskcluster/androidTest/flank-arm-beta.yml create mode 100644 automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml create mode 100644 automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml create mode 100644 automation/taskcluster/androidTest/flank-arm-start-test.yml delete mode 100644 automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml delete mode 100644 automation/taskcluster/androidTest/flank-armeabi-v7a.yml delete mode 100644 automation/taskcluster/androidTest/flank-x86-beta.yml delete mode 100644 automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml delete mode 100644 automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml delete mode 100644 automation/taskcluster/androidTest/flank-x86-start-test.yml diff --git a/automation/taskcluster/androidTest/flank-arm-beta.yml b/automation/taskcluster/androidTest/flank-arm-beta.yml new file mode 100644 index 000000000000..20144b44d346 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-beta.yml @@ -0,0 +1,42 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest + - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest + - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest + - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest + - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest + - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true + diff --git a/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml b/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml new file mode 100644 index 000000000000..1cd97257deb1 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml @@ -0,0 +1,47 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 2 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest + - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest + - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest + - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest + - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest + + device: + - model: Pixel2.arm + version: 28 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: -1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml b/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml new file mode 100644 index 000000000000..a753062da938 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml @@ -0,0 +1,35 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - package org.mozilla.fenix.screenshots + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-arm-start-test.yml b/automation/taskcluster/androidTest/flank-arm-start-test.yml new file mode 100644 index 000000000000..1303eb093772 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-start-test.yml @@ -0,0 +1,41 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest + - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest + - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest + - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest + - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest + - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-arm64-v8a.yml b/automation/taskcluster/androidTest/flank-arm64-v8a.yml index 58db5c740a08..09e2b730fa4b 100644 --- a/automation/taskcluster/androidTest/flank-arm64-v8a.yml +++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml @@ -1,31 +1,16 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ gcloud: results-bucket: fenix_test_artifacts record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel + async: false + num-flaky-test-attempts: 1 - # test and app are the only required args - app: /APP/PATH - test: /TEST/PATH + app: /app/path + test: /test/path - auto-google-login: true + auto-google-login: false use-orchestrator: true environment-variables: clearPackageData: true @@ -33,15 +18,18 @@ gcloud: - /sdcard/screenshots performance-metrics: true + test-targets: + - notPackage org.mozilla.fenix.screenshots + - notPackage org.mozilla.fenix.syncintegration + device: - - model: Pixel2 - version: 30 + - model: Pixel2.arm + version: 30 + locale: en_US flank: project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: -1 - # num-test-runs- the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x + max-test-shards: 50 num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml b/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml deleted file mode 100644 index b1df75ed80e3..000000000000 --- a/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml +++ /dev/null @@ -1,48 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml b/automation/taskcluster/androidTest/flank-armeabi-v7a.yml deleted file mode 100644 index 78a3137c6e71..000000000000 --- a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml +++ /dev/null @@ -1,47 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # test and app are the only required args - app: /APP/PATH - test: /TEST/PATH - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: -1 - # repeat tests - the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - repeat-tests: 1 diff --git a/automation/taskcluster/androidTest/flank-x86-beta.yml b/automation/taskcluster/androidTest/flank-x86-beta.yml deleted file mode 100644 index 92b6ed0d05b5..000000000000 --- a/automation/taskcluster/androidTest/flank-x86-beta.yml +++ /dev/null @@ -1,61 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 20m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest - - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest - - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest - - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest - - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest - - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest - - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: 1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml b/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml deleted file mode 100644 index 226ff61e529e..000000000000 --- a/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml +++ /dev/null @@ -1,77 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 2 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: false - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest - - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest - - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest - - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest - - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest - - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest - - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest - - device: - - model: Pixel2 - version: 28 - locale: en_US - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 - max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. - output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. - full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml b/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml deleted file mode 100644 index fd026eaa3da5..000000000000 --- a/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml +++ /dev/null @@ -1,55 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 20m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - package org.mozilla.fenix.screenshots - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: 1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-x86-start-test.yml b/automation/taskcluster/androidTest/flank-x86-start-test.yml deleted file mode 100644 index d1648955050d..000000000000 --- a/automation/taskcluster/androidTest/flank-x86-start-test.yml +++ /dev/null @@ -1,61 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest - - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest - - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest - - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest - - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest - - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest - - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: 1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml index 4226b9a209d0..30ede8817b25 100644 --- a/automation/taskcluster/androidTest/flank-x86.yml +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -1,32 +1,14 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ gcloud: results-bucket: fenix_test_artifacts record-video: true - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. num-flaky-test-attempts: 1 - # test and app are the only required args - app: /app/path + app: /app/path test: /test/path auto-google-login: false @@ -48,19 +30,7 @@ gcloud: flank: project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. full-junit-result: true diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 20c7efece86b..0a86dffc1993 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -76,16 +76,14 @@ set +e if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then flank_template="${PATH_TEST}/flank-${device_type}.yml" -elif [[ "${device_type}" == "x86-start-test" ]]; then - flank_template="${PATH_TEST}/flank-x86-start-test.yml" elif [[ "${device_type}" == "arm-start-test" ]]; then - flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" -elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" -elif [[ "${device_type}" == "x86-beta-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-beta.yml" -elif [[ "${device_type}" == "x86-legacy-api-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-legacy-api-tests.yml" + flank_template="${PATH_TEST}/flank-arm-start-test.yml" +elif [[ "${device_type}" == "arm-screenshots-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-screenshots-tests.yml" +elif [[ "${device_type}" == "arm-beta-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-beta.yml" +elif [[ "${device_type}" == "arm-legacy-api-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-legacy-api-tests.yml" else echo "FAILURE: flank config file not found!" exitcode=1 diff --git a/taskcluster/ci/startup-test/kind.yml b/taskcluster/ci/startup-test/kind.yml index d852dd3e14ce..00c69f50a287 100644 --- a/taskcluster/ci/startup-test/kind.yml +++ b/taskcluster/ci/startup-test/kind.yml @@ -45,19 +45,11 @@ task-defaults: signing-android-test: signing-android-test-nightly tasks: - nightly-x86: - run: - commands: - - [wget, {artifact-reference: ''}, '-O', app-x86.apk] - - [wget, {artifact-reference: ''}, '-O', android-test.apk] - - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app-x86.apk, android-test.apk, '-1'] - treeherder: - symbol: nightly(startup-x86) nightly-arm: run: commands: - - [wget, {artifact-reference: ''}, '-O', app-arm.apk] + - [wget, {artifact-reference: ''}, '-O', app.apk] - [wget, {artifact-reference: ''}, '-O', android-test.apk] - - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app-arm.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1'] treeherder: symbol: nightly(startup-arm) diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index df185a7ba91b..98b4f60267b6 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -74,7 +74,7 @@ task-defaults: ] run: commands: - - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', app.apk] - [wget, {artifact-reference: ''}, '-O', android-test.apk] secrets: - name: project/mobile/fenix/firebase @@ -101,16 +101,16 @@ task-defaults: worker-type: b-android tasks: - x86-debug: + arm-debug: description: Test Fenix run-on-tasks-for: [github-pull-request, github-push] run-on-git-branches: ["^((?!releases[_/]).+)$"] run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm64-v8a, app.apk, android-test.apk, '50'] treeherder: - symbol: debug(ui-test-x86) - screenshots-x86: + symbol: debug(ui-test-arm) + screenshots-arm: attributes: screenshots: true description: Run UI screenshots tests to keep them up to date @@ -118,10 +118,10 @@ tasks: run-on-git-branches: [main] run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-screenshots-tests, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-screenshots-tests, app.apk, android-test.apk, '1'] treeherder: - symbol: debug(screenshots-x86) - x86-beta: + symbol: debug(screenshots-arm) + arm-beta: attributes: build-type: beta-firebase description: Test Fenix @@ -132,10 +132,10 @@ tasks: signing-android-test: signing-android-test-beta run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-beta-tests, app.apk, android-test.apk, '50'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-beta-tests, app.apk, android-test.apk, '1'] treeherder: - symbol: beta(ui-test-x86-beta) - x86-nightly: + symbol: beta(ui-test-arm-beta) + arm-nightly: attributes: build-type: nightly-firebase description: Test Fenix @@ -145,10 +145,10 @@ tasks: signing-android-test: signing-android-test-nightly run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1'] treeherder: - symbol: nightly(ui-test-x86-nightly) - legacy-x86: + symbol: nightly(ui-test-arm-nightly) + legacy-arm: attributes: legacy: true description: Run select UI tests on older Android API on cron @@ -156,6 +156,6 @@ tasks: run-on-git-branches: [main] run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-legacy-api-tests, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-legacy-api-tests, app.apk, android-test.apk, '-1'] treeherder: - symbol: debug(legacy-x86) + symbol: debug(legacy-arm)