From 566b8598ab914202e03cef6aff1371fd7fb79842 Mon Sep 17 00:00:00 2001 From: Bret Johnson Date: Fri, 6 Sep 2024 10:32:10 -0400 Subject: [PATCH] Improve Android device type check and improve UI Test cake scripts (#24495) * Improve Android device type check to use screen size For Android visual tests, they should be run with the right device image (currently a Nexus 5X API 30). However, there's no easy way to check the device type for Appium. The check that was there before works on CI since the DEVICE_SKIN environment variable is set there (set by the android.cake script). And it can work locally if folks run android.cake then launch VS from that console session to get the env variable - but it doesn't work if VS is run separately (normal case). So update this with a different check. Getting the actual Android device type requires using adb. We might eventually do that, but for now just use a screen size check as a proxy for the device type. Also update the failure message to give good instructions. * Add a "uitest-prepare" target to android.cake uitest-prepare will build the test app, launch the emulator, and deploy the test app. But it won't actually run any tests. * Validate that JAVA_HOME is set Also display the Android image name * Check Android device capabilties for a match * Add uitest-prepare target for ios.cake to match android --- eng/devices/android.cake | 30 ++++++++++++++++--- eng/devices/devices-shared.cake | 8 ++--- eng/devices/ios.cake | 20 +++++++++---- .../tests/TestCases.Shared.Tests/UITest.cs | 15 +++++----- 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/eng/devices/android.cake b/eng/devices/android.cake index 32496bbc145b..0077a33a9ff0 100644 --- a/eng/devices/android.cake +++ b/eng/devices/android.cake @@ -6,6 +6,11 @@ const int DefaultApiLevel = 30; Information("Local Dotnet: {0}", localDotnet); +if (EnvironmentVariable("JAVA_HOME") == null) +{ + throw new Exception("JAVA_HOME environment variable isn't set. Set it to your JDK installation (e.g. \"C:\\Program Files (x86)\\Android\\openjdk\\jdk-17.0.8.101-hotspot\\bin\")."); +} + string DEFAULT_ANDROID_PROJECT = "../../src/Controls/tests/TestCases.Android.Tests/Controls.TestCases.Android.Tests.csproj"; var projectPath = Argument("project", EnvironmentVariable("ANDROID_TEST_PROJECT") ?? DEFAULT_ANDROID_PROJECT); var testDevice = Argument("device", EnvironmentVariable("ANDROID_TEST_DEVICE") ?? $"android-emulator-64_{DefaultApiLevel}"); @@ -65,7 +70,12 @@ Setup(context => Teardown(context => { - CleanUpVirtualDevice(emulatorProcess, avdSettings); + // For the uitest-prepare target, just leave the virtual device running + if (! string.Equals(TARGET, "uitest-prepare", StringComparison.OrdinalIgnoreCase)) + { + CleanUpVirtualDevice(emulatorProcess, avdSettings); + } + }); Task("boot"); @@ -91,7 +101,14 @@ Task("uitest-build") ExecuteBuildUITestApp(testAppProjectPath, testDevice, binlogDirectory, configuration, targetFramework, "", dotnetToolPath); }); +Task("uitest-prepare") + .Does(() => + { + ExecutePrepareUITests(projectPath, testAppProjectPath, testAppPackageName, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, "", androidVersion, dotnetToolPath, testAppInstrumentation); + }); + Task("uitest") + .IsDependentOn("uitest-prepare") .Does(() => { ExecuteUITests(projectPath, testAppProjectPath, testAppPackageName, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, "", androidVersion, dotnetToolPath, testAppInstrumentation); @@ -265,10 +282,11 @@ void ExecuteBuildUITestApp(string appProject, string device, string binDir, stri Information("UI Test app build completed."); } -void ExecuteUITests(string project, string app, string appPackageName, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath, string instrumentation) +void ExecutePrepareUITests(string project, string app, string appPackageName, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath, string instrumentation) { string platform = "android"; - Information("Starting UI Tests..."); + Information("Preparing UI Tests..."); + var testApp = GetTestApplications(app, device, config, tfm, "").FirstOrDefault(); if (string.IsNullOrEmpty(testApp)) @@ -295,7 +313,11 @@ void ExecuteUITests(string project, string app, string appPackageName, string de Information($"Results Directory: {resultsDir}"); InstallApk(testApp, appPackageName, resultsDir, deviceSkin); +} +void ExecuteUITests(string project, string app, string appPackageName, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath, string instrumentation) +{ + string platform = "android"; Information("Build UITests project {0}", project); var name = System.IO.Path.GetFileNameWithoutExtension(project); @@ -442,7 +464,7 @@ void HandleVirtualDevice(AndroidEmulatorToolSettings emuSettings, AndroidAvdMana catch { } // create the new AVD - Information("Creating AVD: {0}...", avdName); + Information("Creating AVD: {0} ({1})...", avdName, avdImage); AndroidAvdCreate(avdName, avdImage, avdSkin, force: true, settings: avdSettings); // start the emulator diff --git a/eng/devices/devices-shared.cake b/eng/devices/devices-shared.cake index 87c3a3e89ac2..42c61bffd3ba 100644 --- a/eng/devices/devices-shared.cake +++ b/eng/devices/devices-shared.cake @@ -21,13 +21,9 @@ string TARGET = Argument("target", "Test"); string DEFAULT_PROJECT = ""; string DEFAULT_APP_PROJECT = ""; -if (string.Equals(TARGET, "uitest", StringComparison.OrdinalIgnoreCase)) -{ - DEFAULT_PROJECT = "../../src/Controls/tests/TestCases.Shared.Tests/Controls.TestCases.Shared.Tests.csproj"; - DEFAULT_APP_PROJECT = "../../src/Controls/tests/TestCases.HostApp/Controls.TestCases.HostApp.csproj"; -} -if (string.Equals(TARGET, "uitest-build", StringComparison.OrdinalIgnoreCase)) +// "uitest", "uitest-build", and "uitest-prepare" all trigger this case +if (TARGET.StartsWith("uitest", StringComparison.OrdinalIgnoreCase)) { DEFAULT_PROJECT = "../../src/Controls/tests/TestCases.Shared.Tests/Controls.TestCases.Shared.Tests.csproj"; DEFAULT_APP_PROJECT = "../../src/Controls/tests/TestCases.HostApp/Controls.TestCases.HostApp.csproj"; diff --git a/eng/devices/ios.cake b/eng/devices/ios.cake index 044bc11b228c..089017b5089f 100644 --- a/eng/devices/ios.cake +++ b/eng/devices/ios.cake @@ -65,7 +65,7 @@ Setup(context => Teardown(context => { - if (!deviceBoot || targetBoot) + if (!deviceBoot || targetBoot || string.Equals(TARGET, "uitest-prepare", StringComparison.OrdinalIgnoreCase)) { return; } @@ -99,11 +99,17 @@ Task("uitest-build") ExecuteBuildUITestApp(testAppProjectPath, testDevice, binlogDirectory, configuration, targetFramework, runtimeIdentifier, dotnetToolPath); }); +Task("uitest-prepare") + .Does(() => + { + ExecutePrepareUITests(projectPath, testAppProjectPath, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, runtimeIdentifier, iosVersion, dotnetToolPath); + }); + Task("uitest") + .IsDependentOn("uitest-prepare") .Does(() => { ExecuteUITests(projectPath, testAppProjectPath, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, runtimeIdentifier, iosVersion, dotnetToolPath); - }); Task("cg-uitest") @@ -203,15 +209,14 @@ void ExecuteTests(string project, string device, string resultsDir, string confi Information("Testing completed."); } -void ExecuteUITests(string project, string app, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath) +void ExecutePrepareUITests(string project, string app, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath) { - Information("Starting UI Tests..."); + Information("Preparing UI Tests..."); var testApp = GetTestApplications(app, device, config, tfm, rid).FirstOrDefault(); Information($"Testing Device: {device}"); Information($"Testing App Project: {app}"); Information($"Testing App: {testApp}"); - Information($"Results Directory: {resultsDir}"); if (string.IsNullOrEmpty(testApp)) { @@ -219,6 +224,11 @@ void ExecuteUITests(string project, string app, string device, string resultsDir } InstallIpa(testApp, "", device, resultsDir, ver, toolPath); +} + +void ExecuteUITests(string project, string app, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath) +{ + Information($"Results Directory: {resultsDir}"); Information("Build UITests project {0}", project); diff --git a/src/Controls/tests/TestCases.Shared.Tests/UITest.cs b/src/Controls/tests/TestCases.Shared.Tests/UITest.cs index 267236295bb7..cddd49b7013b 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/UITest.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/UITest.cs @@ -127,18 +127,18 @@ but both can happen. switch (_testDevice) { case TestDevice.Android: - if (deviceName == "Nexus 5X") - { - environmentName = "android"; - } - else + environmentName = "android"; + var deviceApiLevel = (long)((AppiumApp)App).Driver.Capabilities.GetCapability("deviceApiLevel"); + var deviceScreenSize = (string)((AppiumApp)App).Driver.Capabilities.GetCapability("deviceScreenSize"); + var deviceScreenDensity = (long)((AppiumApp)App).Driver.Capabilities.GetCapability("deviceScreenDensity"); + + if (! (deviceApiLevel == 30 && deviceScreenSize == "1080x1920" && deviceScreenDensity == 420)) { - Assert.Fail($"Android visual tests should be run on an Nexus 5X (API 30) emulator image, but the current device is '{deviceName}'. Follow the steps on the MAUI UI testing wiki."); + Assert.Fail($"Android visual tests should be run on an API30 emulator image with 1080x1920 420dpi screen, but the current device is API {deviceApiLevel} with a {deviceScreenSize} {deviceScreenDensity}dpi screen. Follow the steps on the MAUI UI testing wiki to launch the Android emulator with the right image."); } break; case TestDevice.iOS: - var platformVersion = (string)((AppiumApp)App).Driver.Capabilities.GetCapability("platformVersion"); var device = (string)((AppiumApp)App).Driver.Capabilities.GetCapability("deviceName"); @@ -152,7 +152,6 @@ but both can happen. } else { - Assert.Fail($"iOS visual tests should be run on iPhone Xs (iOS 17.2) or iPhone X (iOS 16.4) simulator images, but the current device is '{deviceName}'. Follow the steps on the MAUI UI testing wiki."); } break;