Added WinAppDriver UI Tests to the Calculator Project.

build/pipelines/azure-pipelines.ci.yaml

@@ -33,6 +33,10 @@ jobs:
     platform: ARM64
     condition: not(eq(variables['Build.Reason'], 'PullRequest'))
 
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x64
+
 - template: ./templates/run-unit-tests.yaml
   parameters:
     platform: x64

build/pipelines/templates/build-single-architecture.yaml

@@ -30,7 +30,7 @@ steps:
     inputs:
       solution: src/Calculator.sln
       vsVersion: 15.0
-      msbuildArgs: /bl:$(Build.BinariesDirectory)\$(BuildConfiguration)\$(BuildPlatform)\Calculator.binlog /p:OutDir=$(Build.BinariesDirectory)\$(BuildConfiguration)\$(BuildPlatform)\ /p:GenerateProjectSpecificOutputFolder=true /p:AppVersion=$(Build.BuildNumber) ${{ parameters.extraMsBuildArgs }}
+      msbuildArgs: /bl:$(Build.BinariesDirectory)\$(BuildConfiguration)\$(BuildPlatform)\Calculator.binlog /p:OutDir=$(Build.BinariesDirectory)\$(BuildConfiguration)\$(BuildPlatform)\ /p:GenerateProjectSpecificOutputFolder=true /p:AppVersion=$(Build.BuildNumber) /t:Publish /p:PublishDir=$(Build.BinariesDirectory)\$(BuildConfiguration)\$(BuildPlatform)\publish\ ${{ parameters.extraMsBuildArgs }}
       platform: $(BuildPlatform)
       configuration: $(BuildConfiguration)
       clean: true

build/pipelines/templates/run-ui-tests.yaml

@@ -0,0 +1,52 @@
+# This template contains jobs to run UI tests using WinAppDriver.
+
+parameters:
+  platform: ''
+
+jobs:
+- job: UITests${{ parameters.platform }}
+  displayName: UITests ${{ parameters.platform }}
+  dependsOn: Build${{ parameters.platform }}
+  condition: succeeded()
+  pool:
+    vmImage: windows-2019
+  variables:
+    skipComponentGovernanceDetection: true
+  steps:
+  - checkout: none
+
+  - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force
+    displayName: Set resolution to 1920x1080
+    continueOnError: true
+
+  - task: DownloadBuildArtifacts@0
+    displayName: Download AppxBundle and CalculatorUITests
+    inputs:
+      artifactName: drop
+      itemPattern: |
+        drop/Release/${{ parameters.platform }}/Calculator/AppPackages/**
+        drop/Release/${{ parameters.platform }}/publish/**
+
+  - task: PowerShell@2
+    displayName: Install certificate
+    inputs:
+      filePath: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/Calculator/AppPackages/Calculator_$(Build.BuildNumber)_Test/Add-AppDevPackage.ps1
+      arguments: -CertificatePath $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/Calculator/AppPackages/Calculator_$(Build.BuildNumber)_Test/Calculator_$(Build.BuildNumber)_${{ parameters.platform }}.cer -Force
+
+  - task: PowerShell@2
+    displayName: Install app
+    inputs:
+      filePath: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/Calculator/AppPackages/Calculator_$(Build.BuildNumber)_Test/Add-AppDevPackage.ps1
+      arguments: -Force
+
+  - powershell: Start-Process -FilePath "C:\Program Files (x86)\Windows Application Driver\WinAppDriver.exe" -Verb RunAs
+    displayName: Start WinAppDriver
+
+  - task: VSTest@2
+    displayName: Run CalculatorUITests
+    inputs:
+      testAssemblyVer2: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/publish/CalculatorUITests.dll
+      vsTestVersion: 16.0
+      runSettingsFile: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/publish/CalculatorUITests.runsettings
+      platform: ${{ parameters.platform }}
+      configuration: Release

src/Calculator.sln

@@ -16,6 +16,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CalcViewModel", "CalcViewMo
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CalculatorUnitTests", "CalculatorUnitTests\CalculatorUnitTests.vcxproj", "{D3BAED2C-4B07-4E1D-8807-9D6499450349}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CalculatorUITests", "CalculatorUITests\CalculatorUITests.csproj", "{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|ARM = Debug|ARM
@@ -28,22 +30,6 @@ Global
 		Release|x86 = Release|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM.ActiveCfg = Debug|ARM
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM.Build.0 = Debug|ARM
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM64.ActiveCfg = Debug|ARM64
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM64.Build.0 = Debug|ARM64
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x64.ActiveCfg = Debug|x64
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x64.Build.0 = Debug|x64
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x86.ActiveCfg = Debug|Win32
-		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x86.Build.0 = Debug|Win32
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM.ActiveCfg = Release|ARM
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM.Build.0 = Release|ARM
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM64.ActiveCfg = Release|ARM64
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM64.Build.0 = Release|ARM64
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|x64.ActiveCfg = Release|x64
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|x64.Build.0 = Release|x64
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|x86.ActiveCfg = Release|Win32
-		{311E866D-8B93-4609-A691-265941FEE101}.Release|x86.Build.0 = Release|Win32
 		{9447424A-0E05-4911-BEB8-E0354405F39A}.Debug|ARM.ActiveCfg = Debug|ARM
 		{9447424A-0E05-4911-BEB8-E0354405F39A}.Debug|ARM.Build.0 = Debug|ARM
 		{9447424A-0E05-4911-BEB8-E0354405F39A}.Debug|ARM.Deploy.0 = Debug|ARM
@@ -68,6 +54,22 @@ Global
 		{9447424A-0E05-4911-BEB8-E0354405F39A}.Release|x86.ActiveCfg = Release|Win32
 		{9447424A-0E05-4911-BEB8-E0354405F39A}.Release|x86.Build.0 = Release|Win32
 		{9447424A-0E05-4911-BEB8-E0354405F39A}.Release|x86.Deploy.0 = Release|Win32
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM.ActiveCfg = Debug|ARM
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM.Build.0 = Debug|ARM
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|ARM64.Build.0 = Debug|ARM64
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x64.ActiveCfg = Debug|x64
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x64.Build.0 = Debug|x64
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x86.ActiveCfg = Debug|Win32
+		{311E866D-8B93-4609-A691-265941FEE101}.Debug|x86.Build.0 = Debug|Win32
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM.ActiveCfg = Release|ARM
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM.Build.0 = Release|ARM
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM64.ActiveCfg = Release|ARM64
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|ARM64.Build.0 = Release|ARM64
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|x64.ActiveCfg = Release|x64
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|x64.Build.0 = Release|x64
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|x86.ActiveCfg = Release|Win32
+		{311E866D-8B93-4609-A691-265941FEE101}.Release|x86.Build.0 = Release|Win32
 		{90E9761D-9262-4773-942D-CAEAE75D7140}.Debug|ARM.ActiveCfg = Debug|ARM
 		{90E9761D-9262-4773-942D-CAEAE75D7140}.Debug|ARM.Build.0 = Debug|ARM
 		{90E9761D-9262-4773-942D-CAEAE75D7140}.Debug|ARM64.ActiveCfg = Debug|ARM64
@@ -100,6 +102,22 @@ Global
 		{D3BAED2C-4B07-4E1D-8807-9D6499450349}.Release|x86.ActiveCfg = Release|Win32
 		{D3BAED2C-4B07-4E1D-8807-9D6499450349}.Release|x86.Build.0 = Release|Win32
 		{D3BAED2C-4B07-4E1D-8807-9D6499450349}.Release|x86.Deploy.0 = Release|Win32
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|ARM.Build.0 = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|ARM64.Build.0 = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|x64.Build.0 = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Debug|x86.Build.0 = Debug|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|ARM.ActiveCfg = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|ARM.Build.0 = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|ARM64.ActiveCfg = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|ARM64.Build.0 = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|x64.ActiveCfg = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|x64.Build.0 = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|x86.ActiveCfg = Release|Any CPU
+		{B2C5ADFF-D6B5-48C1-BB8C-571BFD583D7F}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/CalculatorUITests/CalculatorSession.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using OpenQA.Selenium.Appium;
+using OpenQA.Selenium.Appium.Windows;
+using System;
+
+namespace CalculatorUITests
+{
+    public class CalculatorSession
+    {
+        // Note: append /wd/hub to the URL if you're directing the test at Appium
+        private const string WindowsApplicationDriverUrl = "http://127.0.0.1:4723";
+        private const string CalculatorAppId = "Microsoft.WindowsCalculator.Dev_8wekyb3d8bbwe!App";
+        protected static WindowsDriver<WindowsElement> session;
+
+        public static void Setup(TestContext context)
+        {
+            // Launch Calculator application if it is not yet launched
+            if (session == null)
+            {
+                // Create a new session to bring up an instance of the Calculator application
+                // Note: Multiple calculator windows (instances) share the same process Id
+                var options = new AppiumOptions();
+                options.AddAdditionalCapability("app", CalculatorAppId);
+                options.AddAdditionalCapability("deviceName", "WindowsPC");
+                session = new WindowsDriver<WindowsElement>(new Uri(WindowsApplicationDriverUrl), options);
+                session.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(180);
+                Assert.IsNotNull(session);
+            }
+        }
+
+        public static void TearDown()
+        {
+            // Close the application and delete the session
+            if (session != null)
+            { 
+                session.Quit();
+                session = null;
+            }
+        }
+    }
+}

src/CalculatorUITests/CalculatorUITests.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.0.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="1.4.0" />
+    <PackageReference Include="MSTest.TestFramework" Version="1.4.0" />
+    <PackageReference Include="Appium.WebDriver" Version="4.0.0.6-beta" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="CalculatorUITests.runsettings">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/CalculatorUITests/CalculatorUITests.runsettings

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector uri="datacollector://microsoft/VideoRecorder/1.0" assemblyQualifiedName="Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder.VideoRecorderDataCollector, Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" friendlyName="Screen and Voice Recorder">
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>

src/CalculatorUITests/ScenarioStandard.cs

@@ -0,0 +1,112 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using OpenQA.Selenium.Appium.Windows;
+using System.Threading;
+using System;
+
+namespace CalculatorUITests
+{
+    [TestClass]
+    public class StandardModeTests : CalculatorSession
+    {
+        private static WindowsElement header;
+        private static WindowsElement calculatorResult;
+
+        [TestMethod]
+        public void Addition()
+        {
+            // Find the buttons by their names and click them in sequence to perform 1 + 7 = 8
+            session.FindElementByName("One").Click();
+            session.FindElementByName("Plus").Click();
+            session.FindElementByName("Seven").Click();
+            session.FindElementByName("Equals").Click();
+            Assert.AreEqual("8", GetCalculatorResultText());
+        }
+
+        [TestMethod]
+        public void Division()
+        {
+            // Find the buttons by their accessibility ids and click them in sequence to perform 88 / 11 = 8
+            session.FindElementByAccessibilityId("num8Button").Click();
+            session.FindElementByAccessibilityId("num8Button").Click();
+            session.FindElementByAccessibilityId("divideButton").Click();
+            session.FindElementByAccessibilityId("num1Button").Click();
+            session.FindElementByAccessibilityId("num1Button").Click();
+            session.FindElementByAccessibilityId("equalButton").Click();
+            Assert.AreEqual("8", GetCalculatorResultText());
+        }
+
+        [TestMethod]
+        public void Multiplication()
+        {
+            session.FindElementByAccessibilityId("num9Button").Click();
+            session.FindElementByAccessibilityId("multiplyButton").Click();
+            session.FindElementByAccessibilityId("num9Button").Click();
+            session.FindElementByAccessibilityId("equalButton").Click();
+            Assert.AreEqual("81", GetCalculatorResultText());
+        }
+
+        [TestMethod]
+        public void Subtraction()
+        {
+            // Find the buttons by their accessibility ids using XPath and click them in sequence to perform 9 - 1 = 8
+            session.FindElementByAccessibilityId("num9Button").Click();
+            session.FindElementByAccessibilityId("minusButton").Click();
+            session.FindElementByAccessibilityId("num1Button").Click();
+            session.FindElementByAccessibilityId("equalButton").Click();
+            Assert.AreEqual("8", GetCalculatorResultText());
+        }
+
+
+        [ClassInitialize]
+        public static void ClassInitialize(TestContext context)
+        {
+            // Create session to launch a Calculator window
+            Setup(context);
+            // Identify calculator mode by locating the header
+            try
+            {
+                header = session.FindElementByAccessibilityId("Header");
+            }
+            catch
+            {
+                header = session.FindElementByAccessibilityId("ContentPresenter");
+            }
+
+            // Ensure that calculator is in standard mode
+            if (!header.Text.Equals("Standard", StringComparison.OrdinalIgnoreCase))
+            {
+                session.FindElementByAccessibilityId("TogglePaneButton").Click();
+                Thread.Sleep(TimeSpan.FromSeconds(1));
+                var splitViewPane = session.FindElementByClassName("SplitViewPane");
+                splitViewPane.FindElementByName("Standard Calculator").Click();
+                Thread.Sleep(TimeSpan.FromSeconds(1));
+                Assert.IsTrue(header.Text.Equals("Standard", StringComparison.OrdinalIgnoreCase));
+            }
+
+            // Locate the calculatorResult element
+            calculatorResult = session.FindElementByAccessibilityId("CalculatorResults");
+            Assert.IsNotNull(calculatorResult);
+        }
+
+        [ClassCleanup]
+        public static void ClassCleanup()
+        {
+            TearDown();
+        }
+
+        [TestInitialize]
+        public void Clear()
+        {
+            session.FindElementByName("Clear").Click();
+            Assert.AreEqual("0", GetCalculatorResultText());
+        }
+
+        private string GetCalculatorResultText()
+        {
+            return calculatorResult.Text.Replace("Display is", string.Empty).Trim();
+        }
+    }
+}