From 5b757e8e9f2dc7c5648ad45364c327337f279dcb Mon Sep 17 00:00:00 2001 From: Keith Mahoney <41657372+kmahone@users.noreply.github.com> Date: Tue, 20 Apr 2021 11:51:47 -0700 Subject: [PATCH] Refactor WinUI Helix functionality into reusable WinUI.Helix nuget package (#4678) --- .../MUX-CreateHelixProjFile-Steps.yml | 6 +- .../MUX-ProcessTestResults-Job.yml | 39 +- .../MUX-RunHelixTests-Job.yml | 25 +- build/Helix/AzurePipelinesHelperScripts.ps1 | 175 ----- build/Helix/ConvertWttLogToXUnit.ps1 | 28 - build/Helix/CopyPGOFiles.ps1 | 24 + build/Helix/EnsureMachineState.ps1 | 112 --- build/Helix/GenerateTestProjFile.ps1 | 319 +-------- build/Helix/HelixTestHelpers.cs | 643 ------------------ build/Helix/OutputFailedTestQuery.ps1 | 8 - build/Helix/OutputSubResultsJsonFiles.ps1 | 32 - build/Helix/OutputTestResults.ps1 | 131 ---- build/Helix/PrepareHelixPayload.ps1 | 45 +- build/Helix/ProcessHelixFiles.ps1 | 174 ----- build/Helix/RunTestsInHelix.proj | 2 +- build/Helix/UpdateUnreliableTests.ps1 | 128 ---- build/Helix/packages.config | 1 + build/Helix/runtests.cmd | 106 --- .../CopyVisualTreeVerificationFiles.ps1 | 0 .../InstallTestAppDependencies.ps1 | 0 .../scripts/TestPass-EnsureMachineState.ps1 | 24 + build/Helix/scripts/TestPass-PostRun.ps1 | 13 + build/Helix/scripts/TestPass-PreRun.ps1 | 88 +++ test/IXMPTestApp/WaitForDebugger.cs | 1 + test/MUXControls.Test/TestAssembly.cs | 1 + .../ReleaseTestEnvironment.cs | 1 + test/MUXControlsTestApp/WaitForDebugger.cs | 1 + 27 files changed, 243 insertions(+), 1884 deletions(-) delete mode 100644 build/Helix/AzurePipelinesHelperScripts.ps1 delete mode 100644 build/Helix/ConvertWttLogToXUnit.ps1 create mode 100644 build/Helix/CopyPGOFiles.ps1 delete mode 100644 build/Helix/EnsureMachineState.ps1 delete mode 100644 build/Helix/HelixTestHelpers.cs delete mode 100644 build/Helix/OutputFailedTestQuery.ps1 delete mode 100644 build/Helix/OutputSubResultsJsonFiles.ps1 delete mode 100644 build/Helix/OutputTestResults.ps1 delete mode 100644 build/Helix/ProcessHelixFiles.ps1 delete mode 100644 build/Helix/UpdateUnreliableTests.ps1 delete mode 100644 build/Helix/runtests.cmd rename build/Helix/{ => scripts}/CopyVisualTreeVerificationFiles.ps1 (100%) rename build/Helix/{ => scripts}/InstallTestAppDependencies.ps1 (100%) create mode 100644 build/Helix/scripts/TestPass-EnsureMachineState.ps1 create mode 100644 build/Helix/scripts/TestPass-PostRun.ps1 create mode 100644 build/Helix/scripts/TestPass-PreRun.ps1 diff --git a/build/AzurePipelinesTemplates/MUX-CreateHelixProjFile-Steps.yml b/build/AzurePipelinesTemplates/MUX-CreateHelixProjFile-Steps.yml index 7677d8a50f..a0033c8161 100644 --- a/build/AzurePipelinesTemplates/MUX-CreateHelixProjFile-Steps.yml +++ b/build/AzurePipelinesTemplates/MUX-CreateHelixProjFile-Steps.yml @@ -1,8 +1,8 @@ parameters: condition: '' - testFilePath: '' + testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload' + testFilePattern: '' outputProjFileName: '' - testSuite: '' taefQuery: '' steps: @@ -12,4 +12,4 @@ steps: inputs: targetType: filePath filePath: build\Helix\GenerateTestProjFile.ps1 - arguments: -TestFile '${{ parameters.testFilePath }}' -OutputProjFile '$(Build.ArtifactStagingDirectory)\${{ parameters.outputProjFileName }}' -JobTestSuiteName '${{ parameters.testSuite }}' -TaefPath '$(Build.SourcesDirectory)\build\Helix\packages\taef.redist.wlk.10.31.180822002\build\Binaries\x86' -TaefQuery "${{ parameters.taefQuery }}" \ No newline at end of file + arguments: -TestFilePattern '${{ parameters.testFilePattern }}' -TestBinaryDirectoryPath '${{ parameters.testBinaryDirectoryPath }}' -OutputProjFile '$(Build.ArtifactStagingDirectory)\${{ parameters.outputProjFileName }}' -TaefQuery "${{ parameters.taefQuery }}" -TestNamePrefix $(buildConfiguration).$(buildPlatform) \ No newline at end of file diff --git a/build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml b/build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml index 36f1673d94..ef3ae9f982 100644 --- a/build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml +++ b/build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml @@ -7,23 +7,39 @@ parameters: jobs: - job: ProcessTestResults - condition: succeededOrFailed() + condition: always() dependsOn: ${{ parameters.dependsOn }} pool: vmImage: 'windows-2019' timeoutInMinutes: 120 variables: helixOutputFolder: $(Build.SourcesDirectory)\HelixOutput + winuiHelixVersion: 0.0.2.4 + helixScriptPath: $(Build.SourcesDirectory)\build\Helix\packages\Microsoft.Internal.WinUI.Helix.$(winuiHelixVersion)\scripts\pipeline + ${{if eq(parameters.pgoArtifact, '') }}: + processHelixFilesExtraArgs: '' + ${{if ne(parameters.pgoArtifact, '') }}: + processHelixFilesExtraArgs: -ProcessAllJobs steps: + + - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 + displayName: 'NuGet restore build/Helix/packages.config' + inputs: + restoreSolution: build/Helix/packages.config + feedsToUse: config + nugetConfigPath: nuget.config + restoreDirectory: packages + - task: powershell@2 displayName: 'UpdateUnreliableTests.ps1' condition: succeededOrFailed() env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(HelixApiAccessToken) inputs: targetType: filePath - filePath: build\Helix\UpdateUnreliableTests.ps1 + filePath: $(helixScriptPath)\UpdateUnreliableTests.ps1 arguments: -RerunPassesRequiredToAvoidFailure '${{ parameters.rerunPassesRequiredToAvoidFailure }}' - task: powershell@2 @@ -31,10 +47,11 @@ jobs: condition: succeededOrFailed() env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(HelixApiAccessToken) inputs: targetType: filePath - filePath: build\Helix\OutputTestResults.ps1 - arguments: -MinimumExpectedTestsExecutedCount '${{ parameters.minimumExpectedTestsExecutedCount }}' -CheckJobAttempt $${{ parameters.checkJobAttempt }} + filePath: $(helixScriptPath)\OutputTestResults.ps1 + arguments: -MinimumExpectedTestsExecutedCount '${{ parameters.minimumExpectedTestsExecutedCount }}' - task: powershell@2 displayName: 'ProcessHelixFiles.ps1' @@ -44,19 +61,23 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) inputs: targetType: filePath - filePath: build\Helix\ProcessHelixFiles.ps1 - arguments: -OutputFolder '$(helixOutputFolder)' + filePath: $(helixScriptPath)\ProcessHelixFiles.ps1 + arguments: -OutputFolder '$(helixOutputFolder)' $(processHelixFilesExtraArgs) - ${{if ne(parameters.pgoArtifact, '') }}: - - script: move /y $(helixOutputFolder)\PGO $(Build.ArtifactStagingDirectory) - displayName: 'Move pgc files to PGO artifact' + - task: powershell@2 + displayName: CopyPGOFiles.ps1 + inputs: + targetType: filePath + filePath: build\Helix\CopyPGOFiles.ps1 + arguments: -SourceFolder '$(helixOutputFolder)' -OutputFolder '$(Build.ArtifactStagingDirectory)' - task: PublishBuildArtifacts@1 displayName: 'Publish Helix files' condition: succeededOrFailed() inputs: PathtoPublish: $(helixOutputFolder) - artifactName: drop + artifactName: helixTestOutput - ${{if ne(parameters.pgoArtifact, '') }}: - task: PublishBuildArtifacts@1 diff --git a/build/AzurePipelinesTemplates/MUX-RunHelixTests-Job.yml b/build/AzurePipelinesTemplates/MUX-RunHelixTests-Job.yml index cd52fdb301..d920ae243c 100644 --- a/build/AzurePipelinesTemplates/MUX-RunHelixTests-Job.yml +++ b/build/AzurePipelinesTemplates/MUX-RunHelixTests-Job.yml @@ -2,10 +2,10 @@ parameters: name: 'RunTestsInHelix' dependsOn: '' condition: '' - testSuite: '' + testSuite: DevTestSuite + helixType: test/devtest # If a Pipeline runs this template more than once, this parameter should be unique per build flavor to differentiate the # the different test runs: - helixType: 'test/devtest' artifactName: 'drop' maxParallel: 4 rerunPassesRequiredToAvoidFailure: 5 @@ -42,7 +42,6 @@ jobs: matrix: ${{ parameters.matrix }} variables: artifactsDir: $(Build.SourcesDirectory)\Artifacts - taefPath: $(Build.SourcesDirectory)\build\Helix\packages\taef.redist.wlk.10.31.180822002\build\Binaries\$(buildPlatform) helixCommonArgs: '/binaryLogger:$(Build.SourcesDirectory)/${{parameters.name}}.$(buildPlatform).$(buildConfiguration).binlog /p:HelixBuild=$(Build.BuildId).$(buildPlatform).$(buildConfiguration) /p:Platform=$(buildPlatform) /p:Configuration=$(buildConfiguration) /p:HelixType=${{parameters.helixType}} /p:TestSuite=${{parameters.testSuite}} /p:ProjFilesPath=$(Build.ArtifactStagingDirectory) /p:rerunPassesRequiredToAvoidFailure=${{parameters.rerunPassesRequiredToAvoidFailure}}' @@ -97,33 +96,33 @@ jobs: - template: MUX-CreateHelixProjFile-Steps.yml parameters: condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite')) - testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\Test\MUXControls.Test.dll' + testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)' + testFilePattern: 'MUXControls.Test.dll' outputProjFileName: 'RunTestsInHelix-InteractionTests.proj' - testSuite: '${{ parameters.testSuite }}' taefQuery: ${{ parameters.taefQuery }} - template: MUX-CreateHelixProjFile-Steps.yml parameters: condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite')) - testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\AppxPackages\MUXControlsTestApp_Test\MUXControlsTestApp.appx' + testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)' + testFilePattern: 'MUXControlsTestApp.appx' outputProjFileName: 'RunTestsInHelix-ApiTests.proj' - testSuite: '${{ parameters.testSuite }}' taefQuery: ${{ parameters.taefQuery }} - template: MUX-CreateHelixProjFile-Steps.yml parameters: condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite')) - testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\AppxPackages\IXMPTestApp_Test\IXMPTestApp.appx' + testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)' + testFilePattern: 'IXMPTestApp.appx' outputProjFileName: 'RunTestsInHelix-IXMPTestAppTests.proj' - testSuite: '${{ parameters.testSuite }}' taefQuery: ${{ parameters.taefQuery }} - template: MUX-CreateHelixProjFile-Steps.yml parameters: condition: and(succeeded(),eq('${{ parameters.testSuite }}','NugetTestSuite')) - testFilePath: '$(artifactsDir)\${{ parameters.artifactName }}\$(buildConfiguration)\$(buildPlatform)\Test\MUXControls.ReleaseTest.dll' + testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)' + testFilePattern: 'MUXControls.ReleaseTest.dll' outputProjFileName: 'RunTestsInHelix-NugetTests.proj' - testSuite: '${{ parameters.testSuite }}' taefQuery: ${{ parameters.taefQuery }} - task: PublishBuildArtifacts@1 @@ -134,7 +133,7 @@ jobs: - task: DotNetCoreCLI@2 displayName: 'Run tests in Helix (open queues)' - condition: eq(variables['System.CollectionUri'],'https://dev.azure.com/ms/') + condition: and(succeeded(),eq(variables['System.CollectionUri'],'https://dev.azure.com/ms/')) env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: @@ -145,7 +144,7 @@ jobs: - task: DotNetCoreCLI@2 displayName: 'Run tests in Helix (closed queues)' - condition: ne(variables['System.CollectionUri'],'https://dev.azure.com/ms/') + condition: and(succeeded(),ne(variables['System.CollectionUri'],'https://dev.azure.com/ms/')) env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HelixAccessToken: $(HelixApiAccessToken) diff --git a/build/Helix/AzurePipelinesHelperScripts.ps1 b/build/Helix/AzurePipelinesHelperScripts.ps1 deleted file mode 100644 index 8934a8548d..0000000000 --- a/build/Helix/AzurePipelinesHelperScripts.ps1 +++ /dev/null @@ -1,175 +0,0 @@ -function GetAzureDevOpsBaseUri -{ - Param( - [string]$CollectionUri, - [string]$TeamProject - ) - - return $CollectionUri + $TeamProject -} - -function GetQueryTestRunsUri -{ - Param( - [string]$CollectionUri, - [string]$TeamProject, - [string]$BuildUri, - [switch]$IncludeRunDetails - ) - - if ($IncludeRunDetails) - { - $includeRunDetailsParameter = "&includeRunDetails=true" - } - else - { - $includeRunDetailsParameter = "" - } - - $baseUri = GetAzureDevOpsBaseUri -CollectionUri $CollectionUri -TeamProject $TeamProject - $queryUri = "$baseUri/_apis/test/runs?buildUri=$BuildUri$includeRunDetailsParameter&api-version=5.0" - return $queryUri -} - -function Get-HelixJobTypeFromTestRun -{ - Param ($testRun) - - $testRunSingleResultUri = "$($testRun.url)/results?`$top=1&`$skip=0&api-version=5.1" - $singleTestResult = Invoke-RestMethod -Uri $testRunSingleResultUri -Method Get -Headers $azureDevOpsRestApiHeaders - $count = $singleTestResult.value.Length - if($count -eq 0) - { - # If the count is 0, then results have not yet been reported for this run. - # We only care about completed runs with results, so it is ok to just return 'UNKNOWN' for this run. - return "UNKNOWN" - } - else - { - $info = ConvertFrom-Json $singleTestResult.value.comment - $helixJobId = $info.HelixJobId - $job = Invoke-RestMethodWithRetries "https://helix.dot.net/api/2019-06-17/jobs/${helixJobId}?access_token=${HelixAccessToken}" - return $job.Type - } -} - -function Append-HelixAccessTokenToUrl -{ - Param ([string]$url, [string]$token) - if($url.Contains("?")) - { - $url = "$($url)&access_token=$($token)" - } - else - { - $url = "$($url)?access_token=$($token)" - } - return $url -} - - -# The Helix Rest api is sometimes unreliable. So we call these apis with retry logic. -# Note: The Azure DevOps apis are stable and do not need to be called with this retry logic. -$helixApiRetries = 0 -$helixApiRetriesMax = 10 - -function Download-StringWithRetries -{ - Param ([string]$fileName, [string]$url) - - $result = "" - $done = $false - - while(!($done)) - { - try - { - Write-Host "Downloading $fileName" - $result = (New-Object System.Net.WebClient).DownloadString($url) - $done = $true - } - catch - { - Write-Host "Failed to download $fileName $($PSItem.Exception)" - - $helixApiRetries = $helixApiRetries + 1 - if($helixApiRetries -lt $helixApiRetriesMax) - { - Write-Host "Sleep and retry download of $fileName" - Start-Sleep 60 - } - else - { - throw "Failed to download $fileName" - } - } - } - - return $result -} - -function Invoke-RestMethodWithRetries -{ - Param ([string]$url,$Headers) - - $result = @() - $done = $false - - while(!($done)) - { - try - { - $result = Invoke-RestMethod -Uri $url -Method Get -Headers $Headers - $done = $true - } - catch - { - Write-Host "Failed to invoke Rest method $($PSItem.Exception)" - - $helixApiRetries = $helixApiRetries + 1 - if($helixApiRetries -lt $helixApiRetriesMax) - { - Write-Host "Sleep and retry invoke" - Start-Sleep 60 - } - else - { - throw "Failed to invoke Rest method" - } - } - } - - return $result -} - -function Download-FileWithRetries -{ - Param ([string]$fileurl, [string]$destination) - - $done = $false - - while(!($done)) - { - try - { - Write-Host "Downloading $destination" - $webClient.DownloadFile($fileurl, $destination) - $done = $true - } - catch - { - Write-Host "Failed to download $destination $($PSItem.Exception)" - - $helixApiRetries = $helixApiRetries + 1 - if($helixApiRetries -lt $helixApiRetriesMax) - { - Write-Host "Sleep and retry download of $destination" - Start-Sleep 60 - } - else - { - throw "Failed to download $destination" - } - } - } -} \ No newline at end of file diff --git a/build/Helix/ConvertWttLogToXUnit.ps1 b/build/Helix/ConvertWttLogToXUnit.ps1 deleted file mode 100644 index 13d333d3e7..0000000000 --- a/build/Helix/ConvertWttLogToXUnit.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -Param( - [Parameter(Mandatory = $true)] - [string]$WttInputPath, - - [Parameter(Mandatory = $true)] - [string]$WttSingleRerunInputPath, - - [Parameter(Mandatory = $true)] - [string]$WttMultipleRerunInputPath, - - [Parameter(Mandatory = $true)] - [string]$XUnitOutputPath, - - [Parameter(Mandatory = $true)] - [string]$TestNamePrefix -) - -# Ideally these would be passed as parameters to the script. However ps makes it difficult to deal with string literals containing '&', so we just -# read the values directly from the environment variables -$helixResultsContainerUri = $Env:HELIX_RESULTS_CONTAINER_URI -$helixResultsContainerRsas = $Env:HELIX_RESULTS_CONTAINER_RSAS - -$rerunPassesRequiredToAvoidFailure = $env:rerunPassesRequiredToAvoidFailure - -Add-Type -Language CSharp -ReferencedAssemblies System.Xml,System.Xml.Linq,System.Runtime.Serialization,System.Runtime.Serialization.Json (Get-Content $PSScriptRoot\HelixTestHelpers.cs -Raw) - -$testResultParser = [HelixTestHelpers.TestResultParser]::new($TestNamePrefix, $helixResultsContainerUri, $helixResultsContainerRsas) -$testResultParser.ConvertWttLogToXUnitLog($WttInputPath, $WttSingleRerunInputPath, $WttMultipleRerunInputPath, $XUnitOutputPath, $rerunPassesRequiredToAvoidFailure) \ No newline at end of file diff --git a/build/Helix/CopyPGOFiles.ps1 b/build/Helix/CopyPGOFiles.ps1 new file mode 100644 index 0000000000..44e84dfda2 --- /dev/null +++ b/build/Helix/CopyPGOFiles.ps1 @@ -0,0 +1,24 @@ +Param( + [string]$SourceFolder, + [string]$OutputFolder +) + +$pgcFiles = Get-ChildItem -Recurse $SourceFolder | where { $_.Name.EndsWith(".pgc") } + +foreach($pgcFile in $pgcFiles) +{ + $flavorPath = $pgcFile.Name.Split('.')[0] + $archPath = $pgcFile.Name.Split('.')[1] + $fileName = $pgcFile.Name.Remove(0, $flavorPath.length + $archPath.length + 2) + $fullPath = "$OutputFolder\PGO\$flavorPath\$archPath" + $destination = "$fullPath\$fileName" + + Write-Host "Copying $($pgcFile.Name) to $destination" + + if (-Not (Test-Path $fullPath)) + { + New-Item $fullPath -ItemType Directory + } + Write-Host "Copy $($pgcFile.FullName) to $destination" + Copy-Item $pgcFile.FullName $destination +} \ No newline at end of file diff --git a/build/Helix/EnsureMachineState.ps1 b/build/Helix/EnsureMachineState.ps1 deleted file mode 100644 index e068543cab..0000000000 --- a/build/Helix/EnsureMachineState.ps1 +++ /dev/null @@ -1,112 +0,0 @@ -$scriptDirectory = $script:MyInvocation.MyCommand.Path | Split-Path -Parent - -# List all processes to aid debugging: -Write-Host "All processes running:" -Get-Process - -tasklist /svc - -# Add this test directory as an exclusion for Windows Defender -Write-Host "Add $scriptDirectory as Exclusion Path" -Add-MpPreference -ExclusionPath $scriptDirectory -Write-Host "Add $($env:HELIX_CORRELATION_PAYLOAD) as Exclusion Path" -Add-MpPreference -ExclusionPath $env:HELIX_CORRELATION_PAYLOAD -Get-MpPreference -Get-MpComputerStatus - - -# Minimize all windows: -$shell = New-Object -ComObject "Shell.Application" -$shell.minimizeall() - -# Kill any instances of Windows Security Alert: -$windowTitleToMatch = "*Windows Security Alert*" -$procs = Get-Process | Where {$_.MainWindowTitle -like "*Windows Security Alert*"} -foreach ($proc in $procs) -{ - Write-Host "Found process with '$windowTitleToMatch' title: $proc" - $proc.Kill(); -} - -# Kill processes by name that are known to interfere with our tests: -$processNamesToStop = @("Microsoft.Photos", "WinStore.App", "SkypeApp", "SkypeBackgroundHost", "OneDriveSetup", "OneDrive") -foreach($procName in $processNamesToStop) -{ - Write-Host "Attempting to kill $procName if it is running" - Stop-Process -ProcessName $procName -Verbose -ErrorAction Ignore -} -Write-Host "All processes running after attempting to kill unwanted processes:" -Get-Process - -tasklist /svc - -$platform = $env:testbuildplatform -if(!$platform) -{ - $platform = "x86" -} - -function UninstallApps { - Param([string[]]$appsToUninstall) - - foreach($pkgName in $appsToUninstall) - { - foreach($pkg in (Get-AppxPackage $pkgName).PackageFullName) - { - Write-Output "Removing: $pkg" - Remove-AppxPackage $pkg - } - } -} - -function UninstallTestApps { - Param([string[]]$appsToUninstall) - - foreach($pkgName in $appsToUninstall) - { - foreach($pkg in (Get-AppxPackage $pkgName).PackageFullName) - { - Write-Output "Removing: $pkg" - Remove-AppxPackage $pkg - } - - # Sometimes an app can get into a state where it is no longer returned by Get-AppxPackage, but it is still present - # which prevents other versions of the app from being installed. - # To handle this, we can directly call Remove-AppxPackage against the full name of the package. However, without - # Get-AppxPackage to find the PackageFullName, we just have to manually construct the name. - $packageFullName = "$($pkgName)_1.0.0.0_$($platform)__8wekyb3d8bbwe" - Write-Host "Removing $packageFullName if installed" - Remove-AppxPackage $packageFullName -ErrorVariable appxerror -ErrorAction SilentlyContinue - if($appxerror) - { - foreach($error in $appxerror) - { - # In most cases, Remove-AppxPackage will fail due to the package not being found. Don't treat this as an error. - if(!($error.Exception.Message -match "0x80073CF1")) - { - Write-Error $error - } - } - } - else - { - Write-Host "Sucessfully removed $packageFullName" - } - } -} - -Write-Host "Uninstall AppX packages that are known to cause issues with our tests" -UninstallApps("*Skype*", "*Windows.Photos*") - -Write-Host "Uninstall any of our test apps that may have been left over from previous test runs" -UninstallTestApps("NugetPackageTestApp", "NugetPackageTestAppCX", "IXMPTestApp", "MUXControlsTestApp", "WpfApp") - -Write-Host "Uninstall MUX Framework package that may have been left over from previous test runs" -# We don't want to uninstall all versions of the MUX Framework package, as there may be other apps preinstalled on the system -# that depend on it. We only uninstall the Framework package that corresponds to the version of MUX that we are testing. -[xml]$versionData = (Get-Content "version.props") -$versionMajor = $versionData.GetElementsByTagName("MUXVersionMajor").'#text' -$versionMinor = $versionData.GetElementsByTagName("MUXVersionMinor").'#text' -UninstallApps("Microsoft.UI.Xaml.$versionMajor.$versionMinor") - -Get-Process \ No newline at end of file diff --git a/build/Helix/GenerateTestProjFile.ps1 b/build/Helix/GenerateTestProjFile.ps1 index 483b6d0cc7..b8b1e7f143 100644 --- a/build/Helix/GenerateTestProjFile.ps1 +++ b/build/Helix/GenerateTestProjFile.ps1 @@ -1,319 +1,28 @@ [CmdLetBinding()] Param( [Parameter(Mandatory = $true)] - [string]$TestFile, + [string]$TestFilePattern, [Parameter(Mandatory = $true)] - [string]$OutputProjFile, + [string]$TestBinaryDirectoryPath, [Parameter(Mandatory = $true)] - [string]$JobTestSuiteName, + [string]$OutputProjFile, - [Parameter(Mandatory = $true)] - [string]$TaefPath, + [string]$TestNamePrefix, [string]$TaefQuery ) -Class TestCollection -{ - [string]$Name - [string]$SetupMethodName - [string]$TeardownMethodName - [System.Collections.Generic.Dictionary[string, string]]$Properties - - TestCollection() - { - if ($this.GetType() -eq [TestCollection]) - { - throw "This class should never be instantiated directly; it should only be derived from." - } - } - - TestCollection([string]$name) - { - $this.Init($name) - } - - hidden Init([string]$name) - { - $this.Name = $name - $this.Properties = @{} - } -} - -Class Test : TestCollection -{ - Test([string]$name) - { - $this.Init($name) - } -} - -Class TestClass : TestCollection -{ - [System.Collections.Generic.List[Test]]$Tests - - TestClass([string]$name) - { - $this.Init($name) - $this.Tests = @{} - } -} - -Class TestModule : TestCollection -{ - [System.Collections.Generic.List[TestClass]]$TestClasses - - TestModule([string]$name) - { - $this.Init($name) - $this.TestClasses = @{} - } -} - -function Parse-TestInfo([string]$taefOutput) -{ - enum LineType - { - None - TestModule - TestClass - Test - Setup - Teardown - Property - } - - [string]$testModuleIndentation = " " - [string]$testClassIndentation = " " - [string]$testIndentation = " " - [string]$setupBeginning = "Setup: " - [string]$teardownBeginning = "Teardown: " - [string]$propertyBeginning = "Property[" - - function Get-LineType([string]$line) - { - if ($line.Contains($setupBeginning)) - { - return [LineType]::Setup; - } - elseif ($line.Contains($teardownBeginning)) - { - return [LineType]::Teardown; - } - elseif ($line.Contains($propertyBeginning)) - { - return [LineType]::Property; - } - elseif ($line.StartsWith($testModuleIndentation) -and -not $line.StartsWith("$testModuleIndentation ")) - { - return [LineType]::TestModule; - } - elseif ($line.StartsWith($testClassIndentation) -and -not $line.StartsWith("$testClassIndentation ")) - { - return [LineType]::TestClass; - } - elseif ($line.StartsWith($testIndentation) -and -not $line.StartsWith("$testIndentation ")) - { - return [LineType]::Test; - } - else - { - return [LineType]::None; - } - } - - [string[]]$lines = $taefOutput.Split(@([Environment]::NewLine, "`n"), [StringSplitOptions]::RemoveEmptyEntries) - [System.Collections.Generic.List[TestModule]]$testModules = @() - - [TestModule]$currentTestModule = $null - [TestClass]$currentTestClass = $null - [Test]$currentTest = $null - - [TestCollection]$lastTestCollection = $null - - foreach ($rawLine in $lines) - { - [LineType]$lineType = (Get-LineType $rawLine) - - # We don't need the whitespace around the line anymore, so we'll discard it to make things easier. - [string]$line = $rawLine.Trim() - - if ($lineType -eq [LineType]::TestModule) - { - if ($currentTest -ne $null -and $currentTestClass -ne $null) - { - $currentTestClass.Tests.Add($currentTest) - } - - if ($currentTestClass -ne $null -and $currentTestModule -ne $null) - { - $currentTestModule.TestClasses.Add($currentTestClass) - } - - if ($currentTestModule -ne $null) - { - $testModules.Add($currentTestModule) - } - - $currentTestModule = [TestModule]::new($line) - $currentTestClass = $null - $currentTest = $null - $lastTestCollection = $currentTestModule - } - elseif ($lineType -eq [LineType]::TestClass) - { - if ($currentTest -ne $null -and $currentTestClass -ne $null) - { - $currentTestClass.Tests.Add($currentTest) - } - - if ($currentTestClass -ne $null -and $currentTestModule -ne $null) - { - $currentTestModule.TestClasses.Add($currentTestClass) - } - - $currentTestClass = [TestClass]::new($line) - $currentTest = $null - $lastTestCollection = $currentTestClass - } - elseif ($lineType -eq [LineType]::Test) - { - if ($currentTest -ne $null -and $currentTestClass -ne $null) - { - $currentTestClass.Tests.Add($currentTest) - } - - $currentTest = [Test]::new($line) - $lastTestCollection = $currentTest - } - elseif ($lineType -eq [LineType]::Setup) - { - if ($lastTestCollection -ne $null) - { - $lastTestCollection.SetupMethodName = $line.Replace($setupBeginning, "") - } - } - elseif ($lineType -eq [LineType]::Teardown) - { - if ($lastTestCollection -ne $null) - { - $lastTestCollection.TeardownMethodName = $line.Replace($teardownBeginning, "") - } - } - elseif ($lineType -eq [LineType]::Property) - { - if ($lastTestCollection -ne $null) - { - foreach ($match in [Regex]::Matches($line, "Property\[(.*)\]\s+=\s+(.*)")) - { - [string]$propertyKey = $match.Groups[1].Value; - [string]$propertyValue = $match.Groups[2].Value; - $lastTestCollection.Properties.Add($propertyKey, $propertyValue); - } - } - } - } - - if ($currentTest -ne $null -and $currentTestClass -ne $null) - { - $currentTestClass.Tests.Add($currentTest) - } - - if ($currentTestClass -ne $null -and $currentTestModule -ne $null) - { - $currentTestModule.TestClasses.Add($currentTestClass) - } - - if ($currentTestModule -ne $null) - { - $testModules.Add($currentTestModule) - } - - return $testModules -} - -Write-Verbose "TaefQuery = $TaefQuery" - -$TaefSelectQuery = "" -$TaefQueryToAppend = "" -if($TaefQuery) -{ - $TaefSelectQuery = "/select:`"$TaefQuery`"" - $TaefQueryToAppend = " and $TaefQuery" -} -Write-Verbose "TaefSelectQuery = $TaefSelectQuery" - - -$taefExe = "$TaefPath\te.exe" -[string]$taefOutput = & "$taefExe" /listproperties $TaefSelectQuery $TestFile | Out-String - -[System.Collections.Generic.List[TestModule]]$testModules = (Parse-TestInfo $taefOutput) - -$projFileContent = @" - - -"@ - -foreach ($testModule in $testModules) -{ - foreach ($testClass in $testModules.TestClasses) - { - Write-Host "Generating Helix work item for test class $($testClass.Name)..." - [System.Collections.Generic.List[string]]$testSuiteNames = @() - - $testSuiteExists = $false - $suitelessTestExists = $false - - foreach ($tests in $testClass.Tests) - { - if ($tests.Properties.ContainsKey("TestSuite")) - { - [string]$testSuite = $tests.Properties["TestSuite"] - - if (-not $testSuiteNames.Contains($testSuite)) - { - Write-Host " Found test suite $testSuite. Generating Helix work item for it as well." - $testSuiteNames.Add($testSuite) - } - - $testSuiteExists = $true - } - else - { - $suitelessTestExists = $true - } - } - - if ($suitelessTestExists) - { - $projFileContent += @" - - - 00:30:00 - call %HELIX_CORRELATION_PAYLOAD%\runtests.cmd /select:"(@Name='$($testClass.Name).*'$(if ($testSuiteExists) { "and not @TestSuite='*'" }))$($TaefQueryToAppend)" - -"@ - } - - foreach ($testSuiteName in $testSuiteNames) - { - $projFileContent += @" - - - 00:30:00 - call %HELIX_CORRELATION_PAYLOAD%\runtests.cmd /select:"(@Name='$($testClass.Name).*' and @TestSuite='$testSuiteName')$($TaefQueryToAppend)" - -"@ - } - } -} - -$projFileContent += @" +[xml]$pkgVerData = (Get-Content "$PSScriptRoot\packages.config") +$winuiHelixVer = $pkgVerData.SelectSingleNode("//packages/package[@id=`"Microsoft.Internal.WinUI.Helix`"]").version - - -"@ +$packagesDir = Join-Path (Split-Path -Parent $script:MyInvocation.MyCommand.Path) "packages" +$pipelinesScriptsDir = Join-Path $packagesDir "Microsoft.Internal.WinUI.Helix.$winuiHelixVer\scripts\pipeline" -Set-Content $OutputProjFile $projFileContent -NoNewline -Encoding UTF8 \ No newline at end of file +& $pipelinesScriptsDir\GenerateHelixWorkItems.ps1 -TestFilePattern $TestFilePattern ` + -TestBinaryDirectoryPath $TestBinaryDirectoryPath ` + -OutputProjFile $OutputProjFile ` + -TaefBaseQuery $TaefQuery ` + -TestTimeout "00:30:00" ` + -TestNamePrefix $TestNamePrefix \ No newline at end of file diff --git a/build/Helix/HelixTestHelpers.cs b/build/Helix/HelixTestHelpers.cs deleted file mode 100644 index 706562d9ee..0000000000 --- a/build/Helix/HelixTestHelpers.cs +++ /dev/null @@ -1,643 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.Serialization; -using System.Runtime.Serialization.Json; -using System.Text; -using System.Xml.Linq; - -namespace HelixTestHelpers -{ - public class TestResult - { - public TestResult() - { - Screenshots = new List(); - RerunResults = new List(); - } - - public string Name { get; set; } - public string SourceWttFile { get; set; } - public bool Passed { get; set; } - public bool CleanupPassed { get; set; } - public TimeSpan ExecutionTime { get; set; } - public string Details { get; set; } - - public List Screenshots { get; private set; } - public List RerunResults { get; private set; } - - // Returns true if the test pass rate is sufficient to avoid being counted as a failure. - public bool PassedOrUnreliable(int requiredNumberOfPasses) - { - if(Passed) - { - return true; - } - else - { - if(RerunResults.Count == 1) - { - return RerunResults[0].Passed; - } - else - { - return RerunResults.Where(r => r.Passed).Count() >= requiredNumberOfPasses; - } - } - } - } - - // - // Azure DevOps doesn't currently provide a way to directly report sub-results for tests that failed at least once - // that were run multiple times. To get around that limitation, we'll mark the test as "Skip" since - // that's the only non-pass/fail result we can return, and will then report the information about the - // runs in the "reason" category for the skipped test. In order to save space, we'll make the following - // optimizations for size: - // - // 1. Serialize as JSON, which is more compact than XML; - // 2. Don't serialize values that we don't need; - // 3. Store the URL prefix and suffix for the blob storage URL only once instead of - // storing every log and screenshot URL in its entirety; and - // 4. Store a list of unique error messages and then index into that instead of - // storing every error message in its entirety. - // - // #4 is motivated by the fact that if a test fails multiple times, it probably failed for the same reason - // each time, in which case we'd just be repeating ourselves if we stored every error message each time. - // - // TODO (https://github.com/dotnet/arcade/issues/2773): Once we're able to directly report things in a - // more granular fashion than just a binary pass/fail result, we should do that. - // - [DataContract] - internal class JsonSerializableTestResults - { - [DataMember] - internal string blobPrefix; - - [DataMember] - internal string blobSuffix; - - [DataMember] - internal string[] errors; - - [DataMember] - internal JsonSerializableTestResult[] results; - } - - [DataContract] - internal class JsonSerializableTestResult - { - [DataMember] - internal string outcome; - - [DataMember] - internal int duration; - - [DataMember(EmitDefaultValue = false)] - internal string log; - - [DataMember(EmitDefaultValue = false)] - internal string[] screenshots; - - [DataMember(EmitDefaultValue = false)] - internal int errorIndex; - } - - public class TestPass - { - public TimeSpan TestPassExecutionTime { get; set; } - public List TestResults { get; set; } - - public static TestPass ParseTestWttFile(string fileName, bool cleanupFailuresAreRegressions, bool truncateTestNames) - { - using (var stream = File.OpenRead(fileName)) - { - var doc = XDocument.Load(stream); - var testResults = new List(); - var testExecutionTimeMap = new Dictionary>(); - - TestResult currentResult = null; - long frequency = 0; - long startTime = 0; - long stopTime = 0; - bool inTestCleanup = false; - - bool shouldLogToTestDetails = false; - - long testPassStartTime = 0; - long testPassStopTime = 0; - - Func isScopeData = (elt) => - { - return - elt.Element("Data") != null && - elt.Element("Data").Element("WexContext") != null && - ( - elt.Element("Data").Element("WexContext").Value == "Cleanup" || - elt.Element("Data").Element("WexContext").Value == "TestScope" || - elt.Element("Data").Element("WexContext").Value == "TestScope" || - elt.Element("Data").Element("WexContext").Value == "ClassScope" || - elt.Element("Data").Element("WexContext").Value == "ModuleScope" - ); - }; - - Func isModuleOrClassScopeStart = (elt) => - { - return - elt.Name == "Msg" && - elt.Element("Data") != null && - elt.Element("Data").Element("StartGroup") != null && - elt.Element("Data").Element("WexContext") != null && - (elt.Element("Data").Element("WexContext").Value == "ClassScope" || - elt.Element("Data").Element("WexContext").Value == "ModuleScope"); - }; - - Func isModuleScopeEnd = (elt) => - { - return - elt.Name == "Msg" && - elt.Element("Data") != null && - elt.Element("Data").Element("EndGroup") != null && - elt.Element("Data").Element("WexContext") != null && - elt.Element("Data").Element("WexContext").Value == "ModuleScope"; - }; - - Func isClassScopeEnd = (elt) => - { - return - elt.Name == "Msg" && - elt.Element("Data") != null && - elt.Element("Data").Element("EndGroup") != null && - elt.Element("Data").Element("WexContext") != null && - elt.Element("Data").Element("WexContext").Value == "ClassScope"; - }; - - int testsExecuting = 0; - foreach (XElement element in doc.Root.Elements()) - { - // Capturing the frequency data to record accurate - // timing data. - if (element.Name == "RTI") - { - frequency = Int64.Parse(element.Attribute("Frequency").Value); - } - - // It's possible for a test to launch another test. If that happens, we won't modify the - // current result. Instead, we'll continue operating like normal and expect that we get two - // EndTests nodes before our next StartTests. We'll check that we've actually got a stop time - // before creating a new result. This will result in the two results being squashed - // into one result of the outer test that ran the inner one. - if (element.Name == "StartTest") - { - testsExecuting++; - if (testsExecuting == 1) - { - string testName = element.Attribute("Title").Value; - - if (truncateTestNames) - { - const string xamlNativePrefix = "Windows::UI::Xaml::Tests::"; - const string xamlManagedPrefix = "Windows.UI.Xaml.Tests."; - if (testName.StartsWith(xamlNativePrefix)) - { - testName = testName.Substring(xamlNativePrefix.Length); - } - else if (testName.StartsWith(xamlManagedPrefix)) - { - testName = testName.Substring(xamlManagedPrefix.Length); - } - } - - currentResult = new TestResult() { Name = testName, SourceWttFile = fileName, Passed = true, CleanupPassed = true }; - testResults.Add(currentResult); - startTime = Int64.Parse(element.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value); - inTestCleanup = false; - shouldLogToTestDetails = true; - stopTime = 0; - } - } - else if (currentResult != null && element.Name == "EndTest") - { - testsExecuting--; - - // If any inner test fails, we'll still fail the outer - currentResult.Passed &= element.Attribute("Result").Value == "Pass"; - - // Only gather execution data if this is the outer test we ran initially - if (testsExecuting == 0) - { - stopTime = Int64.Parse(element.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value); - if (!testExecutionTimeMap.Keys.Contains(currentResult.Name)) - testExecutionTimeMap[currentResult.Name] = new List(); - testExecutionTimeMap[currentResult.Name].Add((double)(stopTime - startTime) / frequency); - currentResult.ExecutionTime = TimeSpan.FromSeconds(testExecutionTimeMap[currentResult.Name].Average()); - - startTime = 0; - inTestCleanup = true; - } - } - else if (currentResult != null && - (isModuleOrClassScopeStart(element) || isModuleScopeEnd(element) || isClassScopeEnd(element))) - { - shouldLogToTestDetails = false; - inTestCleanup = false; - } - - // Log-appending methods. - if (currentResult != null && element.Name == "Error") - { - if (shouldLogToTestDetails) - { - currentResult.Details += "\r\n[Error]: " + element.Attribute("UserText").Value; - if (element.Attribute("File") != null && element.Attribute("File").Value != "") - { - currentResult.Details += (" [File " + element.Attribute("File").Value); - if (element.Attribute("Line") != null) - currentResult.Details += " Line: " + element.Attribute("Line").Value; - currentResult.Details += "]"; - } - } - - - // The test cleanup errors will often come after the test claimed to have - // 'passed'. We treat them as errors as well. - if (inTestCleanup) - { - currentResult.CleanupPassed = false; - currentResult.Passed = false; - // In stress mode runs, this test will run n times before cleanup is run. If the cleanup - // fails, we want to fail every test. - if (cleanupFailuresAreRegressions) - { - foreach (var result in testResults.Where(res => res.Name == currentResult.Name)) - { - result.Passed = false; - result.CleanupPassed = false; - } - } - } - } - - if (currentResult != null && element.Name == "Warn") - { - if (shouldLogToTestDetails) - { - currentResult.Details += "\r\n[Warn]: " + element.Attribute("UserText").Value; - } - - if (element.Attribute("File") != null && element.Attribute("File").Value != "") - { - currentResult.Details += (" [File " + element.Attribute("File").Value); - if (element.Attribute("Line") != null) - currentResult.Details += " Line: " + element.Attribute("Line").Value; - currentResult.Details += "]"; - } - } - - if (currentResult != null && element.Name == "Msg") - { - var dataElement = element.Element("Data"); - if (dataElement != null) - { - var supportingInfo = dataElement.Element("SupportingInfo"); - if (supportingInfo != null) - { - var screenshots = supportingInfo.Elements("Item") - .Where(item => GetAttributeValue(item, "Name") == "Screenshot") - .Select(item => GetAttributeValue(item, "Value")); - - foreach(var screenshot in screenshots) - { - string fileNameSuffix = string.Empty; - - if (fileName.Contains("_rerun_multiple")) - { - fileNameSuffix = "_rerun_multiple"; - } - else if (fileName.Contains("_rerun")) - { - fileNameSuffix = "_rerun"; - } - - currentResult.Screenshots.Add(screenshot.Replace(".jpg", fileNameSuffix + ".jpg")); - } - } - } - } - } - - testPassStartTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value); - testPassStopTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").Last().Attribute("TimeStamp").Value); - - var testPassTime = TimeSpan.FromSeconds((double)(testPassStopTime - testPassStartTime) / frequency); - - foreach (TestResult testResult in testResults) - { - if (testResult.Details != null) - { - testResult.Details = testResult.Details.Trim(); - } - } - - var testpass = new TestPass - { - TestPassExecutionTime = testPassTime, - TestResults = testResults - }; - - return testpass; - } - } - - public static TestPass ParseTestWttFileWithReruns(string fileName, string singleRerunFileName, string multipleRerunFileName, bool cleanupFailuresAreRegressions, bool truncateTestNames) - { - TestPass testPass = ParseTestWttFile(fileName, cleanupFailuresAreRegressions, truncateTestNames); - TestPass singleRerunTestPass = File.Exists(singleRerunFileName) ? ParseTestWttFile(singleRerunFileName, cleanupFailuresAreRegressions, truncateTestNames) : null; - TestPass multipleRerunTestPass = File.Exists(multipleRerunFileName) ? ParseTestWttFile(multipleRerunFileName, cleanupFailuresAreRegressions, truncateTestNames) : null; - - List rerunTestResults = new List(); - - if (singleRerunTestPass != null) - { - rerunTestResults.AddRange(singleRerunTestPass.TestResults); - } - - if (multipleRerunTestPass != null) - { - rerunTestResults.AddRange(multipleRerunTestPass.TestResults); - } - - // For each failed test result, we'll check to see whether the test passed at least once upon rerun. - // If so, we'll set PassedOnRerun to true to flag the fact that this is an unreliable test - // rather than a genuine test failure. - foreach (TestResult failedTestResult in testPass.TestResults.Where(r => !r.Passed)) - { - failedTestResult.RerunResults.AddRange(rerunTestResults.Where(r => r.Name == failedTestResult.Name)); - } - - return testPass; - } - - private static string GetAttributeValue(XElement element, string attributeName) - { - if(element.Attribute(attributeName) != null) - { - return element.Attribute(attributeName).Value; - } - - return null; - } - } - - public static class FailedTestDetector - { - public static void OutputFailedTestQuery(string wttInputPath) - { - var testPass = TestPass.ParseTestWttFile(wttInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false); - - List failedTestNames = new List(); - - foreach (var result in testPass.TestResults) - { - if (!result.Passed) - { - failedTestNames.Add(result.Name); - } - } - - if (failedTestNames.Count > 0) - { - string failedTestSelectQuery = "(@Name='"; - - for (int i = 0; i < failedTestNames.Count; i++) - { - failedTestSelectQuery += failedTestNames[i]; - - if (i < failedTestNames.Count - 1) - { - failedTestSelectQuery += "' or @Name='"; - } - } - - failedTestSelectQuery += "')"; - - Console.WriteLine(failedTestSelectQuery); - } - else - { - Console.WriteLine(""); - } - } - } - - public class TestResultParser - { - private string testNamePrefix; - private string helixResultsContainerUri; - private string helixResultsContainerRsas; - - public TestResultParser(string testNamePrefix, string helixResultsContainerUri, string helixResultsContainerRsas) - { - this.testNamePrefix = testNamePrefix; - this.helixResultsContainerUri = helixResultsContainerUri; - this.helixResultsContainerRsas = helixResultsContainerRsas; - } - - public Dictionary GetSubResultsJsonByMethodName(string wttInputPath, string wttSingleRerunInputPath, string wttMultipleRerunInputPath) - { - Dictionary subResultsJsonByMethod = new Dictionary(); - TestPass testPass = TestPass.ParseTestWttFileWithReruns(wttInputPath, wttSingleRerunInputPath, wttMultipleRerunInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false); - - foreach (var result in testPass.TestResults) - { - var methodName = result.Name.Substring(result.Name.LastIndexOf('.') + 1); - - if (!result.Passed) - { - // If a test failed, we'll have rerun it multiple times. We'll record the results of each run - // formatted as JSON. - JsonSerializableTestResults serializableResults = new JsonSerializableTestResults(); - serializableResults.blobPrefix = helixResultsContainerUri; - serializableResults.blobSuffix = helixResultsContainerRsas; - - List errorList = new List(); - errorList.Add(result.Details); - - foreach (TestResult rerunResult in result.RerunResults) - { - errorList.Add(rerunResult.Details); - } - - serializableResults.errors = errorList.Distinct().Where(s => s != null).ToArray(); - - var reason = new XElement("reason"); - List serializableResultList = new List(); - serializableResultList.Add(ConvertToSerializableResult(result, serializableResults.errors)); - - foreach (TestResult rerunResult in result.RerunResults) - { - serializableResultList.Add(ConvertToSerializableResult(rerunResult, serializableResults.errors)); - } - - serializableResults.results = serializableResultList.ToArray(); - - using (MemoryStream stream = new MemoryStream()) - { - DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(JsonSerializableTestResults)); - serializer.WriteObject(stream, serializableResults); - stream.Position = 0; - - using (StreamReader streamReader = new StreamReader(stream)) - { - subResultsJsonByMethod.Add(methodName, streamReader.ReadToEnd()); - } - } - } - } - - return subResultsJsonByMethod; - } - - public void ConvertWttLogToXUnitLog(string wttInputPath, string wttSingleRerunInputPath, string wttMultipleRerunInputPath, string xunitOutputPath, int requiredPassRateThreshold) - { - TestPass testPass = TestPass.ParseTestWttFileWithReruns(wttInputPath, wttSingleRerunInputPath, wttMultipleRerunInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false); - var results = testPass.TestResults; - - int resultCount = results.Count; - int passedCount = results.Where(r => r.Passed).Count(); - - // Since we re-run tests on failure, we'll mark every test that failed at least once as "skipped" rather than "failed". - // If the test failed sufficiently often enough for it to count as a failed test (determined by a property on the - // Azure DevOps job), we'll later mark it as failed during test results processing. - - int failedCount = results.Where(r => !r.PassedOrUnreliable(requiredPassRateThreshold)).Count(); - int skippedCount = results.Where(r => !r.Passed && r.PassedOrUnreliable(requiredPassRateThreshold)).Count(); - - var root = new XElement("assemblies"); - - var assembly = new XElement("assembly"); - assembly.SetAttributeValue("name", "MUXControls.Test.dll"); - assembly.SetAttributeValue("test-framework", "TAEF"); - assembly.SetAttributeValue("run-date", DateTime.Now.ToString("yyyy-mm-dd")); - - // This doesn't need to be completely accurate since it's not exposed anywhere. - // If we need accurate an start time we can probably calculate it from the te.wtl file, but for - // now this is fine. - assembly.SetAttributeValue("run-time", (DateTime.Now - testPass.TestPassExecutionTime).ToString("hh:mm:ss")); - - assembly.SetAttributeValue("total", resultCount); - assembly.SetAttributeValue("passed", passedCount); - assembly.SetAttributeValue("failed", failedCount); - assembly.SetAttributeValue("skipped", skippedCount); - - assembly.SetAttributeValue("time", (int)testPass.TestPassExecutionTime.TotalSeconds); - assembly.SetAttributeValue("errors", 0); - root.Add(assembly); - - var collection = new XElement("collection"); - collection.SetAttributeValue("total", resultCount); - collection.SetAttributeValue("passed", passedCount); - collection.SetAttributeValue("failed", failedCount); - collection.SetAttributeValue("skipped", skippedCount); - collection.SetAttributeValue("name", "Test collection"); - collection.SetAttributeValue("time", (int)testPass.TestPassExecutionTime.TotalSeconds); - assembly.Add(collection); - - foreach (var result in results) - { - var test = new XElement("test"); - test.SetAttributeValue("name", testNamePrefix + "." + result.Name); - - var className = result.Name.Substring(0, result.Name.LastIndexOf('.')); - var methodName = result.Name.Substring(result.Name.LastIndexOf('.') + 1); - test.SetAttributeValue("type", className); - test.SetAttributeValue("method", methodName); - - test.SetAttributeValue("time", result.ExecutionTime.TotalSeconds); - - string resultString = string.Empty; - - if (result.Passed) - { - resultString = "Pass"; - } - else if(result.PassedOrUnreliable(requiredPassRateThreshold)) - { - resultString = "Skip"; - } - else - { - resultString = "Fail"; - } - - - test.SetAttributeValue("result", resultString); - - if (!result.Passed) - { - // If a test failed, we'll have rerun it multiple times. - // We'll save the subresults to a JSON text file that we'll upload to the helix results container - - // this allows it to be as long as we want, whereas the reason field in Azure DevOps has a 4000 character limit. - string subResultsFileName = methodName + "_subresults.json"; - string subResultsFilePath = Path.Combine(Path.GetDirectoryName(wttInputPath), subResultsFileName); - - if (result.PassedOrUnreliable(requiredPassRateThreshold)) - { - var reason = new XElement("reason"); - reason.Add(new XCData(GetUploadedFileUrl(subResultsFileName, helixResultsContainerUri, helixResultsContainerRsas))); - test.Add(reason); - } - else - { - var failure = new XElement("failure"); - var message = new XElement("message"); - message.Add(new XCData(GetUploadedFileUrl(subResultsFileName, helixResultsContainerUri, helixResultsContainerRsas))); - failure.Add(message); - test.Add(failure); - } - } - collection.Add(test); - } - - File.WriteAllText(xunitOutputPath, root.ToString()); - } - - private JsonSerializableTestResult ConvertToSerializableResult(TestResult rerunResult, string[] uniqueErrors) - { - var serializableResult = new JsonSerializableTestResult(); - - serializableResult.outcome = rerunResult.Passed ? "Passed" : "Failed"; - serializableResult.duration = (int)Math.Round(rerunResult.ExecutionTime.TotalMilliseconds); - - if (!rerunResult.Passed) - { - serializableResult.log = Path.GetFileName(rerunResult.SourceWttFile); - - if (rerunResult.Screenshots.Any()) - { - List screenshots = new List(); - - foreach (var screenshot in rerunResult.Screenshots) - { - screenshots.Add(Path.GetFileName(screenshot)); - } - - serializableResult.screenshots = screenshots.ToArray(); - } - - // To conserve space, we'll log the index of the error to index in a list of unique errors rather than - // jotting down every single error in its entirety. We'll add one to the result so we can avoid - // serializing this property when it has the default value of 0. - serializableResult.errorIndex = Array.IndexOf(uniqueErrors, rerunResult.Details) + 1; - } - - return serializableResult; - } - - private string GetUploadedFileUrl(string filePath, string helixResultsContainerUri, string helixResultsContainerRsas) - { - var filename = Path.GetFileName(filePath); - return string.Format("{0}/{1}{2}", helixResultsContainerUri, filename, helixResultsContainerRsas); - } - } -} diff --git a/build/Helix/OutputFailedTestQuery.ps1 b/build/Helix/OutputFailedTestQuery.ps1 deleted file mode 100644 index 3ea7abf80a..0000000000 --- a/build/Helix/OutputFailedTestQuery.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -Param( - [Parameter(Mandatory = $true)] - [string]$WttInputPath -) - -Add-Type -Language CSharp -ReferencedAssemblies System.Xml,System.Xml.Linq,System.Runtime.Serialization,System.Runtime.Serialization.Json (Get-Content $PSScriptRoot\HelixTestHelpers.cs -Raw) - -[HelixTestHelpers.FailedTestDetector]::OutputFailedTestQuery($WttInputPath) \ No newline at end of file diff --git a/build/Helix/OutputSubResultsJsonFiles.ps1 b/build/Helix/OutputSubResultsJsonFiles.ps1 deleted file mode 100644 index 476502e987..0000000000 --- a/build/Helix/OutputSubResultsJsonFiles.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -Param( - [Parameter(Mandatory = $true)] - [string]$WttInputPath, - - [Parameter(Mandatory = $true)] - [string]$WttSingleRerunInputPath, - - [Parameter(Mandatory = $true)] - [string]$WttMultipleRerunInputPath, - - [Parameter(Mandatory = $true)] - [string]$TestNamePrefix -) - -# Ideally these would be passed as parameters to the script. However ps makes it difficult to deal with string literals containing '&', so we just -# read the values directly from the environment variables -$helixResultsContainerUri = $Env:HELIX_RESULTS_CONTAINER_URI -$helixResultsContainerRsas = $Env:HELIX_RESULTS_CONTAINER_RSAS - -Add-Type -Language CSharp -ReferencedAssemblies System.Xml,System.Xml.Linq,System.Runtime.Serialization,System.Runtime.Serialization.Json (Get-Content $PSScriptRoot\HelixTestHelpers.cs -Raw) - -$testResultParser = [HelixTestHelpers.TestResultParser]::new($TestNamePrefix, $helixResultsContainerUri, $helixResultsContainerRsas) -[System.Collections.Generic.Dictionary[string, string]]$subResultsJsonByMethodName = $testResultParser.GetSubResultsJsonByMethodName($WttInputPath, $WttSingleRerunInputPath, $WttMultipleRerunInputPath) - -$subResultsJsonDirectory = [System.IO.Path]::GetDirectoryName($WttInputPath) - -foreach ($methodName in $subResultsJsonByMethodName.Keys) -{ - $subResultsJson = $subResultsJsonByMethodName[$methodName] - $subResultsJsonPath = [System.IO.Path]::Combine($subResultsJsonDirectory, $methodName + "_subresults.json") - Out-File $subResultsJsonPath -Encoding utf8 -InputObject $subResultsJson -} \ No newline at end of file diff --git a/build/Helix/OutputTestResults.ps1 b/build/Helix/OutputTestResults.ps1 deleted file mode 100644 index eee4c5baec..0000000000 --- a/build/Helix/OutputTestResults.ps1 +++ /dev/null @@ -1,131 +0,0 @@ -Param( - [Parameter(Mandatory = $true)] - [int]$MinimumExpectedTestsExecutedCount, - - [string]$AccessToken = $env:SYSTEM_ACCESSTOKEN, - [string]$CollectionUri = $env:SYSTEM_COLLECTIONURI, - [string]$TeamProject = $env:SYSTEM_TEAMPROJECT, - [string]$BuildUri = $env:BUILD_BUILDURI, - [bool]$CheckJobAttempt -) - -$azureDevOpsRestApiHeaders = @{ - "Accept"="application/json" - "Authorization"="Basic $([System.Convert]::ToBase64String([System.Text.ASCIIEncoding]::ASCII.GetBytes(":$AccessToken")))" -} - -. "$PSScriptRoot/AzurePipelinesHelperScripts.ps1" - -Write-Host "Checking test results..." - -$queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -IncludeRunDetails -Write-Host "queryUri = $queryUri" - -$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders -[System.Collections.Generic.List[string]]$failingTests = @() -[System.Collections.Generic.List[string]]$unreliableTests = @() -[System.Collections.Generic.List[string]]$unexpectedResultTest = @() - -[System.Collections.Generic.List[string]]$namesOfProcessedTestRuns = @() -$totalTestsExecutedCount = 0 - -# We assume that we only have one testRun with a given name that we care about -# We only process the last testRun with a given name (based on completedDate) -# The name of a testRun is set to the Helix queue that it was run on (e.g. windows.10.amd64.client19h1.xaml) -# If we have multiple test runs on the same queue that we care about, we will need to re-visit this logic -foreach ($testRun in ($testRuns.value | Sort-Object -Property "completedDate" -Descending)) -{ - if ($CheckJobAttempt) - { - if ($namesOfProcessedTestRuns -contains $testRun.name) - { - Write-Host "Skipping test run '$($testRun.name)', since we have already processed a test run of that name." - continue - } - } - - Write-Host "Processing results from test run '$($testRun.name)'" - $namesOfProcessedTestRuns.Add($testRun.name) - - $totalTestsExecutedCount += $testRun.totalTests - - $testRunResultsUri = "$($testRun.url)/results?api-version=5.0" - $testResults = Invoke-RestMethodWithRetries "$($testRun.url)/results?api-version=5.0" -Headers $azureDevOpsRestApiHeaders - - foreach ($testResult in $testResults.value) - { - $shortTestCaseTitle = $testResult.testCaseTitle -replace "[a-zA-Z0-9]+.[a-zA-Z0-9]+.Windows.UI.Xaml.Tests.MUXControls.","" - - if ($testResult.outcome -eq "Failed") - { - if (-not $failingTests.Contains($shortTestCaseTitle)) - { - $failingTests.Add($shortTestCaseTitle) - } - } - elseif ($testResult.outcome -eq "Warning") - { - if (-not $unreliableTests.Contains($shortTestCaseTitle)) - { - $unreliableTests.Add($shortTestCaseTitle) - } - } - elseif ($testResult.outcome -ne "Passed") - { - # We should only see tests with result "Passed", "Failed" or "Warning" - if (-not $unexpectedResultTest.Contains($shortTestCaseTitle)) - { - $unexpectedResultTest.Add($shortTestCaseTitle) - } - } - } -} - -if ($unreliableTests.Count -gt 0) -{ - Write-Host @" -##vso[task.logissue type=warning;]Unreliable tests: -##vso[task.logissue type=warning;]$($unreliableTests -join "$([Environment]::NewLine)##vso[task.logissue type=warning;]") - -"@ -} - -if ($failingTests.Count -gt 0) -{ - Write-Host @" -##vso[task.logissue type=error;]Failing tests: -##vso[task.logissue type=error;]$($failingTests -join "$([Environment]::NewLine)##vso[task.logissue type=error;]") - -"@ -} - -if ($unexpectedResultTest.Count -gt 0) -{ - Write-Host @" -##vso[task.logissue type=error;]Tests with unexpected results: -##vso[task.logissue type=error;]$($unexpectedResultTest -join "$([Environment]::NewLine)##vso[task.logissue type=error;]") - -"@ -} - -if($totalTestsExecutedCount -lt $MinimumExpectedTestsExecutedCount) -{ - Write-Host "Expected at least $MinimumExpectedTestsExecutedCount tests to be executed." - Write-Host "Actual executed test count is: $totalTestsExecutedCount" - Write-Host "##vso[task.complete result=Failed;]" -} -elseif ($failingTests.Count -gt 0) -{ - Write-Host "At least one test failed." - Write-Host "##vso[task.complete result=Failed;]" -} -elseif ($unreliableTests.Count -gt 0) -{ - Write-Host "All tests eventually passed, but some initially failed." - Write-Host "##vso[task.complete result=Succeeded;]" -} -else -{ - Write-Host "All tests passed." - Write-Host "##vso[task.complete result=Succeeded;]" -} diff --git a/build/Helix/PrepareHelixPayload.ps1 b/build/Helix/PrepareHelixPayload.ps1 index bdfb57be30..a1305e4e3a 100644 --- a/build/Helix/PrepareHelixPayload.ps1 +++ b/build/Helix/PrepareHelixPayload.ps1 @@ -1,12 +1,17 @@ [CmdLetBinding()] Param( - [string]$Platform, - [string]$Configuration, + [ValidateSet("x86", "x64")] + [string]$Platform = "x86", + + [ValidateSet("Debug", "Release")] + [string]$Configuration= "Debug", + [string]$ArtifactName='drop' ) $payloadDir = "HelixPayload\$Configuration\$Platform" + $repoDirectory = Join-Path (Split-Path -Parent $script:MyInvocation.MyCommand.Path) "..\..\" $nugetPackagesDir = Join-Path (Split-Path -Parent $script:MyInvocation.MyCommand.Path) "packages" @@ -17,14 +22,29 @@ If(test-path $payloadDir) } New-Item -ItemType Directory -Force -Path $payloadDir + +[xml]$pkgVerData = (Get-Content "$PSScriptRoot\packages.config") +$winuiHelixVer = $pkgVerData.SelectSingleNode("//packages/package[@id=`"Microsoft.Internal.WinUI.Helix`"]").version +$mitaVer = $pkgVerData.SelectSingleNode("//packages/package[@id=`"microsoft.windows.apps.test`"]").version +$taefVer = $pkgVerData.SelectSingleNode("//packages/package[@id=`"TAEF.Redist.Wlk`"]").version +$muxcustomBuildTasksVer = $pkgVerData.SelectSingleNode("//packages/package[@id=`"MUXCustomBuildTasks`"]").version +$netCoreAppVer = $pkgVerData.SelectSingleNode("//packages/package[@id=`"runtime.win-$Platform.microsoft.netcore.app`"]").version + +Write-Host "winuiHelixVer = $winuiHelixVer" +Write-Host "mitaVer = $mitaVer" +Write-Host "taefVer = $taefVer" +Write-Host "muxcustomBuildTasksVer = $muxcustomBuildTasksVer" +Write-Host "netCoreAppVer = $netCoreAppVer" + # Copy files from nuget packages -Copy-Item "$nugetPackagesDir\microsoft.windows.apps.test.1.0.181203002\lib\netcoreapp2.1\*.dll" $payloadDir -Copy-Item "$nugetPackagesDir\taef.redist.wlk.10.31.180822002\build\Binaries\$Platform\*" $payloadDir -Copy-Item "$nugetPackagesDir\taef.redist.wlk.10.31.180822002\build\Binaries\$Platform\CoreClr\*" $payloadDir +Copy-Item "$nugetPackagesDir\Microsoft.Internal.WinUI.Helix.$winuiHelixVer\scripts\test\*" $payloadDir +Copy-Item "$nugetPackagesDir\microsoft.windows.apps.test.$mitaVer\lib\netcoreapp2.1\*.dll" $payloadDir +Copy-Item "$nugetPackagesDir\taef.redist.wlk.$taefVer\build\Binaries\$Platform\*" $payloadDir +Copy-Item "$nugetPackagesDir\taef.redist.wlk.$taefVer\build\Binaries\$Platform\CoreClr\*" $payloadDir New-Item -ItemType Directory -Force -Path "$payloadDir\.NETCoreApp2.1\" -Copy-Item "$nugetPackagesDir\runtime.win-$Platform.microsoft.netcore.app.2.1.0\runtimes\win-$Platform\lib\netcoreapp2.1\*" "$payloadDir\.NETCoreApp2.1\" -Copy-Item "$nugetPackagesDir\runtime.win-$Platform.microsoft.netcore.app.2.1.0\runtimes\win-$Platform\native\*" "$payloadDir\.NETCoreApp2.1\" -Copy-Item "$nugetPackagesDir\MUXCustomBuildTasks.1.0.71\tools\$platform\WttLog.dll" $payloadDir +Copy-Item "$nugetPackagesDir\runtime.win-$Platform.microsoft.netcore.app.$netCoreAppVer\runtimes\win-$Platform\lib\netcoreapp2.1\*" "$payloadDir\.NETCoreApp2.1\" +Copy-Item "$nugetPackagesDir\runtime.win-$Platform.microsoft.netcore.app.$netCoreAppVer\runtimes\win-$Platform\native\*" "$payloadDir\.NETCoreApp2.1\" +Copy-Item "$nugetPackagesDir\MUXCustomBuildTasks.$muxcustomBuildTasksVer\tools\$platform\WttLog.dll" $payloadDir function Copy-If-Exists { @@ -84,12 +104,5 @@ Copy-Recursively-If-Exists "$repoDirectory\Artifacts\$ArtifactName\$Configuratio # Copy files from the repo New-Item -ItemType Directory -Force -Path "$payloadDir" -Copy-Item "build\helix\ConvertWttLogToXUnit.ps1" "$payloadDir" -Copy-Item "build\helix\OutputFailedTestQuery.ps1" "$payloadDir" -Copy-Item "build\helix\OutputSubResultsJsonFiles.ps1" "$payloadDir" -Copy-Item "build\helix\HelixTestHelpers.cs" "$payloadDir" -Copy-Item "build\helix\runtests.cmd" $payloadDir -Copy-Item "build\helix\InstallTestAppDependencies.ps1" "$payloadDir" -Copy-Item "build\Helix\EnsureMachineState.ps1" "$payloadDir" +Copy-Item "build\helix\scripts\*" "$payloadDir" Copy-Item "version.props" "$payloadDir" -Copy-Item "build\Helix\CopyVisualTreeVerificationFiles.ps1" "$payloadDir" diff --git a/build/Helix/ProcessHelixFiles.ps1 b/build/Helix/ProcessHelixFiles.ps1 deleted file mode 100644 index f305b444d1..0000000000 --- a/build/Helix/ProcessHelixFiles.ps1 +++ /dev/null @@ -1,174 +0,0 @@ -Param( - [string]$AccessToken = $env:SYSTEM_ACCESSTOKEN, - [string]$HelixAccessToken = $env:HelixAccessToken, - [string]$CollectionUri = $env:SYSTEM_COLLECTIONURI, - [string]$TeamProject = $env:SYSTEM_TEAMPROJECT, - [string]$BuildUri = $env:BUILD_BUILDURI, - [string]$OutputFolder = "HelixOutput" -) - -Write-Host "CollectionUri: $CollectionUri" -Write-Host "TeamProject: $TeamProject" -Write-Host "BuildUri: $BuildUri" -Write-Host "OutputFolder: $OutputFolder" - -$helixLinkFile = "$OutputFolder\LinksToHelixTestFiles.html" -$visualTreeVerificationFolder = "$OutputFolder\UpdatedVisualTreeVerificationFiles" - -function Generate-File-Links -{ - Param ([Array[]]$files,[string]$sectionName) - if($files.Count -gt 0) - { - Out-File -FilePath $helixLinkFile -Append -InputObject "
" - Out-File -FilePath $helixLinkFile -Append -InputObject "

$sectionName

" - Out-File -FilePath $helixLinkFile -Append -InputObject "
    " - foreach($file in $files) - { - $url = Append-HelixAccessTokenToUrl $file.Link "{Your-Helix-Access-Token-Here}" - Out-File -FilePath $helixLinkFile -Append -InputObject "
  • $($url)
    • " - } - Out-File -FilePath $helixLinkFile -Append -InputObject "
" - Out-File -FilePath $helixLinkFile -Append -InputObject "
" - } -} - -function Log-Error -{ - Param ([string]$message) - - # We want to log the error slightly differently depending if we are running in AzDO or not. - if($env:TF_BUILD) - { - Write-Host "##vso[task.logissue type=error;]$message" - } - else - { - Write-Error "$message" -ErrorAction Continue - } -} - -function Append-HelixAccessTokenToUrl -{ - Param ([string]$url, [string]$token) - if($token) - { - if($url.Contains("?")) - { - $url = "$($url)&access_token=$($token)" - } - else - { - $url = "$($url)?access_token=$($token)" - } - } - return $url -} - -#Create output directory -New-Item $OutputFolder -ItemType Directory - -$azureDevOpsRestApiHeaders = @{ - "Accept"="application/json" - "Authorization"="Basic $([System.Convert]::ToBase64String([System.Text.ASCIIEncoding]::ASCII.GetBytes(":$AccessToken")))" -} - -. "$PSScriptRoot/AzurePipelinesHelperScripts.ps1" - -$queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -IncludeRunDetails -Write-Host "queryUri = $queryUri" - -$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders -$webClient = New-Object System.Net.WebClient -[System.Collections.Generic.List[string]]$workItems = @() - -foreach ($testRun in $testRuns.value) -{ - $testResults = Invoke-RestMethodWithRetries "$($testRun.url)/results?api-version=5.0" -Headers $azureDevOpsRestApiHeaders - $isTestRunNameShown = $false - - foreach ($testResult in $testResults.value) - { - $info = ConvertFrom-Json $testResult.comment - $helixJobId = $info.HelixJobId - $helixWorkItemName = $info.HelixWorkItemName - - $workItem = "$helixJobId-$helixWorkItemName" - - if (-not $workItems.Contains($workItem)) - { - $workItems.Add($workItem) - $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files?access_token=$HelixAccessToken" - $files = Invoke-RestMethodWithRetries $filesQueryUri - - $screenShots = $files | where { $_.Name.EndsWith(".jpg") } - $dumps = $files | where { $_.Name.EndsWith(".dmp") } - $visualTreeVerificationFiles = $files | where { $_.Name.EndsWith(".xml") -And (-Not $_.Name.Contains('testResults')) } - $pgcFiles = $files | where { $_.Name.EndsWith(".pgc") } - if ($screenShots.Count + $dumps.Count + $visualTreeVerificationFiles.Count + $pgcFiles.Count -gt 0) - { - if(-Not $isTestRunNameShown) - { - Out-File -FilePath $helixLinkFile -Append -InputObject "

$($testRun.name)

" - $isTestRunNameShown = $true - } - Out-File -FilePath $helixLinkFile -Append -InputObject "

$helixWorkItemName

" - Generate-File-Links $screenShots "Screenshots" - Generate-File-Links $dumps "CrashDumps" - Generate-File-Links $visualTreeVerificationFiles "visualTreeVerificationFiles" - Generate-File-Links $pgcFiles "PGC files" - $misc = $files | where { ($screenShots -NotContains $_) -And ($dumps -NotContains $_) -And ($visualTreeVerificationFiles -NotContains $_) -And ($pgcFiles -NotContains $_) } - Generate-File-Links $misc "Misc" - - if( -Not (Test-Path $visualTreeVerificationFolder) ) - { - New-Item $visualTreeVerificationFolder -ItemType Directory - } - foreach($verificationFile in $visualTreeVerificationFiles) - { - $destination = "$visualTreeVerificationFolder\$($verificationFile.Name)" - $fileurl = Append-HelixAccessTokenToUrl $verificationFile.Link $HelixAccessToken - try - { - Download-FileWithRetries $fileurl $destination - } - catch - { - Log-Error "Failed to download $($verificationFile.Name) to $destination : $($_.Exception.Message) -- URL: $($verificationFile.Link)" - } - } - - foreach($pgcFile in $pgcFiles) - { - $flavorPath = $pgcFile.Name.Split('.')[0] - $archPath = $pgcFile.Name.Split('.')[1] - $fileName = $pgcFile.Name.Remove(0, $flavorPath.length + $archPath.length + 2) - $fullPath = "$OutputFolder\PGO\$flavorPath\$archPath" - $destination = "$fullPath\$fileName" - - Write-Host "Copying $($pgcFile.Name) to $destination" - - if (-Not (Test-Path $fullPath)) - { - New-Item $fullPath -ItemType Directory - } - - $fileurl = Append-HelixAccessTokenToUrl $pgcFile.Link $HelixAccessToken - Download-FileWithRetries $fileurl $destination - } - } - } - } -} - -if(Test-Path $visualTreeVerificationFolder) -{ - $verificationFiles = Get-ChildItem $visualTreeVerificationFolder - $prefixList = @() - - foreach($file in $verificationFiles) - { - Write-Host "Copying $($file.Name) to $visualTreeVerificationFolder" - Move-Item $file.FullName "$visualTreeVerificationFolder\$($file.Name)" -Force - } -} diff --git a/build/Helix/RunTestsInHelix.proj b/build/Helix/RunTestsInHelix.proj index d925fd9c2a..08d914ea07 100644 --- a/build/Helix/RunTestsInHelix.proj +++ b/build/Helix/RunTestsInHelix.proj @@ -4,7 +4,7 @@ true true true - $(HelixPreCommands);set testnameprefix=$(Configuration).$(Platform);set testbuildplatform=$(Platform);set rerunPassesRequiredToAvoidFailure=$(rerunPassesRequiredToAvoidFailure) + $(HelixPreCommands);set testbuildplatform=$(Platform);set rerunPassesRequiredToAvoidFailure=$(rerunPassesRequiredToAvoidFailure) diff --git a/build/Helix/UpdateUnreliableTests.ps1 b/build/Helix/UpdateUnreliableTests.ps1 deleted file mode 100644 index 065e0903f1..0000000000 --- a/build/Helix/UpdateUnreliableTests.ps1 +++ /dev/null @@ -1,128 +0,0 @@ -[CmdLetBinding()] -Param( - [Parameter(Mandatory = $true)] - [int]$RerunPassesRequiredToAvoidFailure, - - [string]$AccessToken = $env:SYSTEM_ACCESSTOKEN, - [string]$CollectionUri = $env:SYSTEM_COLLECTIONURI, - [string]$TeamProject = $env:SYSTEM_TEAMPROJECT, - [string]$BuildUri = $env:BUILD_BUILDURI -) - -. "$PSScriptRoot/AzurePipelinesHelperScripts.ps1" - - -$azureDevOpsRestApiHeaders = @{ - "Accept"="application/json" - "Authorization"="Basic $([System.Convert]::ToBase64String([System.Text.ASCIIEncoding]::ASCII.GetBytes(":$AccessToken")))" -} - -$queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -Write-Host "queryUri = $queryUri" - -# To account for unreliable tests, we'll iterate through all of the tests associated with this build, check to see any tests that were unreliable -# (denoted by being marked as "skipped"), and if so, we'll instead mark those tests with a warning and enumerate all of the attempted runs -# with their pass/fail states as well as any relevant error messages for failed attempts. -$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders - -$timesSeenByRunName = @{} - -foreach ($testRun in $testRuns.value) -{ - $testRunResultsUri = "$($testRun.url)/results?api-version=5.0" - - Write-Host "Marking test run `"$($testRun.name)`" as in progress so we can change its results to account for unreliable tests." - Invoke-RestMethod "$($testRun.url)?api-version=5.0" -Method Patch -Body (ConvertTo-Json @{ "state" = "InProgress" }) -Headers $azureDevOpsRestApiHeaders -ContentType "application/json" | Out-Null - - Write-Host "Retrieving test results..." - $testResults = Invoke-RestMethodWithRetries $testRunResultsUri -Method Get -Headers $azureDevOpsRestApiHeaders - - foreach ($testResult in $testResults.value) - { - $testNeedsSubResultProcessing = $false - if ($testResult.outcome -eq "NotExecuted") - { - $testNeedsSubResultProcessing = $true - } - elseif($testResult.outcome -eq "Failed") - { - $testNeedsSubResultProcessing = $testResult.errorMessage -like "*_subresults.json*" - } - - if ($testNeedsSubResultProcessing) - { - Write-Host " Test $($testResult.testCaseTitle) was detected as unreliable. Updating..." - - # The errorMessage field contains a link to the JSON-encoded rerun result data. - $resultsJson = Download-StringWithRetries "Error results" $testResult.errorMessage - $rerunResults = ConvertFrom-Json $resultsJson - [System.Collections.Generic.List[System.Collections.Hashtable]]$rerunDataList = @() - $attemptCount = 0 - $passCount = 0 - $totalDuration = 0 - - foreach ($rerun in $rerunResults.results) - { - $rerunData = @{ - "displayName" = "Attempt #$($attemptCount + 1) - $($testResult.testCaseTitle)"; - "durationInMs" = $rerun.duration; - "outcome" = $rerun.outcome; - } - - if ($rerun.outcome -eq "Passed") - { - $passCount++ - } - - if ($attemptCount -gt 0) - { - $rerunData["sequenceId"] = $attemptCount - } - - Write-Host " Attempt #$($attemptCount + 1): $($rerun.outcome)" - - if ($rerun.outcome -ne "Passed") - { - # We subtract 1 from the error index because we added 1 so we could use 0 - # as a default value not injected into the JSON in order to keep its size down. - # We did this because there's a maximum size enforced for the errorMessage parameter - # in the Azure DevOps REST API. - $fullErrorMessage = @" -Log: $($rerunResults.blobPrefix)/$($rerun.log)$($rerunResults.blobSuffix) - -Error log: -$($rerunResults.errors[$rerun.errorIndex - 1]) -"@ - - $rerunData["errorMessage"] = $fullErrorMessage - } - - $attemptCount++ - $totalDuration += $rerun.duration - $rerunDataList.Add($rerunData) - } - - $overallOutcome = "Warning" - - if ($attemptCount -eq 2) - { - Write-Host " Test $($testResult.testCaseTitle) passed on the immediate rerun, so we'll mark it as unreliable." - } - elseif ($passCount -gt $RerunPassesRequiredToAvoidFailure) - { - Write-Host " Test $($testResult.testCaseTitle) passed on $passCount of $attemptCount attempts, which is greater than or equal to the $RerunPassesRequiredToAvoidFailure passes required to avoid being marked as failed. Marking as unreliable." - } - else - { - Write-Host " Test $($testResult.testCaseTitle) passed on only $passCount of $attemptCount attempts, which is less than the $RerunPassesRequiredToAvoidFailure passes required to avoid being marked as failed. Marking as failed." - $overallOutcome = "Failed" - } - - $updateBody = ConvertTo-Json @(@{ "id" = $testResult.id; "outcome" = $overallOutcome; "errorMessage" = " "; "durationInMs" = $totalDuration; "subResults" = $rerunDataList; "resultGroupType" = "rerun" }) -Depth 5 - Invoke-RestMethod -Uri $testRunResultsUri -Method Patch -Headers $azureDevOpsRestApiHeaders -Body $updateBody -ContentType "application/json" | Out-Null - } - } - - Write-Host "Finished updates. Re-marking test run `"$($testRun.name)`" as completed." - Invoke-RestMethod -Uri "$($testRun.url)?api-version=5.0" -Method Patch -Body (ConvertTo-Json @{ "state" = "Completed" }) -Headers $azureDevOpsRestApiHeaders -ContentType "application/json" | Out-Null -} diff --git a/build/Helix/packages.config b/build/Helix/packages.config index 0220f5be82..e52b99778d 100644 --- a/build/Helix/packages.config +++ b/build/Helix/packages.config @@ -5,4 +5,5 @@ + diff --git a/build/Helix/runtests.cmd b/build/Helix/runtests.cmd deleted file mode 100644 index 4c9f7d078f..0000000000 --- a/build/Helix/runtests.cmd +++ /dev/null @@ -1,106 +0,0 @@ -setlocal ENABLEDELAYEDEXPANSION - -echo %TIME% - -robocopy %HELIX_CORRELATION_PAYLOAD% . /s /NP > NUL - -echo %TIME% - -reg add HKLM\Software\Policies\Microsoft\Windows\Appx /v AllowAllTrustedApps /t REG_DWORD /d 1 /f - -rem enable dump collection for our test apps: -rem note, this script is run from a 32-bit cmd, but we need to set the native reg-key -FOR %%A IN (MUXControlsTestApp.exe,IXMPTestApp.exe,NugetPackageTestApp.exe,NugetPackageTestAppCX.exe,te.exe,te.processhost.exe,AppThatUsesMUXIndirectly.exe,WpfApp.exe) DO ( - %systemroot%\sysnative\cmd.exe /c reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\%%A" /v DumpFolder /t REG_EXPAND_SZ /d %HELIX_DUMP_FOLDER% /f - %systemroot%\sysnative\cmd.exe /c reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\%%A" /v DumpType /t REG_DWORD /d 2 /f - %systemroot%\sysnative\cmd.exe /c reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\%%A" /v DumpCount /t REG_DWORD /d 10 /f -) - -echo %TIME% - -:: kill dhandler, which is a tool designed to handle unexpected windows appearing. But since our tests are -:: expected to show UI we don't want it running. -taskkill -f -im dhandler.exe - -echo %TIME% -powershell -ExecutionPolicy Bypass .\EnsureMachineState.ps1 -echo %TIME% -powershell -ExecutionPolicy Bypass .\InstallTestAppDependencies.ps1 -echo %TIME% - -set testBinaryCandidates=MUXControls.Test.dll MUXControlsTestApp.appx IXMPTestApp.appx MUXControls.ReleaseTest.dll -set testBinaries= -for %%B in (%testBinaryCandidates%) do ( - if exist %%B ( - set "testBinaries=!testBinaries! %%B" - ) -) - -echo %TIME% -te.exe %testBinaries% /enablewttlogging /unicodeOutput:false /sessionTimeout:0:15 /testtimeout:0:10 /screenCaptureOnError %* -echo %TIME% - -powershell -ExecutionPolicy Bypass Get-Process - -move te.wtl te_original.wtl - -copy /y te_original.wtl %HELIX_WORKITEM_UPLOAD_ROOT% -copy /y WexLogFileOutput\*.jpg %HELIX_WORKITEM_UPLOAD_ROOT% -for /f "tokens=* delims=" %%a in ('dir /b *.pgc') do ren "%%a" "%testnameprefix%.%%~na.pgc" -copy /y *.pgc %HELIX_WORKITEM_UPLOAD_ROOT% - -set FailedTestQuery= -for /F "tokens=* usebackq" %%I IN (`powershell -ExecutionPolicy Bypass .\OutputFailedTestQuery.ps1 te_original.wtl`) DO ( - set FailedTestQuery=%%I -) - -rem The first time, we'll just re-run failed tests once. In many cases, tests fail very rarely, such that -rem a single re-run will be sufficient to detect many unreliable tests. -if "%FailedTestQuery%" == "" goto :SkipReruns - -echo %TIME% -te.exe %testBinaries% /enablewttlogging /unicodeOutput:false /sessionTimeout:0:15 /testtimeout:0:10 /screenCaptureOnError /select:"%FailedTestQuery%" -echo %TIME% - -move te.wtl te_rerun.wtl - -copy /y te_rerun.wtl %HELIX_WORKITEM_UPLOAD_ROOT% -copy /y WexLogFileOutput\*.jpg %HELIX_WORKITEM_UPLOAD_ROOT% - -rem If there are still failing tests remaining, we'll run them eight more times, so they'll have been run a total of ten times. -rem If any tests fail all ten times, we can be pretty confident that these are actual test failures rather than unreliable tests. -if not exist te_rerun.wtl goto :SkipReruns - -set FailedTestQuery= -for /F "tokens=* usebackq" %%I IN (`powershell -ExecutionPolicy Bypass .\OutputFailedTestQuery.ps1 te_rerun.wtl`) DO ( - set FailedTestQuery=%%I -) - -if "%FailedTestQuery%" == "" goto :SkipReruns - -echo %TIME% -te.exe %testBinaries% /enablewttlogging /unicodeOutput:false /sessionTimeout:0:15 /testtimeout:0:10 /screenCaptureOnError /testmode:Loop /LoopTest:8 /select:"%FailedTestQuery%" -echo %TIME% - -powershell -ExecutionPolicy Bypass Get-Process - -move te.wtl te_rerun_multiple.wtl - -copy /y te_rerun_multiple.wtl %HELIX_WORKITEM_UPLOAD_ROOT% -copy /y WexLogFileOutput\*.jpg %HELIX_WORKITEM_UPLOAD_ROOT% -powershell -ExecutionPolicy Bypass .\CopyVisualTreeVerificationFiles.ps1 - -:SkipReruns - -powershell -ExecutionPolicy Bypass Get-Process - -echo %TIME% -powershell -ExecutionPolicy Bypass .\OutputSubResultsJsonFiles.ps1 te_original.wtl te_rerun.wtl te_rerun_multiple.wtl %testnameprefix% -powershell -ExecutionPolicy Bypass .\ConvertWttLogToXUnit.ps1 te_original.wtl te_rerun.wtl te_rerun_multiple.wtl testResults.xml %testnameprefix% -echo %TIME% - -copy /y *_subresults.json %HELIX_WORKITEM_UPLOAD_ROOT% - -type testResults.xml - -echo %TIME% \ No newline at end of file diff --git a/build/Helix/CopyVisualTreeVerificationFiles.ps1 b/build/Helix/scripts/CopyVisualTreeVerificationFiles.ps1 similarity index 100% rename from build/Helix/CopyVisualTreeVerificationFiles.ps1 rename to build/Helix/scripts/CopyVisualTreeVerificationFiles.ps1 diff --git a/build/Helix/InstallTestAppDependencies.ps1 b/build/Helix/scripts/InstallTestAppDependencies.ps1 similarity index 100% rename from build/Helix/InstallTestAppDependencies.ps1 rename to build/Helix/scripts/InstallTestAppDependencies.ps1 diff --git a/build/Helix/scripts/TestPass-EnsureMachineState.ps1 b/build/Helix/scripts/TestPass-EnsureMachineState.ps1 new file mode 100644 index 0000000000..4f96f5e5d2 --- /dev/null +++ b/build/Helix/scripts/TestPass-EnsureMachineState.ps1 @@ -0,0 +1,24 @@ +Write-Host "TestPass-EnsureMachineState.ps1" + +# Kill any running test processes +$testProcessNames = @( + "MUXControlsTestApp", + "IXMPTestApp", + "NugetPackageTestApp", + "NugetPackageTestAppCX", + "te", + "te.processhost", + "AppThatUsesMUXIndirectly", + "TextInputHost", + "WpfApp") +foreach($testProcessName in $testProcessNames) +{ + $procs = Get-Process -Name $testProcessName -ErrorAction SilentlyContinue + foreach($proc in $procs) + { + Write-Host "Found running proc: $($proc.Name) - $($proc.Id)" + Write-Host "Killing Process" + Stop-Process $proc -Force + } +} + diff --git a/build/Helix/scripts/TestPass-PostRun.ps1 b/build/Helix/scripts/TestPass-PostRun.ps1 new file mode 100644 index 0000000000..4de35f3f30 --- /dev/null +++ b/build/Helix/scripts/TestPass-PostRun.ps1 @@ -0,0 +1,13 @@ +.\CopyVisualTreeVerificationFiles.ps1 + +$uploadRoot = $env:HELIX_WORKITEM_UPLOAD_ROOT +$testnameprefix = $env:testnameprefix + +$pgcFiles = Get-ChildItem $uploadRoot | where { $_.Name.EndsWith(".pgc") } +foreach($pgcFile in $pgcFiles) +{ + $newName = "$testnameprefix.$($pgcFile.Name)" + $newPath = Join-Path $pgcFile.Directory $newName + Write-Host "Move-Item $($pgcFile.FullName) $newPath" + Move-Item $pgcFile.FullName $newPath +} \ No newline at end of file diff --git a/build/Helix/scripts/TestPass-PreRun.ps1 b/build/Helix/scripts/TestPass-PreRun.ps1 new file mode 100644 index 0000000000..9f296d9d6e --- /dev/null +++ b/build/Helix/scripts/TestPass-PreRun.ps1 @@ -0,0 +1,88 @@ +Write-Host "TestPass-PreRun.ps1" + +$Platform = $env:testbuildplatform +if(!$Platform) +{ + $Platform = "x86" +} + +function UninstallTestApps { + Param([string[]]$appsToUninstall) + + foreach($pkgName in $appsToUninstall) + { + foreach($pkg in (Get-AppxPackage $pkgName).PackageFullName) + { + Write-Output "Removing: $pkg" + Remove-AppxPackage $pkg + } + + # Sometimes an app can get into a state where it is no longer returned by Get-AppxPackage, but it is still present + # which prevents other versions of the app from being installed. + # To handle this, we can directly call Remove-AppxPackage against the full name of the package. However, without + # Get-AppxPackage to find the PackageFullName, we just have to manually construct the name. + $packageFullName = "$($pkgName)_1.0.0.0_$($Platform)__8wekyb3d8bbwe" + Write-Host "Removing $packageFullName if installed" + Remove-AppxPackage $packageFullName -ErrorVariable appxerror -ErrorAction SilentlyContinue + if($appxerror) + { + foreach($error in $appxerror) + { + # In most cases, Remove-AppPackage will fail due to the package not being found. Don't treat this as an error. + if(!($error.Exception.Message -match "0x80073CF1")) + { + Write-Error $error + } + } + } + else + { + Write-Host "Sucessfully removed $packageFullName" + } + } +} + +Write-Host "Uninstall any of our test apps that may have been left over from previous test runs" +UninstallTestApps("NugetPackageTestApp", "NugetPackageTestAppCX", "IXMPTestApp", "MUXControlsTestApp") + +Write-Host "Uninstall MUX Framework package that may have been left over from previous test runs" +# We don't want to uninstall all versions of the MUX Framework package, as there may be other apps preinstalled on the system +# that depend on it. We only uninstall the Framework package that corresponds to the version of MUX that we are testing. +[xml]$versionData = (Get-Content "version.props") +$versionMajor = $versionData.GetElementsByTagName("MUXVersionMajor").'#text' +$versionMinor = $versionData.GetElementsByTagName("MUXVersionMinor").'#text' +UninstallTestApps("Microsoft.UI.Xaml.$versionMajor.$versionMinor") + + +.\InstallTestAppDependencies.ps1 + + + +# If we set the registry from a 32-bit process on a 64-bit machine, we will set the "virtualized" syswow registry. +# For crash dump collection we always want to set the "native" registry, so we make sure to invoke the native cmd.exe +$nativeCmdPath = "$env:SystemRoot\system32\cmd.exe" +if([Environment]::Is64BitOperatingSystem -and ![Environment]::Is64BitProcess) +{ + # The "sysnative" path is a 'magic' path that allows a 32-bit process to invoke the native 64-bit cmd.exe. + $nativeCmdPath = "$env:SystemRoot\sysnative\cmd.exe" +} + +$dumpFolder = $env:HELIX_DUMP_FOLDER +if(!$dumpFolder) +{ + $dumpFolder = "C:\dumps" +} + +function Enable-CrashDumpsForProcesses { + Param([string[]]$namesOfProcessesForDumpCollection) + + foreach($procName in $namesOfProcessesForDumpCollection ) + { + Write-Host "Enabling local crash dumps for $procName" + & $nativeCmdPath /c reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\$procName" /v DumpFolder /t REG_EXPAND_SZ /d $dumpFolder /f + & $nativeCmdPath /c reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\$procName" /v DumpType /t REG_DWORD /d 2 /f + & $nativeCmdPath /c reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\$procName" /v DumpCount /t REG_DWORD /d 3 /f + } +} + +Enable-CrashDumpsForProcesses @("MUXControlsTestApp.exe,IXMPTestApp.exe","NugetPackageTestApp.exe","NugetPackageTestAppCX.exe","AppThatUsesMUXIndirectly.exe","WpfApp.exe") \ No newline at end of file diff --git a/test/IXMPTestApp/WaitForDebugger.cs b/test/IXMPTestApp/WaitForDebugger.cs index 6dbc2da20e..c4d4bc4879 100644 --- a/test/IXMPTestApp/WaitForDebugger.cs +++ b/test/IXMPTestApp/WaitForDebugger.cs @@ -26,6 +26,7 @@ namespace MUXControlsTestApp public class WaitForDebugger { [AssemblyInitialize] + [TestProperty("HelixWorkItemCreation", "CreateWorkItemPerTestClass")] public static void AssemblyInitialize(TestContext testContext) { if (testContext.Properties.Contains("WaitForDebugger") || testContext.Properties.Contains("WaitForAppDebugger")) diff --git a/test/MUXControls.Test/TestAssembly.cs b/test/MUXControls.Test/TestAssembly.cs index d292ffe2be..44308acbe6 100644 --- a/test/MUXControls.Test/TestAssembly.cs +++ b/test/MUXControls.Test/TestAssembly.cs @@ -40,6 +40,7 @@ public class TestAssembly [TestProperty("CoreClrProfile", ".NETCoreApp2.1")] [TestProperty("RunFixtureAs:Assembly", "ElevatedUserOrSystem")] [TestProperty("Hosting:Mode", "UAP")] + [TestProperty("HelixWorkItemCreation", "CreateWorkItemPerTestClass")] public static void AssemblyInitialize(TestContext testContext) { TestEnvironment.AssemblyInitialize(testContext, "MUXControlsTestApp.cer"); diff --git a/test/MUXControlsReleaseTest/MUXControls.ReleaseTest/ReleaseTestEnvironment.cs b/test/MUXControlsReleaseTest/MUXControls.ReleaseTest/ReleaseTestEnvironment.cs index df9ed0afa8..d503c00dc6 100644 --- a/test/MUXControlsReleaseTest/MUXControls.ReleaseTest/ReleaseTestEnvironment.cs +++ b/test/MUXControlsReleaseTest/MUXControls.ReleaseTest/ReleaseTestEnvironment.cs @@ -23,6 +23,7 @@ public class ReleaseTestEnvironment [AssemblyInitialize] [TestProperty("CoreClrProfile", ".NETCoreApp2.1")] [TestProperty("RunFixtureAs:Assembly", "ElevatedUserOrSystem")] + [TestProperty("HelixWorkItemCreation", "CreateWorkItemPerTestClass")] public static void AssemblyInitialize(TestContext testContext) { TestEnvironment.AssemblyInitialize(testContext, TestApplicationInfo.NugetPackageTestAppCX.TestAppPackageName + ".cer"); diff --git a/test/MUXControlsTestApp/WaitForDebugger.cs b/test/MUXControlsTestApp/WaitForDebugger.cs index f413fb5539..ac74edb956 100644 --- a/test/MUXControlsTestApp/WaitForDebugger.cs +++ b/test/MUXControlsTestApp/WaitForDebugger.cs @@ -23,6 +23,7 @@ public class WaitForDebugger { [AssemblyInitialize] [TestProperty("Classification", "Integration")] + [TestProperty("HelixWorkItemCreation", "CreateWorkItemPerTestClass")] public static void AssemblyInitialize(TestContext testContext) { if (testContext.Properties.Contains("WaitForDebugger") || testContext.Properties.Contains("WaitForAppDebugger"))