diff --git a/.azure-pipelines/live-test.yml b/.azure-pipelines/live-test.yml index 90d1d9bfefa3..31a8ee275b84 100644 --- a/.azure-pipelines/live-test.yml +++ b/.azure-pipelines/live-test.yml @@ -1,17 +1,17 @@ parameters: -- name: img_windows_2019 +- name: os_windows_2019 displayName: Windows 2019 Image Version type: string default: windows-2019 -- name: img_windows_2022 +- name: os_windows_2022 displayName: Windows 2022 Image Version type: string default: windows-2022 -- name: img_linux +- name: os_linux displayName: Linux Image Version type: string default: ubuntu-20.04 -- name: img_macos +- name: os_macos displayName: MacOS Image Version type: string default: macOS-11 @@ -55,115 +55,133 @@ schedules: pr: none trigger: none -jobs: -- template: util/live-test-steps.yml - parameters: - name: 'win_ps_5_1_win_2019' - vmImage: ${{ parameters.img_windows_2019 }} - osType: 'Windows' - psVersion: ${{ parameters.win_ps_5_1 }} - dotnetVersion: ${{ parameters.dotnet_sdk_6 }} - -- template: util/live-test-steps.yml - parameters: - name: 'win_ps_5_1_win_2022' - vmImage: ${{ parameters.img_windows_2022 }} - osType: 'Windows' - psVersion: ${{ parameters.win_ps_5_1 }} - dotnetVersion: ${{ parameters.dotnet_sdk_6 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_2_x_win_2019' - vmImage: ${{ parameters.img_windows_2019 }} - osType: 'Windows' - psVersion: ${{ parameters.ps_7_2_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_6 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_3_x_win_2019' - vmImage: ${{ parameters.img_windows_2019 }} - osType: 'Windows' - psVersion: ${{ parameters.ps_7_3_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_latest_win_2019' - vmImage: ${{ parameters.img_windows_2019 }} - osType: 'Windows' - psVersion: ${{ parameters.ps_latest }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_2_x_win_2022' - vmImage: ${{ parameters.img_windows_2022 }} - osType: 'Windows' - psVersion: ${{ parameters.ps_7_2_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_6 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_3_x_win_2022' - vmImage: ${{ parameters.img_windows_2022 }} - osType: 'Windows' - psVersion: ${{ parameters.ps_7_3_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_latest_win_2022' - vmImage: ${{ parameters.img_windows_2022 }} - osType: 'Windows' - psVersion: ${{ parameters.ps_latest }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_2_x_linux' - vmImage: ${{ parameters.img_linux }} - osType: 'Linux' - psVersion: ${{ parameters.ps_7_2_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_6 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_3_x_linux' - vmImage: ${{ parameters.img_linux }} - osType: 'Linux' - psVersion: ${{ parameters.ps_7_3_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_latest_linux' - vmImage: ${{ parameters.img_linux }} - osType: 'Linux' - psVersion: ${{ parameters.ps_latest }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_2_x_macOS' - vmImage: ${{ parameters.img_macos }} - osType: 'MacOS' - psVersion: ${{ parameters.ps_7_2_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_6 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_7_3_x_macOS' - vmImage: ${{ parameters.img_macos }} - osType: 'MacOS' - psVersion: ${{ parameters.ps_7_3_x }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} - -- template: util/live-test-steps.yml - parameters: - name: 'ps_latest_macOS' - vmImage: ${{ parameters.img_macos }} - osType: 'MacOS' - psVersion: ${{ parameters.ps_latest }} - dotnetVersion: ${{ parameters.dotnet_sdk_7 }} +stages: +- stage: Test + displayName: Run Live Test + jobs: + - template: util/live-test-steps.yml + parameters: + name: 'win_ps_5_1_win_2019' + vmImage: ${{ parameters.os_windows_2019 }} + osType: 'Windows' + psVersion: ${{ parameters.win_ps_5_1 }} + dotnetVersion: ${{ parameters.dotnet_sdk_6 }} + + - template: util/live-test-steps.yml + parameters: + name: 'win_ps_5_1_win_2022' + vmImage: ${{ parameters.os_windows_2022 }} + osType: 'Windows' + psVersion: ${{ parameters.win_ps_5_1 }} + dotnetVersion: ${{ parameters.dotnet_sdk_6 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_2_x_win_2019' + vmImage: ${{ parameters.os_windows_2019 }} + osType: 'Windows' + psVersion: ${{ parameters.ps_7_2_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_6 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_3_x_win_2019' + vmImage: ${{ parameters.os_windows_2019 }} + osType: 'Windows' + psVersion: ${{ parameters.ps_7_3_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_latest_win_2019' + vmImage: ${{ parameters.os_windows_2019 }} + osType: 'Windows' + psVersion: ${{ parameters.ps_latest }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_2_x_win_2022' + vmImage: ${{ parameters.os_windows_2022 }} + osType: 'Windows' + psVersion: ${{ parameters.ps_7_2_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_6 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_3_x_win_2022' + vmImage: ${{ parameters.os_windows_2022 }} + osType: 'Windows' + psVersion: ${{ parameters.ps_7_3_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_latest_win_2022' + vmImage: ${{ parameters.os_windows_2022 }} + osType: 'Windows' + psVersion: ${{ parameters.ps_latest }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_2_x_linux' + vmImage: ${{ parameters.os_linux }} + osType: 'Linux' + psVersion: ${{ parameters.ps_7_2_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_6 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_3_x_linux' + vmImage: ${{ parameters.os_linux }} + osType: 'Linux' + psVersion: ${{ parameters.ps_7_3_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_latest_linux' + vmImage: ${{ parameters.os_linux }} + osType: 'Linux' + psVersion: ${{ parameters.ps_latest }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_2_x_macOS' + vmImage: ${{ parameters.os_macos }} + osType: 'MacOS' + psVersion: ${{ parameters.ps_7_2_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_6 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_7_3_x_macOS' + vmImage: ${{ parameters.os_macos }} + osType: 'MacOS' + psVersion: ${{ parameters.ps_7_3_x }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + + - template: util/live-test-steps.yml + parameters: + name: 'ps_latest_macOS' + vmImage: ${{ parameters.os_macos }} + osType: 'MacOS' + psVersion: ${{ parameters.ps_latest }} + dotnetVersion: ${{ parameters.dotnet_sdk_7 }} + +- stage: Report + displayName: Send Report + dependsOn: Test + jobs: + - job: + displayName: Send Live Test Status Report + steps: + - task: PowerShell@2 + displayName: Send live test report + inputs: + pwsh: true + targetType: filePath + filePath: ./tools/TestFx/Live/SendLiveTestReport.ps1 + arguments: $(KustoServicePrincipalTenantId) $(KustoServicePrincipalId) $(KustoServicePrincipalSecret) $(KustoClusterName) $(KustoClusterRegion) $(LiveTestDatabaseName) $(LiveTestTableName) $(Build.BuildId) "$(EmailServiceConnectionString)" "$(EmailFrom)" "$(EmailTo)" diff --git a/.azure-pipelines/util/live-test-steps.yml b/.azure-pipelines/util/live-test-steps.yml index 45b9f36f7649..190d369dd0d5 100644 --- a/.azure-pipelines/util/live-test-steps.yml +++ b/.azure-pipelines/util/live-test-steps.yml @@ -120,6 +120,7 @@ jobs: targetType: filePath filePath: ./tools/TestFx/Live/InvokeLiveTestCITask.ps1 arguments: -UseWindowsPowerShell ('${{ parameters.psVersion }}' -eq '5.1') -ScriptFile './tools/TestFx/Live/InvokeLiveTestScenarios.ps1 -RunPlatform ${{ parameters.osType }} -RunPowerShell ${{ parameters.psVersion }} -PowerShellLatest $(PowerShellLatest) -RepoLocation $(Build.SourcesDirectory) -DataLocation $(DataLocation)' + retryCountOnTaskFailure: 3 - task: PowerShell@2 displayName: Save live test results to Kusto @@ -135,6 +136,6 @@ jobs: - task: PublishPipelineArtifact@1 displayName: Publish live test results to pipeline artifact inputs: - artifact: livetest-os-${{ parameters.vmImage }}-powershell-${{ parameters.name }} + artifact: livetest-os-${{ parameters.vmImage }}-ps-${{ replace(parameters.psVersion, '*', 'x') }} targetPath: $(DataLocation) condition: always() diff --git a/src/BotService/LiveTests/TestLiveScenarios_disabled.ps1 b/src/BotService/LiveTests/TestLiveScenarios.disabled.ps1 similarity index 100% rename from src/BotService/LiveTests/TestLiveScenarios_disabled.ps1 rename to src/BotService/LiveTests/TestLiveScenarios.disabled.ps1 diff --git a/tools/TestFx/Live/SaveLiveTestResult.ps1 b/tools/TestFx/Live/SaveLiveTestResult.ps1 index 28212365268f..a32d1e122ee0 100644 --- a/tools/TestFx/Live/SaveLiveTestResult.ps1 +++ b/tools/TestFx/Live/SaveLiveTestResult.ps1 @@ -69,16 +69,8 @@ if (![string]::IsNullOrWhiteSpace($liveTestResults)) { Export-Csv -Path $_ -Encoding utf8 -NoTypeInformation -Force } - Import-Module "./tools/TestFx/Utilities/KustoUtility.psd1" -Force - Import-KustoDataFromCsv ` - -ServicePrincipalTenantId $ServicePrincipalTenantId ` - -ServicePrincipalId $ServicePrincipalId ` - -ServicePrincipalSecret $ServicePrincipalSecret ` - -ClusterName $ClusterName ` - -ClusterRegion $ClusterRegion ` - -DatabaseName $DatabaseName ` - -TableName $TableName ` - -CsvFile $liveTestResults + Import-Module "./tools/TestFx/Utilities/KustoUtility.psd1" -ArgumentList $ServicePrincipalTenantId, $ServicePrincipalId, $ServicePrincipalSecret, $ClusterName, $ClusterRegion -Force + Import-KustoDataFromCsv -DatabaseName $DatabaseName -TableName $TableName -CsvFile $liveTestResults } else { Write-Host "##[warning]No live test data was found." diff --git a/tools/TestFx/Live/SendLiveTestReport.ps1 b/tools/TestFx/Live/SendLiveTestReport.ps1 new file mode 100644 index 000000000000..934594190a8c --- /dev/null +++ b/tools/TestFx/Live/SendLiveTestReport.ps1 @@ -0,0 +1,97 @@ +param ( + [Parameter(Mandatory, Position = 0)] + [ValidateNotNullOrEmpty()] + [guid] $ServicePrincipalTenantId, + + [Parameter(Mandatory, Position = 1)] + [ValidateNotNullOrEmpty()] + [guid] $ServicePrincipalId, + + [Parameter(Mandatory, Position = 2)] + [ValidateNotNullOrEmpty()] + [string] $ServicePrincipalSecret, + + [Parameter(Mandatory, Position = 3)] + [ValidateNotNullOrEmpty()] + [string] $ClusterName, + + [Parameter(Mandatory, Position = 4)] + [ValidateNotNullOrEmpty()] + [string] $ClusterRegion, + + [Parameter(Mandatory, Position = 5)] + [ValidateNotNullOrEmpty()] + [string] $DatabaseName, + + [Parameter(Mandatory, Position = 6)] + [ValidateNotNullOrEmpty()] + [string] $TableName, + + [Parameter(Mandatory, Position = 7)] + [ValidateNotNullOrEmpty()] + [string] $BuildId, + + [Parameter(Mandatory, Position = 8)] + [ValidateNotNullOrEmpty()] + [string] $EmailServiceConnectionString, + + [Parameter(Mandatory, Position = 9)] + [ValidateNotNullOrEmpty()] + [string] $EmailFrom, + + [Parameter(Mandatory, Position = 10)] + [ValidateNotNullOrEmpty()] + [string] $EmailTo +) + +$utilDir = Join-Path -Path ($PSScriptRoot | Split-Path) -ChildPath "Utilities" + +$kustoUtil = $utilDir | Join-Path -ChildPath "KustoUtility.psd1" +Import-Module $kustoUtil -ArgumentList $ServicePrincipalTenantId, $ServicePrincipalId, $ServicePrincipalSecret, $ClusterName, $ClusterRegion -Force + +$query = @" + $TableName + | where BuildId == $BuildId and IsSuccess == false + | project BuildId, OSVersion, PSVersion, Module, Name, Exception = tostring(Errors["Exception"]), RetryException = tostring(Errors["Retry3Exception"]) +"@ +$errors = Get-KustoQueryData -DatabaseName $DatabaseName -Query $query +$errors + +$emailSvcUtil = $utilDir | Join-Path -ChildPath "EmailServiceUtility.psd1" +Import-Module $emailSvcUtil -ArgumentList $EmailServiceConnectionString, $EmailFrom -Force + +$css = @" + +"@ + +$emailSubject = "Live Test Status Report" + +if ($errors.Count -gt 0) { + $emailBody = $errors | ConvertTo-Html -Property BuildId, OSVersion, PSVersion, Module, Name, Exception, RetryException -Head $css -Title "Live Test Report" -PreContent "