feat: Use separate pipeline tasks for tests

build/ci-build.yml

@@ -18,6 +18,39 @@ parameters:
   - name: 'Package.Version.ManualTrigger'
     type: string
     default: 'preview'
+  - name: 'UnitTests'
+    type: object
+    default: 
+    - name: ActiveDirectory
+    - name: ApiManagement
+    - name: AppService
+    - name: DataFactory
+    - name: DevOps
+    - name: IntegrationAccount
+    - name: KeyVault
+    - name: LogicApps
+    - name: Management
+    - name: Security
+    - name: Sql
+    - name: Storage.Blob
+    - name: Storage.FileShare
+    - name: Storage.Table
+  - name: 'IntegrationTests'
+    type: object
+    default: 
+    - name: ActiveDirectory
+    - name: AppService
+    - name: ARM
+    - name: DevOps
+    - name: IntegrationAccount
+    - name: KeyVault
+    - name: LogicApps
+    - name: Management
+    - name: Security
+    - name: Sql
+    - name: Storage.Blob
+    - name: Storage.FileShare
+    - name: Storage.Table
 
 resources:
   repositories:
@@ -63,8 +96,9 @@ stages:
     dependsOn: Build
     condition: succeeded()
     jobs:
-      - job: RunUnitTests
-        displayName: 'Run unit tests'
+    - ${{ each UnitTest in parameters.UnitTests }}:
+      - job: RunUnitTests_${{replace(UnitTest.name, '.', '_')}}
+        displayName: 'Run unit tests - ${{UnitTest.name}}'
         pool:
           vmImage: '$(Vm.Linux.Image)'
         steps:
@@ -76,14 +110,16 @@ stages:
           - template: 'templates/run-pester-tests.yml'
             parameters:
               projectName: '$(Project).Tests.Unit'
+              testName: '$(Project).${{UnitTest.name}}'
 
   - stage: IntegrationTests
     displayName: Integration Tests
     dependsOn: Build
     condition: succeeded()
     jobs:
-      - job: RunIntegrationTests
-        displayName: 'Run integration tests'
+    - ${{ each IntegrationTest in parameters.IntegrationTests }}:
+      - job: RunIntegrationTests_${{replace(IntegrationTest.name, '.', '_')}}
+        displayName: 'Run integration tests - ${{IntegrationTest.name}}'
         strategy:
           maxParallel: 1
           matrix:
@@ -102,6 +138,7 @@ stages:
           - template: 'templates/run-pester-tests.yml'
             parameters:
               projectName: '$(Project).Tests.Integration'
+              testName: '$(Project).${{IntegrationTest.name}}'
 
   - stage: ReleaseToMyget
     displayName: 'Release to MyGet'

build/psgallery-release.yml

@@ -10,6 +10,39 @@ parameters:
     displayName: 'Prerelease string (ex. -alpha, -alpha1, -BETA, -update20171020) or none'
     type: 'string'
     default: 'none'
+  - name: 'UnitTests'
+    type: object
+    default: 
+    - name: ActiveDirectory
+    - name: ApiManagement
+    - name: AppService
+    - name: DataFactory
+    - name: DevOps
+    - name: IntegrationAccount
+    - name: KeyVault
+    - name: LogicApps
+    - name: Management
+    - name: Security
+    - name: Sql
+    - name: Storage.Blob
+    - name: Storage.FileShare
+    - name: Storage.Table
+  - name: 'IntegrationTests'
+    type: object
+    default: 
+    - name: ActiveDirectory
+    - name: AppService
+    - name: ARM
+    - name: DevOps
+    - name: IntegrationAccount
+    - name: KeyVault
+    - name: LogicApps
+    - name: Management
+    - name: Security
+    - name: Sql
+    - name: Storage.Blob
+    - name: Storage.FileShare
+    - name: Storage.Table
 
 resources:
   repositories:
@@ -56,8 +89,9 @@ stages:
     dependsOn: Build
     condition: succeeded()
     jobs:
-      - job: RunUnitTests
-        displayName: 'Run unit tests'
+    - ${{ each UnitTest in parameters.UnitTests }}:
+      - job: RunUnitTests_${{replace(UnitTest.name, '.', '_')}}
+        displayName: 'Run unit tests - ${{UnitTest.name}}'
         pool:
           vmImage: '$(Vm.Linux.Image)'
         steps:
@@ -69,14 +103,16 @@ stages:
           - template: 'templates/run-pester-tests.yml'
             parameters:
               projectName: '$(Project).Tests.Unit'
+              testName: '$(Project).${{UnitTest.name}}'
 
   - stage: IntegrationTests
     displayName: Integration Tests
     dependsOn: Build
     condition: succeeded()
     jobs:
-      - job: RunIntegrationTests
-        displayName: 'Run integration tests'
+    - ${{ each IntegrationTest in parameters.IntegrationTests }}:
+      - job: RunIntegrationTests_${{replace(IntegrationTest.name, '.', '_')}}
+        displayName: 'Run integration tests - ${{IntegrationTest.name}}'
         strategy:
           maxParallel: 1
           matrix:
@@ -95,6 +131,7 @@ stages:
           - template: 'templates/run-pester-tests.yml'
             parameters:
               projectName: '$(Project).Tests.Integration'
+              testName: '$(Project).${{IntegrationTest.name}}'
 
   - stage: Release
     displayName: 'Release to PowerShell Gallery'

build/templates/run-pester-tests.yml

@@ -1,5 +1,6 @@
 parameters:
   projectName: ''
+  testName: ''
 
 steps:
   - pwsh: 'wget -O - https://aka.ms/install-powershell.sh | sudo bash'
@@ -48,7 +49,7 @@ steps:
       Get-ChildItem -Path ./src -Filter *.psm1 -Recurse -Exclude "*Arcus.Scripting.Security*", "*.All.psm1" |
         % { Write-Host "Import $($_.DirectoryName) module"
             Import-Module -Name $_.DirectoryName -ErrorAction Stop }
-      Invoke-Pester -Script "./src/${{ parameters.projectName }}/" -OutputFile "./pester.test.results.xml" -OutputFormat 'NUnitXML' -EnableExit
+      Invoke-Pester -Script "./src/${{ parameters.projectName }}/${{ parameters.testName }}.tests.ps1" -OutputFile "./pester.test.results.${{ parameters.testName }}.xml" -OutputFormat 'NUnitXML' -EnableExit
     displayName: 'Run Pester tests'
     failOnStderr: true
     env:

src/Arcus.Scripting.Tests.Unit/Arcus.Scripting.ActiveDirectory.ps1 → src/Arcus.Scripting.Tests.Unit/Arcus.Scripting.ActiveDirectory.tests.ps1

@@ -109,6 +109,8 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
 
                 Mock Write-Host {}
 
+                Mock Write-Warning {}
+
                 Mock Get-AzADApplication {
                     $Filter | Should -Be "AppId eq '$ClientId'"
                     return [pscustomobject] @{
@@ -138,7 +140,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 Assert-MockCalled Get-AzADServicePrincipal -Times 1
                 Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Found role '$RoleName' on Active Directory Application '$AppName'" }
                 Assert-MockCalled Get-MgServicePrincipalAppRoleAssignedTo -Times 1
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "No role assignments found" }
+                Assert-MockCalled Write-Warning -Exactly 1 -ParameterFilter { $Message -eq "No role assignments found in Active Directory Application '$AppName'" }
             }
             It "Providing a ClientId that has roles and also assignments should succeed" {
                 # Arrange
@@ -305,7 +307,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 $AppName = 'SomeApp'
                 $AssignRoleToAppName = 'SomeAppToAssignRoleTo'
 
-                Mock Write-Host {}
+                Mock Write-Warning {}
 
                 Mock Get-AzADApplication {
                     if ($Filter -eq "AppId eq '$ClientId'") {
@@ -346,9 +348,9 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 Assert-VerifiableMock
                 Assert-MockCalled Get-AzADApplication -Times 2
                 Assert-MockCalled Get-AzADServicePrincipal -Times 2
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Active Directory Application '$AppName' already contains the role '$RoleName'" }
+                Assert-MockCalled Write-Warning -Exactly 1 -ParameterFilter { $Message -eq "Active Directory Application '$AppName' already contains the role '$RoleName'" }
                 Assert-MockCalled Get-MgServicePrincipalAppRoleAssignedTo -Times 1
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Active Directory Application '$AssignRoleToAppName' already contains a role assignment for the role '$RoleName'" }
+                Assert-MockCalled Write-Warning -Exactly 1 -ParameterFilter { $Message -eq "Active Directory Application '$AssignRoleToAppName' already contains a role assignment for the role '$RoleName'" }
             }
             It "Providing a role that already exists and does not have a role assignment should succeed" {
                 # Arrange
@@ -361,6 +363,8 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
 
                 Mock Write-Host {}
 
+                Mock Write-Warning {}
+
                 Mock Get-AzADApplication {
                     if ($Filter -eq "AppId eq '$ClientId'") {
                         return [pscustomobject] @{
@@ -413,7 +417,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 Assert-VerifiableMock
                 Assert-MockCalled Get-AzADApplication -Times 2
                 Assert-MockCalled Get-AzADServicePrincipal -Times 2
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Active Directory Application '$AppName' already contains the role '$RoleName'" }
+                Assert-MockCalled Write-Warning -Exactly 1 -ParameterFilter { $Message -eq "Active Directory Application '$AppName' already contains the role '$RoleName'" }
                 Assert-MockCalled Get-MgServicePrincipalAppRoleAssignedTo -Times 1
                 Assert-MockCalled Get-MgServicePrincipal -Times 1
                 Assert-MockCalled New-MgServicePrincipalAppRoleAssignment -Times 1
@@ -485,7 +489,6 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 Assert-VerifiableMock
                 Assert-MockCalled Get-AzADApplication -Times 2
                 Assert-MockCalled Get-AzADServicePrincipal -Times 2
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Active Directory Application '$AppName' does not contain the role '$RoleName', adding the role" }
                 Assert-MockCalled Update-AzADApplication -Times 1
                 Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Added Role '$RoleName' to Active Directory Application '$AppName'" }
                 Assert-MockCalled Get-MgServicePrincipalAppRoleAssignedTo -Times 1
@@ -605,7 +608,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 $RemoveRoleFromClientId = '9876'
                 $AppName = 'SomeApp'
 
-                Mock Write-Host {}
+                Mock Write-Warning {}
 
                 Mock Get-AzADApplication {
                     if ($Filter -eq "AppId eq '$ClientId'") {
@@ -630,7 +633,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 Assert-VerifiableMock
                 Assert-MockCalled Get-AzADApplication -Times 2
                 Assert-MockCalled Get-AzADServicePrincipal -Times 2
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Active Directory Application '$AppName' does not contain the role '$RoleName', skipping removal" }
+                Assert-MockCalled Write-Warning -Exactly 1 -ParameterFilter { $Message -eq "Active Directory Application '$AppName' does not contain the role '$RoleName', skipping removal" }
             }
             It "Providing a role that has no role assignment on the Active Directory Application it should be removed from should succeed" {
                 # Arrange
@@ -641,7 +644,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 $AppName = 'SomeApp'
                 $RemoveRoleFromAppName = 'SomeAppToRemoveRoleFrom'
 
-                Mock Write-Host {}
+                Mock Write-Warning {}
 
                 Mock Get-AzADApplication {
                     if ($Filter -eq "AppId eq '$ClientId'") {
@@ -680,7 +683,7 @@ InModuleScope Arcus.Scripting.ActiveDirectory {
                 Assert-MockCalled Get-AzADApplication -Times 2
                 Assert-MockCalled Get-AzADServicePrincipal -Times 2
                 Assert-MockCalled Get-MgServicePrincipalAppRoleAssignedTo -Times 1
-                Assert-MockCalled Write-Host -Exactly 1 -ParameterFilter { $Object -eq "Role '$RoleName' is not assigned to Active Directory Application '$RemoveRoleFromAppName', skipping role assignment removal" }
+                Assert-MockCalled Write-Warning -Exactly 1 -ParameterFilter { $Message -eq "Role '$RoleName' is not assigned to Active Directory Application '$RemoveRoleFromAppName', skipping role assignment removal" }
             }
             It "Providing a role that has a role assignment on the Active Directory Application it should be removed from should succeed" {
                 # Arrange

src/Arcus.Scripting.Tests.Unit/Arcus.Scripting.Tests.Unit.pssproj

@@ -76,7 +76,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="Arcus.Scripting.ActiveDirectory.ps1" />
+    <Compile Include="Arcus.Scripting.ActiveDirectory.tests.ps1" />
     <Compile Include="Arcus.Scripting.LogicApps.tests.ps1" />
     <Compile Include="Arcus.Scripting.Management.tests.ps1" />
     <Compile Include="Arcus.Scripting.ApiManagement.tests.ps1" />