Onboard to code coverage in ADO (#2798)

* Onboard to code coverage * Test stability improvements * Update test script for code coverage --------- Co-authored-by: Keegan Caruso <[email protected]>
AzureAD · Aug 30, 2024 · 42ec510 · 42ec510
1 parent ff63778
commit 42ec510
Show file tree

Hide file tree

Showing 8 changed files with 74 additions and 26 deletions.
diff --git a/azurepipelines-coverage.yml b/azurepipelines-coverage.yml
@@ -0,0 +1,5 @@
+coverage:
+  status:           # Code coverage status will be posted to pull requests based on targets defined below.
+    comments: on    # Off by default. When on, details about coverage for each file changed will be posted as a pull request comment.
+    diff:           # diff coverage is code coverage only for the lines changed in a pull request.
+      target: 75%   # set this to a desired %.
diff --git a/build/CodeCoverage.runsettings b/build/CodeCoverage.runsettings
@@ -7,16 +7,17 @@
 <RunSettings>
     <DataCollectionRunSettings>
         <DataCollectors>
-            <DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0" assemblyQualifiedName="Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector, Microsoft.VisualStudio.TraceCollector, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+            <DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0">
                 <Configuration>
+                    <Format>Cobertura</Format>
                     <CodeCoverage>
 
-                        <!--  
-About include/exclude lists:  
-Empty "Include" clauses imply all; empty "Exclude" clauses imply none.  
-Each element in the list is a regular expression (ECMAScript syntax). See http://msdn.microsoft.com/library/2k3te2cs.aspx.  
-An item must first match at least one entry in the include list to be included.  
-Included items must then not match any entries in the exclude list to remain included.  
+                        <!--
+About include/exclude lists:
+Empty "Include" clauses imply all; empty "Exclude" clauses imply none.
+Each element in the list is a regular expression (ECMAScript syntax). See http://msdn.microsoft.com/library/2k3te2cs.aspx.
+An item must first match at least one entry in the include list to be included.
+Included items must then not match any entries in the exclude list to remain included.
 -->
 
                         <!-- Match assembly file paths: -->
@@ -40,7 +41,17 @@ Included items must then not match any entries in the exclude list to remain inc
                                 <ModulePath>.*\system.identitymodel.tokens.jwt.dll</ModulePath>
                             </Include>
                         </ModulePaths>
-
+                        <Attributes>
+                            <Exclude>
+                                <Attribute>^System\.Diagnostics\.CodeAnalysis\.ExcludeFromCodeCoverageAttribute$</Attribute>
+                                <Attribute>^System\.CodeDom\.Compiler\.GeneratedCodeAttribute$</Attribute>
+                            </Exclude>
+                        </Attributes>
+                        <Sources>
+                            <Exclude>
+                                <Source>.*\.g\.cs$</Source>
+                            </Exclude>
+                        </Sources>
                     </CodeCoverage>
                 </Configuration>
             </DataCollector>

diff --git a/build/dependenciesTest.props b/build/dependenciesTest.props
@@ -3,7 +3,7 @@
     <BannedApiAnalyzersVersion>3.3.4</BannedApiAnalyzersVersion>
     <DotNetCoreAppRuntimeVersion>2.1.30</DotNetCoreAppRuntimeVersion>
     <MicrosoftAzureKeyVaultCryptographyVersion>2.0.5</MicrosoftAzureKeyVaultCryptographyVersion>
-    <MicrosoftNETTestSdkVersion>16.10.0</MicrosoftNETTestSdkVersion>
+    <MicrosoftNETTestSdkVersion>17.11.0</MicrosoftNETTestSdkVersion>
     <NetStandardVersion>2.0.3</NetStandardVersion>
     <NewtonsoftVersion>13.0.3</NewtonsoftVersion>
     <SystemSecurityClaimsVersion>4.3.0</SystemSecurityClaimsVersion>

diff --git a/build/template-Build-run-tests-sign.yml b/build/template-Build-run-tests-sign.yml
@@ -68,7 +68,19 @@ steps:
   inputs:
     targetType: filePath
     filePath: ./$(WilsonSourceDirectory)runTests.ps1
-    arguments: '-buildType $(BuildConfiguration)'
+    arguments: '-buildType $(BuildConfiguration) -runningInCI $True'
+
+- task: PublishTestResults@2
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFormat: 'XUnit'
+    testResultsFiles: '**/*.trx'
+    searchFolder: '$(Agent.TempDirectory)'
+
+- task: PublishCodeCoverageResults@2
+  condition: succeededOrFailed()
+  inputs:
+    summaryFileLocation: '$(Agent.TempDirectory)/**/*.cobertura.xml'
 
 - task: CopyFiles@2
   displayName: 'Copy Files to: [staging]\ProductBinaries'
@@ -174,7 +186,7 @@ steps:
 - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@2
   displayName: 'TSA upload to Codebase: WILSON Stamp: Azure'
   inputs:
-    GdnPublishTsaOnboard: false 
+    GdnPublishTsaOnboard: false
     GdnPublishTsaConfigFile: '$(Build.SourcesDirectory)/build/tsaConfig.json'
   continueOnError: true
   condition: and(succeeded(), eq(variables['PipelineType'], 'legacy'))

diff --git a/runTests.ps1 b/runTests.ps1
@@ -1,4 +1,9 @@
-param([string]$buildType="Debug", [string]$dotnetDir="c:\Program Files\dotnet", [string]$root=$PSScriptRoot, [string]$failBuildOnTest="YES")
+param(
+    [string]$buildType="Debug",
+    [string]$dotnetDir="c:\Program Files\dotnet",
+    [string]$root=$PSScriptRoot,
+    [string]$failBuildOnTest="YES",
+    [bool]$runningInCI=$false)
 
 ################################################# Functions ############################################################
 
@@ -31,7 +36,9 @@ Write-Host "dotnetDir:       " $dotnetDir
 Write-Host "root:            " $root;
 Write-Host "failBuildOnTest: " $failBuildOnTest;
 Write-Host "slnFile:         " $slnFile;
+Write-Host "runningInCI:     " $runningInCI;
 
+$runSettingsPath = $PSScriptRoot + "\build\CodeCoverage.runsettings"
 [xml]$buildConfiguration = Get-Content $PSScriptRoot\buildConfiguration.xml
 $dotnetexe = "$dotnetDir\dotnet.exe";
 $startTime = Get-Date
@@ -41,6 +48,13 @@ Write-Host "dotnetexe:      " $dotnetexe;
 
 $ErrorActionPreference = "Stop"
 
+$tempToUse = $env:TEMP;
+
+if ($runningInCI) {
+    # Temp dir used in ADO
+    $tempToUse = "C:\__w\_temp";
+}
+
 $testProjects = $buildConfiguration.SelectNodes("root/projects/test/project")
 foreach ($testProject in $testProjects)
 {
@@ -50,10 +64,10 @@ foreach ($testProject in $testProjects)
 
         $name = $testProject.name;
         Write-Host ">>> Set-Location $root\test\$name"
-        pushd
+        Push-Location
         Set-Location $root\test\$name
-        Write-Host ">>> Start-Process -Wait -PassThru -NoNewWindow $dotnetexe 'test $name.csproj' --filter category!=nonwindowstests --no-build --no-restore -nodereuse:false -v n -c $buildType"
-        $p = Start-Process -Wait -PassThru -NoNewWindow $dotnetexe "test $name.csproj --filter category!=nonwindowstests --no-build --no-restore -nodereuse:false -v n -c $buildType"
+        Write-Host ">>> Start-Process -Wait -PassThru -NoNewWindow $dotnetexe 'test $name.csproj' --filter category!=nonwindowstests --no-build --no-restore -nodereuse:false -v n -c $buildType --collect ""Code Coverage"" --settings ""$runSettingsPath"" --logger trx --results-directory ""$tempToUse"""
+        $p = Start-Process -Wait -PassThru -NoNewWindow $dotnetexe "test $name.csproj --filter category!=nonwindowstests --no-build --no-restore -nodereuse:false -v n -c $buildType --collect ""Code Coverage"" --settings ""$runSettingsPath"" --logger trx --results-directory ""$tempToUse"""
 
         if($p.ExitCode -ne 0)
         {
@@ -68,7 +82,7 @@ foreach ($testProject in $testProjects)
         }
         $testExitCode = $p.ExitCode + $testExitCode
 
-        popd
+        Pop-Location
 
         WriteSectionFooter("End Test");
     }

diff --git a/test/Microsoft.IdentityModel.Protocols.OpenIdConnect.Tests/ConfigurationManagerTests.cs b/test/Microsoft.IdentityModel.Protocols.OpenIdConnect.Tests/ConfigurationManagerTests.cs
@@ -384,11 +384,12 @@ public async Task RequestRefresh(ConfigurationManagerTheoryData<OpenIdConnectCon
                 configuration = await theoryData.ConfigurationManager.GetConfigurationAsync(CancellationToken.None);
             }
 
-            theoryData.ConfigurationManager.RefreshInterval = theoryData.RefreshInterval;
-            theoryData.ConfigurationManager.MetadataAddress = theoryData.UpdatedMetadataAddress;
             if (theoryData.SleepTimeInMs > 0)
                 Thread.Sleep(theoryData.SleepTimeInMs);
 
+            theoryData.ConfigurationManager.RefreshInterval = theoryData.RefreshInterval;
+            theoryData.ConfigurationManager.MetadataAddress = theoryData.UpdatedMetadataAddress;
+
             theoryData.ConfigurationManager.RequestRefresh();
 
             if (theoryData.SleepTimeInMs > 0)
@@ -433,6 +434,7 @@ public static TheoryData<ConfigurationManagerTheoryData<OpenIdConnectConfigurati
                     ExpectedUpdatedConfiguration = OpenIdConfigData.AADCommonV1Config,
                     RefreshInterval = TimeSpan.MaxValue,
                     RequestRefresh = true,
+                    SleepTimeInMs = 1000,
                     UpdatedMetadataAddress = "AADCommonV2Json"
                 });
 

diff --git a/.../Microsoft.IdentityModel.Protocols.OpenIdConnect.Tests/OpenIdConnectConfigurationTests.cs b/.../Microsoft.IdentityModel.Protocols.OpenIdConnect.Tests/OpenIdConnectConfigurationTests.cs
@@ -7,12 +7,9 @@
 using System.Globalization;
 using System.IO;
 using System.Reflection;
-using System.Security.Cryptography;
 using System.Text;
-using System.Text.Json;
 using Microsoft.IdentityModel.TestUtils;
 using Microsoft.IdentityModel.Tokens;
-using Newtonsoft.Json;
 using Xunit;
 
 namespace Microsoft.IdentityModel.Protocols.OpenIdConnect.Tests

diff --git a/test/Microsoft.IdentityModel.Protocols.Tests/ExtensibilityTests.cs b/test/Microsoft.IdentityModel.Protocols.Tests/ExtensibilityTests.cs
@@ -83,16 +83,23 @@ public async Task ConfigurationManagerUsingCustomClass()
             configuration = await configManager.GetConfigurationAsync();
             TestUtilities.SetField(configManager, "_lastRequestRefresh", DateTimeOffset.UtcNow - TimeSpan.FromHours(1));
             configManager.MetadataAddress = "IssuerMetadata2.json";
-            configManager.RequestRefresh();
 
             // Wait for the refresh to complete.
-            await Task.Delay(250).ContinueWith(_ =>
+            await Task.Delay(500);
+
+            for (int i = 0; i < 5; i++)
             {
-                configuration2 = configManager.GetConfigurationAsync().GetAwaiter().GetResult();
-            });
+                configManager.RequestRefresh();
+                configuration2 = await configManager.GetConfigurationAsync();
+
+                if (IdentityComparer.AreEqual(configuration.Issuer, configuration2.Issuer))
+                    await Task.Delay(1000);
+                else
+                    break;
+            }
 
             if (IdentityComparer.AreEqual(configuration.Issuer, configuration2.Issuer))
-                context.Diffs.Add("IdentityComparer.AreEqual(configuration.Issuer, configuration2.Issuer)");
+                context.Diffs.Add($"Expected: {configuration.Issuer}, to be different from: {configuration2.Issuer}");
 
             TestUtilities.AssertFailIfErrors(context);
         }