PR #636: Fix unstable tests and switch to dotcover

* test: use static random seed in tests this prevents the code coverage from varying depending on how well the random data compresses * ci: use dotcover for code coverage
icsharpcode · May 14, 2021 · 8c0a169 · 8c0a169
1 parent 8e4d144
commit 8c0a169
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 27 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -22,7 +22,9 @@ jobs:
       LIB_PROJ: src/ICSharpCode.SharpZipLib/ICSharpCode.SharpZipLib.csproj
     steps:
     - uses: actions/checkout@v2
-
+      with:
+        fetch-depth: 0
+
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:
@@ -39,46 +41,76 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu, windows, macos]
+        # Windows testing is combined with code coverage
+        os: [ubuntu, macos]
         target: [netcoreapp3.1]
-        include:
-          - os: windows
-            target: net46
     steps:
     - uses: actions/checkout@v2
-
+      with:
+        fetch-depth: 0
+
     - name: Setup .NET Core
       if: matrix.target == 'netcoreapp3.1'
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: '3.1.x'
 
-    # NOTE: This is the temporary fix for https://github.com/actions/virtual-environments/issues/1090
-    - name: Cleanup before restore
-      if: ${{ matrix.os == 'windows' }}
-      run: dotnet clean ICSharpCode.SharpZipLib.sln && dotnet nuget locals all --clear
-
     - name: Restore test dependencies
       run: dotnet restore
 
     - name: Run tests (Debug)
       run: dotnet test -c debug -f ${{ matrix.target }} --no-restore
 
     - name: Run tests (Release)
-      # Only upload code coverage for windows in an attempt to fix the broken code coverage
-      if: ${{ matrix.os == 'windows' }}
-      run: dotnet test -c release -f ${{ matrix.target }} --no-restore --collect="XPlat Code Coverage"
-
-    - name: Run tests with coverage (Release)
-      # Only upload code coverage for windows in an attempt to fix the broken code coverage
-      if: ${{ matrix.os != 'windows' }}
       run: dotnet test -c release -f ${{ matrix.target }} --no-restore
 
+
+  CodeCov:
+    name: Code Coverage
+    runs-on: windows-latest
+    env:
+      DOTCOVER_VER: 2021.1.2
+      DOTCOVER_PKG: jetbrains.dotcover.commandlinetools
+      COVER_SNAPSHOT: SharpZipLib.dcvr
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '3.1.x'
+
+    # NOTE: This is the temporary fix for https://github.com/actions/virtual-environments/issues/1090
+    - name: Cleanup before restore
+      run: dotnet clean ICSharpCode.SharpZipLib.sln && dotnet nuget locals all --clear
+
+    - name: Install codecov
+      run: nuget install -o tools -version ${{env.DOTCOVER_VER}} ${{env.DOTCOVER_PKG}}
+
+    - name: Add dotcover to path
+      run: echo "$(pwd)\tools\${{env.DOTCOVER_PKG}}.${{env.DOTCOVER_VER}}\tools" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+    - name: Run tests with code coverage
+      run: dotcover dotnet --output=${{env.COVER_SNAPSHOT}} --filters=-:ICSharpCode.SharpZipLib.Tests -- test -c release
+
+    - name: Create code coverage report
+      run: dotcover report --source=${{env.COVER_SNAPSHOT}} --reporttype=detailedxml --output=dotcover-report.xml
+
     - name: Upload coverage to Codecov
       uses: codecov/[email protected]
+      with:
+        files: dotcover-report.xml
+
+    - name: Upload coverage snapshot artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: Code coverage snapshot
+        path: ${{env.COVER_SNAPSHOT}}
 
   Pack:
-    needs: [Build, Test]
+    needs: [Build, Test, CodeCov]
     runs-on: windows-latest
     env:
       PKG_SUFFIX: ''

diff --git a/test/ICSharpCode.SharpZipLib.Tests/BZip2/Bzip2Tests.cs b/test/ICSharpCode.SharpZipLib.Tests/BZip2/Bzip2Tests.cs
@@ -12,6 +12,9 @@ namespace ICSharpCode.SharpZipLib.Tests.BZip2
 	[TestFixture]
 	public class BZip2Suite
 	{
+		// Use the same random seed to guarantee all the code paths are followed
+		const int RandomSeed = 4;
+
 		/// <summary>
 		/// Basic compress/decompress test BZip2
 		/// </summary>
@@ -23,7 +26,7 @@ public void BasicRoundTrip()
 			var outStream = new BZip2OutputStream(ms);
 
 			byte[] buf = new byte[10000];
-			var rnd = new Random();
+			var rnd = new Random(RandomSeed);
 			rnd.NextBytes(buf);
 
 			outStream.Write(buf, 0, buf.Length);

diff --git a/test/ICSharpCode.SharpZipLib.Tests/Base/InflaterDeflaterTests.cs b/test/ICSharpCode.SharpZipLib.Tests/Base/InflaterDeflaterTests.cs
@@ -16,6 +16,9 @@ namespace ICSharpCode.SharpZipLib.Tests.Base
 	[TestFixture]
 	public class InflaterDeflaterTestSuite
 	{
+		// Use the same random seed to guarantee all the code paths are followed
+		const int RandomSeed = 5;
+
 		private void Inflate(MemoryStream ms, byte[] original, int level, bool zlib)
 		{
 			byte[] buf2 = new byte[original.Length];
@@ -60,7 +63,7 @@ private MemoryStream Deflate(byte[] data, int level, bool zlib)
 		private static byte[] GetRandomTestData(int size)
 		{
 			byte[] buffer = new byte[size];
-			var rnd = new Random();
+			var rnd = new Random(RandomSeed);
 			rnd.NextBytes(buffer);
 
 			return buffer;
@@ -184,7 +187,7 @@ public async Task InflateDeflateZlibAsync([Range(0, 9)] int level)
 		private int runLevel;
 		private bool runZlib;
 		private long runCount;
-		private readonly Random runRandom = new Random(5);
+		private readonly Random runRandom = new Random(RandomSeed);
 
 		private void DeflateAndInflate(byte[] buffer)
 		{

diff --git a/test/ICSharpCode.SharpZipLib.Tests/ICSharpCode.SharpZipLib.Tests.csproj b/test/ICSharpCode.SharpZipLib.Tests/ICSharpCode.SharpZipLib.Tests.csproj
@@ -8,15 +8,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
     <PackageReference Include="nunit" Version="3.13.1" />
     <PackageReference Include="nunit.console" Version="3.12.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.17.0" />
-    <PackageReference Include="OpenCover" Version="4.6.519" />
     <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
     <PackageReference Include="System.ValueTuple" Version="4.5.0" />
   </ItemGroup>