From 71f95f529924be77e8a0af86afb811897ca6abb1 Mon Sep 17 00:00:00 2001 From: Bas Stottelaar Date: Thu, 14 Jun 2018 12:07:08 +0200 Subject: [PATCH] Add support for an intermediate result. This commits adds the option `IntermediateResult`, which will write the raw coverage result to an intermediate file, and merge the result of the next run with that file. This is useful in multi-project solutions. Eventually, the last coverage run will produce a report with the combined results of all the runs. --- README.md | 10 + src/coverlet.console/Program.cs | 4 + src/coverlet.core/Coverage.cs | 3 +- src/coverlet.core/CoverageResult.cs | 4 - .../Extensions/CoverageResultExtensions.cs | 108 +++++++++++ .../Reporters/CoberturaReporter.cs | 5 + src/coverlet.core/Reporters/IReporter.cs | 1 + src/coverlet.core/Reporters/JsonReporter.cs | 116 +++++++++++- src/coverlet.core/Reporters/LcovReporter.cs | 5 + .../Reporters/OpenCoverReporter.cs | 5 + .../CoverageResultTask.cs | 38 ++++ src/coverlet.msbuild/coverlet.msbuild.targets | 1 + .../CoverageResultExtensionsTests.cs | 176 ++++++++++++++++++ .../Reporters/JsonReporterTests.cs | 34 +++- 14 files changed, 496 insertions(+), 14 deletions(-) create mode 100644 src/coverlet.core/Extensions/CoverageResultExtensions.cs create mode 100644 test/coverlet.core.tests/Extensions/CoverageResultExtensionsTests.cs diff --git a/README.md b/README.md index a0d42c82a..df0b43727 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,16 @@ dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line You can specify multiple values for `ThresholdType` by separating them with commas. Valid values include `line`, `branch` and `method`. +### Intermediate Result + +For combining the results of multiple projects, it is possible to use an intermediate result by using the `CoverletIntermediateResult` property. Coverage output will be merged with an intermediate result before generating the report(s). Ensure that all test runs point to the same intermediate result file. + +```bash +dotnet test /p:CollectCoverage=true /p:CoverletIntermediateResult=intermediate.json +``` + +_Note: When using build automation, ensure that this intermediate result file is removed first. It doesn't make sense to merge with an intermediate result from a different build!_ + ### Excluding From Coverage #### Attributes diff --git a/src/coverlet.console/Program.cs b/src/coverlet.console/Program.cs index f2d333a2f..ca2b8e15e 100644 --- a/src/coverlet.console/Program.cs +++ b/src/coverlet.console/Program.cs @@ -18,6 +18,7 @@ static int Main(string[] args) CommandArgument project = app.Argument("", "The project to test. Defaults to the current directory."); CommandOption config = app.Option("-c|--configuration", "Configuration to use for building the project.", CommandOptionType.SingleValue); + CommandOption intermediateResult = app.Option("-i|--coverage-intermediate-result", "The output path of intermediate result (for merging multiple runs).", CommandOptionType.SingleValue); CommandOption output = app.Option("-o|--coverage-output", "The output path of the generated coverage report", CommandOptionType.SingleValue); CommandOption format = app.Option("-f|--coverage-format", "The format of the coverage report", CommandOptionType.SingleValue); @@ -35,6 +36,9 @@ static int Main(string[] args) dotnetTestArgs.Add("/p:CollectCoverage=true"); + if (intermediateResult.HasValue()) + dotnetTestArgs.Add($"/p:CoverletIntermediateResult={intermediateResult.Value()}"); + if (output.HasValue()) dotnetTestArgs.Add($"/p:CoverletOutput={output.Value()}"); diff --git a/src/coverlet.core/Coverage.cs b/src/coverlet.core/Coverage.cs index db2bcbc1e..d44e9acb3 100644 --- a/src/coverlet.core/Coverage.cs +++ b/src/coverlet.core/Coverage.cs @@ -126,7 +126,8 @@ public CoverageResult GetCoverageResult() } } - modules.Add(result.ModulePath, documents); + // TODO: Module path is not generic across multiple projects referencing the same assemblies. + modules.Add(Path.GetFileName(result.ModulePath), documents); InstrumentationHelper.RestoreOriginalModule(result.ModulePath, _identifier); } diff --git a/src/coverlet.core/CoverageResult.cs b/src/coverlet.core/CoverageResult.cs index 742469e10..af9e1c379 100644 --- a/src/coverlet.core/CoverageResult.cs +++ b/src/coverlet.core/CoverageResult.cs @@ -1,8 +1,4 @@ -using System; using System.Collections.Generic; -using System.IO; - -using Jil; namespace Coverlet.Core { diff --git a/src/coverlet.core/Extensions/CoverageResultExtensions.cs b/src/coverlet.core/Extensions/CoverageResultExtensions.cs new file mode 100644 index 000000000..39ab10f10 --- /dev/null +++ b/src/coverlet.core/Extensions/CoverageResultExtensions.cs @@ -0,0 +1,108 @@ +using Coverlet.Core; + +namespace coverlet.core.Extensions +{ + public static class CoverageResultHelper + { + public static void Merge(this CoverageResult result, CoverageResult other) + { + MergeModules(result.Modules, other.Modules); + } + + private static void MergeModules(Modules result, Modules other) + { + foreach (var keyValuePair in other) + { + if (!result.ContainsKey(keyValuePair.Key)) + { + result[keyValuePair.Key] = keyValuePair.Value; + } + else + { + MergeDocuments(result[keyValuePair.Key], keyValuePair.Value); + } + } + } + + private static void MergeDocuments(Documents result, Documents other) + { + foreach (var keyValuePair in other) + { + if (!result.ContainsKey(keyValuePair.Key)) + { + result[keyValuePair.Key] = keyValuePair.Value; + } + else + { + MergeClasses(result[keyValuePair.Key], keyValuePair.Value); + } + } + } + + private static void MergeClasses(Classes result, Classes other) + { + foreach (var keyValuePair in other) + { + if (!result.ContainsKey(keyValuePair.Key)) + { + result[keyValuePair.Key] = keyValuePair.Value; + } + else + { + MergeMethods(result[keyValuePair.Key], keyValuePair.Value); + } + } + } + + private static void MergeMethods(Methods result, Methods other) + { + foreach (var keyValuePair in other) + { + if (!result.ContainsKey(keyValuePair.Key)) + { + result[keyValuePair.Key] = keyValuePair.Value; + } + else + { + MergeMethod(result[keyValuePair.Key], keyValuePair.Value); + } + } + } + + private static void MergeMethod(Method result, Method other) + { + MergeLines(result.Lines, other.Lines); + MergeBranches(result.Branches, other.Branches); + } + + private static void MergeBranches(Branches result, Branches other) + { + foreach (var keyValuePair in other) + { + if (!result.ContainsKey(keyValuePair.Key)) + { + result[keyValuePair.Key] = keyValuePair.Value; + } + else + { + result[keyValuePair.Key].Hits += keyValuePair.Value.Hits; + } + } + } + + private static void MergeLines(Lines result, Lines other) + { + foreach (var keyValuePair in other) + { + if (!result.ContainsKey(keyValuePair.Key)) + { + result[keyValuePair.Key] = keyValuePair.Value; + } + else + { + result[keyValuePair.Key].Hits += keyValuePair.Value.Hits; + } + } + } + } +} diff --git a/src/coverlet.core/Reporters/CoberturaReporter.cs b/src/coverlet.core/Reporters/CoberturaReporter.cs index 9ba29b2cb..b876614e1 100644 --- a/src/coverlet.core/Reporters/CoberturaReporter.cs +++ b/src/coverlet.core/Reporters/CoberturaReporter.cs @@ -130,6 +130,11 @@ public string Report(CoverageResult result) return Encoding.UTF8.GetString(stream.ToArray()); } + public CoverageResult Read(string data) + { + throw new NotSupportedException("Not supported by this reporter."); + } + private string GetBasePath(Modules modules) { List sources = new List(); diff --git a/src/coverlet.core/Reporters/IReporter.cs b/src/coverlet.core/Reporters/IReporter.cs index dc05d547c..738b42f99 100644 --- a/src/coverlet.core/Reporters/IReporter.cs +++ b/src/coverlet.core/Reporters/IReporter.cs @@ -5,5 +5,6 @@ public interface IReporter string Format { get; } string Extension { get; } string Report(CoverageResult result); + CoverageResult Read(string data); } } \ No newline at end of file diff --git a/src/coverlet.core/Reporters/JsonReporter.cs b/src/coverlet.core/Reporters/JsonReporter.cs index ae5cd51de..e4d510606 100644 --- a/src/coverlet.core/Reporters/JsonReporter.cs +++ b/src/coverlet.core/Reporters/JsonReporter.cs @@ -1,4 +1,6 @@ +using System; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace Coverlet.Core.Reporters { @@ -10,7 +12,119 @@ public class JsonReporter : IReporter public string Report(CoverageResult result) { - return JsonConvert.SerializeObject(result.Modules, Formatting.Indented); + return JsonConvert.SerializeObject(result.Modules, Formatting.Indented, new LinesConverter(), new BranchesConverter()); + } + + public CoverageResult Read(string data) + { + return new CoverageResult + { + Identifier = Guid.NewGuid().ToString(), + Modules = JsonConvert.DeserializeObject(data, new LinesConverter(), new BranchesConverter()) + }; + } + + private class BranchesConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(Branches); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var array = JArray.Load(reader); + var branches = new Branches(); + + foreach (var item in array) + { + var obj = (JObject)item; + + var key = ( + (int)obj["Key"]["Number"], + (int)obj["Key"]["Offset"], + (int)obj["Key"]["EndOffset"], + (int)obj["Key"]["Path"], + (uint)obj["Key"]["Ordinal"]); + var value = new HitInfo { Hits = (int)obj["Value"]["Hits"] }; + + branches.Add(key, value); + } + + return branches; + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var branches = (Branches) value; + var array = new JArray(); + + foreach (var kv in branches) + { + dynamic obj = new JObject(); + + obj.Key = new JObject(); + obj.Key.Number = kv.Key.Number; + obj.Key.Offset = kv.Key.Offset; + obj.Key.EndOffset = kv.Key.EndOffset; + obj.Key.Path = kv.Key.Path; + obj.Key.Ordinal = kv.Key.Ordinal; + + obj.Value = new JObject(); + obj.Value.Hits = kv.Value.Hits; + + array.Add(obj); + } + + array.WriteTo(writer); + } + } + + private class LinesConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(Lines); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var array = JArray.Load(reader); + var lines = new Lines(); + + foreach (var item in array) + { + var obj = (JObject) item; + + var key = (int)obj["Key"]["Line"]; + var value = new HitInfo { Hits = (int)obj["Value"]["Hits"] }; + + lines.Add(key, value); + } + + return lines; + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var lines = (Lines)value; + var array = new JArray(); + + foreach (var kv in lines) + { + dynamic obj = new JObject(); + + obj.Key = new JObject(); + obj.Key.Line = kv.Key; + + obj.Value = new JObject(); + obj.Value.Hits = kv.Value.Hits; + + array.Add(obj); + } + + array.WriteTo(writer); + } } } } \ No newline at end of file diff --git a/src/coverlet.core/Reporters/LcovReporter.cs b/src/coverlet.core/Reporters/LcovReporter.cs index 4f7644a37..2ff7c09b7 100644 --- a/src/coverlet.core/Reporters/LcovReporter.cs +++ b/src/coverlet.core/Reporters/LcovReporter.cs @@ -60,5 +60,10 @@ public string Report(CoverageResult result) return string.Join(Environment.NewLine, lcov); } + + public CoverageResult Read(string data) + { + throw new NotSupportedException("Not supported by this reporter."); + } } } \ No newline at end of file diff --git a/src/coverlet.core/Reporters/OpenCoverReporter.cs b/src/coverlet.core/Reporters/OpenCoverReporter.cs index 7f5061239..3255df439 100644 --- a/src/coverlet.core/Reporters/OpenCoverReporter.cs +++ b/src/coverlet.core/Reporters/OpenCoverReporter.cs @@ -233,5 +233,10 @@ public string Report(CoverageResult result) return Encoding.UTF8.GetString(stream.ToArray()); } + + public CoverageResult Read(string data) + { + throw new NotSupportedException("Not supported by this reporter."); + } } } \ No newline at end of file diff --git a/src/coverlet.msbuild.tasks/CoverageResultTask.cs b/src/coverlet.msbuild.tasks/CoverageResultTask.cs index 0bce1363e..c2561e424 100644 --- a/src/coverlet.msbuild.tasks/CoverageResultTask.cs +++ b/src/coverlet.msbuild.tasks/CoverageResultTask.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Text; +using coverlet.core.Extensions; using ConsoleTables; using Coverlet.Core; using Coverlet.Core.Reporters; @@ -13,10 +14,17 @@ namespace Coverlet.MSbuild.Tasks public class CoverageResultTask : Task { private string _filename; + private string _intermediateResult; private string _format; private int _threshold; private string _thresholdType; + public string IntermediateResult + { + get { return _intermediateResult; } + set { _intermediateResult = value; } + } + [Required] public string Output { @@ -59,6 +67,36 @@ public override bool Execute() Directory.CreateDirectory(directory); } + if (!string.IsNullOrEmpty(_intermediateResult)) + { + Console.WriteLine("\nMerging with intermediate result..."); + var reporter = new ReporterFactory("json").CreateReporter(); + + if (File.Exists(_intermediateResult)) + { + try + { + result.Merge(reporter.Read(File.ReadAllText(_intermediateResult))); + } + catch (Exception) + { + Console.WriteLine(" Unable to read intermediate results, ignoring"); + } + } + + try + { + File.WriteAllText(_intermediateResult, reporter.Report(result)); + } + catch (IOException e) + { + throw new Exception("Unable to write intermediate results", e); + } + + Console.WriteLine($" Intermediate result written to '{_intermediateResult}'"); + } + + Console.WriteLine($"\nWriting report(s)..."); var formats = _format.Split(','); foreach (var format in formats) { diff --git a/src/coverlet.msbuild/coverlet.msbuild.targets b/src/coverlet.msbuild/coverlet.msbuild.targets index e77b7a0c8..e88ef4b74 100644 --- a/src/coverlet.msbuild/coverlet.msbuild.targets +++ b/src/coverlet.msbuild/coverlet.msbuild.targets @@ -22,6 +22,7 @@