diff --git a/Fake.sln b/Fake.sln index b7cf07d8c05..b3281a1d9ef 100644 --- a/Fake.sln +++ b/Fake.sln @@ -179,6 +179,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fake.DotNet.ILMerge", "src\ EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fake.DotNet.Xdt", "src\app\Fake.DotNet.Xdt\Fake.DotNet.Xdt.fsproj", "{32A8D516-30FA-4DD2-BDA3-F8532CE642D1}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fake.DotNet.Testing.Coverlet", "src\app\Fake.DotNet.Testing.Coverlet\Fake.DotNet.Testing.Coverlet.fsproj", "{664A121E-17A2-453E-BC2E-1C59A67875D2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1112,6 +1114,18 @@ Global {32A8D516-30FA-4DD2-BDA3-F8532CE642D1}.Release|x64.Build.0 = Release|Any CPU {32A8D516-30FA-4DD2-BDA3-F8532CE642D1}.Release|x86.ActiveCfg = Release|Any CPU {32A8D516-30FA-4DD2-BDA3-F8532CE642D1}.Release|x86.Build.0 = Release|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Debug|x64.ActiveCfg = Debug|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Debug|x64.Build.0 = Debug|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Debug|x86.ActiveCfg = Debug|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Debug|x86.Build.0 = Debug|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Release|Any CPU.Build.0 = Release|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Release|x64.ActiveCfg = Release|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Release|x64.Build.0 = Release|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Release|x86.ActiveCfg = Release|Any CPU + {664A121E-17A2-453E-BC2E-1C59A67875D2}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1196,6 +1210,7 @@ Global {69E4C443-11D6-41BC-920C-ED9D7B36DDCA} = {7BFFAE76-DEE9-417A-A79B-6A6644C4553A} {D15BF11A-0426-4053-B6EC-C38B86849240} = {7BFFAE76-DEE9-417A-A79B-6A6644C4553A} {32A8D516-30FA-4DD2-BDA3-F8532CE642D1} = {901F162F-8925-4390-89C5-9EE2C343F744} + {664A121E-17A2-453E-BC2E-1C59A67875D2} = {7BFFAE76-DEE9-417A-A79B-6A6644C4553A} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {058A0C5E-2216-4306-8AFB-0AE28320C26A} diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index a646deac25e..70cd6664591 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,10 @@ # Release Notes +## 5.18.2 - 2019-10-26 + +* NEW: Add `Fake.DotNet.Testing.Coverlet`, thanks @Tarmil - https://github.com/fsharp/FAKE/pull/2413 +* BUGFIX: `paket pack` module was broken, thanks @sergey-tihon - https://github.com/fsharp/FAKE/pull/2418 + ## 5.18.1 - 2019-10-22 * BUGFIX: Paket module was broken - https://github.com/fsharp/FAKE/pull/2413 diff --git a/build.fsx b/build.fsx index 91b7760323d..9bd556bd9bb 100644 --- a/build.fsx +++ b/build.fsx @@ -275,6 +275,7 @@ let dotnetAssemblyInfos = "Fake.DotNet.MSBuild", "Running msbuild" "Fake.DotNet.NuGet", "Running NuGet Client and interacting with NuGet Feeds" "Fake.DotNet.Paket", "Running Paket and publishing packages" + "Fake.DotNet.Testing.Coverlet", "Code coverage with Coverlet" "Fake.DotNet.Testing.DotCover", "Code coverage with DotCover" "Fake.DotNet.Testing.Expecto", "Running expecto test runner" "Fake.DotNet.Testing.MSpec", "Running mspec test runner" diff --git a/help/markdown/dotnet-testing-coverlet.md b/help/markdown/dotnet-testing-coverlet.md new file mode 100644 index 00000000000..fca4d1cd18e --- /dev/null +++ b/help/markdown/dotnet-testing-coverlet.md @@ -0,0 +1,77 @@ +# Analyze your code coverage with Coverlet + +From the [project](https://github.com/tonerdo/coverlet): +> Coverlet is a cross platform code coverage framework for .NET, with support for line, branch and method coverage. It works with .NET Framework on Windows and .NET Core on all supported platforms. + +[API-Reference](apidocs/v5/fake-dotnet-testing-coverlet.html) + +## Minimal working example + +To work with Coverlet, you must first: + +* Add the NuGet reference `coverlet.msbuild` to your test projects (and only your test projects!). +* Ensure that these test projects are marked as such with a property `true`. + +Then, Coverlet will run as part of `dotnet test` using `Coverlet.withDotNetTestOptions`: + +```fsharp +#r "paket: +nuget Fake.DotNet.Testing.Coverlet //" +open Fake.DotNet +open Fake.DotNet.Testing + +DotNet.test (fun p -> + { p with + // Your dotnet test configuration here... + Configuration = DotNet.BuildConfiguration.Release + } + |> Coverlet.withDotNetTestOptions (fun p -> + { p with + Output = "coverage.json" + })) + "tests/MyProject.fsproj" +``` + +## Full example + +```fsharp +#r "paket: +nuget Fake.DotNet.Testing.Coverlet //" +open Fake.DotNet +open Fake.DotNet.Testing + +DotNet.test (fun p -> + { p with + // Your dotnet test configuration here... + Configuration = DotNet.BuildConfiguration.Release + } + |> Coverlet.withDotNetTestOptions (fun p -> + { p with + OutputFormat = Coverlet.OutputFormat.OpenCover + Output = "coverage.xml" + Include = [ + // Include all namespaces from assemblies whose name starts with MyProject + "MyProject.*", "*" + ] + Exclude = [ + // Exclude all namespaces from assemblies whose name ends with .Test or .Tests + "*.Tests?", "*" + // Exclude all namespaces starting with System., even in included assemblies + "*", "System.*" + ] + // Exclude assemblies, types and methods marked with these attributes + ExcludeByAttribute = ["MyCustomIgnoreCoverageAttribute"] + // Exclude all code from these files + ExcludeByFile = ["AssemblyInfo.fs"; "Program.fs"] + // Merge results with results from another coverage session + // (which must have OutputFormat = Json) + MergeWith = Some "other-coverage.json" + // Fail if total line coverage is below 80% + Threshold = Some 80 + TresholdType = Coverlet.ThresholdType.Line + ThresholdStat = Coverlet.ThresholdState.Total + // Generate links to SourceLink URLs rather than local paths + UseSourceLink = true + })) + "tests/MyProject.fsproj" +``` diff --git a/help/templates/template.cshtml b/help/templates/template.cshtml index 9e0f72396eb..1f4fedfd92f 100644 --- a/help/templates/template.cshtml +++ b/help/templates/template.cshtml @@ -155,6 +155,8 @@ Testing - OpenCover Testing - DotCover + Testing - + Coverlet NuGet Mage Paket diff --git a/src/app/Fake.DotNet.Paket/Paket.fs b/src/app/Fake.DotNet.Paket/Paket.fs index b62393f1da3..6686610d4ce 100644 --- a/src/app/Fake.DotNet.Paket/Paket.fs +++ b/src/app/Fake.DotNet.Paket/Paket.fs @@ -144,6 +144,7 @@ let internal createProcess (runType:StartType) = |> Arguments.appendIf parameters.IncludeReferencedProjects "--include-referenced-projects" |> List.foldBack (fun t -> Arguments.append ["--exclude"; t]) parameters.ExcludedTemplates |> List.foldBack (fun (id, v) -> Arguments.append ["--specific-version"; id; v]) parameters.SpecificVersions + |> Arguments.append [parameters.OutputPath] |> startPaket parameters.ToolType parameters.ToolPath parameters.WorkingDir parameters.TimeOut | Restore (parameters) -> Arguments.OfArgs ["restore"] diff --git a/src/app/Fake.DotNet.Testing.Coverlet/AssemblyInfo.fs b/src/app/Fake.DotNet.Testing.Coverlet/AssemblyInfo.fs new file mode 100644 index 00000000000..b81ada48e12 --- /dev/null +++ b/src/app/Fake.DotNet.Testing.Coverlet/AssemblyInfo.fs @@ -0,0 +1,19 @@ +// Auto-Generated by FAKE; do not edit +namespace System +open System.Reflection + +[] +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + let [] AssemblyTitle = "FAKE - F# Make Code coverage with Coverlet" + let [] AssemblyProduct = "FAKE - F# Make" + let [] AssemblyVersion = "5.18.0" + let [] AssemblyInformationalVersion = "5.18.0" + let [] AssemblyFileVersion = "5.18.0" + let [] AssemblyMetadata_BuildDate = "2019-10-21" diff --git a/src/app/Fake.DotNet.Testing.Coverlet/Coverlet.fs b/src/app/Fake.DotNet.Testing.Coverlet/Coverlet.fs new file mode 100644 index 00000000000..0eae906531a --- /dev/null +++ b/src/app/Fake.DotNet.Testing.Coverlet/Coverlet.fs @@ -0,0 +1,119 @@ +/// Contains options to run [Coverlet](https://github.com/tonerdo/coverlet) as part of dotnet test. +[] +module Fake.DotNet.Testing.Coverlet + +open Fake.DotNet + +/// The coverage report file format. +type OutputFormat = + | Json + | Lcov + | OpenCover + | Cobertura + | TeamCity + +/// The type of coverage to use when failing under a threshold. +type ThresholdType = + | Line + | Branch + | Method + +/// The statistic to use when failing under a threshold. +type ThresholdStat = + | Minimum + | Total + | Average + +/// Coverlet MSBuild parameters. For more details see: https://github.com/tonerdo/coverlet/blob/master/Documentation/MSBuildIntegration.md +type CoverletParams = + { /// (Required) Format of the generated output. + OutputFormat : OutputFormat + /// (Required) Path to the generated output file, or directory if it ends with a `/`. + Output : string + /// Namespaces to include, as (AssemblyName, Namespace) pairs. Supports `*` and `?` globbing. + Include : (string * string) list + /// Namespaces to exclude, as (AssemblyName, Namespace) pairs. Supports `*` and `?` globbing. + Exclude : (string * string) list + /// Exclude methods, types and assemblies annotated with these attributes. + ExcludeByAttribute : string list + /// Exclude these source files. Supports path globbing. + ExcludeByFile : string list + /// Coverlet json file to merge with the output of this run. + MergeWith : string option + /// Minimum coverage percent. Build fails if the result is below. + Threshold : int option + /// Type of coverage to check against the threshold. + ThresholdType : ThresholdType + /// Coverage statistic to check against the threshold. + ThresholdStat : ThresholdStat + /// Generate results with URL links from SourceLink instead of file paths. + UseSourceLink : bool } + +/// The default parameters. +let private defaults = + { OutputFormat = OutputFormat.Json + Output = "./" + Include = [] + Exclude = [] + ExcludeByAttribute = [] + ExcludeByFile = [] + MergeWith = None + Threshold = None + ThresholdType = ThresholdType.Line + ThresholdStat = ThresholdStat.Minimum + UseSourceLink = false } + +let private outputFormatToString = function + | OutputFormat.Json -> "json" + | OutputFormat.Lcov -> "lcov" + | OutputFormat.OpenCover -> "opencover" + | OutputFormat.Cobertura -> "cobertura" + | OutputFormat.TeamCity -> "teamcity" + +let private namespacesToString = + Seq.map (fun (asm, ns) -> "[" + asm + "]" + ns) + >> String.concat "," + +let private thresholdTypeToString = function + | ThresholdType.Line -> "line" + | ThresholdType.Branch -> "branch" + | ThresholdType.Method -> "method" + +let private thresholdStatToString = function + | ThresholdStat.Minimum -> "minimum" + | ThresholdStat.Total -> "total" + | ThresholdStat.Average -> "average" + +/// Add Coverlet parameters to the MSBuild command. +let withMSBuildArguments (param: CoverletParams -> CoverletParams) (args: MSBuild.CliArguments) = + let param = param defaults + let properties = + [ + "CollectCoverage", "true" + "OutputFormat", outputFormatToString param.OutputFormat + "CoverletOutput", param.Output + if not (List.isEmpty param.Include) then + "Include", namespacesToString param.Include + if not (List.isEmpty param.Exclude) then + "Exclude", namespacesToString param.Exclude + if not (List.isEmpty param.ExcludeByAttribute) then + "ExcludeByAttribute", String.concat "," param.ExcludeByAttribute + if not (List.isEmpty param.ExcludeByFile) then + "ExcludeByFile", String.concat "," param.ExcludeByFile + match param.MergeWith with + | Some f -> "MergeWith", f + | None -> () + match param.Threshold with + | Some t -> + "Threshold", string t + "ThresholdType", thresholdTypeToString param.ThresholdType + "ThresholdStat", thresholdStatToString param.ThresholdStat + | None -> () + if param.UseSourceLink then + "UseSourceLink", "true" + ] + { args with Properties = args.Properties @ properties } + +/// Add Coverlet parameters to the dotnet test command. +let withDotNetTestOptions (param: CoverletParams -> CoverletParams) (options: DotNet.TestOptions) = + { options with MSBuildParams = withMSBuildArguments param options.MSBuildParams } diff --git a/src/app/Fake.DotNet.Testing.Coverlet/Fake.DotNet.Testing.Coverlet.fsproj b/src/app/Fake.DotNet.Testing.Coverlet/Fake.DotNet.Testing.Coverlet.fsproj new file mode 100644 index 00000000000..2f0ac203c23 --- /dev/null +++ b/src/app/Fake.DotNet.Testing.Coverlet/Fake.DotNet.Testing.Coverlet.fsproj @@ -0,0 +1,22 @@ + + + netstandard2.0;net462 + $(DefineConstants);NO_DOTNETCORE_BOOTSTRAP + Fake.DotNet.Testing.Coverlet + Library + + + $(DefineConstants);NETSTANDARD;USE_HTTPCLIENT + + + $(DefineConstants);RELEASE + + + + + + + + + + diff --git a/src/app/Fake.DotNet.Testing.Coverlet/paket.references b/src/app/Fake.DotNet.Testing.Coverlet/paket.references new file mode 100644 index 00000000000..2c8a7ddfd73 --- /dev/null +++ b/src/app/Fake.DotNet.Testing.Coverlet/paket.references @@ -0,0 +1,4 @@ +group netcore + +FSharp.Core +NETStandard.Library \ No newline at end of file diff --git a/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj b/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj index 82bea107198..4f6f55c4fd6 100644 --- a/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj +++ b/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj @@ -11,6 +11,7 @@ + @@ -46,6 +47,7 @@ + diff --git a/src/test/Fake.Core.UnitTests/Fake.DotNet.Paket.fs b/src/test/Fake.Core.UnitTests/Fake.DotNet.Paket.fs index 4dce3ddd732..da0b8092ab4 100644 --- a/src/test/Fake.Core.UnitTests/Fake.DotNet.Paket.fs +++ b/src/test/Fake.Core.UnitTests/Fake.DotNet.Paket.fs @@ -33,7 +33,7 @@ let tests = |> ArgumentHelper.checkIfMono let cmd = args |> Arguments.toStartInfo Expect.equal file expectedPath "Expected paket.exe" - Expect.equal cmd "pack" "expected pack argument" + Expect.equal cmd "pack ./temp" "expected pack command line" testCase "Test push is not missing, #2411" <| fun _ -> let cp = Paket.createProcess (Paket.StartType.PushFile (Paket.PaketPushDefaults(), "testfile")) diff --git a/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.Coverlet.fs b/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.Coverlet.fs new file mode 100644 index 00000000000..196cb9897bc --- /dev/null +++ b/src/test/Fake.Core.UnitTests/Fake.DotNet.Testing.Coverlet.fs @@ -0,0 +1,59 @@ +module Fake.DotNet.Testing.CoverletTests + +open Fake.DotNet +open Fake.DotNet.Testing +open Expecto + +let getProps options = + let args = + MSBuild.CliArguments.Create() + |> Coverlet.withMSBuildArguments options + args.Properties + +[] +let tests = + testList "Fake.DotNet.Testing.Coverlet.Tests" [ + testCase "Test that full properties are converted" <| fun _ -> + let props = getProps (fun p -> + { + OutputFormat = Coverlet.OutputFormat.Cobertura + Output = "coverage.json" + Include = [ + "Incl.Asm1", "Incl.Ns1" + "Incl.Asm2", "Incl.Ns2" + ] + Exclude = [ + "Excl.Asm1", "Excl.Ns1" + "Excl.Asm2", "Excl.Ns2" + ] + ExcludeByAttribute = ["Attr1"; "Attr2"] + ExcludeByFile = ["file1.cs"; "file2.cs"] + MergeWith = Some "prev-coverage.json" + Threshold = Some 80 + ThresholdType = Coverlet.ThresholdType.Branch + ThresholdStat = Coverlet.ThresholdStat.Total + UseSourceLink = true + }) + Expect.containsAll [ + "CollectCoverage", "true" + "CoverletOutput", "coverage.json" + "OutputFormat", "cobertura" + "Include", "[Incl.Asm1]Incl.Ns1,[Incl.Asm2]Incl.Ns2" + "Exclude", "[Excl.Asm1]Excl.Ns1,[Excl.Asm2]Excl.Ns2" + "ExcludeByAttribute", "Attr1,Attr2" + "ExcludeByFile", "file1.cs,file2.cs" + "MergeWith", "prev-coverage.json" + "Threshold", "80" + "ThresholdType", "branch" + "ThresholdStat", "total" + "UseSourceLink", "true" + ] props "expected proper MSBuild properties" + + testCase "Test that default properties are converted" <| fun _ -> + let props = getProps id + Expect.containsAll [ + "CollectCoverage", "true" + "CoverletOutput", "./" + "OutputFormat", "json" + ] props "expected proper MSBuild properties" + ]