Merge pull request #2336 from fsprojects/fix_restrictions

[Ready] Fix/refactor/improve framework restriction system, add performance metrics and pre-load requests

Paket.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26403.7
+VisualStudioVersion = 15.0.26430.6
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{A6A6AF7D-D6E3-442D-9B1E-58CC91879BE1}"
 EndProject
@@ -102,6 +102,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{62D18A
 	EndProjectSection
 EndProject
 Global
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Release|Any CPU = Release|Any CPU
@@ -143,4 +146,7 @@ Global
 		{7C622582-E281-4EAB-AADA-B5893BB89B45} = {ED8079DD-2B06-4030-9F0F-DC548F98E1C4}
 		{7234B9B4-8CF5-4E68-AA29-050C087B9246} = {ED8079DD-2B06-4030-9F0F-DC548F98E1C4}
 	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 EndGlobal

RELEASE_NOTES.md

@@ -1,10 +1,18 @@
-#### 5.0.0-beta005 - 23.05.2017
+#### 5.0.0-beta006 - 24.05.2017
 * BUGFIX: Fixed dotnetcore integration:
   * Paket now properly understands runtime and reference assemblies
   * Paket now understands the runtime graph and restores runtime dependencies
   * New API `InstallModel.GetRuntimeAssemblies` and `InstallModel.GetRuntimeLibraries` can be used to retrieve the correct assets for a particular RID and TFM
 * New command `paket generate-nuspec`
 * New Command: `FixNuspecs` - Can fix a list of nuspec files now
+* New restriction system (https://github.com/fsprojects/Paket/pull/2336):
+  * Paket is now more accurate in calculating restrictions and referencing libraries
+  * Paket will convert (lock-)files to a new syntax (but still understands the old syntax), we might revert this within the beta phase
+  * This should fix a bunch of edge cases and invalid behavior in combination with portable profiles and netstandard
+  * Add support for net403 (required for some portable profiles)
+* PERFORMANCE: Improved performance by pre-loading requests (https://github.com/fsprojects/Paket/pull/2336)
+* PERFORMANCE: Report performance in a more detailed way (https://github.com/fsprojects/Paket/pull/2336)
+* BREAKING CHANGE: Paket simplify no longer support simplifying restrictions (https://github.com/fsprojects/Paket/pull/2336)
 * BREAKING CHANGE: Paket.PowerShell is no longer supported
 * BREAKING CHANGE: `InstallModel` API changed and Paket.Core.dll users might need to adapt
 * DEPRECATED: `FixNuspec` function is now obsolete, use `FixNuspecs` instead

build.fsx

@@ -288,7 +288,7 @@ let mergePaketTool () =
     let result =
         ExecProcess (fun info ->
             info.FileName <- currentDirectory </> "packages" </> "build" </> "ILRepack" </> "tools" </> "ILRepack.exe"
-            info.Arguments <- sprintf "/verbose /lib:%s /ver:%s /out:%s %s" buildDir release.AssemblyVersion paketFile toPack
+            info.Arguments <- sprintf "/lib:%s /ver:%s /out:%s %s" buildDir release.AssemblyVersion paketFile toPack
             ) (TimeSpan.FromMinutes 5.)
 
     if result <> 0 then failwithf "Error during ILRepack execution."
@@ -305,6 +305,8 @@ Target "MergePaketTool" (fun _ ->
 
 Target "RunIntegrationTests" (fun _ ->
     mergePaketTool ()
+    // improves the speed of the test-suite by disabling the runtime resolution.
+    System.Environment.SetEnvironmentVariable("PAKET_DISABLE_RUNTIME_RESOLUTION", "true")
     !! integrationTestAssemblies    
     |> NUnit3 (fun p ->
         { p with

docs/content/dependencies-file.md

@@ -92,6 +92,10 @@ Sometimes you don't want to generate dependencies for older framework versions.
 
     nuget Example >= 2.0 // only .NET 3.5 and .NET 4.0
 
+It means
+
+> Paket, I only compile for 'net35' and 'net40', please leave out all other stuff I don't need to compile for this set of frameworks.
+
 #### Automatic framework detection
 
 Paket can detect the target frameworks from your project and then limit the installation to these target frameworks. You can control this in the [`paket.dependencies` file](dependencies-file.html):

docs/content/nuget-dependencies.md

@@ -183,6 +183,16 @@ Sometimes you don't want to generate dependencies for older framework versions.
     nuget Example >= 2.0 framework: net35, net40  // .NET 3.5 and .NET 4.0
     nuget Example >= 2.0 framework: >= net45      // .NET 4.5 and above
 
+> Note: This feature is deprecated and can be seen as an expert feature. 
+> Using framework restrictions on single packages might make you projects uncompilable.
+> The recommended way is to globally (on a group) specifiy the frameworks you want to compile for.
+
+This feature basically tells paket to only consider the specified frameworks for this package.
+It means 
+
+> Paket I use 'Example' only to compile against 'net35' and 'net40'.
+> I never need this package to compile for another framework like 'net45'."
+
 ### Putting the version no. into the path
 
 If you need to be NuGet compatible and want to have the version no. in the package path you can do the following:

docs/content/references-files.md

@@ -66,6 +66,18 @@ Sometimes you don't want to generate dependencies for older framework versions.
     Newtonsoft.Json framework: net35, net40
     DotNetZip framework: >= net45           
 
+> Note: This feature is deprecated and can be seen as an expert feature. 
+> Using framework restrictions on single packages might make you projects uncompilable.
+> The recommended way is to globally (on a group) specifiy the frameworks you want to compile for.
+
+This feature basically tells paket to only consider the specified frameworks for this package on this particular project.
+The resolution will not be modified when restrictions are specified in the reference file.
+It means 
+
+> In this project 'Newtonsoft.Json' is only required when compiling against 'net35' or 'net40'.
+> I never need this package to compile for another framework like 'net45', therefore don't add references for other frameworks.
+
+
 ## Redirects settings
 
 You can instruct Paket to create assembly binding redirects for NuGet packages:

integrationtests/Paket.IntegrationTests/ConvertFromNuGetSpecs.fs

@@ -44,13 +44,13 @@ let ``#1225 should convert simple C# project with non-matching framework restric
     requirement.Name |> shouldEqual (PackageName "Castle.Core")
     requirement.VersionRequirement.ToString() |> shouldEqual "3.3.3"
     requirement.ResolverStrategyForTransitives |> shouldEqual None
-    requirement.Settings.FrameworkRestrictions  |> getRestrictionList |> shouldEqual [FrameworkRestriction.AtLeast(FrameworkIdentifier.DotNetFramework(FrameworkVersion.V3_5))]
+    requirement.Settings.FrameworkRestrictions  |> getExplicitRestriction |> shouldEqual (FrameworkRestriction.AtLeast(FrameworkIdentifier.DotNetFramework(FrameworkVersion.V3_5)))
 
     let requirement2 = depsFile.GetGroup(Constants.MainDependencyGroup).Packages.Tail.Head
     requirement2.Name |> shouldEqual (PackageName "Newtonsoft.Json")
     requirement2.VersionRequirement.ToString() |> shouldEqual "7.0.1"
     requirement2.ResolverStrategyForTransitives |> shouldEqual None
-    requirement2.Settings.FrameworkRestrictions  |> getRestrictionList |> shouldEqual [FrameworkRestriction.AtLeast(FrameworkIdentifier.DotNetFramework(FrameworkVersion.V4_Client))]
+    requirement2.Settings.FrameworkRestrictions  |> getExplicitRestriction |> shouldEqual (FrameworkRestriction.AtLeast(FrameworkIdentifier.DotNetFramework(FrameworkVersion.V4)))
 
 [<Test>]
 let ``#1217 should replace packages.config files in project``() = 

integrationtests/Paket.IntegrationTests/FrameworkRestrictionsSpecs.fs

@@ -1,4 +1,5 @@
 module Paket.IntegrationTests.FrameworkRestrictionsSpecs
+
 open Fake
 open Paket
 open System
@@ -13,8 +14,8 @@ open Paket.Requirements
 let ``#140 windsor should resolve framework dependent dependencies``() =
     let lockFile = update "i000140-resolve-framework-restrictions"
     lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "TaskParallelLibrary"].Settings.FrameworkRestrictions
-    |> getRestrictionList
-    |> shouldEqual [FrameworkRestriction.Exactly(DotNetFramework(FrameworkVersion.V3_5))]
+    |> getExplicitRestriction
+    |> shouldEqual (FrameworkRestriction.Between(DotNetFramework(FrameworkVersion.V3_5), DotNetFramework(FrameworkVersion.V4)))
 
 [<Test>]
 let ``#1182 framework restrictions overwrite each other``() =
@@ -27,20 +28,19 @@ let ``#1182 framework restrictions overwrite each other``() =
 [<Platform("Mono")>] // PATH TOO LONG on Windows...
 let ``#1190 paket add nuget should handle transitive dependencies``() = 
     paket "add nuget xunit version 2.1.0" "i001190-transitive-dependencies-with-restr" |> ignore
-
     let lockFile = LockFile.LoadFrom(Path.Combine(scenarioTempPath "i001190-transitive-dependencies-with-restr","paket.lock"))
     lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "xunit.abstractions"].Settings.FrameworkRestrictions
-    |> getRestrictionList
-    |> shouldContain (FrameworkRestriction.AtLeast(DotNetFramework(FrameworkVersion.V4_5)))
+    |> getExplicitRestriction
+    |> fun res -> res.ToString() |> shouldEqual "|| (>= dnx451) (>= dnxcore50) (>= portable-net45+win8+wp8+wpa81)"
 
 [<Test>]
 let ``#1190 paket add nuget should handle transitive dependencies with restrictions``() = 
     paket "add nuget xunit version 2.1.0" "i001190-transitive-deps" |> ignore
 
     let lockFile = LockFile.LoadFrom(Path.Combine(scenarioTempPath "i001190-transitive-deps","paket.lock"))
     lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "xunit.abstractions"].Settings.FrameworkRestrictions
-    |> getRestrictionList
-    |> shouldEqual []
+    |> getExplicitRestriction
+    |> shouldEqual FrameworkRestriction.NoRestriction
 
 
 [<Test>]
@@ -56,24 +56,29 @@ let ``#1213 framework dependencies propagate``() =
     let lockFile = update "i001213-framework-propagation"
 
     lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "Newtonsoft.Json"].Settings.FrameworkRestrictions
-    |> getRestrictionList
-    |> shouldEqual []
+    |> getExplicitRestriction
+    |> shouldEqual FrameworkRestriction.NoRestriction
 
 [<Test>]
 let ``#1215 framework dependencies propagate``() = 
     let lockFile = update "i001215-framework-propagation-no-restriction"
 
     lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "Microsoft.Bcl.Async"].Settings.FrameworkRestrictions
-    |> getRestrictionList
-    |> shouldEqual []
+    |> getExplicitRestriction
+    |> shouldEqual FrameworkRestriction.NoRestriction
 
 [<Test>]
 let ``#1232 framework dependencies propagate``() = 
     let lockFile = update "i001232-sql-lite"
-
-    lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "System.Data.SQLite.Core"].Settings.FrameworkRestrictions
-    |> getRestrictionList
-    |> shouldContain (FrameworkRestriction.Exactly(DotNetFramework(FrameworkVersion.V4_Client)))
+    let restriction =
+        lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "System.Data.SQLite.Core"].Settings.FrameworkRestrictions
+        |> getExplicitRestriction
+    (FrameworkRestriction.Between(DotNetFramework(FrameworkVersion.V4), DotNetFramework(FrameworkVersion.V4_5))).IsSubsetOf restriction
+    |> shouldEqual true
+    (FrameworkRestriction.Between(DotNetFramework(FrameworkVersion.V4_5), DotNetFramework(FrameworkVersion.V4_5_1))).IsSubsetOf restriction
+    |> shouldEqual true
+    (FrameworkRestriction.AtLeast(DotNetFramework(FrameworkVersion.V4_6))).IsSubsetOf restriction
+    |> shouldEqual true
 
 [<Test>]
 let ``#1494 detect platform 5.0``() = 

integrationtests/Paket.IntegrationTests/PackSpecs.fs

@@ -11,6 +11,8 @@ open System.IO.Compression
 open Paket.Domain
 open Paket
 
+let getDependencies = Paket.NuGet.NuGetPackageCache.getDependencies
+
 [<Test>]
 let ``#1234 empty assembly name``() = 
     let outPath = Path.Combine(scenarioTempPath "i001234-missing-assemblyname","out")
@@ -96,9 +98,9 @@ let ``#1429 pack deps from template``() =
         NuGetV2.getDetailsFromLocalNuGetPackage false None outPath "" (PackageName "PaketBug") (SemVer.Parse "1.0.0.0")
         |> Async.RunSynchronously
 
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldNotContain (PackageName "PaketBug2") // it's not packed in same round
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldNotContain (PackageName "PaketBug")
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldNotContain (PackageName "PaketBug2") // it's not packed in same round
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldNotContain (PackageName "PaketBug")
 
     File.Delete(Path.Combine(scenarioTempPath "i001429-pack-deps","PaketBug","paket.template"))
 
@@ -112,9 +114,9 @@ let ``#1429 pack deps``() =
         NuGetV2.getDetailsFromLocalNuGetPackage false None outPath "" (PackageName "PaketBug") (SemVer.Parse "1.0.0.0")
         |> Async.RunSynchronously
 
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "PaketBug2")
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldNotContain (PackageName "PaketBug")
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "PaketBug2")
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldNotContain (PackageName "PaketBug")
 
     File.Delete(Path.Combine(scenarioTempPath "i001429-pack-deps","PaketBug","paket.template"))
 
@@ -128,8 +130,8 @@ let ``#1429 pack deps using minimum-from-lock-file``() =
         NuGetV2.getDetailsFromLocalNuGetPackage false None outPath "" (PackageName "PaketBug") (SemVer.Parse "1.0.0.0")
         |> Async.RunSynchronously
 
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
-    let packageName, versionRequirement, restrictions = details.Dependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
+    let packageName, versionRequirement, restrictions = details |> getDependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
     versionRequirement |> shouldNotEqual (VersionRequirement.AllReleases)
 
     File.Delete(Path.Combine(scenarioTempPath "i001429-pack-deps-minimum-from-lock","PaketBug","paket.template"))
@@ -144,8 +146,8 @@ let ``#1429 pack deps without minimum-from-lock-file uses dependencies file rang
         NuGetV2.getDetailsFromLocalNuGetPackage false None outPath "" (PackageName "PaketBug") (SemVer.Parse "1.0.0.0")
         |> Async.RunSynchronously
 
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
-    let packageName, versionRequirement, restrictions = details.Dependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
+    let packageName, versionRequirement, restrictions = details |> getDependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
     versionRequirement |> shouldEqual (VersionRequirement.Parse "1.2.3")
 
     File.Delete(Path.Combine(scenarioTempPath "i001429-pack-deps-minimum-from-lock","PaketBug","paket.template"))
@@ -160,8 +162,8 @@ let ``#1429 pack deps without minimum-from-lock-file uses specifc dependencies f
         NuGetV2.getDetailsFromLocalNuGetPackage false None outPath "" (PackageName "PaketBug") (SemVer.Parse "1.0.0.0")
         |> Async.RunSynchronously
 
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
-    let packageName, versionRequirement, restrictions = details.Dependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
+    let packageName, versionRequirement, restrictions = details |> getDependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
     versionRequirement |> shouldEqual (VersionRequirement.Parse "[2.3.4]")
 
     File.Delete(Path.Combine(scenarioTempPath "i001429-pack-deps-specific","PaketBug","paket.template"))
@@ -176,8 +178,8 @@ let ``#1429 pack deps with minimum-from-lock-file uses specifc dependencies file
         NuGetV2.getDetailsFromLocalNuGetPackage false None outPath "" (PackageName "PaketBug") (SemVer.Parse "1.0.0.0")
         |> Async.RunSynchronously
 
-    details.Dependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
-    let packageName, versionRequirement, restrictions = details.Dependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
+    details |> getDependencies |> List.map (fun (x,_,_) -> x) |> shouldContain (PackageName "MySql.Data")
+    let packageName, versionRequirement, restrictions = details |> getDependencies |> List.filter (fun (x,_,_) -> x = PackageName "MySql.Data") |> List.head 
     versionRequirement |> shouldEqual (VersionRequirement.Parse "[2.3.4]")
 
     File.Delete(Path.Combine(scenarioTempPath "i001429-pack-deps-specific","PaketBug","paket.template"))

integrationtests/Paket.IntegrationTests/TestHelper.fs

@@ -10,6 +10,7 @@ open System
 open System.IO
 
 let scenarios = System.Collections.Generic.List<_>()
+let isLiveUnitTesting = AppDomain.CurrentDomain.GetAssemblies() |> Seq.exists (fun a -> a.GetName().Name = "Microsoft.CodeAnalysis.LiveUnitTesting.Runtime")
 
 let paketToolPath = FullName(__SOURCE_DIRECTORY__ + "../../../bin/paket.exe")
 let integrationTestPath = FullName(__SOURCE_DIRECTORY__ + "../../../integrationtests/scenarios")
@@ -26,6 +27,7 @@ let cleanupAllScenarios() =
     scenarios.Clear()
 
 let prepare scenario =
+    if isLiveUnitTesting then Assert.Inconclusive("Integration tests are disabled when in a Live-Unit-Session")
     if scenarios.Count > 10 then
         cleanupAllScenarios()
 

integrationtests/Paket.IntegrationTests/UpdatePackageSpecs.fs

@@ -87,12 +87,12 @@ let ``#1117 can understand portable``() =
     let lockFile = LockFile.LoadFrom(Path.Combine(scenarioTempPath "i001117-aws","paket.lock"))
     let restrictions = lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "PCLStorage"].Settings.FrameworkRestrictions
     match restrictions with
-    | FrameworkRestrictionList l -> l.ToString() |> shouldEqual ("[portable-net45+win8+wp8+wpa81]")
+    | ExplicitRestriction l -> l.ToString() |> shouldEqual ("&& (< net45) (>= portable-net45+win8+wp8+wpa81)")
     | _ -> failwith "wrong"
 
     let restrictions = lockFile.Groups.[Constants.MainDependencyGroup].Resolution.[PackageName "Microsoft.Bcl.Async"].Settings.FrameworkRestrictions
     match restrictions with
-    | FrameworkRestrictionList l -> l.ToString() |> shouldEqual ("[portable-net45+win8+wp8+wpa81]")
+    | ExplicitRestriction l -> l.ToString() |> shouldEqual ("&& (< portable-net45+monoandroid+monotouch+xamarinios+xamarinmac+win8+wp8+wpa81) (>= portable-net45+win8+wp8+wpa81)")
     | _ -> failwith "wrong"
 
 [<Test>]

integrationtests/scenarios/i000346-aliases/before/MyClassLibrary/MyClassLibrary/MyClassLibrary.csprojtemplate

@@ -62,7 +62,7 @@
         </Reference>
       </ItemGroup>
     </When>
-    <When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And $(TargetFrameworkVersion) == 'v4.0'">
+    <When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And ($(TargetFrameworkVersion) == 'v4.0' Or $(TargetFrameworkVersion) == 'v4.0.3')">
       <ItemGroup>
         <Reference Include="Dapper">
           <HintPath>..\..\packages\Dapper\lib\net40\Dapper.dll</HintPath>

integrationtests/scenarios/i001145-excludes/before/MyClassLibrary/MyClassLibrary/MyClassLibrary.csprojtemplate

@@ -62,7 +62,7 @@
         </Reference>
       </ItemGroup>
     </When>
-    <When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And $(TargetFrameworkVersion) == 'v4.0'">
+    <When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And ($(TargetFrameworkVersion) == 'v4.0' Or $(TargetFrameworkVersion) == 'v4.0.3')">
       <ItemGroup>
         <Reference Include="Dapper">
           <HintPath>..\..\packages\Dapper\lib\net40\Dapper.dll</HintPath>