make restore fast by copying and comparing the lockfile.

.paket/Paket.Restore.targets

@@ -5,6 +5,9 @@
     <!-- Mark that this target file has been loaded.  -->
     <IsPaketRestoreTargetsFileLoaded>true</IsPaketRestoreTargetsFileLoaded>
     <PaketToolsPath>$(MSBuildThisFileDirectory)</PaketToolsPath>
+    <PaketRootPath>$(MSBuildThisFileDirectory)..\</PaketRootPath>
+    <PaketRestoreCacheFile>$(PaketRootPath)paket-files\paket.restore.cached</PaketRestoreCacheFile>
+    <PaketLockFilePath>$(PaketRootPath)paket.lock</PaketLockFilePath>
     <MonoPath Condition="'$(MonoPath)' == '' And Exists('/Library/Frameworks/Mono.framework/Commands/mono')">/Library/Frameworks/Mono.framework/Commands/mono</MonoPath>
     <MonoPath Condition="'$(MonoPath)' == ''">mono</MonoPath>
     <!-- Paket command -->
@@ -16,29 +19,74 @@
     <PaketBootStrapperExePath Condition=" '$(PaketBootStrapperExePath)' == '' ">$(PaketToolsPath)paket.bootstrapper.exe</PaketBootStrapperExePath>
     <PaketBootStrapperCommand Condition=" '$(OS)' == 'Windows_NT'">"$(PaketBootStrapperExePath)"</PaketBootStrapperCommand>
     <PaketBootStrapperCommand Condition=" '$(OS)' != 'Windows_NT' ">$(MonoPath) --runtime=v4.0.30319 "$(PaketBootStrapperExePath)"</PaketBootStrapperCommand>
-    
+
     <!-- Disable automagic references for F# dotnet sdk -->
     <!-- This will not do anything for other project types -->
     <!-- see https://github.com/fsharp/fslang-design/blob/master/RFCs/FS-1032-fsharp-in-dotnet-sdk.md -->
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
-    <DisableImplicitSystemValueTupleReference>true</DisableImplicitSystemValueTupleReference>    
+    <DisableImplicitSystemValueTupleReference>true</DisableImplicitSystemValueTupleReference>
   </PropertyGroup>
 
   <Target Name="PaketRestore" BeforeTargets="_GenerateDotnetCliToolReferenceSpecs;_GenerateProjectRestoreGraphPerFramework;_GenerateRestoreGraphWalkPerFramework;CollectPackageReferences" >
 
-    <Exec Command='$(PaketBootStrapperCommand) ' Condition="Exists('$(PaketBootStrapperExePath)') AND !(Exists('$(PaketExePath)'))" ContinueOnError="false" />
-    <Exec Command='$(PaketCommand) restore --project "$(MSBuildProjectFullPath)" --target-framework $(TargetFramework)' Condition="$(TargetFramework) != ''" ContinueOnError="false" />
-    <Exec Command='$(PaketCommand) restore --project "$(MSBuildProjectFullPath)" --target-framework "$(TargetFrameworks)"' Condition="$(TargetFramework) == ''" ContinueOnError="false" />
+    <!-- Step 1 Check if lockfile is properly restored -->
+    <PropertyGroup>
+      <PaketRestoreRequired>true</PaketRestoreRequired>
+    </PropertyGroup>
+
+    <PropertyGroup Condition="Exists('$(PaketRestoreCacheFile)') ">
+      <PaketRestoreCachedHash>$([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)'))</PaketRestoreCachedHash>
+      <PaketRestoreLockFileHash>$([System.IO.File]::ReadAllText('$(PaketLockFilePath)'))</PaketRestoreLockFileHash>
+      <PaketRestoreRequired>true</PaketRestoreRequired>
+      <PaketRestoreRequired Condition=" '$(PaketRestoreLockFileHash)' == '$(PaketRestoreCachedHash)' ">false</PaketRestoreRequired>
+      <PaketRestoreRequired Condition=" '$(PaketRestoreLockFileHash)' == '' ">true</PaketRestoreRequired>
+    </PropertyGroup>
+
+    <!-- Do a global restore if required -->
+    <Exec Command='$(PaketBootStrapperCommand)' Condition="Exists('$(PaketBootStrapperExePath)') AND !(Exists('$(PaketExePath)'))" ContinueOnError="false" />
+    <Exec Command='$(PaketCommand) restore' Condition=" '$(PaketRestoreRequired)' == 'true' " ContinueOnError="false" />
 
+    <!-- Step 2 Detect project specific changes -->
     <PropertyGroup>
-      <PaketReferencesFilePath>$(MSBuildProjectDirectory)/obj/$(MSBuildProjectFile).references</PaketReferencesFilePath>
+      <PaketReferencesCachedFilePath>$(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).paket.references.cached</PaketReferencesCachedFilePath>
+      <!-- MyProject.fsproj.paket.references has the highest precedence -->
+      <PaketOriginalReferencesFilePath>$(MSBuildProjectFullPath).paket.references</PaketOriginalReferencesFilePath>
+      <!-- MyProject.paket.references -->
+      <PaketOriginalReferencesFilePath Condition=" !Exists('$(PaketOriginalReferencesFilePath)')">$(MSBuildProjectDirectory)\$(MSBuildProjectName).paket.references</PaketOriginalReferencesFilePath>
+      <!-- paket.references -->
+      <PaketOriginalReferencesFilePath Condition=" !Exists('$(PaketOriginalReferencesFilePath)')">$(MSBuildProjectDirectory)\paket.references</PaketOriginalReferencesFilePath>
+      <PaketReferencesFilePath>$(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).$(TargetFramework).references</PaketReferencesFilePath>
+      <PaketRestoreRequired>true</PaketRestoreRequired>
+      <PaketRestoreRequiredReason>references-file-or-cache-not-found</PaketRestoreRequiredReason>
     </PropertyGroup>
 
-    <ReadLinesFromFile File="$(PaketReferencesFilePath)" >
+    <!-- Step 2 a Detect changes in references file -->
+    <PropertyGroup Condition="Exists('$(PaketOriginalReferencesFilePath)') AND Exists('$(PaketReferencesCachedFilePath)') ">
+      <PaketRestoreCachedHash>$([System.IO.File]::ReadAllText('$(PaketReferencesCachedFilePath)'))</PaketRestoreCachedHash>
+      <PaketRestoreReferencesFileHash>$([System.IO.File]::ReadAllText('$(PaketOriginalReferencesFilePath)'))</PaketRestoreReferencesFileHash>
+      <PaketRestoreRequiredReason>references-file</PaketRestoreRequiredReason>
+      <PaketRestoreRequired Condition=" '$(PaketRestoreReferencesFileHash)' == '$(PaketRestoreCachedHash)' ">false</PaketRestoreRequired>
+    </PropertyGroup>
+
+    <!-- Step 2 b detect relevant changes in project file (new targetframework) -->
+    <PropertyGroup Condition=" !Exists('$(PaketReferencesFilePath)') AND '$(TargetFramework)' != '' ">
+      <PaketRestoreRequired>true</PaketRestoreRequired>
+      <PaketRestoreRequiredReason>target-framework</PaketRestoreRequiredReason>
+    </PropertyGroup>
+
+    <!-- Step 3 Restore project specific stuff if required -->
+    <Warning Condition=" '$(PaketRestoreRequired)' == 'true' "  Text="Detected a change ('$(PaketRestoreRequiredReason)') in the project file '$(MSBuildProjectFullPath)', calling paket restore" />
+    <Exec Command='$(PaketCommand) restore --project "$(MSBuildProjectFullPath)"' Condition=" '$(PaketRestoreRequired)' == 'true' " ContinueOnError="false" />
+
+    <!-- This shouldn't actually happen, but just to be sure. -->
+    <Error Condition=" !Exists('$(PaketReferencesFilePath)') AND '$(TargetFramework)' != '' " Text="A paket file for the framework '$(TargetFramework)' is missing. Please delete 'paket-files/paket.restore.cached' and call 'paket restore'." />
+
+    <!-- Step 4 forward all msbuild properties (PackageReference, DotNetCliToolReference) to msbuild -->
+    <ReadLinesFromFile Condition="Exists('$(PaketReferencesFilePath)')" File="$(PaketReferencesFilePath)" >
       <Output TaskParameter="Lines" ItemName="PaketReferencesFileLines"/>
     </ReadLinesFromFile>
 
-    <ItemGroup Condition=" '@(PaketReferencesFileLines)' != '' " >
+    <ItemGroup Condition=" Exists('$(PaketReferencesFilePath)') AND '@(PaketReferencesFileLines)' != '' " >
       <PaketReferencesFileLinesInfo Include="@(PaketReferencesFileLines)" >
         <PackageName>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0])</PackageName>
         <PackageVersion>$([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1])</PackageVersion>
@@ -80,7 +128,7 @@
 
   <Target Name="PaketOverrideNuspec" AfterTargets="GenerateNuspec" Condition="('$(IsPackable)' == '' Or '$(IsPackable)' == 'true') And Exists('$(MSBuildProjectDirectory)/obj/$(MSBuildProjectFile).references')" >
     <PropertyGroup>
-      <PaketReferencesFilePath>$(MSBuildProjectDirectory)/obj/$(MSBuildProjectFile).references</PaketReferencesFilePath>
+      <PaketProjectFile>$(MSBuildProjectDirectory)/$(MSBuildProjectFile)</PaketProjectFile>
       <ContinuePackingAfterGeneratingNuspec>true</ContinuePackingAfterGeneratingNuspec>
       <UseNewPack>false</UseNewPack>
       <UseNewPack Condition=" '$(NuGetToolVersion)' != '4.0.0' ">true</UseNewPack>
@@ -90,7 +138,7 @@
       <_NuspecFiles Include="$(BaseIntermediateOutputPath)*.nuspec"/>
     </ItemGroup>
 
-    <Exec Command='$(PaketCommand) fix-nuspecs files "@(_NuspecFiles)" references-file "$(PaketReferencesFilePath)" ' Condition="@(_NuspecFiles) != ''" />
+    <Exec Command='$(PaketCommand) fix-nuspecs files "@(_NuspecFiles)" project-file "$(PaketProjectFile)" ' Condition="@(_NuspecFiles) != ''" />
 
     <ConvertToAbsolutePath Condition="@(_NuspecFiles) != ''" Paths="@(_NuspecFiles)">
       <Output TaskParameter="AbsolutePaths" PropertyName="NuspecFileAbsolutePath" />

.paket/paket.targets

@@ -6,6 +6,8 @@
     <RestorePackages Condition=" '$(RestorePackages)' == '' ">true</RestorePackages>
     <PaketToolsPath>$(MSBuildThisFileDirectory)</PaketToolsPath>
     <PaketRootPath>$(MSBuildThisFileDirectory)..\</PaketRootPath>
+    <PaketLockFilePath>$(PaketRootPath)paket.lock</PaketLockFilePath>
+    <PaketRestoreCacheFile>$(PaketRootPath)paket-files\paket.restore.cached</PaketRestoreCacheFile>
     <MonoPath Condition="'$(MonoPath)' == '' And Exists('/Library/Frameworks/Mono.framework/Commands/mono')">/Library/Frameworks/Mono.framework/Commands/mono</MonoPath>
     <MonoPath Condition="'$(MonoPath)' == ''">mono</MonoPath>
   </PropertyGroup>
@@ -48,11 +50,23 @@
     <BuildDependsOn Condition="$(RestorePackages) == 'true'">RestorePackages; $(BuildDependsOn);</BuildDependsOn>
   </PropertyGroup>
   <Target Name="RestorePackages">
+    <PropertyGroup>
+      <PaketRestoreRequired>true</PaketRestoreRequired>
+    </PropertyGroup>
+
+    <PropertyGroup Condition="Exists('$(PaketRestoreCacheFile)') ">
+      <PaketRestoreCachedHash>$([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)'))</PaketRestoreCachedHash>
+      <PaketRestoreLockFileHash>$([System.IO.File]::ReadAllText('$(PaketLockFilePath)'))</PaketRestoreLockFileHash>
+      <PaketRestoreRequired>true</PaketRestoreRequired>
+      <PaketRestoreRequired Condition=" '$(PaketRestoreLockFileHash)' == '$(PaketRestoreCachedHash)' ">false</PaketRestoreRequired>
+      <PaketRestoreRequired Condition=" '$(PaketRestoreLockFileHash)' == '' ">true</PaketRestoreRequired>
+    </PropertyGroup>
+
     <Exec Command="$(RestoreCommand)" 
           IgnoreStandardErrorWarningFormat="true" 
           WorkingDirectory="$(PaketRootPath)" 
           ContinueOnError="false" 
-          Condition=" Exists('$(PaketReferences)') AND '$(PaketReferences)' != '' " 
+          Condition=" '$(PaketRestoreRequired)' == 'true' AND Exists('$(PaketReferences)') AND '$(PaketReferences)' != '' " 
     />
   </Target>
 </Project>

Paket.sln

@@ -110,6 +110,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{62D18A
 	EndProjectSection
 EndProject
 Global
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Release|Any CPU = Release|Any CPU

RELEASE_NOTES.md

@@ -1,3 +1,6 @@
+#### 5.92.0 - 26.08.2017
+* PERFORMANCE: Make restore faster - https://github.com/fsprojects/Paket/pull/2675
+
 #### 5.91.0 - 26.08.2017
 * BUGFIX: fix a bug in the runtime parser - https://github.com/fsprojects/Paket/pull/2665
 * BUGFIX: Add props to correct Paket.Restore.targets - https://github.com/fsprojects/Paket/pull/2665

build.fsx

@@ -168,6 +168,9 @@ Target "Build" (fun _ ->
                 "SourceLinkCreate"   , "true"
         ] "Rebuild"
         |> ignore
+
+    // DogFood newly build paket.exe for the dotnet build
+    setEnvironVar "PaketExePath" (Path.GetFullPath (buildDir @@ "paket.exe"))
 )
 
 let assertExitCodeZero x = 
@@ -183,11 +186,14 @@ let runCmdIn workDir exe =
 let dotnet workDir = runCmdIn workDir "dotnet"
 
 Target "DotnetRestoreTools" (fun _ ->
+    // DogFood newly build paket.exe for the dotnet restore for the tools.
+    setEnvironVar "PaketExePath" (Path.GetFullPath (buildDir @@ "paket.exe"))
     DotNetCli.Restore (fun c ->
         { c with
             Project = currentDirectory </> "tools" </> "tools.fsproj"
-            ToolPath = dotnetExePath 
+            ToolPath = dotnetExePath
         })
+    setEnvironVar "PaketExePath" null
 )
 
 Target "DotnetRestore" (fun _ ->
@@ -230,6 +236,8 @@ Target "DotnetPackage" (fun _ ->
 // Run the unit tests using test runner
 
 Target "RunTests" (fun _ ->
+    // Stop bootstrapping in from here (tests should use whatever they want to test).
+    setEnvironVar "PaketExePath" null
     !! testAssemblies
     |> NUnit3 (fun p ->
         { p with

src/Paket.Core/Common/Constants.fs

@@ -14,6 +14,8 @@ let [<Literal>] GithubReleaseDownloadUrl  = "https://github.com/fsprojects/Paket
 let [<Literal>] LockFileName              = "paket.lock"
 /// 'paket.local'
 let [<Literal>] LocalFileName             = "paket.local"
+/// 'paket.restore.sha512'
+let [<Literal>] RestoreHashFile           = "paket.restore.cached"
 /// 'paket.dependencies'
 let [<Literal>] DependenciesFileName      = "paket.dependencies"
 /// '.paket'

src/Paket.Core/Common/Utils.fs

@@ -1071,6 +1071,14 @@ let removeComment (text:string) =
         | p1, p2 -> stripComment (min p1 p2) 
     remove 0
 
+let getSha512Stream (stream:Stream) =
+    use hasher = System.Security.Cryptography.SHA512.Create() :> System.Security.Cryptography.HashAlgorithm
+    Convert.ToBase64String(hasher.ComputeHash(stream))
+
+let getSha512File (filePath:string) =
+    use stream = File.OpenRead(filePath)
+    getSha512Stream stream
+
 // adapted from MiniRx
 // http://minirx.codeplex.com/
 [<AutoOpen>]

src/Paket.Core/Dependencies/NuGetCache.fs

@@ -316,10 +316,7 @@ let rec ExtractPackageToUserFolder(fileName:string, packageName:PackageName, ver
 
             let cachedHashFile = Path.Combine(Constants.NuGetCacheFolder,fi.Name + ".sha512")
             if not <| File.Exists cachedHashFile then
-                use stream = File.OpenRead(fileName)
-                let packageSize = stream.Length
-                use hasher = System.Security.Cryptography.SHA512.Create() :> System.Security.Cryptography.HashAlgorithm
-                let packageHash = Convert.ToBase64String(hasher.ComputeHash(stream))
+                let packageHash = getSha512File fileName
                 File.WriteAllText(cachedHashFile,packageHash)
 
             File.Copy(cachedHashFile,targetPackageFileName + ".sha512")

src/Paket.Core/Dependencies/PackageResolver.fs

@@ -1271,7 +1271,7 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou
     // Flag to ensure that we don't hide underlying exceptions in the finally block.
     let mutable exceptionThrown = false
     try
-#if DEBUG
+#if DEBUG && !DOTNETCORE
         let mutable results = None
         let mutable error = None
         // Increase stack size, because we have no tail-call-elimination