Merge pull request #2362 from fsprojects/fix_retry_logic

Improve preloading-request logic

RELEASE_NOTES.md

@@ -1,3 +1,8 @@
+#### Unreleased 5.0.0-rc004
+* PERFORMANCE: Limit the number of concurrent requests to 7 (https://github.com/fsprojects/Paket/pull/2362)
+* PERFORMANCE: Report how often the pre-loading feature worked (https://github.com/fsprojects/Paket/pull/2362)
+* PERFORMANCE: Request queue can now re-prioritize on-demand (https://github.com/fsprojects/Paket/pull/2362)
+
 #### 5.0.0-rc003 - 09.06.2017
 * Internals: Started proper dotnetcore integration (disabled by default, can be enabled via setting `PAKET_DISABLE_RUNTIME_RESOLUTION` to `false`):
   * Paket now properly understands runtime and reference assemblies

integrationtests/scenarios/i001427-content-none/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0 content:none
+nuget Costura.Fody ~> 1.3.3.0 content:none
+nuget Fody ~> 2.0.10 content:none

integrationtests/scenarios/i001427-content-once-remove/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0 content: once
+nuget Costura.Fody ~> 1.3.3.0 content: once
+nuget Fody ~> 2.0.10 content: once

integrationtests/scenarios/i001427-content-once-stable/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0 content: once
+nuget Costura.Fody ~> 1.3.3.0 content: once
+nuget Fody ~> 2.0.10 content: once

integrationtests/scenarios/i001427-content-once/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0 content: once
+nuget Costura.Fody ~> 1.3.3.0 content: once
+nuget Fody ~> 2.0.10 content: once

integrationtests/scenarios/i001427-content-true/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0
+nuget Costura.Fody ~> 1.3.3.0
+nuget Fody ~> 2.0.10

integrationtests/scenarios/i001427-ref-content-once/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0
+nuget Costura.Fody ~> 1.3.3.0
+nuget Fody ~> 2.0.10

integrationtests/scenarios/i001522-copy-content/before/paket.dependencies

@@ -1,4 +1,5 @@
 source https://www.nuget.org/api/v2/
 framework >= net45
 
-nuget Costura.Fody ~> 1.3.3.0 content: once, copy_content_to_output_dir: always
+nuget Costura.Fody ~> 1.3.3.0 content: once, copy_content_to_output_dir: always
+nuget Fody ~> 2.0.10 content: once, copy_content_to_output_dir: always

src/Paket.Core/Common/Profile.fs

@@ -10,6 +10,7 @@ type BlockReason =
 type Category =
     | ResolverAlgorithm
     | ResolverAlgorithmBlocked of BlockReason
+    | ResolverAlgorithmNotBlocked of BlockReason
     | NuGetRequest
     | NuGetDownload
     | FileIO
@@ -18,6 +19,9 @@ type Event = { Category: Category; Duration : TimeSpan }
 let events =
     System.Collections.Concurrent.ConcurrentBag<Event>()
 
+let trackEvent cat =
+    events.Add({ Category = cat; Duration = TimeSpan() })
+
 let startCategory cat =
     let cw = Stopwatch.StartNew()
     let mutable wasDisposed = false

src/Paket.Core/Dependencies/NuGetV2.fs

@@ -18,6 +18,7 @@ open Paket.Xml
 open Paket.PackageSources
 open Paket.Requirements
 open FSharp.Polyfill
+open System.Runtime.ExceptionServices
 
 let rec private followODataLink auth url =
     async {
@@ -268,7 +269,7 @@ let getDetailsFromNuGetViaOData auth nugetURL (packageName:PackageName) (version
         try
             let! result = getDetailsFromNuGetViaODataFast auth nugetURL packageName version
             if urlSimilarToTfsOrVsts nugetURL && result |> NuGet.NuGetPackageCache.getDependencies |> List.isEmpty then
-                // TODO: There is a bug in VSTS, so we can't trust this protocol. Remvoe when VSTS is fixed
+                // TODO: There is a bug in VSTS, so we can't trust this protocol. Remove when VSTS is fixed
                 // TODO: TFS has the same bug
                 return! queryPackagesProtocol packageName
             else
@@ -651,7 +652,7 @@ let rec private getPackageDetails alternativeProjectRoot root force (sources:Pac
                     nugetSource.Url
                     packageName
                     version
-            return Some(source,result)  }
+            return Choice1Of2(source,result)  }
 
         let tryV3 source nugetSource force = async {
             if nugetSource.Url.Contains("myget.org") || nugetSource.Url.Contains("nuget.org") || nugetSource.Url.Contains("visualstudio.com") || nugetSource.Url.Contains("/nuget/v3/") then
@@ -664,25 +665,25 @@ let rec private getPackageDetails alternativeProjectRoot root force (sources:Pac
                             url
                             packageName
                             version
-                    return Some(source,result)
+                    return Choice1Of2(source,result)
                 | _ ->
                     let! result = NuGetV3.GetPackageDetails force nugetSource packageName version
-                    return Some(source,result)
+                    return Choice1Of2(source,result)
             else
                 let! result = NuGetV3.GetPackageDetails force nugetSource packageName version
-                return Some(source,result) }
+                return Choice1Of2(source,result) }
 
         let getPackageDetails force =
             // helper to work through the list sequentially
-            let rec trySelectFirst workLeft =
+            let rec trySelectFirst errors workLeft =
                 async {
                     match workLeft with
                     | work :: rest ->
                         let! r = work
                         match r with
-                        | Some result -> return Some result
-                        | None -> return! trySelectFirst rest
-                    | [] -> return None
+                        | Choice1Of2 result -> return Choice1Of2 result
+                        | Choice2Of2 error -> return! trySelectFirst (error::errors) rest
+                    | [] -> return Choice2Of2 errors
                 }
             sources
             |> List.sortBy (fun source ->
@@ -720,33 +721,39 @@ let rec private getPackageDetails alternativeProjectRoot root force (sources:Pac
 
                     | LocalNuGet(path,Some _) ->
                         let! result = getDetailsFromLocalNuGetPackage true alternativeProjectRoot root path packageName version
-                        return Some(source,result)
+                        return Choice1Of2(source,result)
                     | LocalNuGet(path,None) ->
                         let! result = getDetailsFromLocalNuGetPackage false alternativeProjectRoot root path packageName version
-                        return Some(source,result)
+                        return Choice1Of2(source,result)
                 with e ->
                     if verbose then
                         verbosefn "Source '%O' exception: %O" source e
-                    return None })
-            |> trySelectFirst
+                    let capture = ExceptionDispatchInfo.Capture e
+                    return Choice2Of2 capture })
+            |> trySelectFirst []
 
         let! maybePackageDetails = getPackageDetails force
         let! source,nugetObject =
             async {
+                let fallback () =
+                    match sources |> List.map (fun (s:PackageSource) -> s.ToString()) with
+                    | [source] ->
+                        failwithf "Couldn't get package details for package %O %O on %O." packageName version source
+                    | [] ->
+                        failwithf "Couldn't get package details for package %O %O, because no sources were specified." packageName version
+                    | sources ->
+                        failwithf "Couldn't get package details for package %O %O on any of %A." packageName version sources
+
                 match maybePackageDetails with
-                | None ->
-                    let! m = getPackageDetails true
-                    match m with
-                    | None ->
-                        match sources |> List.map (fun (s:PackageSource) -> s.ToString()) with
-                        | [source] ->
-                            return failwithf "Couldn't get package details for package %O %O on %O." packageName version source
-                        | [] ->
-                            return failwithf "Couldn't get package details for package %O %O, because no sources were specified." packageName version
-                        | sources ->
-                            return failwithf "Couldn't get package details for package %O %O on any of %A." packageName version sources
-                    | Some packageDetails -> return packageDetails
-                | Some packageDetails -> return packageDetails
+                | Choice2Of2 ([]) -> return fallback()
+                | Choice2Of2 (h::restError) ->
+                    for error in restError do
+                        if not verbose then
+                            // Otherwise the error was already mentioned above
+                            traceWarnfn "Ignoring: %s" error.SourceException.Message
+                    h.Throw()
+                    return fallback()
+                | Choice1Of2 packageDetails -> return packageDetails
             }
 
         let encodeURL (url:string) =

src/Paket.Core/Dependencies/Nuget.fs

@@ -106,25 +106,29 @@ let getDetailsFromCacheOr force nugetURL (packageName:PackageName) (version:SemV
     async {
         if not force && cacheFile.Exists then
             let json = File.ReadAllText(cacheFile.FullName)
-            try
-                let cachedObject = JsonConvert.DeserializeObject<NuGetPackageCache> json
+            let cacheResult =
+                try
+                    let cachedObject = JsonConvert.DeserializeObject<NuGetPackageCache> json
 
-                if (PackageName cachedObject.PackageName <> packageName) ||
-                  (cachedObject.Version <> version.Normalize())
-                then
-                    traceVerbose (sprintf "Invalidating Cache '%s:%s' <> '%s:%s'" cachedObject.PackageName cachedObject.Version packageName.Name (version.Normalize()))
+                    if (PackageName cachedObject.PackageName <> packageName) ||
+                      (cachedObject.Version <> version.Normalize())
+                    then
+                        traceVerbose (sprintf "Invalidating Cache '%s:%s' <> '%s:%s'" cachedObject.PackageName cachedObject.Version packageName.Name (version.Normalize()))
+                        cacheFile.Delete()
+                        None
+                    else
+                        Some cachedObject
+                with
+                | exn ->
                     cacheFile.Delete()
-                    return! get()
-                else
-                    return cachedObject
-            with
-            | exn ->
-                cacheFile.Delete()
-                if verbose then
-                    traceWarnfn "Error while loading cache: %O" exn
-                else
-                    traceWarnfn "Error while loading cache: %s" exn.Message
-                return! get()
+                    if verbose then
+                        traceWarnfn "Error while loading cache: %O" exn
+                    else
+                        traceWarnfn "Error while loading cache: %s" exn.Message
+                    None
+            match cacheResult with
+            | Some res -> return res
+            | None -> return! get()
         else
             return! get()
     }