Skip to content

Commit

Permalink
Improve speed of myget
Browse files Browse the repository at this point in the history 
- Remove magic strings for known nuget sources
- Improve error handling of new system.net.http library
- Do not convert v3 sources to v2 sources when parsing them -> this lead
  to a huge performance bottleneck for myget.
  • Loading branch information
@matthid
matthid committed Jul 9, 2017
1 parent 3720adb commit b2e477e
Show file tree
Hide file tree
Showing 8 changed files with 190 additions and 93 deletions.
46 changes: 45 additions & 1 deletion src/Paket.Core/Common/Async.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,20 @@ module AsyncExtensions =
open System
open System.Threading.Tasks
open System.Threading

open System.Runtime.ExceptionServices

// This uses a trick to get the underlying OperationCanceledException
let inline getCancelledException (completedTask:Task) (waitWithAwaiter) =
let fallback = new TaskCanceledException(completedTask) :> OperationCanceledException
// sadly there is no other public api to retrieve it, but to call .GetAwaiter().GetResult().
try waitWithAwaiter()
// should not happen, but just in case...
fallback
with
| :? OperationCanceledException as o -> o
| other ->
// shouldn't happen, but just in case...
new TaskCanceledException(fallback.Message, other) :> OperationCanceledException
type Microsoft.FSharp.Control.Async with
/// Runs both computations in parallel and returns the result as a tuple.
static member Parallel (a : Async<'a>, b : Async<'b>) : Async<'a * 'b> =
Expand All @@ -25,6 +38,37 @@ module AsyncExtensions =
let! b'' = b'
return (a'',b'')
}
static member AwaitTaskWithoutAggregate (task:Task<'T>) : Async<'T> =
Async.FromContinuations(fun (cont, econt, ccont) ->
let continuation (completedTask : Task<_>) =
if completedTask.IsCanceled then
let cancelledException =
getCancelledException completedTask (fun () -> completedTask.GetAwaiter().GetResult() |> ignore)
econt (cancelledException)
elif completedTask.IsFaulted then
if completedTask.Exception.InnerExceptions.Count = 1 then
econt completedTask.Exception.InnerExceptions.[0]
else
econt completedTask.Exception
else
cont completedTask.Result
task.ContinueWith(Action<Task<'T>>(continuation)) |> ignore)
static member AwaitTaskWithoutAggregate (task:Task) : Async<unit> =
Async.FromContinuations(fun (cont, econt, ccont) ->
let continuation (completedTask : Task) =
if completedTask.IsCanceled then
let cancelledException =
getCancelledException completedTask (fun () -> completedTask.GetAwaiter().GetResult() |> ignore)
econt (cancelledException)
elif completedTask.IsFaulted then
if completedTask.Exception.InnerExceptions.Count = 1 then
econt completedTask.Exception.InnerExceptions.[0]
else
econt completedTask.Exception
else
cont ()
task.ContinueWith(Action<Task>(continuation)) |> ignore)

static member awaitTaskWithToken (fallBack:unit -> 'T) (item: Task<'T>) : Async<'T> =
async {
let! ct = Async.CancellationToken
Expand Down
41 changes: 27 additions & 14 deletions src/Paket.Core/Common/Utils.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,6 @@ let normalizeFeedUrl (source:string) =
| "http://nuget.org/api/v2" -> Constants.DefaultNuGetStream.Replace("https","http")
| "https://www.nuget.org/api/v2" -> Constants.DefaultNuGetStream
| "http://www.nuget.org/api/v2" -> Constants.DefaultNuGetStream.Replace("https","http")
| url when url.EndsWith("/api/v3/index.json") -> url.Replace("/api/v3/index.json","")
| source -> source

#if NETSTANDARD1_6
Expand Down Expand Up @@ -385,12 +384,24 @@ let getDefaultProxyFor =
| Some p -> if p.GetProxy uri <> uri then p else getDefault()
| None -> getDefault())


exception RequestReturnedError of statusCode:HttpStatusCode * content:Stream * mediaType:string
let failIfNoSuccess (resp:HttpResponseMessage) = async {
if not resp.IsSuccessStatusCode then
if verbose then
tracefn "Request failed with '%d': '%s'" (int resp.StatusCode) (resp.RequestMessage.RequestUri.ToString())
let mem = new MemoryStream()
do! resp.Content.CopyToAsync(mem) |> Async.AwaitTaskWithoutAggregate
mem.Position <- 0L
raise <| RequestReturnedError(resp.StatusCode, mem, resp.Content.Headers.ContentType.MediaType)
() }
type HttpClient with
member x.DownloadFileTaskAsync (uri : Uri, tok : CancellationToken, filePath : string) =
async {
let! response = x.GetAsync(uri, tok) |> Async.AwaitTask
let! response = x.GetAsync(uri, tok) |> Async.AwaitTaskWithoutAggregate
do! failIfNoSuccess response
use fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)
do! response.Content.CopyToAsync(fileStream) |> Async.AwaitTask
do! response.Content.CopyToAsync(fileStream) |> Async.AwaitTaskWithoutAggregate
fileStream.Flush()
} |> Async.StartAsTask
member x.DownloadFileTaskAsync (uri : string, tok : CancellationToken, filePath : string) = x.DownloadFileTaskAsync(Uri uri, tok, filePath)
Expand All @@ -400,8 +411,9 @@ type HttpClient with
x.DownloadFileTaskAsync(uri, CancellationToken.None, filePath).GetAwaiter().GetResult()
member x.DownloadStringTaskAsync (uri : Uri, tok : CancellationToken) =
async {
let! response = x.GetAsync(uri, tok) |> Async.AwaitTask
let! result = response.Content.ReadAsStringAsync() |> Async.AwaitTask
let! response = x.GetAsync(uri, tok) |> Async.AwaitTaskWithoutAggregate
do! failIfNoSuccess response
let! result = response.Content.ReadAsStringAsync() |> Async.AwaitTaskWithoutAggregate
return result
} |> Async.StartAsTask
member x.DownloadStringTaskAsync (uri : string, tok : CancellationToken) = x.DownloadStringTaskAsync(Uri uri, tok)
Expand All @@ -412,8 +424,9 @@ type HttpClient with

member x.DownloadDataTaskAsync(uri : Uri, tok : CancellationToken) =
async {
let! response = x.GetAsync(uri, tok) |> Async.AwaitTask
let! result = response.Content.ReadAsByteArrayAsync() |> Async.AwaitTask
let! response = x.GetAsync(uri, tok) |> Async.AwaitTaskWithoutAggregate
do! failIfNoSuccess response
let! result = response.Content.ReadAsByteArrayAsync() |> Async.AwaitTaskWithoutAggregate
return result
} |> Async.StartAsTask
member x.DownloadDataTaskAsync (uri : string, tok : CancellationToken) = x.DownloadDataTaskAsync(Uri uri, tok)
Expand Down Expand Up @@ -491,7 +504,7 @@ let downloadFromUrl (auth:Auth option, url : string) (filePath: string) =
if verbose then
verbosefn "Starting download from '%O'" url
use _ = Profile.startCategory Profile.Category.NuGetDownload
let task = client.DownloadFileTaskAsync (Uri url, tok, filePath) |> Async.AwaitTask
let task = client.DownloadFileTaskAsync (Uri url, tok, filePath) |> Async.AwaitTaskWithoutAggregate
do! task
with
| exn ->
Expand All @@ -510,7 +523,7 @@ let getFromUrl (auth:Auth option, url : string, contentType : string) =
if verbose then
verbosefn "Starting request to '%O'" url
use _ = Profile.startCategory Profile.Category.NuGetRequest
return! client.DownloadStringTaskAsync (Uri url, tok) |> Async.AwaitTask
return! client.DownloadStringTaskAsync (Uri url, tok) |> Async.AwaitTaskWithoutAggregate
with
| exn ->
return raise <| Exception(sprintf "Could not retrieve data from '%s'" url, exn)
Expand All @@ -530,7 +543,7 @@ let getXmlFromUrl (auth:Auth option, url : string) =
if verbose then
verbosefn "Starting request to '%O'" url
use _ = Profile.startCategory Profile.Category.NuGetRequest
return! client.DownloadStringTaskAsync (Uri url, tok) |> Async.AwaitTask
return! client.DownloadStringTaskAsync (Uri url, tok) |> Async.AwaitTaskWithoutAggregate
with
| exn ->
return raise <| Exception(sprintf "Could not retrieve data from '%s'" url, exn)
Expand Down Expand Up @@ -569,12 +582,12 @@ let safeGetFromUrl (auth:Auth option, url : string, contentType : string) =
if verbose then
verbosefn "Starting request to '%O'" uri
use _ = Profile.startCategory Profile.Category.NuGetRequest
let! raw = client.DownloadStringTaskAsync(uri, tok) |> Async.awaitTaskWithToken (fun () -> failwithf "Uri '%O' failed to respond and cancellation was requested." uri)
let! raw = client.DownloadStringTaskAsync(uri, tok) |> Async.AwaitTaskWithoutAggregate
return SuccessResponse raw
with
| :? WebException as w ->
match w.Response with
| :? HttpWebResponse as wr when wr.StatusCode = HttpStatusCode.NotFound -> return NotFound
| RequestReturnedError(statusCode, content, mediaType) as w ->
match statusCode with
| HttpStatusCode.NotFound -> return NotFound
| _ ->
if verbose then
Logging.verbosefn "Error while retrieving '%s': %O" url w
Expand Down
97 changes: 53 additions & 44 deletions src/Paket.Core/Dependencies/NuGet.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,12 @@ let tryNuGetV3 (auth, nugetV3Url, package:PackageName) =

let rec private getPackageDetails alternativeProjectRoot root force (sources:PackageSource list) packageName (version:SemVerInfo) : Async<PackageResolver.PackageDetails> =
async {
let inCache =
sources
|> Seq.choose(fun source ->
NuGetCache.tryGetDetailsFromCache force source.Url packageName version |> Option.map (fun details -> source, details))
|> Seq.tryHead

let tryV2 (nugetSource:NugetSource) force =
NuGetV2.getDetailsFromNuGet
force
Expand Down Expand Up @@ -156,55 +162,61 @@ let rec private getPackageDetails alternativeProjectRoot root force (sources:Pac
| ODataSearchResult.EmptyResult -> return! trySelectFirst rest
| [] -> return None
}
sources
|> List.sortBy (fun source ->
match source with // put local caches to the end
| LocalNuGet(_,Some _) -> true
| _ -> false)
|> List.map (fun source -> source, async {
try
match inCache with
| Some (source, ODataSearchResult.Match result) -> async { return Some (source, result) }
| _ ->
sources
|> List.sortBy (fun source ->
// put local caches to the end
// prefer nuget gallery
match source with
| NuGetV2 nugetSource ->
return! tryV2 nugetSource force
| NuGetV3 nugetSource when NuGetV2.urlSimilarToTfsOrVsts nugetSource.Url ->
match NuGetV3.calculateNuGet2Path nugetSource.Url with
| Some url ->
let nugetSource : NugetSource =
{ Url = url
Authentication = nugetSource.Authentication }
| LocalNuGet(_,Some _) -> 10
| s when s.NuGetType = KnownNuGetSources.OfficialNuGetGallery -> 1
| _ -> 3)
|> List.map (fun source -> source, async {
try
match source with
| NuGetV2 nugetSource ->
return! tryV2 nugetSource force
| _ ->
return! tryV3 nugetSource force
| NuGetV3 nugetSource ->
try
return! tryV3 nugetSource force
with
| exn ->
| NuGetV3 nugetSource when urlSimilarToTfsOrVsts nugetSource.Url ->
match NuGetV3.calculateNuGet2Path nugetSource.Url with
| Some url ->
let nugetSource : NugetSource =
{ Url = url
Authentication = nugetSource.Authentication }
return! tryV2 nugetSource force
| _ ->
raise exn
return! tryV3 nugetSource force

| LocalNuGet(path,hasCache) ->
return! NuGetLocal.getDetailsFromLocalNuGetPackage hasCache.IsSome alternativeProjectRoot root path packageName version
with
| :? System.IO.IOException as exn ->
// Handling IO exception here for less noise in output: https://github.com/fsprojects/Paket/issues/2480
if verbose then
traceWarnfn "I/O error for source '%O': %O" source exn
else
traceWarnfn "I/O error for source '%O': %s" source exn.Message
return EmptyResult
| e ->
traceWarnfn "Source '%O' exception: %O" source e
//let capture = ExceptionDispatchInfo.Capture e
return EmptyResult })
|> trySelectFirst
| NuGetV3 nugetSource ->
try
return! tryV3 nugetSource force
with
| exn ->
match NuGetV3.calculateNuGet2Path nugetSource.Url with
| Some url ->
let nugetSource : NugetSource =
{ Url = url
Authentication = nugetSource.Authentication }
return! tryV2 nugetSource force
| _ ->
raise exn
return! tryV3 nugetSource force

| LocalNuGet(path,hasCache) ->
return! NuGetLocal.getDetailsFromLocalNuGetPackage hasCache.IsSome alternativeProjectRoot root path packageName version
with
| :? System.IO.IOException as exn ->
// Handling IO exception here for less noise in output: https://github.com/fsprojects/Paket/issues/2480
if verbose then
traceWarnfn "I/O error for source '%O': %O" source exn
else
traceWarnfn "I/O error for source '%O': %s" source exn.Message
return EmptyResult
| e ->
traceWarnfn "Source '%O' exception: %O" source e
//let capture = ExceptionDispatchInfo.Capture e
return EmptyResult })
|> trySelectFirst

let! maybePackageDetails = getPackageDetails force
let! source,nugetObject =
Expand Down Expand Up @@ -347,11 +359,8 @@ let GetVersions force alternativeProjectRoot root (sources, packageName:PackageN
| Some v3Url -> return (getVersionsCached "V3" tryNuGetV3 (nugetSource, auth, v3Url, packageName)) :: v2Feeds
| NuGetV3 source ->
let! versionsAPI = PackageSources.getNuGetV3Resource source AllVersionsAPI
let req = tryNuGetV3
(source.Authentication |> Option.map toBasicAuth,
versionsAPI,
packageName)
return [ req ]
let auth = source.Authentication |> Option.map toBasicAuth
return [ getVersionsCached "V3" tryNuGetV3 (nugetSource, auth, versionsAPI, packageName) ]
| LocalNuGet(path,Some _) ->
return [ NuGetLocal.getAllVersionsFromLocalPath (true, path, packageName, alternativeProjectRoot, root) ]
| LocalNuGet(path,None) ->
Expand Down
60 changes: 34 additions & 26 deletions src/Paket.Core/Dependencies/NuGetCache.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,37 @@ module ODataSearchResult =
match x with
| EmptyResult -> failwithf "Cannot call get on 'EmptyResult'"
| Match r -> r

let tryGetDetailsFromCache force nugetURL (packageName:PackageName) (version:SemVerInfo) : ODataSearchResult option =
let cacheFile, oldFiles = getCacheFiles NuGetPackageCache.CurrentCacheVersion nugetURL packageName version
oldFiles |> Seq.iter (fun f -> File.Delete f)
if not force && cacheFile.Exists then
let json = File.ReadAllText(cacheFile.FullName)
let cacheResult =
try
let cachedObject = JsonConvert.DeserializeObject<NuGetPackageCache> json
if (PackageName cachedObject.PackageName <> packageName) ||
(cachedObject.Version <> version.Normalize())
then
traceVerbose (sprintf "Invalidating Cache '%s:%s' <> '%s:%s'" cachedObject.PackageName cachedObject.Version packageName.Name (version.Normalize()))
cacheFile.Delete()
None
else
Some cachedObject
with
| exn ->
cacheFile.Delete()
if verbose then
traceWarnfn "Error while loading cache: %O" exn
else
traceWarnfn "Error while loading cache: %s" exn.Message
None
match cacheResult with
| Some res -> Some (ODataSearchResult.Match res)
| None -> None
else
None

let getDetailsFromCacheOr force nugetURL (packageName:PackageName) (version:SemVerInfo) (get : unit -> ODataSearchResult Async) : ODataSearchResult Async =
let cacheFile, oldFiles = getCacheFiles NuGetPackageCache.CurrentCacheVersion nugetURL packageName version
oldFiles |> Seq.iter (fun f -> File.Delete f)
Expand All @@ -181,32 +212,9 @@ let getDetailsFromCacheOr force nugetURL (packageName:PackageName) (version:SemV
return result
}
async {
if not force && cacheFile.Exists then
let json = File.ReadAllText(cacheFile.FullName)
let cacheResult =
try
let cachedObject = JsonConvert.DeserializeObject<NuGetPackageCache> json
if (PackageName cachedObject.PackageName <> packageName) ||
(cachedObject.Version <> version.Normalize())
then
traceVerbose (sprintf "Invalidating Cache '%s:%s' <> '%s:%s'" cachedObject.PackageName cachedObject.Version packageName.Name (version.Normalize()))
cacheFile.Delete()
None
else
Some cachedObject
with
| exn ->
cacheFile.Delete()
if verbose then
traceWarnfn "Error while loading cache: %O" exn
else
traceWarnfn "Error while loading cache: %s" exn.Message
None
match cacheResult with
| Some res -> return ODataSearchResult.Match res
| None -> return! get()
else
return! get()
match tryGetDetailsFromCache force nugetURL packageName version with
| None -> return! get()
| Some res -> return res
}


Expand Down
Loading

0 comments on commit b2e477e

Please sign in to comment.