From e4cf862710f347dd98e0c053b988d2f4f8c89f4d Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Sun, 3 Jun 2018 14:25:31 +0200 Subject: [PATCH 1/2] initial scetch around reporting at lease every 10 seconds what's going on (still missing if AsyncRead 'hangs') --- src/Paket.Core/Dependencies/NuGet.fs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/Paket.Core/Dependencies/NuGet.fs b/src/Paket.Core/Dependencies/NuGet.fs index 32f0dc73ce..b9d7dc59b0 100644 --- a/src/Paket.Core/Dependencies/NuGet.fs +++ b/src/Paket.Core/Dependencies/NuGet.fs @@ -892,6 +892,10 @@ let private downloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverr request.UseDefaultCredentials <- true request.Proxy <- NetUtils.getDefaultProxyFor source.Url + + let lastSpeedMeasure = Stopwatch.StartNew() + let mutable readSinceLastMeasure = 0L + use! httpResponse = request.AsyncGetResponse() use httpResponseStream = httpResponse.GetResponseStream() @@ -901,17 +905,34 @@ let private downloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverr let bytesRead = ref -1 use fileStream = File.Create(targetFileName) + let printProgress = not Console.IsOutputRedirected + let mutable pos = 0L + let length = try Some httpResponseStream.Length with :? System.NotSupportedException -> None while !bytesRead <> 0 do + if printProgress && lastSpeedMeasure.Elapsed > TimeSpan.FromSeconds(10.) then + // report speed and progress + let speed = int (float readSinceLastMeasure * 8. / float lastSpeedMeasure.ElapsedMilliseconds) + let percent = + match length with + | Some l -> sprintf "%d" (int (pos * 100L / l)) + | None -> "Unknown" + tracefn "Still downloading from %O to %s (%d kbit/s, %s %%)" !downloadUrl targetFileName speed percent + readSinceLastMeasure <- 0L + lastSpeedMeasure.Restart() let! bytes = httpResponseStream.AsyncRead(buffer, 0, bufferSize) bytesRead := bytes do! fileStream.AsyncWrite(buffer, 0, !bytesRead) + readSinceLastMeasure <- readSinceLastMeasure + int64 bytes + pos <- pos + int64 bytes match (httpResponse :?> HttpWebResponse).StatusCode with | HttpStatusCode.OK -> () | statusCode -> failwithf "HTTP status code was %d - %O" (int statusCode) statusCode - - tracefn "Download of %O %O%s done in %s." packageName version groupString (Utils.TimeSpanToReadableString sw.Elapsed) + + let speed = int (float pos * 8. / float lastSpeedMeasure.ElapsedMilliseconds) + let size = pos / (1024L * 1024L) + tracefn "Download of %O %O%s done in %s. (%d kbit/s, %d MB)" packageName version groupString (Utils.TimeSpanToReadableString sw.Elapsed) speed size try if downloadLicense && not (String.IsNullOrWhiteSpace nugetPackage.LicenseUrl) then From a467619896282edc76fbc88bccf85d65325a7b25 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Sun, 3 Jun 2018 14:46:42 +0200 Subject: [PATCH 2/2] timeout asynchronous operations --- src/Paket.Core/Common/NetUtils.fs | 4 +++- src/Paket.Core/Dependencies/NuGet.fs | 16 +++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Paket.Core/Common/NetUtils.fs b/src/Paket.Core/Common/NetUtils.fs index f3bfe52a81..79678d6ea3 100644 --- a/src/Paket.Core/Common/NetUtils.fs +++ b/src/Paket.Core/Common/NetUtils.fs @@ -539,6 +539,8 @@ let rec private _safeGetFromUrl (auth:Auth option, url : string, contentType : s let uri = Uri url use client = createHttpClient (url,auth) let! tok = Async.CancellationToken + let tokSource = System.Threading.CancellationTokenSource.CreateLinkedTokenSource(tok) + tokSource.CancelAfter(60000) if notNullOrEmpty contentType then addAcceptHeader client contentType @@ -546,7 +548,7 @@ let rec private _safeGetFromUrl (auth:Auth option, url : string, contentType : s if verbose then verbosefn "Starting request to '%O'" uri use _ = Profile.startCategory Profile.Category.NuGetRequest - let! raw = client.DownloadStringTaskAsync(uri, tok) |> Async.AwaitTaskWithoutAggregate + let! raw = client.DownloadStringTaskAsync(uri, tokSource.Token) |> Async.AwaitTaskWithoutAggregate return SuccessResponse raw with diff --git a/src/Paket.Core/Dependencies/NuGet.fs b/src/Paket.Core/Dependencies/NuGet.fs index b9d7dc59b0..b539447725 100644 --- a/src/Paket.Core/Dependencies/NuGet.fs +++ b/src/Paket.Core/Dependencies/NuGet.fs @@ -896,11 +896,11 @@ let private downloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverr let lastSpeedMeasure = Stopwatch.StartNew() let mutable readSinceLastMeasure = 0L - use! httpResponse = request.AsyncGetResponse() - + let! child = Async.StartChild(request.AsyncGetResponse(), 60000) + use! httpResponse = child use httpResponseStream = httpResponse.GetResponseStream() - let bufferSize = 4096 + let bufferSize = 1024 * 10 let buffer : byte [] = Array.zeroCreate bufferSize let bytesRead = ref -1 @@ -909,6 +909,7 @@ let private downloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverr let mutable pos = 0L let length = try Some httpResponseStream.Length with :? System.NotSupportedException -> None + let! tok = Async.CancellationToken while !bytesRead <> 0 do if printProgress && lastSpeedMeasure.Elapsed > TimeSpan.FromSeconds(10.) then // report speed and progress @@ -920,9 +921,14 @@ let private downloadAndExtractPackage(alternativeProjectRoot, root, isLocalOverr tracefn "Still downloading from %O to %s (%d kbit/s, %s %%)" !downloadUrl targetFileName speed percent readSinceLastMeasure <- 0L lastSpeedMeasure.Restart() - let! bytes = httpResponseStream.AsyncRead(buffer, 0, bufferSize) + // if there is no response for a minute -> abort + + let s = System.Threading.CancellationTokenSource.CreateLinkedTokenSource(tok) + s.CancelAfter (60000) + let! bytes = httpResponseStream.ReadAsync(buffer, 0, bufferSize, s.Token) |> Async.AwaitTaskWithoutAggregate + //let! bytes = httpResponseStream.AsyncRead(buffer, 0, bufferSize) bytesRead := bytes - do! fileStream.AsyncWrite(buffer, 0, !bytesRead) + do! fileStream.WriteAsync(buffer, 0, !bytesRead, tok) |> Async.AwaitTaskWithoutAggregate readSinceLastMeasure <- readSinceLastMeasure + int64 bytes pos <- pos + int64 bytes