From a9b4a3c2ded76388b920f13e0b250bf1eb09b604 Mon Sep 17 00:00:00 2001 From: Kendrick Coleman Date: Fri, 6 Feb 2015 16:11:18 -0500 Subject: [PATCH 1/3] fix for no content-length in header for download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using a 3rd party S3 storage service (not AWS), there are instances when the server returns a header without ‘content-length’ specified and instead uses 'transfer-encoding': 'chunked’. This workout around will create a writeable stream to save a file and still emit progress total, but at 99%. When the transfer ends, the progress will add 1 to be complete at 100% --- lib/index.js | 92 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/lib/index.js b/lib/index.js index 5e732be..4ac2b26 100644 --- a/lib/index.js +++ b/lib/index.js @@ -460,47 +460,67 @@ Client.prototype.downloadFile = function(params) { if (statusCode >= 300) { handleError(new Error("http status code " + statusCode)); return; - } - var contentLength = parseInt(headers['content-length'], 10); - downloader.progressTotal = contentLength; - downloader.progressAmount = 0; - downloader.emit('progress'); - downloader.emit('httpHeaders', statusCode, headers, resp); - var eTag = cleanETag(headers.etag); - var eTagCount = getETagCount(eTag); + } + if (headers['content-length'] == undefined) { + var outStream = fs.createWriteStream(localFile); + outStream.on('error', handleError); + downloader.progressTotal = 0 + downloader.progressAmount = -1; + request.on('httpData', function(chunk) { + downloader.progressTotal += chunk.length; + downloader.progressAmount += chunk.length; + downloader.emit('progress'); + outStream.write(chunk); + }) + + request.on('httpDone', function() { + if (errorOccurred) return; + downloader.progressAmount += 1; + downloader.emit('progress'); + outStream.end(); + }) + } else { + var contentLength = parseInt(headers['content-length'], 10); + downloader.progressTotal = contentLength; + downloader.progressAmount = 0; + downloader.emit('progress'); + downloader.emit('httpHeaders', statusCode, headers, resp); + var eTag = cleanETag(headers.etag); + var eTagCount = getETagCount(eTag); - var outStream = fs.createWriteStream(localFile); - var multipartETag = new MultipartETag({size: contentLength, count: eTagCount}); - var httpStream = resp.httpResponse.createUnbufferedStream(); + var outStream = fs.createWriteStream(localFile); + var multipartETag = new MultipartETag({size: contentLength, count: eTagCount}); + var httpStream = resp.httpResponse.createUnbufferedStream(); - httpStream.on('error', handleError); - outStream.on('error', handleError); + httpStream.on('error', handleError); + outStream.on('error', handleError); - hashCheckPend.go(function(cb) { - multipartETag.on('end', function() { - if (multipartETag.bytes !== contentLength) { - handleError(new Error("Downloaded size does not match Content-Length")); - return; - } - if (eTagCount === 1 && !multipartETag.anyMatch(eTag)) { - handleError(new Error("ETag does not match MD5 checksum")); - return; - } - cb(); + hashCheckPend.go(function(cb) { + multipartETag.on('end', function() { + if (multipartETag.bytes !== contentLength) { + handleError(new Error("Downloaded size does not match Content-Length")); + return; + } + if (eTagCount === 1 && !multipartETag.anyMatch(eTag)) { + handleError(new Error("ETag does not match MD5 checksum")); + return; + } + cb(); + }); + }); + multipartETag.on('progress', function() { + downloader.progressAmount = multipartETag.bytes; + downloader.emit('progress'); + }); + outStream.on('close', function() { + if (errorOccurred) return; + hashCheckPend.wait(cb); }); - }); - multipartETag.on('progress', function() { - downloader.progressAmount = multipartETag.bytes; - downloader.emit('progress'); - }); - outStream.on('close', function() { - if (errorOccurred) return; - hashCheckPend.wait(cb); - }); - httpStream.pipe(multipartETag); - httpStream.pipe(outStream); - multipartETag.resume(); + httpStream.pipe(multipartETag); + httpStream.pipe(outStream); + multipartETag.resume(); + } }); request.send(handleError); From 4329df2753b3edd3d22579d978f45f3f2963b0ac Mon Sep 17 00:00:00 2001 From: Kendrick Coleman Date: Fri, 6 Feb 2015 16:37:58 -0500 Subject: [PATCH 2/3] forgot to add the callback --- lib/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/index.js b/lib/index.js index 4ac2b26..e06e0fc 100644 --- a/lib/index.js +++ b/lib/index.js @@ -478,6 +478,7 @@ Client.prototype.downloadFile = function(params) { downloader.progressAmount += 1; downloader.emit('progress'); outStream.end(); + cb(); }) } else { var contentLength = parseInt(headers['content-length'], 10); From 0b7ecefe8169898e80609e05e812689b3ba4a8f1 Mon Sep 17 00:00:00 2001 From: Kendrick Coleman Date: Thu, 8 Oct 2015 21:48:32 -0400 Subject: [PATCH 3/3] added test to allow other endpoints tested and works with both s3 and s3-compatible storage Signed-off-by: Kendrick Coleman --- test/test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test.js b/test/test.js index a5559ca..4b60b52 100644 --- a/test/test.js +++ b/test/test.js @@ -36,6 +36,7 @@ function createClient() { s3Options: { accessKeyId: process.env.S3_KEY, secretAccessKey: process.env.S3_SECRET, + endpoint: process.env.S3_ENDPOINT, }, }); }