Skip to content

Commit

Permalink
chore(downloader): replace workflow with a single request (#158)
Browse files Browse the repository at this point in the history 
- add method to downloader to getFile
- add method to file manager to use new downloader getFile
- rewrite updateBinary function to use new file manager method
- add a couple of unit tests around downloader getFile
- use es6 promises, change travis to use node 6 minimum
- remove old method to request file in file manager and downloader
  • Loading branch information
@cnishina @sjelin
cnishina authored and sjelin committed Dec 16, 2016
1 parent 10b2823 commit c0a65af
Show file tree
Hide file tree
Showing 5 changed files with 199 additions and 180 deletions.
47 changes: 21 additions & 26 deletions lib/cmds/update.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,8 @@ function update(options: Options): void {
// permissions
if (standalone) {
let binary = binaries[StandAlone.id];
FileManager.toDownload(binary, outputDir, proxy, ignoreSSL).then((value: boolean) => {
if (value) {
Downloader.downloadBinary(binary, outputDir, proxy, ignoreSSL);
} else {
FileManager.downloadFile(binary, outputDir, proxy, ignoreSSL).then((downloaded: boolean) => {
if (!downloaded) {
logger.info(
binary.name + ': file exists ' +
path.resolve(outputDir, binary.filename(Config.osType(), Config.osArch())));
Expand All @@ -131,21 +129,21 @@ function update(options: Options): void {
}
if (chrome) {
let binary = binaries[ChromeDriver.id];
updateBinary(binary, outputDir, proxy, ignoreSSL).done();
updateBinary(binary, outputDir, proxy, ignoreSSL);
}
if (gecko) {
let binary = binaries[GeckoDriver.id];
updateBinary(binary, outputDir, proxy, ignoreSSL).done();
updateBinary(binary, outputDir, proxy, ignoreSSL);
}
if (ie) {
let binary = binaries[IEDriver.id];
binary.arch = Config.osArch(); // Win32 or x64
updateBinary(binary, outputDir, proxy, ignoreSSL).done();
updateBinary(binary, outputDir, proxy, ignoreSSL);
}
if (ie32) {
let binary = binaries[IEDriver.id];
binary.arch = 'Win32';
updateBinary(binary, outputDir, proxy, ignoreSSL).done();
updateBinary(binary, outputDir, proxy, ignoreSSL);
}
if (android) {
let binary = binaries[AndroidSDK.id];
Expand Down Expand Up @@ -179,27 +177,24 @@ function update(options: Options): void {
}
}

function updateBinary(
binary: Binary, outputDir: string, proxy: string, ignoreSSL: boolean): q.Promise<any> {
return FileManager.toDownload(binary, outputDir, proxy, ignoreSSL).then((value: boolean) => {
if (value) {
let deferred = q.defer();
Downloader.downloadBinary(
function updateBinary(binary: Binary, outputDir: string, proxy: string, ignoreSSL: boolean) {
FileManager
.downloadFile(
binary, outputDir, proxy, ignoreSSL,
(binary: Binary, outputDir: string, fileName: string) => {
unzip(binary, outputDir, fileName);
deferred.resolve();
});
return deferred.promise;
} else {
logger.info(
binary.name + ': file exists ' +
path.resolve(outputDir, binary.filename(Config.osType(), Config.osArch())));
let fileName = binary.filename(Config.osType(), Config.osArch());
unzip(binary, outputDir, fileName);
logger.info(binary.name + ': v' + binary.versionCustom + ' up to date');
}
});
})
.then(downloaded => {
if (!downloaded) {
// The file did not have to download, we should unzip it.
logger.info(
binary.name + ': file exists ' +
path.resolve(outputDir, binary.filename(Config.osType(), Config.osArch())));
let fileName = binary.filename(Config.osType(), Config.osArch());
unzip(binary, outputDir, fileName);
logger.info(binary.name + ': v' + binary.versionCustom + ' up to date');
}
});
}

function unzip<T extends Binary>(binary: T, outputDir: string, fileName: string): void {
Expand Down
204 changes: 86 additions & 118 deletions lib/files/downloader.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import * as fs from 'fs';
import * as http from 'http';
import * as path from 'path';
import * as q from 'q';
import * as request from 'request';
Expand All @@ -15,32 +14,6 @@ let logger = new Logger('downloader');
* The file downloader.
*/
export class Downloader {
/**
* Download the binary file.
* @param binary The binary of interest.
* @param outputDir The directory where files are downloaded and stored.
* @param opt_proxy The proxy for downloading files.
* @param opt_ignoreSSL To ignore SSL.
* @param opt_callback Callback method to be executed after the file is downloaded.
*/
static downloadBinary(
binary: Binary, outputDir: string, opt_proxy?: string, opt_ignoreSSL?: boolean,
opt_callback?: Function): void {
logger.info(binary.name + ': downloading version ' + binary.version());
var url = binary.url(Config.osType(), Config.osArch());
if (!url) {
logger.error(binary.name + ' v' + binary.version() + ' is not available for your system.');
return;
}
Downloader.httpGetFile_(
url, binary.filename(Config.osType(), Config.osArch()), outputDir, opt_proxy, opt_ignoreSSL,
(filePath: string) => {
if (opt_callback) {
opt_callback(binary, outputDir, filePath);
}
});
}

/**
* Resolves proxy based on values set
* @param fileUrl The url to download the file.
Expand Down Expand Up @@ -80,60 +53,29 @@ export class Downloader {
return undefined;
}

static httpHeadContentLength(fileUrl: string, opt_proxy?: string, opt_ignoreSSL?: boolean):
q.Promise<any> {
let deferred = q.defer();
if (opt_ignoreSSL) {
logger.info('ignoring SSL certificate');
}

let options = {
method: 'GET',
url: fileUrl,
strictSSL: !opt_ignoreSSL,
rejectUnauthorized: !opt_ignoreSSL,
proxy: Downloader.resolveProxy_(fileUrl, opt_proxy)
};

request(options).on('response', (response) => {
if (response.headers['Location']) {
let urlLocation = response.headers['Location'];
deferred.resolve(Downloader.httpHeadContentLength(urlLocation, opt_proxy, opt_ignoreSSL));
} else if (response.headers['content-length']) {
let contentLength = response.headers['content-length'];
deferred.resolve(contentLength);
}
response.destroy();
});
return deferred.promise;
}

/**
* Ceates the GET request for the file name.
* @param fileUrl The url to download the file.
* @param fileName The name of the file to download.
* @param opt_proxy The proxy to connect to to download files.
* @param opt_ignoreSSL To ignore SSL.
* Http get the file. Check the content length of the file before writing the file.
* If the content length does not match, remove it and download the file.
*
* @param binary The binary of interest.
* @param fileName The file name.
* @param outputDir The directory where files are downloaded and stored.
* @param contentLength The content length of the existing file.
* @param opt_proxy The proxy for downloading files.
* @param opt_ignoreSSL Should the downloader ignore SSL.
* @param opt_callback Callback method to be executed after the file is downloaded.
* @returns Promise<any> Resolves true = downloaded. Resolves false = not downloaded.
* Rejected with an error.
*/
static httpGetFile_(
fileUrl: string, fileName: string, outputDir: string, opt_proxy?: string,
opt_ignoreSSL?: boolean, callback?: Function): void {
logger.info('curl -o ' + outputDir + '/' + fileName + ' ' + fileUrl);
static getFile(
binary: Binary, fileUrl: string, fileName: string, outputDir: string, contentLength: number,
opt_proxy?: string, opt_ignoreSSL?: boolean, callback?: Function): Promise<any> {
// logger.info('curl -o ' + outputDir + '/' + fileName + ' ' + fileUrl);
// let contentLength = 0;
let filePath = path.resolve(outputDir, fileName);
let file = fs.createWriteStream(filePath);
let contentLength = 0;

interface Options {
url: string;
timeout: number;
strictSSL?: boolean;
rejectUnauthorized?: boolean;
proxy?: string;
headers?: {[key: string]: any};
[key: string]: any;
}
let file: any;

let options: Options = {
let options: IOptions = {
url: fileUrl,
// default Linux can be anywhere from 20-120 seconds
// increasing this arbitrarily to 4 minutes
Expand All @@ -153,46 +95,72 @@ export class Downloader {
}
}

request(options)
.on('response',
(response) => {
if (response.statusCode !== 200) {
fs.unlinkSync(filePath);
logger.error('Error: Got code ' + response.statusCode + ' from ' + fileUrl);
}
contentLength = response.headers['content-length'];
})
.on('error',
(error) => {
if ((error as any).code === 'ETIMEDOUT') {
logger.error('Connection timeout downloading: ' + fileUrl);
logger.error('Default timeout is 4 minutes.');

} else if ((error as any).connect) {
logger.error('Could not connect to the server to download: ' + fileUrl);
}
logger.error(error);
fs.unlinkSync(filePath);
})
.pipe(file);

file.on('close', function() {
fs.stat(filePath, function(err, stats) {
if (err) {
logger.error('Error: Got error ' + err + ' from ' + fileUrl);
return;
}
if (stats.size != contentLength) {
logger.error(
'Error: corrupt download for ' + fileName +
'. Please re-run webdriver-manager update');
fs.unlinkSync(filePath);
return;
}
if (callback) {
callback(filePath);
}
});
});
let req: request.Request = null;
let resContentLength: number;

return new Promise<boolean>((resolve, reject) => {
req = request(options);
req.on('response', response => {
if (response.statusCode === 200) {
resContentLength = +response.headers['content-length'];
if (contentLength === resContentLength) {
// if the size is the same, do not download and stop here
response.destroy();
resolve(false);
} else {
// only pipe if the headers are different length
file = fs.createWriteStream(filePath);
req.pipe(file);
file.on('close', () => {
fs.stat(filePath, (error, stats) => {
if (error) {
(error as any).msg = 'Error: Got error ' + error + ' from ' + fileUrl;
reject(error);
}
if (stats.size != resContentLength) {
(error as any).msg = 'Error: corrupt download for ' + fileName +
'. Please re-run webdriver-manager update';
fs.unlinkSync(filePath);
reject(error);
}
if (callback) {
callback(binary, outputDir, fileName);
}
});
resolve(true);
});
}

} else {
let error = new Error();
(error as any).msg =
'Expected response code 200, received: ' + response.statusCode;
reject(error);
}
});
req.on('error', error => {
if ((error as any).code === 'ETIMEDOUT') {
(error as any).msg = 'Connection timeout downloading: ' + fileUrl +
'. Default timeout is 4 minutes.';
} else if ((error as any).connect) {
(error as any).msg = 'Could not connect to the server to download: ' + fileUrl;
}
reject(error);
});
})
.catch(error => {
logger.error((error as any).msg);
});
}
}

interface IOptions {
method?: string;
url: string;
timeout: number;
strictSSL?: boolean;
rejectUnauthorized?: boolean;
proxy?: string;
headers?: {[key: string]: any};
[key: string]: any;
}
Loading

0 comments on commit c0a65af

Please sign in to comment.