Skip to content

Commit

Permalink
Rewrite downloader
Browse files Browse the repository at this point in the history
The downloader is now similar to the WOF downloader in that it uses
child_process.exec and curl to download data. This seems to be more
reliable than using request and piping to a file.

There were also issues with the progress bar package, so rather than
sort them out, this allows us to simply remove them.

This change probably will help fix #26,
but it's not 100% certain.
  • Loading branch information
orangejulius committed Dec 13, 2016
1 parent 27bcef9 commit 6a4dcc3
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 21 deletions.
40 changes: 21 additions & 19 deletions lib/tasks/download.js
Original file line number Diff line number Diff line change
@@ -1,30 +1,32 @@
'use strict';

var fs = require('fs-extra'),
util = require('util'),
request = require('request'),
_ = require('lodash'),
progress = require('../../util/streamProgressBar'),
logger = require( 'pelias-logger' ).get( 'geonames' );
const child_process = require('child_process');
const logger = require( 'pelias-logger' ).get( 'geonames' );

// use datapath setting from your config file
var settings = require('pelias-config').generate();
var basepath = settings.imports.geonames.datapath;
const settings = require('pelias-config').generate();
const basepath = settings.imports.geonames.datapath;

module.exports = function (filename) {
const remoteFilePath = `http://download.geonames.org/export/dump/${filename}.zip`;
const localFileName = `${basepath}/${filename}.zip`;

var remoteFilePath = util.format( 'http://download.geonames.org/export/dump/%s.zip', filename );
var localFileName = util.format( '%s/%s.zip', basepath, filename );
logger.info( 'downloading datafile from:', remoteFilePath );

fs.mkdirs('data', function(error) {
if( error ){
logger.error( error );
return;
}
logger.info( 'downloading datafile from:', remoteFilePath );
const command = `curl ${remoteFilePath} > ${localFileName}`;

request.get( remoteFilePath )
.pipe( progress( _.padEnd( localFileName, 30 ) ) )
.pipe( fs.createWriteStream( localFileName ) );
const job = child_process.exec(command);

job.stdout.on('data', (data) => {
process.stdout.write(data);
});

job.stderr.on('data', (data) => {
process.stderr.write(data);
});

job.on('close', (code) => {
console.log(`Geonames download finished with exit code ${code}`);
process.exitCode = code;
});
};
2 changes: 0 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,6 @@
"pelias-logger": "0.1.0",
"pelias-model": "4.4.0",
"pelias-wof-admin-lookup": "2.8.0",
"progress": "^1.1.5",
"progress-stream": "^1.2.0",
"request": "^2.34.0",
"through2": "^2.0.1",
"through2-filter": "^2.0.0"
Expand Down

0 comments on commit 6a4dcc3

Please sign in to comment.