From b0b7f420d60546c7347fa2eb7f3eed09899c26f8 Mon Sep 17 00:00:00 2001 From: Kevin Samson Date: Tue, 17 Sep 2024 13:13:59 +0400 Subject: [PATCH] Add verbose mode for progress tracking --- .gitignore | 5 ++++ main.js | 21 ++++++++++------ progress.js | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 8cbe63a..6e35a1c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ Downloads json node_modules +progress.json +progress.txt +progress*.txt +progress*.json +memories_history.json \ No newline at end of file diff --git a/main.js b/main.js index b04814a..8ef1296 100644 --- a/main.js +++ b/main.js @@ -22,10 +22,11 @@ program .option( "-f ", "Filepath to memories_history.json", - "./json/memories_history.json" + "./memories_history.json" ) .option("-o ", "Download directory", "Downloads") - .option("-l", "Preserve location data as file metadata", false); + .option("-l", "Preserve location data as file metadata", false) + .option("-v", "Verbose mode", false); program.parse(); const options = program.opts(); @@ -59,7 +60,7 @@ try { } var queue = new Queue(maxConcurrentDownloads); -var progress = new Progress(downloads.length, progressBarLength); +var progress = new Progress(downloads.length, progressBarLength, options.v); function main() { // Create download directory @@ -83,7 +84,7 @@ function main() { queue .enqueue(() => getDownloadLink(url, body, fileName, fileTime)) .then((result) => { - progress.cdnLinkSucceeded(true); + progress.cdnLinkSucceeded(true,result[0]); // Download the file queue @@ -91,14 +92,16 @@ function main() { downloadMemory(result[0], result[1], result[2], lat, long) ) .then((success) => { - progress.downloadSucceeded(true); + progress.downloadSucceeded(true,result[0]); }) .catch((err) => { - progress.downloadSucceeded(false); + progress.downloadSucceeded(false),result[0]; + progress.logError(err); }); }) .catch((err) => { - progress.cdnLinkSucceeded(false); + progress.cdnLinkSucceeded(false,downloads[i]["Download Link"]); + progress.logError("CDN link error: " + downloads[i]["Download Link"]); }); } } @@ -223,6 +226,9 @@ const downloadMemory = (downloadUrl, fileName, fileTime, lat = "", long = "") => }); } else { console.log("download error", res.statusCode, res.statusMessage); + progress.logError( + `Download error: ${res.statusCode} ${res.statusMessage}` + ); if (maxRetries > 0) { download(maxRetries - 1); } else { @@ -233,6 +239,7 @@ const downloadMemory = (downloadUrl, fileName, fileTime, lat = "", long = "") => req.on("error", function () { console.log("request error"); + progress.logError("Request error" + downloadUrl); req.destroy(); if (maxRetries > 0) { download(maxRetries - 1); diff --git a/progress.js b/progress.js index e150c54..b47a4a5 100644 --- a/progress.js +++ b/progress.js @@ -1,22 +1,87 @@ +import fs from 'fs'; + export default class Progress { - constructor(total, progressBarLength) { + constructor(total, progressBarLength, verbose) { this.total = total; + this.verbose = verbose; this.progressBarLength = progressBarLength; this.cdnLinks = { success: 0, fails: 0 }; this.downloads = { success: 0, fails: 0 }; this.printProgress(true); + let progressFile = './progress.json'; + let logfile = './progress.txt'; + if(verbose) { + let counter = 1; + while (fs.existsSync(progressFile)) { + progressFile = `./progress${counter}.json`; + counter++; + } + fs.writeFileSync(progressFile, JSON.stringify(this, null, 2)); + let logCounter = 1; + while (fs.existsSync(logfile)) { + logfile = `./progress${logCounter}.txt`; + logCounter++; + } + } + this.logFile = logfile; + this.processFile = progressFile; } - cdnLinkSucceeded(didSucceed) { + cdnLinkSucceeded(didSucceed, url) { + if(this.verbose){ + if (didSucceed) { + const progressData = JSON.parse(fs.readFileSync(this.processFile)); + if (!progressData.cdnLinks.CDNSuccessUrls) { + progressData.cdnLinks.CDNSuccessUrls = []; + } + progressData.cdnLinks.CDNSuccessUrls.push(url); + progressData.cdnLinks.success++; + fs.writeFileSync(this.processFile, JSON.stringify(progressData, null, 2)); + + } else { + const progressData = JSON.parse(fs.readFileSync(this.processFile)); + if (!progressData.cdnLinks.CDNFailedUrls) { + progressData.cdnLinks.CDNFailedUrls = []; + } + progressData.cdnLinks.CDNFailedUrls.push(url); + progressData.cdnLinks.fails++; + fs.writeFileSync(this.processFile, JSON.stringify(progressData, null, 2)); + } + } didSucceed ? this.cdnLinks.success++ : this.cdnLinks.fails++; this.printProgress(); } - downloadSucceeded(didSucceed) { + downloadSucceeded(didSucceed, url) { + if(this.verbose){ + if (didSucceed) { + const progressData = JSON.parse(fs.readFileSync(this.processFile)); + if (!progressData.downloads.downloadSuccessUrls) { + progressData.downloads.downloadSuccessUrls = []; + } + progressData.downloads.downloadSuccessUrls.push(url); + progressData.downloads.success++; + fs.writeFileSync(this.processFile, JSON.stringify(progressData, null, 2)); + } else { + const progressData = JSON.parse(fs.readFileSync(this.processFile)); + if (!progressData.downloads.downloadFailedUrls) { + progressData.downloads.downloadFailedUrls = []; + } + progressData.downloads.downloadFailedUrls.push(url); + progressData.downloads.fails++; + fs.writeFileSync(this.processFile, JSON.stringify(progressData, null, 2)); + } + } didSucceed ? this.downloads.success++ : this.downloads.fails++; this.printProgress(); } + logError(message) { + const time = new Date().toISOString(); + const logMessage = `${time}: ${message}\n`; + fs.appendFileSync(this.logFile, logMessage); + } + printProgress(firstRun) { if (!firstRun) process.stdout.moveCursor(0, -2);