From 75bd67da3e63b0b5953b24da22d91cd649fa3bcb Mon Sep 17 00:00:00 2001 From: "S. Amir Mohammad Najafi" Date: Wed, 21 Dec 2022 13:39:36 +0330 Subject: [PATCH 1/2] fix(uniquely/flight-crawler): prevent set storage repeated --- services/flight-crawler/src/crawl.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/flight-crawler/src/crawl.ts b/services/flight-crawler/src/crawl.ts index 9078b4756..e4f7cafa1 100644 --- a/services/flight-crawler/src/crawl.ts +++ b/services/flight-crawler/src/crawl.ts @@ -10,10 +10,12 @@ import type {FetchOptions} from '@alwatr/fetch'; export async function crawlAllJobs(): Promise { logger.logMethod('crawlAllJobs'); const jobList = (await storageClient.getStorage()).data; - for (const jobId in jobList) { - if (!Object.prototype.hasOwnProperty.call(jobList, jobId)) continue; + const jobKey = Object.keys(jobList); + let updated = false; + + for (let i = 0; i < jobKey.length; i++) { try { - const job = jobList[jobId]; + const job = jobList[jobKey[i]]; const oldResultList = job.resultList; const resultList = await crawl(job.detail); job.resultList = resultList; @@ -23,11 +25,14 @@ export async function crawlAllJobs(): Promise { logger.logOther(`Notified to ${config.notifier.to}!`); } await storageClient.set(job); + updated = true; } catch (err) { logger.error('crawlAllJobs', 'crawling_failed', err); } } + // for updating meta + if (updated === false) await storageClient.set(jobList[jobKey[jobKey.length - 1]]); } async function crawl(detail: JobDetail): Promise> { From bfc87ce851bfdf9fa31c687fb3fcf78e011c1cf1 Mon Sep 17 00:00:00 2001 From: "S. Amir Mohammad Najafi" Date: Thu, 22 Dec 2022 09:56:26 +0330 Subject: [PATCH 2/2] fix(flight-crawler): set in storage single time --- services/flight-crawler/src/crawl.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/flight-crawler/src/crawl.ts b/services/flight-crawler/src/crawl.ts index e4f7cafa1..6252b5848 100644 --- a/services/flight-crawler/src/crawl.ts +++ b/services/flight-crawler/src/crawl.ts @@ -10,12 +10,12 @@ import type {FetchOptions} from '@alwatr/fetch'; export async function crawlAllJobs(): Promise { logger.logMethod('crawlAllJobs'); const jobList = (await storageClient.getStorage()).data; - const jobKey = Object.keys(jobList); + const jobKeyList = Object.keys(jobList); let updated = false; - for (let i = 0; i < jobKey.length; i++) { + for (let i = 0; i < jobKeyList.length; i++) { try { - const job = jobList[jobKey[i]]; + const job = jobList[jobKeyList[i]]; const oldResultList = job.resultList; const resultList = await crawl(job.detail); job.resultList = resultList; @@ -23,16 +23,16 @@ export async function crawlAllJobs(): Promise { const message = makeMessage(job); await notify(config.notifier.to, message); logger.logOther(`Notified to ${config.notifier.to}!`); + await storageClient.set(job); + updated = true; } - await storageClient.set(job); - updated = true; } catch (err) { logger.error('crawlAllJobs', 'crawling_failed', err); } } // for updating meta - if (updated === false) await storageClient.set(jobList[jobKey[jobKey.length - 1]]); + if (updated === false) await storageClient.set(jobList[jobKeyList[jobKeyList.length - 1]]); } async function crawl(detail: JobDetail): Promise> {