fix(uniquely/flight-crawler): prevent set storage repeated (#536)

Alwatr · Dec 22, 2022 · 0299b57 · 0299b57
2 parents 3f02c06 + 0a89675
commit 0299b57
Showing 1 changed file with 9 additions and 4 deletions.
diff --git a/services/flight-crawler/src/crawl.ts b/services/flight-crawler/src/crawl.ts
@@ -10,24 +10,29 @@ import type {FetchOptions} from '@alwatr/fetch';
 export async function crawlAllJobs(): Promise<void> {
   logger.logMethod('crawlAllJobs');
   const jobList = (await storageClient.getStorage()).data;
-  for (const jobId in jobList) {
-    if (!Object.prototype.hasOwnProperty.call(jobList, jobId)) continue;
+  const jobKeyList = Object.keys(jobList);
+  let updated = false;
+
+  for (let i = 0; i < jobKeyList.length; i++) {
     try {
-      const job = jobList[jobId];
+      const job = jobList[jobKeyList[i]];
       const oldResultList = job.resultList;
       const resultList = await crawl(job.detail);
       job.resultList = resultList;
       if (differentObject(job.resultList, oldResultList)) {
         const message = makeMessage(job);
         await notify(config.notifier.to, message);
         logger.logOther(`Notified to ${config.notifier.to}!`);
+        await storageClient.set(job);
+        updated = true;
       }
-      await storageClient.set(job);
     }
     catch (err) {
       logger.error('crawlAllJobs', 'crawling_failed', err);
     }
   }
+  // for updating meta
+  if (updated === false) await storageClient.set(jobList[jobKeyList[jobKeyList.length - 1]]);
 }
 
 async function crawl(detail: JobDetail): Promise<Array<JobResult>> {