increase retry attempts

parlemonde · Jul 26, 2024 · 4365530 · 4365530
1 parent 5f28d4c
commit 4365530
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 8 deletions.
diff --git a/src/archive.ts b/src/archive.ts
@@ -20,7 +20,7 @@ const SELECTORS = {
 
 const PHASES = [1, 2, 3];
 
-async function gotoWithRetry(page: Page, url: string, maxAttempts = 3): Promise<void> {
+async function gotoWithRetry(page: Page, url: string, maxAttempts = 5): Promise<void> {
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
       await page.goto(url, { 
@@ -31,12 +31,12 @@ async function gotoWithRetry(page: Page, url: string, maxAttempts = 3): Promise<
     } catch (error) {
       if (attempt === maxAttempts) throw error;
       logger.info(`Navigation attempt ${attempt} failed, retrying in 30 seconds...`);
-      await sleep(30000);  // Wait 30 seconds before retrying
+      await sleep(30000); 
     }
   }
 }
 
-async function clickWithRetry(page: Page, url: string, maxAttempts = 3): Promise<void> {
+async function clickWithRetry(page: Page, url: string, maxAttempts = 5): Promise<void> {
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
       await page.click(url);
@@ -215,6 +215,7 @@ export async function archiveWebsite() {
     for (let i = 0; i < villageCount; i++) {
       const villageName = await selectVillage(page, i + 1);
       if (villageName == '') {
+        logger.success(`An error occurred when selecting a village !`);
         continue
       }
       logger.info(villageName);
@@ -223,6 +224,7 @@ export async function archiveWebsite() {
         await sleep(20000);
         await archivePage(dirPath, page, resources, villageName, phase)
       }
+      logger.info(`Village ${villageName} fully archived`);
     }
 
     await handleAllCssFiles(resources, dirPath);

diff --git a/src/village.ts b/src/village.ts
@@ -10,7 +10,7 @@ const SELECTORS = {
   CONFIRM_BUTTON: 'div.MuiDialog-root button.MuiButton-containedSecondary',
 };
 
-async function gotoWithRetry(page: Page, url: string, maxAttempts = 3): Promise<void> {
+async function gotoWithRetry(page: Page, url: string, maxAttempts = 5): Promise<void> {
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
       await page.goto(url, { 
@@ -21,13 +21,13 @@ async function gotoWithRetry(page: Page, url: string, maxAttempts = 3): Promise<
     } catch (error) {
       if (attempt === maxAttempts) throw error;
       logger.info(`Navigation attempt ${attempt} failed, retrying in 30 seconds...`);
-      await sleep(30000);  // Wait 30 seconds before retrying
+      await sleep(30000);
     }
   }
 }
 
 
-async function waitForSelectorWithRetry(page: Page, selector: string, maxAttempts = 3): Promise<void> {
+async function waitForSelectorWithRetry(page: Page, selector: string, maxAttempts = 5): Promise<void> {
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
       await page.waitForSelector(selector, { visible: true, timeout: 60000 });
@@ -36,8 +36,6 @@ async function waitForSelectorWithRetry(page: Page, selector: string, maxAttempt
       if (attempt === maxAttempts) throw error;
       logger.info(`Selector ${selector} not found on attempt ${attempt}, retrying...`);
       await page.reload({ waitUntil: 'domcontentloaded' });
-      const html = await page.evaluate(() => document.documentElement.outerHTML);
-      logger.info(`ERROR ---> ${html}`);
       await sleep(30000);
     }
   }