limit interrupt fix: after self-interrupting, only look at local pend…

…ing list (for redis state)

logging: don't log CF check errors, do log when errorCount is reset

crawler.js

@@ -412,7 +412,7 @@ class Crawler {
     }
 
     if (interrupt) {
-      this.crawlState.setDrain();
+      this.crawlState.setDrain(true);
       this.exitCode = 11;
     }
   }
@@ -625,7 +625,10 @@ class Crawler {
 
     try {
       await page.goto(url, gotoOpts);
-      this.errorCount = 0;
+      if (this.errorCount > 0) {
+        this.statusLog(`Page loaded, resetting error count ${this.errorCount} to 0`);
+        this.errorCount = 0;
+      }
     } catch (e) {
       let msg = e.message || "";
       if (!msg.startsWith("net::ERR_ABORTED") || !ignoreAbort) {
@@ -719,7 +722,8 @@ class Crawler {
         await this.sleep(5500);
       }
     } catch (e) {
-      console.warn(e);
+      //console.warn("Check CF failed, ignoring");
+      //console.warn(e);
     }
   }
 

util/state.js

@@ -6,10 +6,12 @@ class BaseState
 {
   constructor() {
     this.drainMax = 0;
+    this.localOnly = false;
   }
 
-  async setDrain() {
-    this.drainMax = (await this.numPending()) + (await this.numDone());
+  async setDrain(localOnly = false) {
+    this.drainMax = (await this.numRealPending()) + (await this.numDone());
+    this.localOnly = localOnly;
   }
 
   async size() {
@@ -29,6 +31,14 @@ class BaseState
 
     return seed.isIncluded(data.url, data.depth, data.extraHops);
   }
+
+  numPending(localPending = 0) {
+    if (this.localOnly) {
+      return localPending;
+    }
+
+    return this.numRealPending();
+  }
 }
 
 
@@ -146,7 +156,7 @@ class MemoryCrawlState extends BaseState
     return this.seenList.size;
   }
 
-  async numPending() {
+  async numRealPending() {
     return this.pending.size;
   }
 }
@@ -386,7 +396,7 @@ return 0;
     return await this.redis.scard(this.skey);
   }
 
-  async numPending() {
+  async numRealPending() {
     const res = await this.redis.hlen(this.pkey);
 
     // reset pendings