fix(walk): premature walk termination

DonIsaac · Aug 26, 2024 · 94dd167 · 94dd167
1 parent 73201ce
commit 94dd167
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 9 deletions.
diff --git a/src/cmd/runner.rs b/src/cmd/runner.rs
@@ -99,12 +99,14 @@ impl Runner {
                     result
                 });
 
+                trace!("Starting API key collector thread");
                 let collector_handle = thread::spawn(move || collector.collect());
                 if let Err(error) = Self::join(&url, collector_handle, walk_handle) {
                     errors.push(error)
                 }
             });
 
+            debug!("Sending stop signal to API key collector");
             key_sender
                 .send(ApiKeyMessage::Stop)
                 .into_diagnostic()
@@ -125,18 +127,12 @@ impl Runner {
     ) -> Result<()> {
         let _url = url.as_ref();
 
+        debug!("Waiting for collector thread to finish...");
         collector_handle
             .join()
             .expect("ApiKeyCollector thread should have joined successfully");
 
-        // match walk_result {
-        //     Ok(_) => {
-        //         info!(target: "keyhunter::main", "Done scraping for {url}");
-        //     }
-        //     Err(e) => {
-        //         error!(target: "keyhunter::main", "[run] Failed to scrape for '{url}': {e}");
-        //     }
-        // }
+        debug!("Waiting for walker thread to finish...");
         walk_handle
             .join()
             .expect("WebsiteWalker thread should have joined successfully")

diff --git a/src/walk/website/walk.rs b/src/walk/website/walk.rs
@@ -130,7 +130,7 @@ impl WebsiteWalker {
 
     fn visit_many(&mut self, urls: Vec<Url>) -> Result<(), Error> {
         let pages_to_visit = self.reserve_walk_count(urls.len());
-        if pages_to_visit == 0 {
+        if pages_to_visit == 0 && *self.in_progress.get_mut().unwrap() == 0 {
             debug!("Finishing walk, No more pages to visit");
             self.finish();
             return Ok(());