Update website_crawler.py to keep clean urls (#114)

vectara · Aug 21, 2024 · cbdfac4 · cbdfac4
1 parent 42eddb0
commit cbdfac4
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/crawlers/website_crawler.py b/crawlers/website_crawler.py
@@ -79,7 +79,6 @@ def crawl(self) -> None:
                                            pos_regex=self.pos_regex, neg_regex=self.neg_regex, 
                                            indexer=self.indexer, visited=set(), verbose=self.indexer.verbose)
                 urls = clean_urls(urls_set, keep_query_params)
-                urls = list(set(urls_set))
             else:
                 logging.info(f"Unknown pages_source: {self.cfg.website_crawler.pages_source}")
                 return