Adding multiprocessing to preloading

Reichenbachian · Feb 15, 2018 · 03a1fc0 · 03a1fc0
1 parent b79b51a
commit 03a1fc0
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/adlframework/datasource.py b/adlframework/datasource.py
@@ -69,8 +69,9 @@ def __init__(self, retrieval, DataEntity, controllers=[], ignore_retrieval_cache
 		self.initialize_retrieval(ignore_retrieval_cache)
 		self.__prefilter()
 		if preload_memory:
-			for id_ in tqdm(self._entity_ids, leave=False):
-				self.process_id(id_, just_cache=True)
+			process_wrap = lambda x: self.process_id(x, just_cache=True)
+			with Pool(workers) as p:
+      			tqdm.tqdm(p.imap(process_wrap, self._entity_ids), total=len(self._entity_ids))
 			self.cache.save()
 
 		if self.workers > 1: