From 03a1fc026eb8a048f7f9ed8d0a93e62fd92bff70 Mon Sep 17 00:00:00 2001 From: localhost Date: Thu, 15 Feb 2018 15:06:37 -0500 Subject: [PATCH] Adding multiprocessing to preloading --- adlframework/datasource.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/adlframework/datasource.py b/adlframework/datasource.py index c1f1b68..462ef01 100644 --- a/adlframework/datasource.py +++ b/adlframework/datasource.py @@ -69,8 +69,9 @@ def __init__(self, retrieval, DataEntity, controllers=[], ignore_retrieval_cache self.initialize_retrieval(ignore_retrieval_cache) self.__prefilter() if preload_memory: - for id_ in tqdm(self._entity_ids, leave=False): - self.process_id(id_, just_cache=True) + process_wrap = lambda x: self.process_id(x, just_cache=True) + with Pool(workers) as p: + tqdm.tqdm(p.imap(process_wrap, self._entity_ids), total=len(self._entity_ids)) self.cache.save() if self.workers > 1: