Merge pull request #197 from caracal-pipeline/issue-195

fixes #195 (RAM usage in for-loops)

stimela/kitchen/recipe.py

@@ -7,7 +7,7 @@
 from collections.abc import Mapping
 import rich.table
 
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from stimela.config import EmptyDictDefault, EmptyListDefault
 import stimela
@@ -1214,12 +1214,11 @@ def _run(self, params, subst=None, backend={}) -> Dict[str, Any]:
                 with ProcessPoolExecutor(num_workers) as pool:
                     # submit each iterant to pool
                     futures = [pool.submit(self._iterate_loop_worker, *args, raise_exc=False) for args in loop_worker_args]
-                    stats = [f.result() for f in futures]
                     # update task stats, since they're recorded independently within each step, as well
                     # as get any exceptions from the nesting
                     errors = []
                     nfail = 0
-                    for f in futures:
+                    for f in as_completed(futures):
                         stats, outputs, exc, tb = f.result()
                         task_stats.add_missing_stats(stats)
                         if exc is not None:

stimela/task_stats.py

@@ -146,9 +146,9 @@ def add(self, other: "TaskStatsDatum"):
     def peak(self, other: "TaskStatsDatum"):
         for f in _taskstats_sample_names + other.extras:
             if hasattr(self, f):
-                self.extras.append(f)
                 setattr(self, f, max(getattr(self, f, -1e-9999), getattr(other, f)))
             else:
+                self.extras.append(f)
                 setattr(self, f, getattr(other, f))
 
     def averaged(self):