From 6e78b42099480767f331e29f8b762c8a7eeeaabd Mon Sep 17 00:00:00 2001 From: David Ackerman Date: Tue, 11 Jun 2024 15:36:13 -0400 Subject: [PATCH] fix: :bug: fix how file stats are updated previously file stat writing would overwrite existing stats; this fix prevents that by appending the new stats --- dacapo/store/file_stats_store.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/dacapo/store/file_stats_store.py b/dacapo/store/file_stats_store.py index 72cf9df58..2efdd51a2 100644 --- a/dacapo/store/file_stats_store.py +++ b/dacapo/store/file_stats_store.py @@ -87,6 +87,7 @@ def store_training_stats(self, run_name, stats): ) else: # current stats are behind DB--drop DB + existing_stats = None logger.warning( f"Overwriting previous training stats for run {run_name}" ) @@ -94,7 +95,7 @@ def store_training_stats(self, run_name, stats): # store all new stats self.__store_training_stats( - stats, store_from_iteration, stats.trained_until(), run_name + existing_stats, stats, store_from_iteration, stats.trained_until(), run_name ) def retrieve_training_stats(self, run_name): @@ -174,11 +175,12 @@ def delete_training_stats(self, run_name: str) -> None: """ self.__delete_training_stats(run_name) - def __store_training_stats(self, stats, begin, end, run_name): + def __store_training_stats(self, existing_stats, stats, begin, end, run_name): """ Store the training statistics for a specific run. Args: + existing_stats (Stats): The statistics object containing the training stats that are already stored. stats (Stats): The statistics object containing the training stats. begin (int): The starting index of the iteration stats to store. end (int): The ending index of the iteration stats to store. @@ -190,10 +192,15 @@ def __store_training_stats(self, stats, begin, end, run_name): """ docs = converter.unstructure(stats.iteration_stats[begin:end]) - for doc in docs: - doc.update({"run_name": run_name}) if docs: + if existing_stats: + # prepend existing stats to new stats + docs = converter.unstructure(existing_stats.iteration_stats) + docs + + for doc in docs: + doc.update({"run_name": run_name}) + file_store = self.training_stats / run_name with file_store.open("wb") as fd: pickle.dump(docs, fd)