only global rank 0 can log tensorboard data; avoid multi gpu/node rac…

…e for the log directory (#296)
microsoft · Jul 21, 2020 · 1f97242 · 1f97242
1 parent 376818e
commit 1f97242
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/deepspeed/pt/deepspeed_light.py b/deepspeed/pt/deepspeed_light.py
@@ -144,12 +144,12 @@ def __init__(self,
         self._configure_with_arguments(args, mpu)
         self._do_sanity_check()
 
+        self._init_distributed(dist_init_required)
+
         self.sample_count = 0
-        if self.tensorboard_enabled():
+        if self.tensorboard_enabled() and self.global_rank == 0:
             self.summary_writer = self.get_summary_writer()
 
-        self._init_distributed(dist_init_required)
-
         # Configure distributed model
         self._configure_distributed_model(model)