From 1f972427fc6c9b22da97ca8b89c194bfb3988d6f Mon Sep 17 00:00:00 2001 From: Arash Ashari Date: Tue, 21 Jul 2020 11:11:07 -0700 Subject: [PATCH] only global rank 0 can log tensorboard data; avoid multi gpu/node race for the log directory (#296) --- deepspeed/pt/deepspeed_light.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deepspeed/pt/deepspeed_light.py b/deepspeed/pt/deepspeed_light.py index ce27d7d68f49..376196d3423e 100755 --- a/deepspeed/pt/deepspeed_light.py +++ b/deepspeed/pt/deepspeed_light.py @@ -144,12 +144,12 @@ def __init__(self, self._configure_with_arguments(args, mpu) self._do_sanity_check() + self._init_distributed(dist_init_required) + self.sample_count = 0 - if self.tensorboard_enabled(): + if self.tensorboard_enabled() and self.global_rank == 0: self.summary_writer = self.get_summary_writer() - self._init_distributed(dist_init_required) - # Configure distributed model self._configure_distributed_model(model)