diff --git a/deepspeed/pt/deepspeed_light.py b/deepspeed/pt/deepspeed_light.py
index ce27d7d68f49..376196d3423e 100755
--- a/deepspeed/pt/deepspeed_light.py
+++ b/deepspeed/pt/deepspeed_light.py
@@ -144,12 +144,12 @@ def __init__(self,
         self._configure_with_arguments(args, mpu)
         self._do_sanity_check()
 
+        self._init_distributed(dist_init_required)
+
         self.sample_count = 0
-        if self.tensorboard_enabled():
+        if self.tensorboard_enabled() and self.global_rank == 0:
             self.summary_writer = self.get_summary_writer()
 
-        self._init_distributed(dist_init_required)
-
         # Configure distributed model
         self._configure_distributed_model(model)