apply dynamic loss scaling (#161)

microsoft · Feb 9, 2022 · 068e656 · 068e656
1 parent 41f4fdf
commit 068e656
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_3_config_release.json b/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_3_config_release.json
@@ -19,7 +19,7 @@
   "gradient_clipping": 1.0,
   "fp16": {
     "enabled": true,
-    "loss_scale": 1024,
+    "loss_scale": 0,
     "loss_scale_window": 1000,
     "hysteresis": 2,
     "min_loss_scale": 1

diff --git a/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_infinity_config.json b/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_infinity_config.json
@@ -30,7 +30,7 @@
   "gradient_clipping": 1.0,
   "fp16": {
     "enabled": true,
-    "loss_scale": 1024,
+    "loss_scale": 0,
     "loss_scale_window": 1000,
     "hysteresis": 2,
     "min_loss_scale": 1

diff --git a/Megatron-LM/scripts/ds_zero-offload_10B_config.json b/Megatron-LM/scripts/ds_zero-offload_10B_config.json
@@ -18,7 +18,7 @@
   "gradient_clipping": 1.0,
   "fp16": {
     "enabled": true,
-    "loss_scale": 4096,
+    "loss_scale": 0,
     "loss_scale_window": 1000,
     "hysteresis": 2,
     "min_loss_scale": 1