Fix addcmul_

will-rice · May 15, 2020 · 3e0f062 · 3e0f062
1 parent fc2a4c8
commit 3e0f062
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/transformers/optimization.py b/src/transformers/optimization.py
@@ -153,7 +153,7 @@ def step(self, closure=None):
                 # Decay the first and second moment running average coefficient
                 # In-place operations to update the averages at the same time
                 exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1)
-                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, 1.0 - beta2)
+                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2)
                 denom = exp_avg_sq.sqrt().add_(group["eps"])
 
                 step_size = group["lr"]