From 3e0f06210646a440509efa718b30d18322d6a830 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 May 2020 17:44:17 -0400 Subject: [PATCH] Fix addcmul_ --- src/transformers/optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/optimization.py b/src/transformers/optimization.py index 13175b7c311507..5af1120c818eed 100644 --- a/src/transformers/optimization.py +++ b/src/transformers/optimization.py @@ -153,7 +153,7 @@ def step(self, closure=None): # Decay the first and second moment running average coefficient # In-place operations to update the averages at the same time exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, 1.0 - beta2) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2) denom = exp_avg_sq.sqrt().add_(group["eps"]) step_size = group["lr"]