From 3ad7dab79c0d850722836fec56fe44df17e037bf Mon Sep 17 00:00:00 2001 From: marcoyang Date: Thu, 25 Aug 2022 11:38:04 +0800 Subject: [PATCH] Add clamping operation in Eve optimizer for all scalar weights to avoid non stable training in some scenarios. The clamping range is set to (-10,2). Note that this change may cause unexpected effect if you resume training from a model that is trained without clamping. --- egs/librispeech/ASR/pruned_transducer_stateless2/optim.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/optim.py b/egs/librispeech/ASR/pruned_transducer_stateless2/optim.py index 432bf82207..041a81f45b 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless2/optim.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless2/optim.py @@ -164,6 +164,10 @@ def step(self, closure=None): p.mul_(1 - (weight_decay * is_above_target_rms)) p.addcdiv_(exp_avg, denom, value=-step_size) + # Constrain the range of scalar weights + if p.numel() == 1: + p.clamp_(min=-10, max=2) + return loss