From 00bd2c44d218f8828132b003eb0a8c0cb21fb187 Mon Sep 17 00:00:00 2001 From: carefree0910 Date: Sun, 21 Mar 2021 07:50:59 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8FOptimized=20`Transformer`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `highway` fcnn head should be used in order to prevent rank collapse --- cflearn/models/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cflearn/models/transformer.py b/cflearn/models/transformer.py index 51ce7ad16..d2e483413 100644 --- a/cflearn/models/transformer.py +++ b/cflearn/models/transformer.py @@ -2,7 +2,7 @@ @ModelBase.register("transformer") -@ModelBase.register_pipe("transformer", head="linear") +@ModelBase.register_pipe("transformer", head="fcnn", head_config="highway") class Transformer(ModelBase): pass