Fix non-negligible difference between GPT2 and TFGP2 (#13679)

Co-authored-by: ydshieh <[email protected]>

src/transformers/models/gpt2/modeling_tf_gpt2.py

@@ -174,7 +174,7 @@ def __init__(self, n_state, config, **kwargs):
         nx = config.n_embd
         self.c_fc = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name="c_fc")
         self.c_proj = TFConv1D(nx, n_state, initializer_range=config.initializer_range, name="c_proj")
-        self.act = get_tf_activation("gelu")
+        self.act = get_tf_activation(config.activation_function)
         self.dropout = tf.keras.layers.Dropout(config.resid_pdrop)
 
     def call(self, x, training=False):