diff --git a/src/transformers/models/gemma/modeling_gemma.py b/src/transformers/models/gemma/modeling_gemma.py
index 2d93c43425f99a..baa0839e658c9c 100644
--- a/src/transformers/models/gemma/modeling_gemma.py
+++ b/src/transformers/models/gemma/modeling_gemma.py
@@ -174,15 +174,13 @@ def __init__(self, config):
         self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
         if config.hidden_activation is None:
             logger.warning_once(
-                "Gemma's activation function should be approximate GeLU and not exact GeLU.\n"
-                "Changing the activation function to `gelu_pytorch_tanh`."
-                f"if you want to use the legacy `{config.hidden_act}`, "
-                f"edit the `model.config` to set `hidden_activation={config.hidden_act}` "
-                "  instead of `hidden_act`. See https://github.com/huggingface/transformers/pull/29402 for more details."
+                "`config.hidden_act` is ignored, you should use `config.hidden_activation` instead.\n"
+                "Gemma's activation function will be set to `gelu_pytorch_tanh`. Please, use\n"
+                "`config.hidden_activation` if you want to override this behaviour.\n"
+                "See https://github.com/huggingface/transformers/pull/29402 for more details."
             )
-            hidden_activation = "gelu_pytorch_tanh"
-        else:
-            hidden_activation = config.hidden_activation
+            config.hidden_activation = "gelu_pytorch_tanh"
+        hidden_activation = config.hidden_activation
         self.act_fn = ACT2FN[hidden_activation]
 
     def forward(self, x):