diff --git a/src/transformers/models/zamba/configuration_zamba.py b/src/transformers/models/zamba/configuration_zamba.py
index b741546b02f..5f0c153703b 100644
--- a/src/transformers/models/zamba/configuration_zamba.py
+++ b/src/transformers/models/zamba/configuration_zamba.py
@@ -135,7 +135,7 @@ def __init__(
         rms_norm_eps=1e-5,
         use_cache=True,
         num_logits_to_keep=1,
-        pad_token_id=None,
+        pad_token_id=0,
         bos_token_id=1,
         eos_token_id=2,
         max_position_embeddings=4096,
diff --git a/tests/models/zamba/test_modeling_zamba.py b/tests/models/zamba/test_modeling_zamba.py
index 086a651aa08..b1931254f58 100644
--- a/tests/models/zamba/test_modeling_zamba.py
+++ b/tests/models/zamba/test_modeling_zamba.py
@@ -646,8 +646,6 @@ def setUpClass(cls):
             model_id, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, use_mamba_kernels=False
         )
         cls.tokenizer = AutoTokenizer.from_pretrained(model_id)
-        cls.tokenizer.add_special_tokens({"pad_token": "[PAD]"})
-        cls.model.resize_token_embeddings(len(cls.tokenizer))
 
     @slow
     def test_simple_generate(self):