diff --git a/vllm/utils.py b/vllm/utils.py
index f88c52731b3b2..4899b9074060e 100644
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -205,7 +205,7 @@ def get_ip() -> str:
 
 
 def get_distributed_init_method(ip: str, port: int) -> str:
-    return f"tcp://{ip}:{port}"
+    return f"tcp://[{ip}]:{port}"
 
 
 def get_open_port() -> int: