Add a timeout for socket connection (#2791)

src/Orleans/Configuration/MessagingConfiguration.cs

@@ -12,6 +12,10 @@ namespace Orleans.Runtime.Configuration
     /// </summary>
     public interface IMessagingConfiguration
     {
+        /// <summary>
+        /// The OpenConnectionTimeout attribute specifies the timeout before a connection open is assumed to have failed
+        /// </summary>
+        TimeSpan OpenConnectionTimeout { get; set; }
         /// <summary>
         /// The ResponseTimeout attribute specifies the default timeout before a request is assumed to have failed.
         /// </summary>
@@ -92,6 +96,7 @@ public interface IMessagingConfiguration
     [Serializable]
     public class MessagingConfiguration : IMessagingConfiguration
     {
+        public TimeSpan OpenConnectionTimeout { get; set; }
         public TimeSpan ResponseTimeout { get; set; }
         public int MaxResendCount { get; set; }
         public bool ResendOnTimeout { get; set; }
@@ -138,6 +143,7 @@ internal MessagingConfiguration(bool isSilo)
         {
             isSiloConfig = isSilo;
 
+            OpenConnectionTimeout = Constants.DEFAULT_OPENCONNECTION_TIMEOUT;
             ResponseTimeout = Constants.DEFAULT_RESPONSE_TIMEOUT;
             MaxResendCount = 0;
             ResendOnTimeout = DEFAULT_RESEND_ON_TIMEOUT;

src/Orleans/Messaging/GatewayConnection.cs

@@ -158,16 +158,16 @@ public void Connect()
                         }
                         lastConnect = DateTime.UtcNow;
                         Socket = new Socket(Silo.Endpoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
-                        Socket.Connect(Silo.Endpoint);
+                        SocketManager.Connect(Socket, Silo.Endpoint, MsgCenter.MessagingConfiguration.OpenConnectionTimeout);
                         NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
                         MsgCenter.OnGatewayConnectionOpen();
                         SocketManager.WriteConnectionPreamble(Socket, MsgCenter.ClientId);  // Identifies this client
                         Log.Info(ErrorCode.ProxyClient_Connected, "Connected to gateway at address {0} on trial {1}.", Address, i);
                         return;
                     }
-                    catch (Exception)
+                    catch (Exception ex)
                     {
-                        Log.Warn(ErrorCode.ProxyClient_CannotConnect, "Unable to connect to gateway at address {0} on trial {1}.", Address, i);
+                        Log.Warn(ErrorCode.ProxyClient_CannotConnect, $"Unable to connect to gateway at address {Address} on trial {i} (Exception: {ex.Message})");
                         MarkAsDisconnected(Socket);
                     }
                 }

src/Orleans/Messaging/SocketManager.cs

@@ -2,18 +2,21 @@
 using System.Collections.Generic;
 using System.Net;
 using System.Net.Sockets;
+using System.Threading;
 using Orleans.Runtime.Configuration;
 
 namespace Orleans.Runtime
 {
     internal class SocketManager
     {
         private readonly LRU<IPEndPoint, Socket> cache;
+        private TimeSpan connectionTimeout;
 
         private const int MAX_SOCKETS = 200;
 
         internal SocketManager(IMessagingConfiguration config)
         {
+            connectionTimeout = config.OpenConnectionTimeout;
             cache = new LRU<IPEndPoint, Socket>(MAX_SOCKETS, config.MaxSocketAge, SendingSocketCreator);
             cache.RaiseFlushEvent += FlushHandler;
         }
@@ -60,7 +63,7 @@ private Socket SendingSocketCreator(IPEndPoint target)
             var s = new Socket(target.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
             try
             {
-                s.Connect(target);
+                Connect(s, target, connectionTimeout);
                 // Prep the socket so it will reset on close and won't Nagle
                 s.LingerState = new LingerOption(true, 0);
                 s.NoDelay = true;
@@ -171,6 +174,28 @@ internal void Stop()
             cache.Clear();
         }
 
+        /// <summary>
+        /// Connect the socket to the target endpoint
+        /// </summary>
+        /// <param name="s">The socket</param>
+        /// <param name="endPoint">The target endpoint</param>
+        /// <param name="connectionTimeout">The timeout value to use when opening the connection</param>
+        /// <exception cref="TimeoutException">When the connection could not be established in time</exception>
+        internal static void Connect(Socket s, IPEndPoint endPoint, TimeSpan connectionTimeout)
+        {
+            var signal = new AutoResetEvent(false);
+            var e = new SocketAsyncEventArgs();
+            e.RemoteEndPoint = endPoint;
+            e.Completed += (sender, eventArgs) => signal.Set();
+            s.ConnectAsync(e);
+
+            if (!signal.WaitOne(connectionTimeout))
+                throw new TimeoutException($"Connection to {endPoint} could not be established in {connectionTimeout}");
+
+            if (e.SocketError != SocketError.Success || !s.Connected)
+                throw new OrleansException($"Could not connect to {endPoint}: {e.SocketError}");
+        }
+
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")]
         internal static void CloseSocket(Socket s)
         {

src/Orleans/Runtime/Constants.cs

@@ -52,7 +52,9 @@ internal class Constants
         public static readonly GrainId SiloDirectConnectionId = GrainId.GetSystemGrainId(new Guid("01111111-1111-1111-1111-111111111111"));
 
         internal const long ReminderTableGrainId = 12345;
-
+
+        public static TimeSpan DEFAULT_OPENCONNECTION_TIMEOUT = TimeSpan.FromSeconds(5);
+
         /// <summary>
         /// The default timeout before a request is assumed to have failed.
         /// </summary>