Add SO_REUSEPORT support to socket

Introduce new feature for Twemproxy running on newer kernels. With this
feature you are able to do upgrades, restarts without any downtime. Kernel
does load balancing between processes with the same host:port pairs.

Also, helps running on more than single core.

Signed-off-by: Donatas Abraitis <[email protected]>

src/nc_conf.c

@@ -78,6 +78,10 @@ static struct command conf_commands[] = {
       conf_set_bool,
       offsetof(struct conf_pool, tcpkeepalive) },
 
+    { string("reuseport"),
+      conf_set_bool,
+      offsetof(struct conf_pool, reuseport) },
+
     { string("redis_auth"),
       conf_set_string,
       offsetof(struct conf_pool, redis_auth) },
@@ -199,6 +203,7 @@ conf_pool_init(struct conf_pool *cp, struct string *name)
 
     cp->redis = CONF_UNSET_NUM;
     cp->tcpkeepalive = CONF_UNSET_NUM;
+    cp->reuseport = CONF_UNSET_NUM;
     cp->redis_db = CONF_UNSET_NUM;
     cp->preconnect = CONF_UNSET_NUM;
     cp->auto_eject_hosts = CONF_UNSET_NUM;
@@ -287,6 +292,7 @@ conf_pool_each_transform(void *elem, void *data)
     sp->hash_tag = cp->hash_tag;
 
     sp->tcpkeepalive = cp->tcpkeepalive ? 1 : 0;
+    sp->reuseport = cp->reuseport ? 1 : 0;
 
     sp->redis = cp->redis ? 1 : 0;
     sp->timeout = cp->timeout;
@@ -1242,6 +1248,10 @@ conf_validate_pool(struct conf *cf, struct conf_pool *cp)
         cp->tcpkeepalive = CONF_DEFAULT_TCPKEEPALIVE;
     }
 
+    if (cp->reuseport == CONF_UNSET_NUM) {
+	cp->reuseport = CONF_DEFAULT_REUSEPORT;
+    }
+
     if (cp->redis_db == CONF_UNSET_NUM) {
         cp->redis_db = CONF_DEFAULT_REDIS_DB;
     }

src/nc_conf.h

@@ -55,6 +55,7 @@
 #define CONF_DEFAULT_SERVER_CONNECTIONS      1
 #define CONF_DEFAULT_KETAMA_PORT             11211
 #define CONF_DEFAULT_TCPKEEPALIVE            false
+#define CONF_DEFAULT_REUSEPORT		      false
 
 struct conf_listen {
     struct string   pname;   /* listen: as "hostname:port" */
@@ -95,6 +96,7 @@ struct conf_pool {
     int                server_failure_limit;  /* server_failure_limit: */
     struct array       server;                /* servers: conf_server[] */
     unsigned           valid:1;               /* valid? */
+    int                reuseport;             /* set SO_REUSEPORT to socket */
 };
 
 struct conf {

src/nc_proxy.c

@@ -142,6 +142,16 @@ proxy_listen(struct context *ctx, struct conn *p)
         return NC_ERROR;
     }
 
+    if (pool->reuseport) {
+        status = nc_set_reuseport(p->sd);
+        if (status < 0) {
+            log_error("reuse of port '%.*s' for listening on p %d failed: %s",
+                      pool->addrstr.len, pool->addrstr.data, p->sd,
+                      strerror(errno));
+            return NC_ERROR;
+        }
+    }
+
     status = bind(p->sd, p->addr, p->addrlen);
     if (status < 0) {
         log_error("bind on p %d to addr '%.*s' failed: %s", p->sd,
@@ -294,7 +304,7 @@ proxy_accept(struct context *ctx, struct conn *p)
                 return NC_OK;
             }
 
-            /* 
+            /*
              * Workaround of https://github.com/twitter/twemproxy/issues/97
              *
              * We should never reach here because the check for conn_ncurr_cconn()

src/nc_server.h

@@ -121,6 +121,7 @@ struct server_pool {
     unsigned           preconnect:1;         /* preconnect? */
     unsigned           redis:1;              /* redis? */
     unsigned           tcpkeepalive:1;       /* tcpkeepalive? */
+    unsigned           reuseport:1;          /* set SO_REUSEPORT to socket */
 };
 
 void server_ref(struct conn *conn, void *owner);

src/nc_stats.c

@@ -827,7 +827,7 @@ stats_listen(struct stats *st)
         log_error("socket failed: %s", strerror(errno));
         return NC_ERROR;
     }
-
+    nc_set_reuseport(st->sd);
     status = nc_set_reuseaddr(st->sd);
     if (status < 0) {
         log_error("set reuseaddr on m %d failed: %s", st->sd, strerror(errno));

src/nc_util.c

@@ -75,6 +75,20 @@ nc_set_reuseaddr(int sd)
     return setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &reuse, len);
 }
 
+int
+nc_set_reuseport(int sd)
+{
+    int reuse;
+    socklen_t len;
+
+    reuse = 1;
+    len = sizeof(reuse);
+#ifdef SO_REUSEPORT
+    return setsockopt(sd, SOL_SOCKET, SO_REUSEPORT, &reuse, len);
+#endif
+    return -1;
+}
+
 /*
  * Disable Nagle algorithm on TCP socket.
  *

src/nc_util.h

@@ -81,6 +81,7 @@
 int nc_set_blocking(int sd);
 int nc_set_nonblocking(int sd);
 int nc_set_reuseaddr(int sd);
+int nc_set_reuseport(int sd);
 int nc_set_tcpnodelay(int sd);
 int nc_set_linger(int sd, int timeout);
 int nc_set_sndbuf(int sd, int size);