socket: Set receive and send window size

This is experimental. We're hoping this improve performance on
high speed links like 10GigE.

Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

2 files changed, 72 insertions, 0 deletions

diff --git a/transport/socket/src/socket.c b/transport/socket/src/socket.c
index 49312f7b2..65c17763d 100644
--- a/transport/socket/src/socket.c
+++ b/transport/socket/src/socket.c
@@ -996,6 +996,27 @@ socket_connect (transport_t *this)
                         goto unlock;
                 }
 
+                /* Cant help if setting socket options fails. We can continue
+                 * working nonetheless.
+                 */
+                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
+                                &priv->windowsize,
+                                sizeof (priv->windowsize)) < 0) {
+                        gf_log (this->xl->name, GF_LOG_ERROR,
+                                "setting receive window size failed: %d: %d: "
+                                "%s", priv->sock, priv->windowsize,
+                                strerror (errno));
+                }
+
+                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
+                                &priv->windowsize,
+                                sizeof (priv->windowsize)) < 0) {
+                        gf_log (this->xl->name, GF_LOG_ERROR,
+                                "setting send window size failed: %d: %d: "
+                                "%s", priv->sock, priv->windowsize,
+                                strerror (errno));
+                }
+
                 if (!priv->bio) {
                         ret = __socket_nonblock (priv->sock);
 
@@ -1106,6 +1127,27 @@ socket_listen (transport_t *this)
                         goto unlock;
                 }
 
+                /* Cant help if setting socket options fails. We can continue
+                 * working nonetheless.
+                 */
+                if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF,
+                                &priv->windowsize,
+                                sizeof (priv->windowsize)) < 0) {
+                        gf_log (this->xl->name, GF_LOG_ERROR,
+                                "setting receive window size failed: %d: %d: "
+                                "%s", priv->sock, priv->windowsize,
+                                strerror (errno));
+                }
+
+                if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF,
+                                &priv->windowsize,
+                                sizeof (priv->windowsize)) < 0) {
+                        gf_log (this->xl->name, GF_LOG_ERROR,
+                                "setting send window size failed: %d: %d: "
+                                "%s", priv->sock, priv->windowsize,
+                                strerror (errno));
+                }
+
                 if (!priv->bio) {
                         ret = __socket_nonblock (priv->sock);
 
@@ -1279,6 +1321,8 @@ socket_init (transport_t *this)
         socket_private_t *priv = NULL;
         gf_boolean_t      tmp_bool = 0;
         char             *nb_connect = NULL;
+        uint64_t          windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
+        char             *wsizestr = NULL;
 
         if (this->private) {
                 gf_log (this->xl->name, GF_LOG_DEBUG,
@@ -1320,6 +1364,14 @@ socket_init (transport_t *this)
                 }
         }
 
+        if (dict_get_str (this->xl->options, "window-size", &wsizestr) == 0) {
+                if (gf_string2bytesize (wsizestr, &windowsize) != 0) {
+                        gf_log (this->xl->name, GF_LOG_ERROR,
+                                "invalid number format: %s", wsizestr);
+                        return -1;
+                }
+        }
+        priv->windowsize = (int)windowsize;
         this->private = priv;
 
         return 0;
@@ -1380,6 +1432,11 @@ struct volume_options options[] = {
                     "unix", "inet-sdp" },
           .type  = GF_OPTION_TYPE_STR 
         },
+        { .key   = {"transport.window-size"},
+          .type  = GF_OPTION_TYPE_SIZET,
+          .min   = GF_MIN_SOCKET_WINDOW_SIZE,
+          .max   = GF_MAX_SOCKET_WINDOW_SIZE,
+        },
 
         { .key = {NULL} }
 };
diff --git a/transport/socket/src/socket.h b/transport/socket/src/socket.h
index 4f34e1686..50d7d0303 100644
--- a/transport/socket/src/socket.h
+++ b/transport/socket/src/socket.h
@@ -38,6 +38,20 @@
 
 #define GF_DEFAULT_SOCKET_LISTEN_PORT 6996
 
+/* This is the size set through setsockopt for
+ * both the TCP receive window size and the
+ * send buffer size.
+ * Till the time iobuf size becomes configurable, this size is set to include
+ * two iobufs + the GlusterFS protocol headers.
+ * Linux allows us to over-ride the max values for the system.
+ * Should we over-ride them? Because if we set a value larger than the default
+ * setsockopt will fail. Having larger values might be beneficial for
+ * IB links.
+ */
+#define GF_DEFAULT_SOCKET_WINDOW_SIZE   (512 * GF_UNIT_KB)
+#define GF_MAX_SOCKET_WINDOW_SIZE       (1 * GF_UNIT_MB)
+#define GF_MIN_SOCKET_WINDOW_SIZE       (128 * GF_UNIT_KB)
+
 typedef enum {
         SOCKET_PROTO_STATE_NADA = 0,
         SOCKET_PROTO_STATE_HEADER_COMING,
@@ -101,6 +115,7 @@ typedef struct {
                 int                  pending_count;
         } incoming;
         pthread_mutex_t        lock;
+        int                    windowsize;
 } socket_private_t;