From 4e89da5054837a2266fe1732a309c341dd6cd663 Mon Sep 17 00:00:00 2001 From: Shehjar Tikoo Date: Tue, 26 May 2009 09:02:29 +0000 Subject: socket: New option: Set receive and send buffer size ..and hope for a chance to improve performance on high speed links like 10GigE. Signed-off-by: Anand V. Avati --- transport/socket/src/socket.c | 64 +++++++++++++++++++++++++++++++++++++++++-- transport/socket/src/socket.h | 15 ++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) (limited to 'transport') diff --git a/transport/socket/src/socket.c b/transport/socket/src/socket.c index 5ff30fe5adc..9035eafc62a 100644 --- a/transport/socket/src/socket.c +++ b/transport/socket/src/socket.c @@ -996,6 +996,27 @@ socket_connect (transport_t *this) goto unlock; } + /* Cant help if setting socket options fails. We can continue + * working nonetheless. + */ + if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF, + &priv->windowsize, + sizeof (priv->windowsize)) < 0) { + gf_log (this->xl->name, GF_LOG_ERROR, + "setting receive window size failed: %d: %d: " + "%s", priv->sock, priv->windowsize, + strerror (errno)); + } + + if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF, + &priv->windowsize, + sizeof (priv->windowsize)) < 0) { + gf_log (this->xl->name, GF_LOG_ERROR, + "setting send window size failed: %d: %d: " + "%s", priv->sock, priv->windowsize, + strerror (errno)); + } + if (!priv->bio) { ret = __socket_nonblock (priv->sock); @@ -1106,6 +1127,27 @@ socket_listen (transport_t *this) goto unlock; } + /* Cant help if setting socket options fails. We can continue + * working nonetheless. + */ + if (setsockopt (priv->sock, SOL_SOCKET, SO_RCVBUF, + &priv->windowsize, + sizeof (priv->windowsize)) < 0) { + gf_log (this->xl->name, GF_LOG_ERROR, + "setting receive window size failed: %d: %d: " + "%s", priv->sock, priv->windowsize, + strerror (errno)); + } + + if (setsockopt (priv->sock, SOL_SOCKET, SO_SNDBUF, + &priv->windowsize, + sizeof (priv->windowsize)) < 0) { + gf_log (this->xl->name, GF_LOG_ERROR, + "setting send window size failed: %d: %d: " + "%s", priv->sock, priv->windowsize, + strerror (errno)); + } + if (!priv->bio) { ret = __socket_nonblock (priv->sock); @@ -1278,7 +1320,8 @@ socket_init (transport_t *this) { socket_private_t *priv = NULL; gf_boolean_t tmp_bool = 0; - char *nb_connect = NULL; + uint64_t windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE; + char *optstr = NULL; if (this->private) { gf_log (this->xl->name, GF_LOG_DEBUG, @@ -1303,10 +1346,10 @@ socket_init (transport_t *this) INIT_LIST_HEAD (&priv->ioq); if (dict_get (this->xl->options, "non-blocking-io")) { - nb_connect = data_to_str (dict_get (this->xl->options, + optstr = data_to_str (dict_get (this->xl->options, "non-blocking-io")); - if (gf_string2boolean (nb_connect, &tmp_bool) == -1) { + if (gf_string2boolean (optstr, &tmp_bool) == -1) { gf_log (this->xl->name, GF_LOG_ERROR, "'non-blocking-io' takes only boolean options," " not taking any action"); @@ -1320,6 +1363,16 @@ socket_init (transport_t *this) } } + optstr = NULL; + if (dict_get_str (this->xl->options, "transport.window-size", + &optstr) == 0) { + if (gf_string2bytesize (optstr, &windowsize) != 0) { + gf_log (this->xl->name, GF_LOG_ERROR, + "invalid number format: %s", optstr); + return -1; + } + } + priv->windowsize = (int)windowsize; this->private = priv; return 0; @@ -1384,6 +1437,11 @@ struct volume_options options[] = { { .key = {"non-blocking-io"}, .type = GF_OPTION_TYPE_BOOL }, + { .key = {"transport.window-size"}, + .type = GF_OPTION_TYPE_SIZET, + .min = GF_MIN_SOCKET_WINDOW_SIZE, + .max = GF_MAX_SOCKET_WINDOW_SIZE, + }, { .key = {NULL} } }; diff --git a/transport/socket/src/socket.h b/transport/socket/src/socket.h index 4f34e16863a..50d7d0303bb 100644 --- a/transport/socket/src/socket.h +++ b/transport/socket/src/socket.h @@ -38,6 +38,20 @@ #define GF_DEFAULT_SOCKET_LISTEN_PORT 6996 +/* This is the size set through setsockopt for + * both the TCP receive window size and the + * send buffer size. + * Till the time iobuf size becomes configurable, this size is set to include + * two iobufs + the GlusterFS protocol headers. + * Linux allows us to over-ride the max values for the system. + * Should we over-ride them? Because if we set a value larger than the default + * setsockopt will fail. Having larger values might be beneficial for + * IB links. + */ +#define GF_DEFAULT_SOCKET_WINDOW_SIZE (512 * GF_UNIT_KB) +#define GF_MAX_SOCKET_WINDOW_SIZE (1 * GF_UNIT_MB) +#define GF_MIN_SOCKET_WINDOW_SIZE (128 * GF_UNIT_KB) + typedef enum { SOCKET_PROTO_STATE_NADA = 0, SOCKET_PROTO_STATE_HEADER_COMING, @@ -101,6 +115,7 @@ typedef struct { int pending_count; } incoming; pthread_mutex_t lock; + int windowsize; } socket_private_t; -- cgit