summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMilind Changire <mchangir@redhat.com>2017-04-11 12:30:06 +0530
committerRaghavendra G <rgowdapp@redhat.com>2017-04-12 05:44:01 -0400
commit6b8df081b46ac4f485c86a5052fc30472e74bfbb (patch)
treef472b4d48d2f70a274b019f695db09ee6bde4f40
parent1612355327fa5f86078b9dbcf7a38e4e0c63e205 (diff)
rpc: add options to manage socket keepalive lifespan
Problem: Default values for handling socket timeouts for brick responses are insufficient for aggressive applications such as databases. Solution: Add 1:1 gluster options for keepalive, keepalive-idle, keepalive-interval and keepalive-timeout as per the socket level options available as per tcp(7) man page. Default values for options are NOT agressive and continue to be values which result in default timeout when only the keep alive option is turned on. These options are Linux specific and will not be applicable to the *BSDs. Change-Id: I2a08ecd949ca8ceb3e090d336ad634341e2dbf14 BUG: 1426059 Signed-off-by: Milind Changire <mchangir@redhat.com> Reviewed-on: https://review.gluster.org/16731 Smoke: Gluster Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.org> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
-rw-r--r--libglusterfs/src/globals.h2
-rw-r--r--rpc/rpc-transport/socket/src/socket.c135
-rw-r--r--rpc/rpc-transport/socket/src/socket.h5
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-set.c56
4 files changed, 155 insertions, 43 deletions
diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 24ce0683f7a..d0eb37716e1 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -80,6 +80,8 @@
#define GD_OP_VERSION_3_10_1 31001 /* Op-version for GlusterFS 3.10.1 */
+#define GD_OP_VERSION_3_10_2 31002 /* Op-version for GlusterFS 3.10.2 */
+
#define GD_OP_VERSION_4_0_0 40000 /* Op-version for GlusterFS 4.0.0 */
#define GD_OP_VER_PERSISTENT_AFR_XATTRS GD_OP_VERSION_3_6_0
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index d062cbbc4fb..e6e73bf8f22 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -838,8 +838,8 @@ __socket_nodelay (int fd)
static int
-__socket_keepalive (int fd, int family, int keepalive_intvl,
- int keepalive_idle, int timeout)
+__socket_keepalive (int fd, int family, int keepaliveintvl,
+ int keepaliveidle, int keepalivecnt, int timeout)
{
int on = 1;
int ret = -1;
@@ -852,16 +852,16 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
goto err;
}
- if (keepalive_intvl == GF_USE_DEFAULT_KEEPALIVE)
+ if (keepaliveintvl == GF_USE_DEFAULT_KEEPALIVE)
goto done;
#if !defined(GF_LINUX_HOST_OS) && !defined(__NetBSD__)
#if defined(GF_SOLARIS_HOST_OS) || defined(__FreeBSD__)
- ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &keepalive_intvl,
- sizeof (keepalive_intvl));
+ ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &keepaliveintvl,
+ sizeof (keepaliveintvl));
#else
- ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepalive_intvl,
- sizeof (keepalive_intvl));
+ ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepaliveintvl,
+ sizeof (keepaliveintvl));
#endif
if (ret == -1) {
gf_log ("socket", GF_LOG_WARNING,
@@ -872,20 +872,20 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
if (family != AF_INET && family != AF_INET6)
goto done;
- ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle,
- sizeof (keepalive_idle));
+ ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepaliveidle,
+ sizeof (keepaliveidle));
if (ret == -1) {
gf_log ("socket", GF_LOG_WARNING,
"failed to set keep idle %d on socket %d, %s",
- keepalive_idle, fd, strerror(errno));
+ keepaliveidle, fd, strerror(errno));
goto err;
}
- ret = setsockopt (fd, IPPROTO_TCP , TCP_KEEPINTVL, &keepalive_intvl,
- sizeof (keepalive_intvl));
+ ret = setsockopt (fd, IPPROTO_TCP , TCP_KEEPINTVL, &keepaliveintvl,
+ sizeof (keepaliveintvl));
if (ret == -1) {
gf_log ("socket", GF_LOG_WARNING,
"failed to set keep interval %d on socket %d, %s",
- keepalive_intvl, fd, strerror(errno));
+ keepaliveintvl, fd, strerror(errno));
goto err;
}
@@ -901,12 +901,23 @@ __socket_keepalive (int fd, int family, int keepalive_intvl,
goto err;
}
#endif
+#if defined(TCP_KEEPCNT)
+ ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPCNT, &keepalivecnt,
+ sizeof (keepalivecnt));
+ if (ret == -1) {
+ gf_log ("socket", GF_LOG_WARNING, "failed to set "
+ "TCP_KEEPCNT %d on socket %d, %s", keepalivecnt, fd,
+ strerror(errno));
+ goto err;
+ }
+#endif
#endif
done:
- gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, "
- "interval %d, idle: %d, timeout: %d", fd, keepalive_intvl,
- keepalive_idle, timeout);
+ gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket: %d, "
+ "(idle: %d, interval: %d, max-probes: %d, timeout: %d)",
+ fd, keepaliveidle, keepaliveintvl, keepalivecnt,
+ timeout);
err:
return ret;
@@ -2700,6 +2711,7 @@ socket_server_event_handler (int fd, int idx, void *data,
new_sockaddr.ss_family,
priv->keepaliveintvl,
priv->keepaliveidle,
+ priv->keepalivecnt,
priv->timeout);
if (ret == -1)
gf_log (this->name, GF_LOG_WARNING,
@@ -3104,6 +3116,7 @@ socket_connect (rpc_transport_t *this, int port)
sa_family,
priv->keepaliveintvl,
priv->keepaliveidle,
+ priv->keepalivecnt,
priv->timeout);
if (ret == -1)
gf_log (this->name, GF_LOG_ERROR,
@@ -3745,6 +3758,9 @@ reconfigure (rpc_transport_t *this, dict_t *options)
int ret = 0;
uint64_t windowsize = 0;
uint32_t timeout = 0;
+ int keepaliveidle = GF_KEEPALIVE_TIME;
+ int keepaliveintvl = GF_KEEPALIVE_INTERVAL;
+ int keepalivecnt = GF_KEEPALIVE_COUNT;
GF_VALIDATE_OR_GOTO ("socket", this, out);
GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@@ -3773,12 +3789,30 @@ reconfigure (rpc_transport_t *this, dict_t *options)
else
priv->keepalive = 1;
- if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
- &timeout) == 0) {
+ if (dict_get_int32 (this->options, "transport.tcp-user-timeout",
+ &(priv->timeout)) != 0)
priv->timeout = timeout;
- gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
- "transport.tcp-user-timeout=%d", timeout);
- }
+ gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+ "transport.tcp-user-timeout=%d", priv->timeout);
+
+ if (dict_get_int32 (this->options, "transport.socket.keepalive-time",
+ &(priv->keepaliveidle)) != 0)
+ priv->keepaliveidle = keepaliveidle;
+ gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+ "transport.socket.keepalive-time=%d", priv->keepaliveidle);
+
+ if (dict_get_int32 (this->options,
+ "transport.socket.keepalive-interval",
+ &(priv->keepaliveintvl)) != 0)
+ priv->keepaliveintvl = keepaliveintvl;
+ gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+ "transport.socket.keepalive-interval=%d", priv->keepaliveintvl);
+
+ if (dict_get_int32 (this->options, "transport.socket.keepalive-count",
+ &(priv->keepalivecnt)) != 0)
+ priv->keepalivecnt = keepalivecnt;
+ gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+ "transport.socket.keepalive-count=%d", priv->keepalivecnt);
optstr = NULL;
if (dict_get_str (this->options, "tcp-window-size",
@@ -3921,8 +3955,10 @@ socket_init (rpc_transport_t *this)
gf_boolean_t tmp_bool = 0;
uint64_t windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
char *optstr = NULL;
- uint32_t keepalive = 0;
uint32_t timeout = 0;
+ int keepaliveidle = GF_KEEPALIVE_TIME;
+ int keepaliveintvl = GF_KEEPALIVE_INTERVAL;
+ int keepalivecnt = GF_KEEPALIVE_COUNT;
uint32_t backlog = 0;
int session_id = 0;
int32_t cert_depth = DEFAULT_VERIFY_DEPTH;
@@ -4014,8 +4050,9 @@ socket_init (rpc_transport_t *this)
optstr = NULL;
/* Enable Keep-alive by default. */
priv->keepalive = 1;
- priv->keepaliveintvl = 2;
- priv->keepaliveidle = 20;
+ priv->keepaliveintvl = GF_KEEPALIVE_INTERVAL;
+ priv->keepaliveidle = GF_KEEPALIVE_TIME;
+ priv->keepalivecnt = GF_KEEPALIVE_COUNT;
if (dict_get_str (this->options, "transport.socket.keepalive",
&optstr) == 0) {
if (gf_string2boolean (optstr, &tmp_bool) == -1) {
@@ -4029,24 +4066,29 @@ socket_init (rpc_transport_t *this)
priv->keepalive = 0;
}
- if (dict_get_uint32 (this->options,
- "transport.socket.keepalive-interval",
- &keepalive) == 0) {
- priv->keepaliveintvl = keepalive;
- }
+ if (dict_get_int32 (this->options, "transport.tcp-user-timeout",
+ &(priv->timeout)) != 0)
+ priv->timeout = timeout;
+ gf_log (this->name, GF_LOG_DEBUG, "Configued "
+ "transport.tcp-user-timeout=%d", priv->timeout);
- if (dict_get_uint32 (this->options,
- "transport.socket.keepalive-time",
- &keepalive) == 0) {
- priv->keepaliveidle = keepalive;
+ if (dict_get_int32 (this->options,
+ "transport.socket.keepalive-time",
+ &(priv->keepaliveidle)) != 0) {
+ priv->keepaliveidle = keepaliveidle;
}
- if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
- &timeout) == 0) {
- priv->timeout = timeout;
+ if (dict_get_int32 (this->options,
+ "transport.socket.keepalive-interval",
+ &(priv->keepaliveintvl)) != 0) {
+ priv->keepaliveintvl = keepaliveintvl;
}
- gf_log (this->name, GF_LOG_DEBUG, "Configued "
- "transport.tcp-user-timeout=%d", priv->timeout);
+
+ if (dict_get_int32 (this->options, "transport.socket.keepalive-count",
+ &(priv->keepalivecnt)) != 0)
+ priv->keepalivecnt = keepalivecnt;
+ gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
+ "transport.keepalivecnt=%d", keepalivecnt);
if (dict_get_uint32 (this->options,
"transport.socket.listen-backlog",
@@ -4446,21 +4488,30 @@ struct volume_options options[] = {
},
{ .key = {"transport.tcp-user-timeout"},
.type = GF_OPTION_TYPE_INT,
+ .default_value = "0"
},
{ .key = {"transport.socket.nodelay"},
- .type = GF_OPTION_TYPE_BOOL
+ .type = GF_OPTION_TYPE_BOOL,
+ .default_value = "1"
},
{ .key = {"transport.socket.lowlat"},
.type = GF_OPTION_TYPE_BOOL
},
{ .key = {"transport.socket.keepalive"},
- .type = GF_OPTION_TYPE_BOOL
+ .type = GF_OPTION_TYPE_BOOL,
+ .default_value = "1"
},
{ .key = {"transport.socket.keepalive-interval"},
- .type = GF_OPTION_TYPE_INT
+ .type = GF_OPTION_TYPE_INT,
+ .default_value = "2"
},
{ .key = {"transport.socket.keepalive-time"},
- .type = GF_OPTION_TYPE_INT
+ .type = GF_OPTION_TYPE_INT,
+ .default_value = "20"
+ },
+ { .key = {"transport.socket.keepalive-count"},
+ .type = GF_OPTION_TYPE_INT,
+ .default_value = "9"
},
{ .key = {"transport.socket.listen-backlog"},
.type = GF_OPTION_TYPE_INT
diff --git a/rpc/rpc-transport/socket/src/socket.h b/rpc/rpc-transport/socket/src/socket.h
index 8528bdeba8d..6c8875f7fb7 100644
--- a/rpc/rpc-transport/socket/src/socket.h
+++ b/rpc/rpc-transport/socket/src/socket.h
@@ -50,6 +50,10 @@
#define GF_MIN_SOCKET_WINDOW_SIZE (0)
#define GF_USE_DEFAULT_KEEPALIVE (-1)
+#define GF_KEEPALIVE_TIME (20)
+#define GF_KEEPALIVE_INTERVAL (2)
+#define GF_KEEPALIVE_COUNT (9)
+
typedef enum {
SP_STATE_NADA = 0,
SP_STATE_COMPLETE,
@@ -224,6 +228,7 @@ typedef struct {
int keepalive;
int keepaliveidle;
int keepaliveintvl;
+ int keepalivecnt;
int timeout;
uint32_t backlog;
gf_boolean_t read_fail_log;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index ca32552fb0d..e5818a1aa15 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1680,6 +1680,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
{ .key = "network.ping-timeout",
.voltype = "protocol/client",
.op_version = 1,
+ .value = "42",
.flags = OPT_FLAG_CLIENT_OPT
},
{ .key = "network.tcp-window-size",
@@ -1723,12 +1724,40 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.voltype = "protocol/client",
.op_version = GD_OP_VERSION_3_7_0,
},
+ { .key = "client.tcp-user-timeout",
+ .voltype = "protocol/client",
+ .option = "transport.tcp-user-timeout",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "0", /* 0 - implies "use system default" */
+ .flags = OPT_FLAG_CLIENT_OPT
+ },
+ { .key = "client.keepalive-time",
+ .voltype = "protocol/client",
+ .option = "transport.socket.keepalive-time",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "20",
+ .flags = OPT_FLAG_CLIENT_OPT
+ },
+ { .key = "client.keepalive-interval",
+ .voltype = "protocol/client",
+ .option = "transport.socket.keepalive-interval",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "2",
+ .flags = OPT_FLAG_CLIENT_OPT
+ },
+ { .key = "client.keepalive-count",
+ .voltype = "protocol/client",
+ .option = "transport.socket.keepalive-count",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "9",
+ .flags = OPT_FLAG_CLIENT_OPT
+ },
/* Server xlator options */
{ .key = "network.ping-timeout",
.voltype = "protocol/server",
- .option = "transport.tcp-user-timeout",
.op_version = GD_OP_VERSION_3_7_0,
+ .value = "42",
},
{ .key = "network.tcp-window-size",
.voltype = "protocol/server",
@@ -1754,6 +1783,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.voltype = "protocol/server",
.option = "transport.socket.keepalive",
.type = NO_DOC,
+ .value = "1",
.op_version = 1
},
{ .key = "server.allow-insecure",
@@ -1841,6 +1871,30 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.voltype = "protocol/server",
.op_version = GD_OP_VERSION_3_7_0,
},
+ { .key = "server.tcp-user-timeout",
+ .voltype = "protocol/server",
+ .option = "transport.tcp-user-timeout",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "0", /* 0 - implies "use system default" */
+ },
+ { .key = "server.keepalive-time",
+ .voltype = "protocol/server",
+ .option = "transport.socket.keepalive-time",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "20",
+ },
+ { .key = "server.keepalive-interval",
+ .voltype = "protocol/server",
+ .option = "transport.socket.keepalive-interval",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "2",
+ },
+ { .key = "server.keepalive-count",
+ .voltype = "protocol/server",
+ .option = "transport.socket.keepalive-count",
+ .op_version = GD_OP_VERSION_3_10_2,
+ .value = "9",
+ },
/* Generic transport options */
{ .key = SSL_OWN_CERT_OPT,