From 6b3704990257643da54100d8581856a7d2c72f86 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 17 Feb 2015 12:12:11 +0100 Subject: socket: use TCP_USER_TIMEOUT to detect client failures quicker Use the network.ping-timeout to set the TCP_USER_TIMEOUT socket option (see 'man 7 tcp'). The option sets the transport.tcp-user-timeout option that is handled in the rpc/socket layer on the protocol/server side. This socket option makes detecting unclean disconnected clients more reliable. When the socket gets closed, any locks that the client held are been released. This makes it possible to reduce the fail-over time for applications that run on systems that became unreachable due to a network partition or general system error client-side (kernel panic, hang, ...). It is not trivial to create a test-case for this at the moment. We need a client that unclean disconnects and an other client that tries to take over the lock from the disconnected client. URL: http://supercolony.gluster.org/pipermail/gluster-devel/2014-May/040755.html Change-Id: I5e5f540a49abfb5f398291f1818583a63a5f4bb4 BUG: 1129787 Signed-off-by: Niels de Vos Reviewed-on: http://review.gluster.org/8065 Tested-by: Gluster Build System Reviewed-by: soumya k Reviewed-by: Santosh Pradhan Reviewed-by: Kaleb KEITHLEY --- rpc/rpc-lib/src/rpc-transport.c | 7 +++- rpc/rpc-lib/src/rpc-transport.h | 2 +- rpc/rpc-transport/socket/src/socket.c | 55 +++++++++++++++++++++---- rpc/rpc-transport/socket/src/socket.h | 1 + xlators/mgmt/glusterd/src/glusterd-handler.c | 12 ++++-- xlators/mgmt/glusterd/src/glusterd-volume-set.c | 5 +++ xlators/protocol/server/src/server.c | 6 +++ 7 files changed, 76 insertions(+), 12 deletions(-) diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c index f6774b72353..23fbf37360d 100644 --- a/rpc/rpc-lib/src/rpc-transport.c +++ b/rpc/rpc-lib/src/rpc-transport.c @@ -572,7 +572,7 @@ out: //why call it if you dont set it. int rpc_transport_keepalive_options_set (dict_t *options, int32_t interval, - int32_t time) + int32_t time, int32_t timeout) { int ret = -1; @@ -588,6 +588,11 @@ rpc_transport_keepalive_options_set (dict_t *options, int32_t interval, "transport.socket.keepalive-time", time); if (ret) goto out; + + ret = dict_set_int32 (options, + "transport.tcp-user-timeout", timeout); + if (ret) + goto out; out: return ret; } diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h index d1c650e7ec1..d0572a16333 100644 --- a/rpc/rpc-lib/src/rpc-transport.h +++ b/rpc/rpc-lib/src/rpc-transport.h @@ -306,7 +306,7 @@ rpc_transport_pollin_destroy (rpc_transport_pollin_t *pollin); int rpc_transport_keepalive_options_set (dict_t *options, int32_t interval, - int32_t time); + int32_t time, int32_t timeout); int rpc_transport_unix_options_build (dict_t **options, char *filepath, diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c index 2b61eb417d2..054a1c0b1db 100644 --- a/rpc/rpc-transport/socket/src/socket.c +++ b/rpc/rpc-transport/socket/src/socket.c @@ -31,9 +31,15 @@ #include "xdr-nfs3.h" #include "rpcsvc.h" +/* for TCP_USER_TIMEOUT */ +#if !defined(TCP_USER_TIMEOUT) && defined(GF_LINUX_HOST_OS) +#include +#else +#include +#endif + #include #include -#include #include #include #define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR) @@ -857,10 +863,12 @@ __socket_nodelay (int fd) static int -__socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle) +__socket_keepalive (int fd, int family, int keepalive_intvl, + int keepalive_idle, int timeout) { int on = 1; int ret = -1; + int timeout_ms = timeout * 1000; ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof (on)); if (ret == -1) { @@ -890,7 +898,7 @@ __socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle) goto done; ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle, - sizeof (keepalive_intvl)); + sizeof (keepalive_idle)); if (ret == -1) { gf_log ("socket", GF_LOG_WARNING, "failed to set keep idle %d on socket %d, %s", @@ -905,11 +913,23 @@ __socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle) keepalive_intvl, fd, strerror(errno)); goto err; } + +#if defined(TCP_USER_TIMEOUT) + ret = setsockopt (fd, IPPROTO_TCP , TCP_USER_TIMEOUT, &timeout_ms, + sizeof (timeout_ms)); + if (ret == -1) { + gf_log ("socket", GF_LOG_WARNING, "failed to set " + "TCP_USER_TIMEOUT %d on socket %d, %s", timeout_ms, fd, + strerror(errno)); + goto err; + } +#endif #endif done: - gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, interval " - "%d, idle: %d", fd, keepalive_intvl, keepalive_idle); + gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, " + "interval %d, idle: %d, timeout: %d", fd, keepalive_intvl, + keepalive_idle, timeout); err: return ret; @@ -2642,7 +2662,8 @@ socket_server_event_handler (int fd, int idx, void *data, ret = __socket_keepalive (new_sock, new_sockaddr.ss_family, priv->keepaliveintvl, - priv->keepaliveidle); + priv->keepaliveidle, + priv->timeout); if (ret == -1) gf_log (this->name, GF_LOG_WARNING, "Failed to set keep-alive: %s", @@ -2986,7 +3007,8 @@ socket_connect (rpc_transport_t *this, int port) ret = __socket_keepalive (priv->sock, sa_family, priv->keepaliveintvl, - priv->keepaliveidle); + priv->keepaliveidle, + priv->timeout); if (ret == -1) gf_log (this->name, GF_LOG_ERROR, "Failed to set keep-alive: %s", @@ -3577,6 +3599,7 @@ reconfigure (rpc_transport_t *this, dict_t *options) char *optstr = NULL; int ret = 0; uint64_t windowsize = 0; + uint32_t timeout = 0; GF_VALIDATE_OR_GOTO ("socket", this, out); GF_VALIDATE_OR_GOTO ("socket", this->private, out); @@ -3605,6 +3628,13 @@ reconfigure (rpc_transport_t *this, dict_t *options) else priv->keepalive = 1; + if (dict_get_uint32 (this->options, "transport.tcp-user-timeout", + &timeout) == 0) { + priv->timeout = timeout; + gf_log (this->name, GF_LOG_DEBUG, "Reconfigued " + "transport.tcp-user-timeout=%d", timeout); + } + optstr = NULL; if (dict_get_str (this->options, "tcp-window-size", &optstr) == 0) { @@ -3659,6 +3689,7 @@ socket_init (rpc_transport_t *this) uint64_t windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE; char *optstr = NULL; uint32_t keepalive = 0; + uint32_t timeout = 0; uint32_t backlog = 0; int session_id = 0; int32_t cert_depth = 1; @@ -3771,6 +3802,13 @@ socket_init (rpc_transport_t *this) priv->keepaliveidle = keepalive; } + if (dict_get_uint32 (this->options, "transport.tcp-user-timeout", + &timeout) == 0) { + priv->timeout = timeout; + } + gf_log (this->name, GF_LOG_DEBUG, "Configued " + "transport.tcp-user-timeout=%d", priv->timeout); + if (dict_get_uint32 (this->options, "transport.socket.listen-backlog", &backlog) == 0) { @@ -4027,6 +4065,9 @@ struct volume_options options[] = { .min = GF_MIN_SOCKET_WINDOW_SIZE, .max = GF_MAX_SOCKET_WINDOW_SIZE }, + { .key = {"transport.tcp-user-timeout"}, + .type = GF_OPTION_TYPE_INT, + }, { .key = {"transport.socket.nodelay"}, .type = GF_OPTION_TYPE_BOOL }, diff --git a/rpc/rpc-transport/socket/src/socket.h b/rpc/rpc-transport/socket/src/socket.h index 6a8ab870ab7..2a84e264b81 100644 --- a/rpc/rpc-transport/socket/src/socket.h +++ b/rpc/rpc-transport/socket/src/socket.h @@ -216,6 +216,7 @@ typedef struct { int keepalive; int keepaliveidle; int keepaliveintvl; + int timeout; uint32_t backlog; gf_boolean_t read_fail_log; gf_boolean_t ssl_enabled; /* outbound I/O */ diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index cc97baf6f21..1c33f3febb3 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -2994,7 +2994,8 @@ out: } int -glusterd_transport_keepalive_options_get (int *interval, int *time) +glusterd_transport_keepalive_options_get (int *interval, int *time, + int *timeout) { int ret = 0; xlator_t *this = NULL; @@ -3008,6 +3009,9 @@ glusterd_transport_keepalive_options_get (int *interval, int *time) ret = dict_get_int32 (this->options, "transport.socket.keepalive-time", time); + ret = dict_get_int32 (this->options, + "transport.tcp-user-timeout", + timeout); return 0; } @@ -3018,6 +3022,7 @@ glusterd_transport_inet_options_build (dict_t **options, const char *hostname, dict_t *dict = NULL; int32_t interval = -1; int32_t time = -1; + int32_t timeout = -1; int ret = 0; GF_ASSERT (options); @@ -3044,10 +3049,11 @@ glusterd_transport_inet_options_build (dict_t **options, const char *hostname, } /* Set keepalive options */ - glusterd_transport_keepalive_options_get (&interval, &time); + glusterd_transport_keepalive_options_get (&interval, &time, &timeout); if ((interval > 0) || (time > 0)) - ret = rpc_transport_keepalive_options_set (dict, interval, time); + ret = rpc_transport_keepalive_options_set (dict, interval, + time, timeout); *options = dict; out: gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret); diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 77f6853dd51..b0a3b0c8c39 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -842,6 +842,11 @@ struct volopt_map_entry glusterd_volopt_map[] = { }, /* Server xlator options */ + { .key = "network.ping-timeout", + .voltype = "protocol/server", + .option = "transport.tcp-user-timeout", + .op_version = GD_OP_VERSION_3_7_0, + }, { .key = "network.tcp-window-size", .voltype = "protocol/server", .op_version = 1 diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 023f2a6234f..aea88b623cc 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -1198,6 +1198,12 @@ struct volume_options options[] = { { .key = {"volume-filename.*"}, .type = GF_OPTION_TYPE_PATH, }, + { .key = {"transport.tcp-user-timeout"}, + .type = GF_OPTION_TYPE_TIME, + .min = 0, + .max = 1013, + .default_value = "42", /* default like network.ping-timeout */ + }, { .key = {"transport.*"}, .type = GF_OPTION_TYPE_ANY, }, -- cgit