summaryrefslogtreecommitdiffstats
path: root/rpc/rpc-lib/src
diff options
context:
space:
mode:
authorKrishnan Parthasarathi <kparthas@redhat.com>2014-04-07 10:57:45 +0530
committerAnand Avati <avati@redhat.com>2014-04-29 14:23:51 -0700
commitc61bc1f9e5874cb8380ec6398680fc71aea233b4 (patch)
treec02fee67ebcc7c43a6338d34c5700916e672c110 /rpc/rpc-lib/src
parent91ab65f812ec3b674f53230eacbd0d71964d3d01 (diff)
glusterd: Ping timer implmentation
This patch refactors the existing client ping timer implementation, and makes use of the common code for implementing both client ping timer and the glusterd ping timer. A new gluster rpc program for ping is introduced. The ping timer is only started for peers that have this new program. The deafult glusterd ping timeout is 30 seconds. It is configurable by setting the option 'ping-timeout' in glusterd.vol . Also, this patch introduces changes in the glusterd-handshake path. The client programs for a peer are now set in the callback of dump_versions, for both the older handshake and the newer op-version handshake. This is the only place in the handshake process where we know what programs a peer supports. Change-Id: I035815ac13449ca47080ecc3253c0a9afbe9016a BUG: 1038261 Signed-off-by: Vijaikumar M <vmallika@redhat.com> Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com> Reviewed-on: http://review.gluster.org/5202 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Anand Avati <avati@redhat.com>
Diffstat (limited to 'rpc/rpc-lib/src')
-rw-r--r--rpc/rpc-lib/src/Makefile.am4
-rw-r--r--rpc/rpc-lib/src/rpc-clnt-ping.c266
-rw-r--r--rpc/rpc-lib/src/rpc-clnt-ping.h18
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.c15
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.h2
-rw-r--r--rpc/rpc-lib/src/rpcsvc.c29
-rw-r--r--rpc/rpc-lib/src/xdr-common.h1
7 files changed, 331 insertions, 4 deletions
diff --git a/rpc/rpc-lib/src/Makefile.am b/rpc/rpc-lib/src/Makefile.am
index ee2f27b259b..6a098c9a8ce 100644
--- a/rpc/rpc-lib/src/Makefile.am
+++ b/rpc/rpc-lib/src/Makefile.am
@@ -2,13 +2,13 @@ lib_LTLIBRARIES = libgfrpc.la
libgfrpc_la_SOURCES = auth-unix.c rpcsvc-auth.c rpcsvc.c auth-null.c \
rpc-transport.c xdr-rpc.c xdr-rpcclnt.c rpc-clnt.c auth-glusterfs.c \
- rpc-drc.c $(CONTRIBDIR)/sunrpc/xdr_sizeof.c
+ rpc-drc.c $(CONTRIBDIR)/sunrpc/xdr_sizeof.c rpc-clnt-ping.c
libgfrpc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
libgfrpc_la_LDFLAGS = -version-info $(LIBGFRPC_LT_VERSION) $(GF_LDFLAGS)
noinst_HEADERS = rpcsvc.h rpc-transport.h xdr-common.h xdr-rpc.h xdr-rpcclnt.h \
- rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h
+ rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h rpc-clnt-ping.h
AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-I$(top_srcdir)/rpc/xdr/src \
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
new file mode 100644
index 00000000000..b3bd5e11deb
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -0,0 +1,266 @@
+/*
+ Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "rpc-clnt.h"
+#include "rpc-clnt-ping.h"
+#include "byte-order.h"
+#include "xdr-rpcclnt.h"
+#include "rpc-transport.h"
+#include "protocol-common.h"
+#include "mem-pool.h"
+#include "xdr-rpc.h"
+#include "rpc-common-xdr.h"
+
+
+char *clnt_ping_procs[GF_DUMP_MAXVALUE] = {
+ [GF_DUMP_PING] = "NULL",
+};
+struct rpc_clnt_program clnt_ping_prog = {
+ .progname = "GF-DUMP",
+ .prognum = GLUSTER_DUMP_PROGRAM,
+ .progver = GLUSTER_DUMP_VERSION,
+ .procnames = clnt_ping_procs,
+};
+
+void
+rpc_clnt_ping_timer_expired (void *rpc_ptr)
+{
+ struct rpc_clnt *rpc = NULL;
+ rpc_transport_t *trans = NULL;
+ rpc_clnt_connection_t *conn = NULL;
+ int disconnect = 0;
+ int transport_activity = 0;
+ struct timespec timeout = {0, };
+ struct timeval current = {0, };
+
+ rpc = (struct rpc_clnt*) rpc_ptr;
+ conn = &rpc->conn;
+ trans = conn->trans;
+
+ if (!trans) {
+ gf_log ("ping-timer", GF_LOG_WARNING,
+ "transport not initialized");
+ goto out;
+ }
+
+ pthread_mutex_lock (&conn->lock);
+ {
+ if (conn->ping_timer) {
+ gf_timer_call_cancel (rpc->ctx,
+ conn->ping_timer);
+ conn->ping_timer = NULL;
+ rpc_clnt_unref (rpc);
+ }
+ gettimeofday (&current, NULL);
+
+ if (((current.tv_sec - conn->last_received.tv_sec) <
+ conn->ping_timeout)
+ || ((current.tv_sec - conn->last_sent.tv_sec) <
+ conn->ping_timeout)) {
+ transport_activity = 1;
+ }
+
+ if (transport_activity) {
+ gf_log (trans->name, GF_LOG_TRACE,
+ "ping timer expired but transport activity "
+ "detected - not bailing transport");
+ timeout.tv_sec = conn->ping_timeout;
+ timeout.tv_nsec = 0;
+
+ rpc_clnt_ref (rpc);
+ conn->ping_timer =
+ gf_timer_call_after (rpc->ctx, timeout,
+ rpc_clnt_ping_timer_expired,
+ (void *) rpc);
+ if (conn->ping_timer == NULL) {
+ gf_log (trans->name, GF_LOG_WARNING,
+ "unable to setup ping timer");
+ rpc_clnt_unref (rpc);
+ }
+ } else {
+ conn->ping_started = 0;
+ disconnect = 1;
+ }
+ }
+ pthread_mutex_unlock (&conn->lock);
+
+ if (disconnect) {
+ gf_log (trans->name, GF_LOG_CRITICAL,
+ "server %s has not responded in the last %d "
+ "seconds, disconnecting.",
+ trans->peerinfo.identifier,
+ conn->ping_timeout);
+
+ rpc_transport_disconnect (conn->trans);
+ }
+
+out:
+ return;
+}
+
+int
+rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
+ void *myframe)
+{
+ struct rpc_clnt *rpc = NULL;
+ xlator_t *this = NULL;
+ rpc_clnt_connection_t *conn = NULL;
+ call_frame_t *frame = NULL;
+
+ if (!myframe) {
+ gf_log (THIS->name, GF_LOG_WARNING,
+ "frame with the request is NULL");
+ goto out;
+ }
+
+ frame = myframe;
+ this = frame->this;
+ rpc = frame->local;
+ frame->local = NULL; /* Prevent STACK_DESTROY from segfaulting */
+ conn = &rpc->conn;
+
+ if (req->rpc_status == -1) {
+ pthread_mutex_lock (&conn->lock);
+ {
+ if (conn->ping_timer != NULL) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "socket or ib related error");
+ gf_timer_call_cancel (rpc->ctx,
+ conn->ping_timer);
+ conn->ping_timer = NULL;
+ rpc_clnt_unref (rpc);
+ } else {
+ /* timer expired and transport bailed out */
+ gf_log (this->name, GF_LOG_WARNING,
+ "timer must have expired");
+ }
+ conn->ping_started = 0;
+ }
+ pthread_mutex_unlock (&conn->lock);
+ }
+out:
+ if (frame)
+ STACK_DESTROY (frame->root);
+ if (rpc)
+ rpc_clnt_unref (rpc);
+ return 0;
+}
+
+int
+rpc_clnt_ping (struct rpc_clnt *rpc)
+{
+ call_frame_t *frame = NULL;
+ int32_t ret = -1;
+
+ frame = create_frame (THIS, THIS->ctx->pool);
+ if (!frame)
+ goto fail;
+
+ frame->local = rpc_clnt_ref (rpc);
+
+ ret = rpc_clnt_submit (rpc, &clnt_ping_prog,
+ GF_DUMP_PING, rpc_clnt_ping_cbk, NULL, 0,
+ NULL, 0, NULL, frame, NULL, 0, NULL, 0, NULL);
+ if (ret) {
+ gf_log (THIS->name, GF_LOG_ERROR,
+ "failed to start ping timer");
+ }
+
+ return ret;
+
+fail:
+ if (frame) {
+ STACK_DESTROY (frame->root);
+ }
+
+ return ret;
+
+}
+
+void
+rpc_clnt_start_ping (void *rpc_ptr)
+{
+ struct rpc_clnt *rpc = NULL;
+ rpc_clnt_connection_t *conn = NULL;
+ struct timespec timeout = {0, };
+ int frame_count = 0;
+
+ rpc = (struct rpc_clnt*) rpc_ptr;
+ conn = &rpc->conn;
+
+ if (conn->ping_timeout == 0) {
+ gf_log (THIS->name, GF_LOG_INFO, "ping timeout is 0, returning");
+ return;
+ }
+
+ pthread_mutex_lock (&conn->lock);
+ {
+ if (conn->ping_started) {
+ pthread_mutex_unlock (&conn->lock);
+ return;
+ }
+
+ if (conn->ping_timer) {
+ gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
+ conn->ping_timer = NULL;
+ rpc_clnt_unref (rpc);
+ }
+
+ if (conn->saved_frames)
+ /* treat the case where conn->saved_frames is NULL
+ as no pending frames */
+ frame_count = conn->saved_frames->count;
+
+ if ((frame_count == 0) || !conn->connected) {
+ gf_log (THIS->name, GF_LOG_DEBUG,
+ "returning as transport is already disconnected"
+ " OR there are no frames (%d || %d)",
+ frame_count, !conn->connected);
+
+ pthread_mutex_unlock (&conn->lock);
+ return;
+ }
+
+ if (frame_count < 0) {
+ gf_log (THIS->name, GF_LOG_WARNING,
+ "saved_frames->count is %"PRId64,
+ conn->saved_frames->count);
+ conn->saved_frames->count = 0;
+ }
+
+ timeout.tv_sec = conn->ping_timeout;
+ timeout.tv_nsec = 0;
+
+ rpc_clnt_ref (rpc);
+ conn->ping_timer =
+ gf_timer_call_after (rpc->ctx, timeout,
+ rpc_clnt_ping_timer_expired,
+ (void *) rpc);
+
+ if (conn->ping_timer == NULL) {
+ gf_log (THIS->name, GF_LOG_WARNING,
+ "unable to setup ping timer");
+ rpc_clnt_unref (rpc);
+ pthread_mutex_unlock (&conn->lock);
+ return;
+ } else {
+ conn->ping_started = 1;
+ }
+ }
+ pthread_mutex_unlock (&conn->lock);
+
+ rpc_clnt_ping(rpc);
+}
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.h b/rpc/rpc-lib/src/rpc-clnt-ping.h
new file mode 100644
index 00000000000..e7fbf3ced9d
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.h
@@ -0,0 +1,18 @@
+/*
+ Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+void
+rpc_clnt_start_ping (void *rpc_ptr);
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 3106342e970..46ee0eebcd0 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -17,6 +17,7 @@
#define RPC_CLNT_DEFAULT_REQUEST_COUNT 512
#include "rpc-clnt.h"
+#include "rpc-clnt-ping.h"
#include "byte-order.h"
#include "xdr-rpcclnt.h"
#include "rpc-transport.h"
@@ -552,6 +553,7 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
gf_timer_call_cancel (clnt->ctx, conn->ping_timer);
conn->ping_timer = NULL;
conn->ping_started = 0;
+ rpc_clnt_unref (clnt);
}
}
pthread_mutex_unlock (&conn->lock);
@@ -1000,6 +1002,17 @@ rpc_clnt_connection_init (struct rpc_clnt *clnt, glusterfs_ctx_t *ctx,
}
conn->rpc_clnt = clnt;
+ ret = dict_get_int32 (options, "ping-timeout",
+ &conn->ping_timeout);
+ if (ret >= 0) {
+ gf_log (name, GF_LOG_DEBUG,
+ "setting ping-timeout to %d", conn->ping_timeout);
+ } else {
+ gf_log (name, GF_LOG_INFO,
+ "defaulting ping-timeout to 30secs");
+ conn->ping_timeout = 30;
+ }
+
trans = rpc_transport_load (ctx, options, name);
if (!trans) {
gf_log (name, GF_LOG_WARNING, "loading of new rpc-transport"
@@ -1592,6 +1605,7 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
goto out;
}
+ rpc_clnt_start_ping (rpc);
ret = 0;
out:
@@ -1734,6 +1748,7 @@ rpc_clnt_disable (struct rpc_clnt *rpc)
gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
conn->ping_timer = NULL;
conn->ping_started = 0;
+ rpc_clnt_unref (rpc);
}
trans = conn->trans;
conn->trans = NULL;
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index bd4820605d8..f439d538761 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -145,6 +145,7 @@ struct rpc_clnt_connection {
struct timeval last_received;
int32_t ping_started;
char *name;
+ int32_t ping_timeout;
};
typedef struct rpc_clnt_connection rpc_clnt_connection_t;
@@ -186,7 +187,6 @@ typedef struct rpc_clnt {
char disabled;
} rpc_clnt_t;
-
struct rpc_clnt *rpc_clnt_new (dict_t *options, glusterfs_ctx_t *ctx,
char *name, uint32_t reqpool_size);
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index 1c13048f223..be9f9a861f0 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1844,6 +1844,32 @@ out:
}
static int
+rpcsvc_ping (rpcsvc_request_t *req)
+{
+ char rsp_buf[8 * 1024] = {0,};
+ gf_common_rsp rsp = {0,};
+ struct iovec iov = {0,};
+ int ret = -1;
+ uint32_t ping_rsp_len = 0;
+
+ ping_rsp_len = xdr_sizeof ((xdrproc_t) xdr_gf_common_rsp,
+ &rsp);
+
+ iov.iov_base = rsp_buf;
+ iov.iov_len = ping_rsp_len;
+
+ ret = xdr_serialize_generic (iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+ if (ret < 0) {
+ ret = RPCSVC_ACTOR_ERROR;
+ } else {
+ rsp.op_ret = 0;
+ rpcsvc_submit_generic (req, &iov, 1, NULL, 0, NULL);
+ }
+
+ return 0;
+}
+
+static int
rpcsvc_dump (rpcsvc_request_t *req)
{
char rsp_buf[8 * 1024] = {0,};
@@ -2585,6 +2611,7 @@ out:
rpcsvc_actor_t gluster_dump_actors[] = {
[GF_DUMP_NULL] = {"NULL", GF_DUMP_NULL, NULL, NULL, 0, DRC_NA},
[GF_DUMP_DUMP] = {"DUMP", GF_DUMP_DUMP, rpcsvc_dump, NULL, 0, DRC_NA},
+ [GF_DUMP_PING] = {"PING", GF_DUMP_PING, rpcsvc_ping, NULL, 0, DRC_NA},
[GF_DUMP_MAXVALUE] = {"MAXVALUE", GF_DUMP_MAXVALUE, NULL, NULL, 0, DRC_NA},
};
@@ -2594,5 +2621,5 @@ struct rpcsvc_program gluster_dump_prog = {
.prognum = GLUSTER_DUMP_PROGRAM,
.progver = GLUSTER_DUMP_VERSION,
.actors = gluster_dump_actors,
- .numactors = 2,
+ .numactors = sizeof (gluster_dump_actors) / sizeof (gluster_dump_actors[0]) - 1,
};
diff --git a/rpc/rpc-lib/src/xdr-common.h b/rpc/rpc-lib/src/xdr-common.h
index 631f4b284ad..39b8f240a1e 100644
--- a/rpc/rpc-lib/src/xdr-common.h
+++ b/rpc/rpc-lib/src/xdr-common.h
@@ -29,6 +29,7 @@
enum gf_dump_procnum {
GF_DUMP_NULL,
GF_DUMP_DUMP,
+ GF_DUMP_PING,
GF_DUMP_MAXVALUE,
};