From e134ef2493e8517a6f341416c11230c2bb5bcd6c Mon Sep 17 00:00:00 2001 From: Yaniv Kaul Date: Thu, 8 Nov 2018 07:50:48 +0200 Subject: rpc-clnt*: several code changes to reduce conn lock times Assorted code refactoring to reduce lock contention. Also, took the opportunity to reorder structs more properly. Removed dead code. Hopefully, no functional changes. Compile-tested only! updates: bz#1193929 Signed-off-by: Yaniv Kaul Change-Id: I5de6124ad071fd5e2c31832364d602b5f6d6fe28 --- rpc/rpc-lib/src/rpc-clnt-ping.c | 34 +++++++++++++++------------------- rpc/rpc-lib/src/rpc-clnt.c | 27 ++------------------------- rpc/rpc-lib/src/rpc-clnt.h | 19 ++++++++----------- 3 files changed, 25 insertions(+), 55 deletions(-) (limited to 'rpc') diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c index a98a83dd8c8..7c2026f9086 100644 --- a/rpc/rpc-lib/src/rpc-clnt-ping.c +++ b/rpc/rpc-lib/src/rpc-clnt-ping.c @@ -108,7 +108,6 @@ rpc_clnt_ping_timer_expired(void *rpc_ptr) rpc_transport_t *trans = NULL; rpc_clnt_connection_t *conn = NULL; int disconnect = 0; - int transport_activity = 0; struct timespec current = { 0, }; @@ -123,28 +122,22 @@ rpc_clnt_ping_timer_expired(void *rpc_ptr) goto out; } + clock_gettime(CLOCK_REALTIME, ¤t); pthread_mutex_lock(&conn->lock); { unref = rpc_clnt_remove_ping_timer_locked(rpc); - clock_gettime(CLOCK_REALTIME, ¤t); if (((current.tv_sec - conn->last_received.tv_sec) < conn->ping_timeout) || ((current.tv_sec - conn->last_sent.tv_sec) < conn->ping_timeout)) { - transport_activity = 1; - } - - if (transport_activity) { gf_log(trans->name, GF_LOG_TRACE, "ping timer expired but transport activity " "detected - not bailing transport"); - if (__rpc_clnt_rearm_ping_timer(rpc, rpc_clnt_ping_timer_expired) == -1) { gf_log(trans->name, GF_LOG_WARNING, "unable to setup ping timer"); } - } else { conn->ping_started = 0; disconnect = 1; @@ -198,14 +191,16 @@ rpc_clnt_ping_cbk(struct rpc_req *req, struct iovec *iov, int count, timespec_sub(&local->submit_time, &now, &delta); latency_msec = delta.tv_sec * 1000 + delta.tv_nsec / 1000000; + gf_log(THIS->name, GF_LOG_DEBUG, "Ping latency is %" PRIu64 "ms", + latency_msec); + call_notify = _gf_true; + pthread_mutex_lock(&conn->lock); { - gf_log(THIS->name, GF_LOG_DEBUG, "Ping latency is %" PRIu64 "ms", - latency_msec); - - call_notify = _gf_true; + unref = rpc_clnt_remove_ping_timer_locked(local->rpc); if (req->rpc_status == -1) { - unref = rpc_clnt_remove_ping_timer_locked(local->rpc); + conn->ping_started = 0; + pthread_mutex_unlock(&conn->lock); if (unref) { gf_log(this->name, GF_LOG_WARNING, "socket or ib related error"); @@ -214,19 +209,20 @@ rpc_clnt_ping_cbk(struct rpc_req *req, struct iovec *iov, int count, /* timer expired and transport bailed out */ gf_log(this->name, GF_LOG_WARNING, "socket disconnected"); } - conn->ping_started = 0; - goto unlock; + goto after_unlock; } - unref = rpc_clnt_remove_ping_timer_locked(local->rpc); if (__rpc_clnt_rearm_ping_timer(local->rpc, rpc_clnt_start_ping) == -1) { + /* unlock before logging error */ + pthread_mutex_unlock(&conn->lock); gf_log(this->name, GF_LOG_WARNING, "failed to set the ping timer"); + } else { + /* just unlock the mutex */ + pthread_mutex_unlock(&conn->lock); } } -unlock: - pthread_mutex_unlock(&conn->lock); - +after_unlock: if (call_notify) { ret = local->rpc->notifyfn(local->rpc, this, RPC_CLNT_PING, (void *)(uintptr_t)latency_msec); diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c index 9b8e79070ec..56bde4ed8ab 100644 --- a/rpc/rpc-lib/src/rpc-clnt.c +++ b/rpc/rpc-lib/src/rpc-clnt.c @@ -67,13 +67,13 @@ __saved_frames_put(struct saved_frames *frames, void *frame, } /* THIS should be saved and set back */ - memset(saved_frame, 0, sizeof(*saved_frame)); INIT_LIST_HEAD(&saved_frame->list); saved_frame->capital_this = THIS; saved_frame->frame = frame; saved_frame->rpcreq = rpcreq; gettimeofday(&saved_frame->saved_at, NULL); + memset(&saved_frame->rsp, 0, sizeof(rpc_transport_rsp_t)); if (_is_lock_fop(saved_frame)) list_add_tail(&saved_frame->list, &frames->lk_sf.list); @@ -780,8 +780,7 @@ is_rpc_clnt_disconnected(rpc_clnt_connection_t *conn) pthread_mutex_lock(&conn->lock); { - if (conn->disconnected == _gf_false) - disconnected = _gf_false; + disconnected = conn->disconnected; } pthread_mutex_unlock(&conn->lock); @@ -1849,28 +1848,6 @@ rpc_clnt_unref(struct rpc_clnt *rpc) return rpc; } -char -rpc_clnt_is_disabled(struct rpc_clnt *rpc) -{ - rpc_clnt_connection_t *conn = NULL; - char disabled = 0; - - if (!rpc) { - goto out; - } - - conn = &rpc->conn; - - pthread_mutex_lock(&conn->lock); - { - disabled = rpc->disabled; - } - pthread_mutex_unlock(&conn->lock); - -out: - return disabled; -} - void rpc_clnt_disable(struct rpc_clnt *rpc) { diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h index 28b6f0e7f4d..d122382e446 100644 --- a/rpc/rpc-lib/src/rpc-clnt.h +++ b/rpc/rpc-lib/src/rpc-clnt.h @@ -133,24 +133,23 @@ struct rpc_clnt_connection { gf_timer_t *timer; gf_timer_t *ping_timer; struct rpc_clnt *rpc_clnt; - char connected; - gf_boolean_t disconnected; struct saved_frames *saved_frames; - int32_t frame_timeout; struct timespec last_sent; struct timespec last_received; - int32_t ping_started; - char *name; - int32_t ping_timeout; uint64_t pingcnt; uint64_t msgcnt; uint64_t cleanup_gen; + char *name; + int32_t ping_started; + int32_t frame_timeout; + int32_t ping_timeout; + gf_boolean_t disconnected; + char connected; }; typedef struct rpc_clnt_connection rpc_clnt_connection_t; struct rpc_req { rpc_clnt_connection_t *conn; - uint32_t xid; struct iovec req[2]; int reqcnt; struct iobref *req_iobref; @@ -163,6 +162,7 @@ struct rpc_req { int procnum; fop_cbk_fn_t cbkfn; void *conn_private; + uint32_t xid; }; typedef struct rpc_clnt { @@ -183,8 +183,8 @@ typedef struct rpc_clnt { glusterfs_ctx_t *ctx; gf_atomic_t refcount; int auth_value; - char disabled; xlator_t *owner; + char disabled; } rpc_clnt_t; struct rpc_clnt * @@ -256,9 +256,6 @@ rpc_clnt_disable(struct rpc_clnt *rpc); void rpc_clnt_disconnect(struct rpc_clnt *rpc); -char -rpc_clnt_is_disabled(struct rpc_clnt *rpc); - int rpc_clnt_mgmt_pmap_signout(glusterfs_ctx_t *ctx, char *brick_name); -- cgit