summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMohit Agrawal <moagrawa@redhat.com>2017-08-14 10:15:45 +0530
committerJeff Darcy <jeff@pl.atyp.us>2017-11-28 13:02:06 +0000
commit04fb94c99160ae1c6dea02624fefd47eb48da810 (patch)
tree43dcdc56e7834734523a899a810a3544ef5f4dc9
parent0805771ad3e676224760a8bbfe7a265d0e132a7e (diff)
rpc: Eliminate conn->lock contention by using more granular locks
rpc_clnt_submit() acquires conn->lock before call to rpc_transport_submit_request() and subsequent queuing of frame into saved_frames list. However, as part of handling RPC_TRANSPORT_MSG_RECEIVED and RPC_TRANSPORT_MSG_SENT notifications in rpc_clnt_notify(), conn->lock is again used to atomically update conn->last_received and conn->last_sent event timestamps. So when conn->lock is acquired as part of submitting a request, a parallel POLLIN notification gets blocked at rpc layer until the request submission completes and the lock is released. To get around this, this patch call clock_gettime (instead to call gettimeofday) to update event timestamps in conn->last_received and conn->last_sent and to call clock_gettime don't need to call mutex_lock because it (clock_gettime) is thread safe call. Note: Run fio on vm after apply the patch, iops is improved after apply the patch. Change-Id: I347b5031d61c426b276bc5e07136a7172645d763 BUG: 1467614 Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
-rw-r--r--rpc/rpc-lib/src/rpc-clnt-ping.c4
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.c12
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.h4
3 files changed, 6 insertions, 14 deletions
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
index 2f11341b0d5..40077e7aefd 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.c
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -112,7 +112,7 @@ rpc_clnt_ping_timer_expired (void *rpc_ptr)
rpc_clnt_connection_t *conn = NULL;
int disconnect = 0;
int transport_activity = 0;
- struct timeval current = {0, };
+ struct timespec current = {0, };
int unref = 0;
rpc = (struct rpc_clnt*) rpc_ptr;
@@ -129,7 +129,7 @@ rpc_clnt_ping_timer_expired (void *rpc_ptr)
{
unref = rpc_clnt_remove_ping_timer_locked (rpc);
- gettimeofday (&current, NULL);
+ clock_gettime (CLOCK_REALTIME, &current);
if (((current.tv_sec - conn->last_received.tv_sec) <
conn->ping_timeout)
|| ((current.tv_sec - conn->last_sent.tv_sec) <
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index adb8e3d4a60..cb59408dcf6 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -965,11 +965,7 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
case RPC_TRANSPORT_MSG_RECEIVED:
{
- pthread_mutex_lock (&conn->lock);
- {
- gettimeofday (&conn->last_received, NULL);
- }
- pthread_mutex_unlock (&conn->lock);
+ clock_gettime (CLOCK_REALTIME, &conn->last_received);
pollin = data;
if (pollin->is_reply)
@@ -984,11 +980,7 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
case RPC_TRANSPORT_MSG_SENT:
{
- pthread_mutex_lock (&conn->lock);
- {
- gettimeofday (&conn->last_sent, NULL);
- }
- pthread_mutex_unlock (&conn->lock);
+ clock_gettime (CLOCK_REALTIME, &conn->last_sent);
ret = 0;
break;
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 952ecbce1a1..428ffb80650 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -143,8 +143,8 @@ struct rpc_clnt_connection {
gf_boolean_t disconnected;
struct saved_frames *saved_frames;
int32_t frame_timeout;
- struct timeval last_sent;
- struct timeval last_received;
+ struct timespec last_sent;
+ struct timespec last_received;
int32_t ping_started;
char *name;
int32_t ping_timeout;