rpc: fix possible deadlock left behind in d448fd1

See http://review.gluster.org/9613 for more details.

Change-Id: I05ac0267b8c6f4e9b354acbbdf5469835455fb10
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/10821
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>

3 files changed, 21 insertions, 23 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.c b/rpc/rpc-lib/src/rpc-clnt-ping.c
index 0429bd31120..64d319d0057 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.c
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.c
@@ -70,8 +70,8 @@ __rpc_clnt_rearm_ping_timer (struct rpc_clnt *rpc, gf_timer_cbk_t cbk)
 }
 
 /* Must be called under conn->lock */
-static int
-__rpc_clnt_remove_ping_timer (struct rpc_clnt *rpc)
+int
+rpc_clnt_remove_ping_timer_locked (struct rpc_clnt *rpc)
 {
         rpc_clnt_connection_t *conn  = &rpc->conn;
         gf_timer_t            *timer = NULL;
@@ -117,7 +117,7 @@ rpc_clnt_ping_timer_expired (void *rpc_ptr)
 
         pthread_mutex_lock (&conn->lock);
         {
-                unref = __rpc_clnt_remove_ping_timer (rpc);
+                unref = rpc_clnt_remove_ping_timer_locked (rpc);
 
                 gettimeofday (&current, NULL);
                 if (((current.tv_sec - conn->last_received.tv_sec) <
@@ -188,7 +188,7 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
         pthread_mutex_lock (&conn->lock);
         {
                 if (req->rpc_status == -1) {
-                        unref = __rpc_clnt_remove_ping_timer (rpc);
+                        unref = rpc_clnt_remove_ping_timer_locked (rpc);
                         if (unref) {
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "socket or ib related error");
@@ -203,7 +203,7 @@ rpc_clnt_ping_cbk (struct rpc_req *req, struct iovec *iov, int count,
                         goto unlock;
                 }
 
-                unref = __rpc_clnt_remove_ping_timer (rpc);
+                unref = rpc_clnt_remove_ping_timer_locked (rpc);
                 if (__rpc_clnt_rearm_ping_timer (rpc,
                                                  rpc_clnt_start_ping) == -1) {
                         gf_log (this->name, GF_LOG_WARNING,
@@ -275,7 +275,7 @@ rpc_clnt_start_ping (void *rpc_ptr)
 
         pthread_mutex_lock (&conn->lock);
         {
-                unref = __rpc_clnt_remove_ping_timer (rpc);
+                unref = rpc_clnt_remove_ping_timer_locked (rpc);
 
                 if (conn->saved_frames) {
                         GF_ASSERT (conn->saved_frames->count >= 0);
diff --git a/rpc/rpc-lib/src/rpc-clnt-ping.h b/rpc/rpc-lib/src/rpc-clnt-ping.h
index bf97c0690d2..d92e5054190 100644
--- a/rpc/rpc-lib/src/rpc-clnt-ping.h
+++ b/rpc/rpc-lib/src/rpc-clnt-ping.h
@@ -12,3 +12,5 @@
 #define RPC_DEFAULT_PING_TIMEOUT 30
 void
 rpc_clnt_check_and_start_ping (struct rpc_clnt *rpc_ptr);
+int
+rpc_clnt_remove_ping_timer_locked (struct rpc_clnt *rpc);
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 567ff35b86a..48bd35ce084 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -495,6 +495,7 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
 {
         struct saved_frames    *saved_frames = NULL;
         struct rpc_clnt         *clnt  = NULL;
+        int                     unref = 0;
 
         if (!conn) {
                 goto out;
@@ -516,12 +517,7 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
 
                 conn->connected = 0;
 
-                if (conn->ping_timer) {
-                        gf_timer_call_cancel (clnt->ctx, conn->ping_timer);
-                        conn->ping_timer = NULL;
-                        conn->ping_started = 0;
-                        rpc_clnt_unref (clnt);
-                }
+                unref = rpc_clnt_remove_ping_timer_locked (clnt);
                 /*reset rpc msgs stats*/
                 conn->pingcnt = 0;
                 conn->msgcnt = 0;
@@ -529,6 +525,8 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
         pthread_mutex_unlock (&conn->lock);
 
         saved_frames_destroy (saved_frames);
+        if (unref)
+                rpc_clnt_unref (clnt);
 
 out:
         return 0;
@@ -1709,6 +1707,7 @@ rpc_clnt_disable (struct rpc_clnt *rpc)
 {
         rpc_clnt_connection_t *conn = NULL;
         rpc_transport_t       *trans = NULL;
+        int                    unref = 0;
 
         if (!rpc) {
                 goto out;
@@ -1731,12 +1730,7 @@ rpc_clnt_disable (struct rpc_clnt *rpc)
                 }
                 conn->connected = 0;
 
-                if (conn->ping_timer) {
-                        gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
-                        conn->ping_timer = NULL;
-                        conn->ping_started = 0;
-                        rpc_clnt_unref (rpc);
-                }
+                unref = rpc_clnt_remove_ping_timer_locked (rpc);
                 trans = conn->trans;
                 conn->trans = NULL;
 
@@ -1747,6 +1741,9 @@ rpc_clnt_disable (struct rpc_clnt *rpc)
                 rpc_transport_disconnect (trans);
         }
 
+        if (unref)
+                rpc_clnt_unref (rpc);
+
 out:
         return;
 }
@@ -1756,6 +1753,7 @@ rpc_clnt_disconnect (struct rpc_clnt *rpc)
 {
         rpc_clnt_connection_t *conn  = NULL;
         rpc_transport_t       *trans = NULL;
+        int                    unref = 0;
 
         if (!rpc)
                 goto out;
@@ -1776,11 +1774,7 @@ rpc_clnt_disconnect (struct rpc_clnt *rpc)
                 }
                 conn->connected = 0;
 
-                if (conn->ping_timer) {
-                        gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
-                        conn->ping_timer = NULL;
-                        conn->ping_started = 0;
-                }
+                unref = rpc_clnt_remove_ping_timer_locked (rpc);
                 trans = conn->trans;
         }
         pthread_mutex_unlock (&conn->lock);
@@ -1788,6 +1782,8 @@ rpc_clnt_disconnect (struct rpc_clnt *rpc)
         if (trans) {
                 rpc_transport_disconnect (trans);
         }
+        if (unref)
+                rpc_clnt_unref (rpc);
 
 out:
         return;