rpc-clnt: separate out connection_cleanup() from destroy()v3.3.0qa30

noticed that there are possibilities where one would like to do a
connection_cleanup() before destroying a RPC connection itself, also
current code is such that, rpc_clnt_connection_cleanup() does
rpc_clnt_ref() and unref(), creating a race window/double unref
possibilities in the code.

by separating out the functions, this race window/double fault can be
prevented.

Change-Id: I7ebd3392efa891232857b6db9108b0b19e40fc12
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 802403
Reviewed-on: http://review.gluster.com/2979
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

6 files changed, 33 insertions, 6 deletions

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 68c51162101..053a7c2461a 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -815,8 +815,11 @@ cleanup_and_exit (int signum)
         exit (0);
 #if 0
         /* TODO: Properly do cleanup_and_exit(), with synchronization */
-        if (ctx->mgmt)
+        if (ctx->mgmt) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&ctx->mgmt->conn);
                 rpc_clnt_unref (ctx->mgmt);
+        }
 
         /* call fini() of each xlator */
         trav = NULL;
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 5867c322bd0..428a82b76aa 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1573,7 +1573,6 @@ rpc_clnt_destroy (struct rpc_clnt *rpc)
                 rpc_transport_unref (rpc->conn.trans);
         }
 
-        rpc_clnt_connection_cleanup (&rpc->conn);
         rpc_clnt_reconnect_cleanup (&rpc->conn);
         saved_frames_destroy (rpc->conn.saved_frames);
         pthread_mutex_destroy (&rpc->lock);
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index 17e7e127141..bce40bb1dd4 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -228,6 +228,8 @@ rpc_clnt_ref (struct rpc_clnt *rpc);
 struct rpc_clnt *
 rpc_clnt_unref (struct rpc_clnt *rpc);
 
+int rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn);
+
 void rpc_clnt_set_connected (rpc_clnt_connection_t *conn);
 
 void rpc_clnt_unset_connected (rpc_clnt_connection_t *conn);
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 4b84039dfe6..6bfcd6c0025 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -888,6 +888,9 @@ glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
         glusterd_peerctx_t      *peerctx = NULL;
 
         if (peerinfo->rpc) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&peerinfo->rpc->conn);
+
                 peerctx = peerinfo->rpc->mydata;
                 peerinfo->rpc->mydata = NULL;
                 peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
@@ -1299,6 +1302,9 @@ glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
         GF_ASSERT (brickinfo);
 
         if (brickinfo->rpc) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&brickinfo->rpc->conn);
+
                 rpc_clnt_unref (brickinfo->rpc);
                 brickinfo->rpc = NULL;
         }
diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c
index 0a0d2636649..16507d069d3 100644
--- a/xlators/nfs/server/src/nlm4.c
+++ b/xlators/nfs/server/src/nlm4.c
@@ -315,8 +315,14 @@ nlm_set_rpc_clnt (rpc_clnt_t *rpc_clnt, char *caller_name)
         ret = 0;
 ret:
         UNLOCK (&nlm_client_list_lk);
-        if (rpc_clnt_old)
+
+        if (rpc_clnt_old) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&rpc_clnt_old->conn);
+
                 rpc_clnt_unref (rpc_clnt_old);
+        }
+
         if (old_name)
                 GF_FREE (old_name);
         return ret;
@@ -340,8 +346,12 @@ nlm_unset_rpc_clnt (rpc_clnt_t *rpc)
         if (rpc_clnt == NULL) {
                 return -1;
         }
-        if (rpc_clnt)
+        if (rpc_clnt) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&rpc_clnt->conn);
+
                 rpc_clnt_unref (rpc_clnt);
+        }
         return 0;
 }
 
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 078e6e5d044..93394ebfbfb 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2199,6 +2199,9 @@ client_destroy_rpc (xlator_t *this)
                 goto out;
 
         if (conf->rpc) {
+                /* cleanup the saved-frames before last unref */
+                rpc_clnt_connection_cleanup (&conf->rpc->conn);
+
                 conf->rpc = rpc_clnt_unref (conf->rpc);
                 ret = 0;
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -2434,8 +2437,12 @@ fini (xlator_t *this)
         this->private = NULL;
 
         if (conf) {
-                if (conf->rpc)
-                       rpc_clnt_unref (conf->rpc);
+                if (conf->rpc) {
+                        /* cleanup the saved-frames before last unref */
+                        rpc_clnt_connection_cleanup (&conf->rpc->conn);
+
+                        rpc_clnt_unref (conf->rpc);
+                }
 
                 /* Saved Fds */
                 /* TODO: */