glusterd: fix double-free of rebalance process' rpc object

Change-Id: I0c79c4de47a160b1ecf3a8994eedc02e3f5002a9
BUG: 1225318
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/10872
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.org/10932
Tested-by: NetBSD Build System
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 74 insertions, 9 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 28d8b9a77ce..bc51a96f1ad 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -196,7 +196,6 @@ function is_bad_test ()
     local name=$1
     for bt in ./tests/basic/uss.t                       \
               ./tests/basic/tier/tier.t \
-              ./tests/bugs/glusterd/bug-974007.t        \
               ./tests/basic/quota-anon-fd-nfs.t \
 	      ; do
         [ x"$name" = x"$bt" ] && return 0 # bash: zero means true/success
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 9111c07b8fc..f126e8f0b1f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -149,10 +149,7 @@ __glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
 
                 glusterd_store_perform_node_state_store (volinfo);
 
-                if (defrag->rpc) {
-                        glusterd_rpc_clnt_unref (priv, defrag->rpc);
-                        defrag->rpc = NULL;
-                }
+                glusterd_defrag_rpc_put (defrag);
                 if (defrag->cbk_fn)
                         defrag->cbk_fn (volinfo,
                                         volinfo->rebal.defrag_status);
@@ -339,8 +336,9 @@ glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
                 goto out;
 
         //rpc obj for rebalance process already in place.
-        if (defrag->rpc) {
+        if (glusterd_defrag_rpc_get (defrag)) {
                 ret = 0;
+                glusterd_defrag_rpc_put (defrag);
                 goto out;
         }
         GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 59b0e5f072e..231c2f720ba 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -2173,6 +2173,8 @@ glusterd_brick_op (call_frame_t *frame, xlator_t *this,
                 }
                 if (!ret)
                         pending_bricks++;
+
+                glusterd_pending_node_put_rpc (pending_node);
         }
 
         gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for operation "
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 43539643c0b..6f36478aeca 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -1658,10 +1658,15 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
                         goto out;
 
                 brick_count++;
+                glusterd_pending_node_put_rpc (pending_node);
         }
 
+        pending_node = NULL;
         ret = 0;
 out:
+        if (pending_node)
+                glusterd_pending_node_put_rpc (pending_node);
+
         if (rsp_dict)
                 dict_unref (rsp_dict);
         gf_log (this->name, GF_LOG_DEBUG, "Sent op req to %d bricks",
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index deb0123c435..1c2299db016 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3677,8 +3677,9 @@ gd_check_and_update_rebalance_info (glusterd_volinfo_t *old_volinfo,
         new = &(new_volinfo->rebal);
 
         //Disconnect from rebalance process
-        if (old->defrag && old->defrag->rpc) {
+        if (glusterd_defrag_rpc_get (old->defrag)) {
                 rpc_transport_disconnect (old->defrag->rpc->conn.trans);
+                glusterd_defrag_rpc_put (old->defrag);
         }
 
         if (!gf_uuid_is_null (old->rebalance_id) &&
@@ -3960,6 +3961,39 @@ out:
 }
 
 struct rpc_clnt*
+glusterd_defrag_rpc_get (glusterd_defrag_info_t *defrag)
+{
+        struct rpc_clnt *rpc = NULL;
+
+        if (!defrag)
+                return NULL;
+
+        LOCK (&defrag->lock);
+        {
+                rpc = rpc_clnt_ref (defrag->rpc);
+        }
+        UNLOCK (&defrag->lock);
+        return rpc;
+}
+
+struct rpc_clnt*
+glusterd_defrag_rpc_put (glusterd_defrag_info_t *defrag)
+{
+        struct rpc_clnt *rpc = NULL;
+
+        if (!defrag)
+                return NULL;
+
+        LOCK (&defrag->lock);
+        {
+                rpc = rpc_clnt_unref (defrag->rpc);
+                defrag->rpc = rpc;
+        }
+        UNLOCK (&defrag->lock);
+        return rpc;
+}
+
+struct rpc_clnt*
 glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
 {
         struct rpc_clnt *rpc = NULL;
@@ -3981,8 +4015,8 @@ glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
                 rpc = svc->conn.rpc;
         } else if (pending_node->type == GD_NODE_REBALANCE) {
                 volinfo = pending_node->node;
-                if (volinfo->rebal.defrag)
-                        rpc = volinfo->rebal.defrag->rpc;
+                rpc = glusterd_defrag_rpc_get (volinfo->rebal.defrag);
+
         } else if (pending_node->type == GD_NODE_SNAPD) {
                 volinfo = pending_node->node;
                 rpc = volinfo->snapd.svc.conn.rpc;
@@ -3994,6 +4028,23 @@ out:
         return rpc;
 }
 
+void
+glusterd_pending_node_put_rpc (glusterd_pending_node_t *pending_node)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        switch (pending_node->type) {
+        case GD_NODE_REBALANCE:
+                volinfo = pending_node->node;
+                glusterd_defrag_rpc_put (volinfo->rebal.defrag);
+                break;
+
+        default:
+                break;
+        }
+
+}
+
 int32_t
 glusterd_unlink_file (char *sockfpath)
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 8975e53f91b..246d3ed8689 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -180,6 +180,9 @@ glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len);
 struct rpc_clnt*
 glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node);
 
+void
+glusterd_pending_node_put_rpc (glusterd_pending_node_t *pending_node);
+
 int
 glusterd_remote_hostname_get (rpcsvc_request_t *req,
                               char *remote_host, int len);
@@ -650,6 +653,13 @@ void
 glusterd_list_add_order (struct cds_list_head *new, struct cds_list_head *head,
                         int (*compare)(struct cds_list_head *,
                                        struct cds_list_head *));
+
+struct rpc_clnt*
+glusterd_defrag_rpc_get (glusterd_defrag_info_t *defrag);
+
+struct rpc_clnt*
+glusterd_defrag_rpc_put (glusterd_defrag_info_t *defrag);
+
 int
 glusterd_disallow_op_for_tier (glusterd_volinfo_t *volinfo, glusterd_op_t op,
                                int cmd);