libglusterfs: Implementation of sync lock as recursive lock to avoid crash.

Problem : In glusterd,we are using big lock which is implemented based on sync
task frame work for thread synchronization and rcu lock for data consistency.
sync task frame work swap the threads  if there is no worker poll threads
available,due to this rcu lock and rcu unlock was happening in different threads
(urcu-bp will not allow this),resulting into glusterd crash.

fix : To avoid releasing the sync lock(big lock) in between rcu critical
section,implemented sync lock as recursive lock.

More details:
link : http://www.spinics.net/lists/gluster-devel/msg14632.html

Change-Id: I2b56c1caf3f0470f219b1adcaf62cce29cdc6b88
BUG: 1216942
Signed-off-by: anand <anekkunt@redhat.com>
Reviewed-on: http://review.gluster.org/10285
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Tested-by: NetBSD Build System
Reviewed-by: Vijay Bellur <vbellur@redhat.com>
(cherry picked from commit ada6b3a8800867934af57a57d5312f5a5d8374f0)
Reviewed-on: http://review.gluster.org/10432
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Vijay Bellur <vbellur@redhat.com>

5 files changed, 5 insertions, 33 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
index e07d4100d00..539efb5900d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -378,14 +378,11 @@ gd_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
-
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_LOCK,
                                         gd_mgmt_v3_lock_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
-        synclock_lock (&conf->big_lock);
 out:
         GF_FREE (req.dict.dict_val);
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
@@ -653,14 +650,11 @@ gd_mgmt_v3_pre_validate_req (glusterd_op_t op, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
-
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_PRE_VALIDATE,
                                         gd_mgmt_v3_pre_validate_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_v3_pre_val_req);
-        synclock_lock (&conf->big_lock);
 out:
         GF_FREE (req.dict.dict_val);
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
@@ -911,14 +905,11 @@ gd_mgmt_v3_brick_op_req (glusterd_op_t op, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
-
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_BRICK_OP,
                                         gd_mgmt_v3_brick_op_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_v3_brick_op_req);
-        synclock_lock (&conf->big_lock);
 out:
         GF_FREE (req.dict.dict_val);
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
@@ -1154,14 +1145,11 @@ gd_mgmt_v3_commit_req (glusterd_op_t op, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
-
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_COMMIT,
                                         gd_mgmt_v3_commit_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_v3_commit_req);
-        synclock_lock (&conf->big_lock);
 out:
         GF_FREE (req.dict.dict_val);
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
@@ -1376,14 +1364,11 @@ gd_mgmt_v3_post_validate_req (glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
-
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_POST_VALIDATE,
                                         gd_mgmt_v3_post_validate_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_v3_post_val_req);
-        synclock_lock (&conf->big_lock);
 out:
         GF_FREE (req.dict.dict_val);
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
@@ -1588,14 +1573,11 @@ gd_mgmt_v3_unlock (glusterd_op_t op, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
-
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_UNLOCK,
                                         gd_mgmt_v3_unlock_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
-        synclock_lock (&conf->big_lock);
 out:
         GF_FREE (req.dict.dict_val);
         gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 20aab8dfe27..4c6521f79ea 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -6910,6 +6910,6 @@ int
 glusterd_op_sm_init ()
 {
         CDS_INIT_LIST_HEAD (&gd_op_sm_queue);
-        synclock_init (&gd_op_sm_lock);
+        synclock_init (&gd_op_sm_lock, SYNC_LOCK_DEFAULT);
         return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 22d2d78144d..deb38a7dea9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -406,14 +406,12 @@ gd_syncop_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_LOCK,
                                         gd_syncop_mgmt_v3_lock_cbk,
                                         (xdrproc_t)
                                         xdr_gd1_mgmt_v3_lock_req);
-        synclock_lock (&conf->big_lock);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -503,14 +501,12 @@ gd_syncop_mgmt_v3_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_v3_prog,
                                         GLUSTERD_MGMT_V3_UNLOCK,
                                         gd_syncop_mgmt_v3_unlock_cbk,
                                         (xdrproc_t)
                                         xdr_gd1_mgmt_v3_unlock_req);
-        synclock_lock (&conf->big_lock);
 out:
         gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
@@ -598,13 +594,11 @@ gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
         gf_uuid_copy (req.uuid, my_uuid);
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_prog,
                                         GLUSTERD_MGMT_CLUSTER_LOCK,
                                         gd_syncop_mgmt_lock_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
-        synclock_lock (&conf->big_lock);
         return ret;
 }
 
@@ -689,13 +683,11 @@ gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
         gf_uuid_copy (req.uuid, my_uuid);
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, &peerid,
                                         &gd_mgmt_prog,
                                         GLUSTERD_MGMT_CLUSTER_UNLOCK,
                                         gd_syncop_mgmt_unlock_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
-        synclock_lock (&conf->big_lock);
         return ret;
 }
 
@@ -826,12 +818,10 @@ gd_syncop_mgmt_stage_op (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (peerinfo->rpc, req, args, &peerid,
                                         &gd_mgmt_prog, GLUSTERD_MGMT_STAGE_OP,
                                         gd_syncop_stage_op_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
-        synclock_lock (&conf->big_lock);
 out:
         gd_stage_op_req_free (req);
         return ret;
@@ -1108,12 +1098,10 @@ gd_syncop_mgmt_commit_op (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
 
         gf_uuid_copy (peerid, peerinfo->uuid);
 
-        synclock_unlock (&conf->big_lock);
         ret = gd_syncop_submit_request (peerinfo->rpc, req, args, &peerid,
                                         &gd_mgmt_prog, GLUSTERD_MGMT_COMMIT_OP,
                                         gd_syncop_commit_op_cbk,
                                         (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
-        synclock_lock (&conf->big_lock);
 out:
         gd_commit_op_req_free (req);
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 9d2ce35a193..e4cea90545e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -2730,6 +2730,7 @@ glusterd_spawn_daemons (void *opaque)
         gf_boolean_t    start_bricks = !conf->restart_done;
         int             ret             = -1;
 
+        synclock_lock (&conf->big_lock);
         if (start_bricks) {
                 glusterd_restart_bricks (conf);
                 conf->restart_done = _gf_true;
@@ -9085,7 +9086,8 @@ glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
         this = THIS;
         priv = this->private;
 
-        synclock_lock (&priv->big_lock);
+        /* synclock_lock must be called from within synctask, @fn must call it before
+         * it starts with its work*/
         ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
                             opaque);
         if (ret)
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 361cc007327..3735c62abc1 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1554,7 +1554,7 @@ init (xlator_t *this)
         conf->gfs_mgmt = &gd_brick_prog;
         strncpy (conf->workdir, workdir, PATH_MAX);
 
-        synclock_init (&conf->big_lock);
+        synclock_init (&conf->big_lock, SYNC_LOCK_RECURSIVE);
         pthread_mutex_init (&conf->xprt_lock, NULL);
         INIT_LIST_HEAD (&conf->xprt_list);