summaryrefslogtreecommitdiffstats
path: root/xlators/mgmt/glusterd/src
diff options
context:
space:
mode:
authorKrishnan Parthasarathi <kparthas@redhat.com>2013-04-02 07:56:25 +0530
committerAnand Avati <avati@redhat.com>2013-04-12 13:47:46 -0700
commitf34343d3751cd73e8eabe6d5544fb1f58b316595 (patch)
tree869aa908771b3708f4ad9a7a7ec57a623239b9a5 /xlators/mgmt/glusterd/src
parent732cd267c924554a638519cff0df146b2688d6e8 (diff)
glusterd: big lock - a coarse-grained locking to prevent races
There are primarily three lists that are part of glusterd process, that are concurrently accessed. Namely, priv->volumes, priv->peers and volinfo->bricks_list. Big-lock approach ----------------- WHAT IS IT? Big lock is a coarse-grained lock which protects all three lists, mentioned above, from racy access. HOW DOES IT WORK? At any given point in time, glusterd's thread(s) are in execution _iff_ there is a preceding, inbound network event. Of course, the sigwaiter thread and timer thread are exceptions. A network event is an external trigger to glusterd, via the epoll thread, in the form of POLLIN and POLLERR. As long as we take the big-lock at all such entry points and yield it when we are done, we are guaranteed that all the network events, accessing the global lists, are serialised. This amounts to holding the big lock at - all the handlers of all the actors in glusterd. (POLLIN) - all the cbks in glusterd. (POLLIN) - rpc_notify (DISCONNECT event), if we access/modify one of the three lists. (POLLERR) In the case of synctask'ized volume operations, we must remember that, if we held the big lock for the entire duration of the handler, we may block other non-synctask rpc actors from executing. For eg, volume-start would block in PMAP SIGNIN, if done incorrectly. To prevent this, we need to yield the big lock, when we yield the synctask, and reacquire on waking up of the synctask. Change-Id: Ib929f9905b55fb6c3fc27fefb497a26dba058e4f BUG: 948686 Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com> Reviewed-on: http://review.gluster.org/4784 Reviewed-by: Jeff Darcy <jdarcy@redhat.com> Tested-by: Gluster Build System <jenkins@build.gluster.com>
Diffstat (limited to 'xlators/mgmt/glusterd/src')
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-brick-ops.c15
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-geo-rep.c10
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-handler.c281
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-handshake.c63
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-pmap.c43
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-quota.c10
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-rebalance.c25
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-replace-brick.c11
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-rpc-ops.c106
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-sm.c19
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-syncop.c24
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c40
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.h2
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-ops.c57
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.c45
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.h11
16 files changed, 656 insertions, 106 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index d21facb0ad0..92e1b2622df 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -335,7 +335,7 @@ out:
/* Handler functions */
int
-glusterd_handle_add_brick (rpcsvc_request_t *req)
+__glusterd_handle_add_brick (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -540,9 +540,14 @@ out:
return ret;
}
+int
+glusterd_handle_add_brick (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+}
int
-glusterd_handle_remove_brick (rpcsvc_request_t *req)
+__glusterd_handle_remove_brick (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -848,6 +853,12 @@ out:
return ret;
}
+int
+glusterd_handle_remove_brick (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_remove_brick);
+}
/* op-sm */
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index 8426cdaaf0e..7e2ab833f9e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -37,7 +37,7 @@ static char *gsync_reserved_opts[] = {
};
int
-glusterd_handle_gsync_set (rpcsvc_request_t *req)
+__glusterd_handle_gsync_set (rpcsvc_request_t *req)
{
int32_t ret = 0;
dict_t *dict = NULL;
@@ -155,6 +155,12 @@ out:
}
+int
+glusterd_handle_gsync_set (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_gsync_set);
+}
+
/*****
*
* glusterd_urltransform* internal API
@@ -1343,7 +1349,9 @@ glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
runner_add_arg (&runner, op_name);
if (op_value)
runner_add_arg (&runner, op_value);
+ synclock_unlock (&priv->big_lock);
ret = runner_run (&runner);
+ synclock_lock (&priv->big_lock);
if (ret) {
gf_log ("", GF_LOG_WARNING, "gsyncd failed to "
"%s %s option for %s %s peers",
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 328971c2d5c..09cab9d1ea7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -54,6 +54,30 @@
#include <lvm2app.h>
#endif
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event,
+ void *data, rpc_clnt_notify_t notify_fn)
+{
+ glusterd_conf_t *priv = THIS->private;
+ int ret = -1;
+ synclock_lock (&priv->big_lock);
+ ret = notify_fn (rpc, mydata, event, data);
+ synclock_unlock (&priv->big_lock);
+ return ret;
+}
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn)
+{
+ glusterd_conf_t *priv = THIS->private;
+ int ret = -1;
+
+ synclock_lock (&priv->big_lock);
+ ret = actor_fn (req);
+ synclock_unlock (&priv->big_lock);
+
+ return ret;
+}
+
static int
glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t uuid,
char *hostname, int port,
@@ -500,7 +524,7 @@ out:
}
int
-glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_lock (rpcsvc_request_t *req)
{
gd1_mgmt_cluster_lock_req lock_req = {{0},};
int32_t ret = -1;
@@ -554,6 +578,13 @@ out:
}
int
+glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cluster_lock);
+}
+
+int
glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
glusterd_op_t op, uuid_t uuid,
char *buf_val, size_t buf_len,
@@ -604,7 +635,7 @@ out:
}
int
-glusterd_handle_stage_op (rpcsvc_request_t *req)
+__glusterd_handle_stage_op (rpcsvc_request_t *req)
{
int32_t ret = -1;
glusterd_req_ctx_t *req_ctx = NULL;
@@ -649,7 +680,14 @@ glusterd_handle_stage_op (rpcsvc_request_t *req)
}
int
-glusterd_handle_commit_op (rpcsvc_request_t *req)
+glusterd_handle_stage_op (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_stage_op);
+}
+
+
+int
+__glusterd_handle_commit_op (rpcsvc_request_t *req)
{
int32_t ret = -1;
glusterd_req_ctx_t *req_ctx = NULL;
@@ -700,7 +738,13 @@ out:
}
int
-glusterd_handle_cli_probe (rpcsvc_request_t *req)
+glusterd_handle_commit_op (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_commit_op);
+}
+
+int
+__glusterd_handle_cli_probe (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf1_cli_probe_req cli_req = {0,};
@@ -788,7 +832,13 @@ out:
}
int
-glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+glusterd_handle_cli_probe (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_cli_probe);
+}
+
+int
+__glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf1_cli_deprobe_req cli_req = {0,};
@@ -877,7 +927,13 @@ out:
}
int
-glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_cli_deprobe);
+}
+
+int
+__glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf1_cli_peer_list_req cli_req = {0,};
@@ -925,7 +981,14 @@ out:
}
int
-glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_list_friends);
+}
+
+int
+__glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -978,9 +1041,16 @@ out:
return ret;
}
+int
+glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_get_volume);
+}
+
#ifdef HAVE_BD_XLATOR
int
-glusterd_handle_cli_bd_op (rpcsvc_request_t *req)
+__glusterd_handle_cli_bd_op (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = { {0,} };
@@ -1044,10 +1114,16 @@ out:
return ret;
}
+
+int
+glusterd_handle_cli_bd_op (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_cli_bd_op);
+}
#endif
int
-glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+__glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
{
int ret = -1;
dict_t *dict = NULL;
@@ -1144,7 +1220,14 @@ out:
}
int
-glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_uuid_reset);
+}
+
+int
+__glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
{
int ret = -1;
dict_t *dict = NULL;
@@ -1203,6 +1286,13 @@ out:
return ret;
}
+int
+glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_list_volume);
+}
+
int32_t
glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
char *err_str, size_t err_len)
@@ -1215,7 +1305,7 @@ glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
}
int
-glusterd_handle_reset_volume (rpcsvc_request_t *req)
+__glusterd_handle_reset_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -1280,9 +1370,15 @@ out:
return ret;
}
+int
+glusterd_handle_reset_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_reset_volume);
+}
int
-glusterd_handle_set_volume (rpcsvc_request_t *req)
+__glusterd_handle_set_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -1382,7 +1478,13 @@ out:
}
int
-glusterd_handle_sync_volume (rpcsvc_request_t *req)
+glusterd_handle_set_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_set_volume);
+}
+
+int
+__glusterd_handle_sync_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -1471,6 +1573,12 @@ out:
}
int
+glusterd_handle_sync_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_sync_volume);
+}
+
+int
glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
char *op_errstr, dict_t *dict)
{
@@ -1497,7 +1605,7 @@ glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
}
int
-glusterd_handle_fsm_log (rpcsvc_request_t *req)
+__glusterd_handle_fsm_log (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf1_cli_fsm_log_req cli_req = {0,};
@@ -1554,6 +1662,12 @@ out:
}
int
+glusterd_handle_fsm_log (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_fsm_log);
+}
+
+int
glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
{
@@ -1592,7 +1706,7 @@ glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
}
int
-glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
{
gd1_mgmt_cluster_unlock_req unlock_req = {{0}, };
int32_t ret = -1;
@@ -1644,6 +1758,13 @@ out:
}
int
+glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cluster_unlock);
+}
+
+int
glusterd_op_stage_send_resp (rpcsvc_request_t *req,
int32_t op, int32_t status,
char *op_errstr, dict_t *rsp_dict)
@@ -1724,7 +1845,7 @@ out:
}
int
-glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
{
int32_t ret = -1;
gd1_mgmt_friend_req friend_req = {{0},};
@@ -1763,7 +1884,14 @@ out:
}
int
-glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_incoming_friend_req);
+}
+
+int
+__glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
{
int32_t ret = -1;
gd1_mgmt_friend_req friend_req = {{0},};
@@ -1801,6 +1929,14 @@ out:
}
int
+glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_incoming_unfriend_req);
+
+}
+
+int
glusterd_handle_friend_update_delete (dict_t *dict)
{
char *hostname = NULL;
@@ -1846,7 +1982,7 @@ out:
}
int
-glusterd_handle_friend_update (rpcsvc_request_t *req)
+__glusterd_handle_friend_update (rpcsvc_request_t *req)
{
int32_t ret = -1;
gd1_mgmt_friend_update friend_req = {{0},};
@@ -1983,7 +2119,14 @@ out:
}
int
-glusterd_handle_probe_query (rpcsvc_request_t *req)
+glusterd_handle_friend_update (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_friend_update);
+}
+
+int
+__glusterd_handle_probe_query (rpcsvc_request_t *req)
{
int32_t ret = -1;
xlator_t *this = NULL;
@@ -2077,8 +2220,13 @@ out:
return ret;
}
+int glusterd_handle_probe_query (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_probe_query);
+}
+
int
-glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -2146,7 +2294,14 @@ out:
}
int
-glusterd_handle_getwd (rpcsvc_request_t *req)
+glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_profile_volume);
+}
+
+int
+__glusterd_handle_getwd (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf1_cli_getwd_rsp rsp = {0,};
@@ -2171,9 +2326,14 @@ glusterd_handle_getwd (rpcsvc_request_t *req)
return ret;
}
+int
+glusterd_handle_getwd (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_getwd);
+}
int
-glusterd_handle_mount (rpcsvc_request_t *req)
+__glusterd_handle_mount (rpcsvc_request_t *req)
{
gf1_cli_mount_req mnt_req = {0,};
gf1_cli_mount_rsp rsp = {0,};
@@ -2236,7 +2396,13 @@ glusterd_handle_mount (rpcsvc_request_t *req)
}
int
-glusterd_handle_umount (rpcsvc_request_t *req)
+glusterd_handle_mount (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_mount);
+}
+
+int
+__glusterd_handle_umount (rpcsvc_request_t *req)
{
gf1_cli_umount_req umnt_req = {0,};
gf1_cli_umount_rsp rsp = {0,};
@@ -2249,9 +2415,11 @@ glusterd_handle_umount (rpcsvc_request_t *req)
gf_boolean_t dir_ok = _gf_false;
char *pdir = NULL;
char *t = NULL;
+ glusterd_conf_t *priv = NULL;
GF_ASSERT (req);
GF_ASSERT (this);
+ priv = this->private;
ret = xdr_to_generic (req->msg[0], &umnt_req,
(xdrproc_t)xdr_gf1_cli_umount_req);
@@ -2294,7 +2462,9 @@ glusterd_handle_umount (rpcsvc_request_t *req)
runner_add_args (&runner, "umount", umnt_req.path, NULL);
if (umnt_req.lazy)
runner_add_arg (&runner, "-l");
+ synclock_unlock (&priv->big_lock);
rsp.op_ret = runner_run (&runner);
+ synclock_lock (&priv->big_lock);
if (rsp.op_ret == 0) {
if (realpath (umnt_req.path, mntp))
rmdir (mntp);
@@ -2323,6 +2493,12 @@ glusterd_handle_umount (rpcsvc_request_t *req)
}
int
+glusterd_handle_umount (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_umount);
+}
+
+int
glusterd_friend_remove (uuid_t uuid, char *hostname)
{
int ret = 0;
@@ -2527,7 +2703,9 @@ glusterd_friend_add (const char *hoststr, int port,
if (!restore) {
ret = glusterd_store_peerinfo (*friend);
if (ret == 0) {
+ synclock_unlock (&conf->big_lock);
ret = glusterd_friend_rpc_create (this, *friend, args);
+ synclock_lock (&conf->big_lock);
}
else {
gf_log (this->name, GF_LOG_ERROR,
@@ -2932,7 +3110,7 @@ out:
}
int
-glusterd_handle_status_volume (rpcsvc_request_t *req)
+__glusterd_handle_status_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
uint32_t cmd = 0;
@@ -3004,7 +3182,14 @@ out:
}
int
-glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+glusterd_handle_status_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_status_volume);
+}
+
+int
+__glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -3073,9 +3258,15 @@ out:
}
int
-glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
- rpc_clnt_event_t event,
- void *data)
+glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_clearlocks_volume);
+}
+
+int
+__glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
{
xlator_t *this = NULL;
glusterd_conf_t *conf = NULL;
@@ -3114,9 +3305,16 @@ glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
}
int
-glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
- rpc_clnt_event_t event,
- void *data)
+glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
+{
+ return glusterd_big_locked_notify (rpc, mydata, event, data,
+ __glusterd_brick_rpc_notify);
+}
+
+int
+__glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
{
xlator_t *this = NULL;
glusterd_conf_t *conf = NULL;
@@ -3155,6 +3353,14 @@ glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
}
int
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
+{
+ return glusterd_big_locked_notify (rpc, mydata, event, data,
+ __glusterd_nodesvc_rpc_notify);
+}
+
+int
glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx)
{
int ret = -1;
@@ -3196,9 +3402,8 @@ out:
}
int
-glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
- rpc_clnt_event_t event,
- void *data)
+__glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
{
xlator_t *this = NULL;
glusterd_conf_t *conf = NULL;
@@ -3317,6 +3522,14 @@ out:
}
int
+glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
+{
+ return glusterd_big_locked_notify (rpc, mydata, event, data,
+ __glusterd_peer_rpc_notify);
+}
+
+int
glusterd_null (rpcsvc_request_t *req)
{
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index 8b964702669..c41a9459870 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -109,7 +109,7 @@ out:
}
int
-server_getspec (rpcsvc_request_t *req)
+__server_getspec (rpcsvc_request_t *req)
{
int32_t ret = -1;
int32_t op_errno = 0;
@@ -289,8 +289,14 @@ fail:
return 0;
}
+int
+server_getspec (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __server_getspec);
+}
+
int32_t
-server_event_notify (rpcsvc_request_t *req)
+__server_event_notify (rpcsvc_request_t *req)
{
int32_t ret = -1;
int32_t op_errno = 0;
@@ -350,6 +356,12 @@ fail:
return 0;
}
+int32_t
+server_event_notify (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __server_event_notify);
+}
+
int
gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
char *peerid)
@@ -383,7 +395,7 @@ out:
}
int
-glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+__glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
{
dict_t *dict = NULL;
xlator_t *this = NULL;
@@ -459,7 +471,14 @@ out:
}
int
-glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_mgmt_hndsk_versions);
+}
+
+int
+__glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
{
dict_t *clnt_dict = NULL;
xlator_t *this = NULL;
@@ -527,6 +546,12 @@ out:
return ret;
}
+int
+glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_mgmt_hndsk_versions_ack);
+}
rpcsvc_actor_t gluster_handshake_actors[] = {
[GF_HNDSK_NULL] = {"NULL", GF_HNDSK_NULL, NULL, NULL, 0},
@@ -699,7 +724,7 @@ out:
}
int
-glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
int ret = -1;
@@ -778,7 +803,15 @@ out:
}
int
-glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_mgmt_hndsk_version_ack_cbk);
+}
+
+int
+__glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
int ret = -1;
@@ -883,6 +916,14 @@ out:
}
int
+glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_mgmt_hndsk_version_cbk);
+}
+
+int
glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx)
{
call_frame_t *frame = NULL;
@@ -985,7 +1026,7 @@ out:
}
int
-glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
int ret = -1;
@@ -1092,6 +1133,14 @@ out:
int
+glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_peer_dump_version_cbk);
+}
+
+int
glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
glusterd_peerctx_t *peerctx)
{
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index cb29462c9b7..aab6744a418 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -280,7 +280,7 @@ out:
}
int
-gluster_pmap_portbybrick (rpcsvc_request_t *req)
+__gluster_pmap_portbybrick (rpcsvc_request_t *req)
{
pmap_port_by_brick_req args = {0,};
pmap_port_by_brick_rsp rsp = {0,};
@@ -314,7 +314,14 @@ fail:
int
-gluster_pmap_brickbyport (rpcsvc_request_t *req)
+gluster_pmap_portbybrick (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __gluster_pmap_portbybrick);
+}
+
+
+int
+__gluster_pmap_brickbyport (rpcsvc_request_t *req)
{
pmap_brick_by_port_req args = {0,};
pmap_brick_by_port_rsp rsp = {0,};
@@ -340,6 +347,14 @@ fail:
return 0;
}
+
+int
+gluster_pmap_brickbyport (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __gluster_pmap_brickbyport);
+}
+
+
static int
glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
gf_boolean_t value)
@@ -350,7 +365,7 @@ glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
}
int
-gluster_pmap_signup (rpcsvc_request_t *req)
+__gluster_pmap_signup (rpcsvc_request_t *req)
{
pmap_signup_req args = {0,};
pmap_signup_rsp rsp = {0,};
@@ -376,7 +391,13 @@ fail:
}
int
-gluster_pmap_signin (rpcsvc_request_t *req)
+gluster_pmap_signup (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __gluster_pmap_signup);
+}
+
+int
+__gluster_pmap_signin (rpcsvc_request_t *req)
{
pmap_signin_req args = {0,};
pmap_signin_rsp rsp = {0,};
@@ -407,9 +428,15 @@ fail:
}
+int
+gluster_pmap_signin (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __gluster_pmap_signin);
+}
+
int
-gluster_pmap_signout (rpcsvc_request_t *req)
+__gluster_pmap_signout (rpcsvc_request_t *req)
{
pmap_signout_req args = {0,};
pmap_signout_rsp rsp = {0,};
@@ -440,6 +467,12 @@ fail:
return 0;
}
+int
+gluster_pmap_signout (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __gluster_pmap_signout);
+}
+
rpcsvc_actor_t gluster_pmap_actors[] = {
[GF_PMAP_NULL] = {"NULL", GF_PMAP_NULL, NULL, NULL, 0},
[GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK,
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index e049982a2f1..31826719947 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -25,7 +25,7 @@
#include <sys/wait.h>
int
-glusterd_handle_quota (rpcsvc_request_t *req)
+__glusterd_handle_quota (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -112,6 +112,12 @@ out:
return ret;
}
+int
+glusterd_handle_quota (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_quota);
+}
+
int32_t
glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
{
@@ -257,7 +263,9 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
"-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
mountdir, NULL);
+ synclock_unlock (&priv->big_lock);
ret = runner_run_reuse (&runner);
+ synclock_lock (&priv->big_lock);
if (ret == -1) {
runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
runner_end (&runner);
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index de8f39b7d91..165af4e62fc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -70,9 +70,10 @@ out:
return ret;
}
+
int32_t
-glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
- rpc_clnt_event_t event, void *data)
+__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
{
glusterd_volinfo_t *volinfo = NULL;
glusterd_defrag_info_t *defrag = NULL;
@@ -160,6 +161,14 @@ glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
return ret;
}
+int32_t
+glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event, void *data)
+{
+ return glusterd_big_locked_notify (rpc, mydata, event,
+ data, __glusterd_defrag_notify);
+}
+
int
glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
size_t len, int cmd, defrag_cbk_fn_t cbk,
@@ -274,8 +283,10 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
goto out;
}
+ synclock_unlock (&priv->big_lock);
ret = glusterd_rpc_create (&defrag->rpc, options,
glusterd_defrag_notify, volinfo);
+ synclock_lock (&priv->big_lock);
if (ret) {
gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
goto out;
@@ -326,8 +337,10 @@ glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
goto out;
}
+ synclock_unlock (&priv->big_lock);
ret = glusterd_rpc_create (&defrag->rpc, options,
glusterd_defrag_notify, volinfo);
+ synclock_lock (&priv->big_lock);
if (ret) {
gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
goto out;
@@ -376,7 +389,7 @@ out:
}
int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -461,6 +474,12 @@ out:
return 0;
}
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
+}
+
int
glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 0afe2203c85..c1506033bff 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -70,7 +70,7 @@ out:
}
int
-glusterd_handle_replace_brick (rpcsvc_request_t *req)
+__glusterd_handle_replace_brick (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -171,6 +171,13 @@ out:
return ret;
}
+int
+glusterd_handle_replace_brick (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_replace_brick);
+}
+
static int
glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t **brickinfo)
@@ -750,7 +757,7 @@ rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
if (volinfo->memory_accounting)
runner_add_arg (&runner, "--mem-accounting");
- ret = runner_run (&runner);
+ ret = runner_run_nowait (&runner);
if (ret) {
pmap_registry_remove (THIS, 0, brickinfo->path,
GF_PMAP_PORT_BRICKSERVER, NULL);
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 1249728576f..01fb7777558 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -185,7 +185,21 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
}
int
-glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe, fop_cbk_fn_t fn)
+{
+ glusterd_conf_t *priv = THIS->private;
+ int ret = -1;
+
+ synclock_lock (&priv->big_lock);
+ ret = fn (req, iov, count, myframe);
+ synclock_unlock (&priv->big_lock);
+
+ return ret;
+}
+
+int
+__glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_probe_rsp rsp = {{0},};
@@ -287,7 +301,16 @@ out:
}
int
-glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
+glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_probe_cbk);
+}
+
+
+int
+__glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_friend_rsp rsp = {{0},};
@@ -380,7 +403,15 @@ out:
}
int
-glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
+glusterd_friend_add_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_friend_add_cbk);
+}
+
+int
+__glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_friend_rsp rsp = {{0},};
@@ -473,8 +504,16 @@ respond:
return ret;
}
+int
+glusterd_friend_remove_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_friend_remove_cbk);
+}
+
int32_t
-glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
int ret = -1;
@@ -506,8 +545,16 @@ out:
return ret;
}
+int
+glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_friend_update_cbk);
+}
+
int32_t
-glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_cluster_lock_rsp rsp = {{0},};
@@ -572,7 +619,15 @@ out:
}
int32_t
-glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_cluster_lock_cbk);
+}
+
+int32_t
+__glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_cluster_lock_rsp rsp = {{0},};
@@ -634,7 +689,15 @@ out:
}
int32_t
-glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_cluster_unlock_cbk);
+}
+
+int32_t
+__glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_stage_op_rsp rsp = {{0},};
@@ -753,7 +816,15 @@ out:
}
int32_t
-glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_stage_op_cbk);
+}
+
+int32_t
+__glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_commit_op_rsp rsp = {{0},};
@@ -911,7 +982,13 @@ out:
return ret;
}
-
+int32_t
+glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_commit_op_cbk);
+}
int32_t
glusterd_rpc_probe (call_frame_t *frame, xlator_t *this,
@@ -1276,7 +1353,7 @@ out:
}
int32_t
-glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+__glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
gd1_mgmt_brick_op_rsp rsp = {0};
@@ -1378,6 +1455,14 @@ out:
}
int32_t
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe)
+{
+ return glusterd_big_locked_cbk (req, iov, count, myframe,
+ __glusterd_brick_op_cbk);
+}
+
+int32_t
glusterd_brick_op (call_frame_t *frame, xlator_t *this,
void *data)
{
@@ -1542,3 +1627,4 @@ struct rpc_clnt_program gd_peer_prog = {
.numproc = GLUSTERD_FRIEND_MAXVALUE,
};
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index 5a38fdfecb5..a82ca2e176f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -1075,8 +1075,25 @@ glusterd_friend_sm ()
ret = 0;
out:
- if (quorum_action)
+ if (quorum_action) {
+ /* When glusterd is restarted, it needs to wait until the 'friends' view
+ * of the volumes settle, before it starts any of the internal daemons.
+ *
+ * Every friend that was part of the cluster, would send its
+ * cluster-view, 'our' way. For every friend, who belongs to
+ * a partition which has a different cluster-view from our
+ * partition, we may update our cluster-view. For subsequent
+ * friends from that partition would agree with us, if the first
+ * friend wasn't rejected. For every first friend, whom we agreed with,
+ * we would need to start internal daemons/bricks belonging to the
+ * new volumes.
+ * glusterd_spawn_daemons calls functions that are idempotent. ie,
+ * the functions spawn process(es) only if they are not started yet.
+ *
+ * */
+ glusterd_spawn_daemons ((void*) _gf_false);
glusterd_do_quorum_action ();
+ }
return ret;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 5bdf5917d4b..d364596326b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -18,6 +18,16 @@
#include "glusterd-op-sm.h"
#include "glusterd-utils.h"
+static inline void
+gd_synctask_barrier_wait (struct syncargs *args, int count)
+{
+ glusterd_conf_t *conf = THIS->private;
+
+ synclock_unlock (&conf->big_lock);
+ synctask_barrier_wait (args, count);
+ synclock_lock (&conf->big_lock);
+}
+
static void
gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
char *op_errstr)
@@ -664,7 +674,7 @@ gd_lock_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
gd_syncop_mgmt_lock (peerinfo->rpc, &args, MY_UUID, peer_uuid);
peer_cnt++;
}
- synctask_barrier_wait((&args), peer_cnt);
+ gd_synctask_barrier_wait((&args), peer_cnt);
ret = args.op_ret;
if (ret) {
gf_asprintf (op_errstr, "Another transaction could be "
@@ -739,7 +749,7 @@ stage_done:
op, req_dict, op_ctx);
peer_cnt++;
}
- synctask_barrier_wait((&args), peer_cnt);
+ gd_synctask_barrier_wait((&args), peer_cnt);
ret = args.op_ret;
if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
*op_errstr = gf_strdup (errstr);
@@ -814,7 +824,7 @@ commit_done:
op, req_dict, op_ctx);
peer_cnt++;
}
- synctask_barrier_wait((&args), peer_cnt);
+ gd_synctask_barrier_wait((&args), peer_cnt);
ret = args.op_ret;
if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
*op_errstr = gf_strdup (errstr);
@@ -851,7 +861,7 @@ gd_unlock_op_phase (struct list_head *peers, glusterd_op_t op, int op_ret,
list_del_init (&peerinfo->op_peers_list);
peer_cnt++;
}
- synctask_barrier_wait((&args), peer_cnt);
+ gd_synctask_barrier_wait((&args), peer_cnt);
ret = args.op_ret;
if (ret) {
gf_log (this->name, GF_LOG_ERROR, "Failed to unlock "
@@ -887,8 +897,10 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op
int ret = -1;
rpc_clnt_t *rpc = NULL;
dict_t *rsp_dict = NULL;
+ glusterd_conf_t *conf = NULL;
this = THIS;
+ conf = this->private;
rsp_dict = dict_new ();
if (!rsp_dict) {
ret = -1;
@@ -928,8 +940,12 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict, char **op
"due to rpc failure.");
goto out;
}
+ /*This is to ensure that the brick_op_cbk is able to take
+ * the big lock*/
+ synclock_unlock (&conf->big_lock);
ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
op_ctx, op_errstr);
+ synclock_lock (&conf->big_lock);
if (ret)
goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 6c5d0e2f074..e2d46a6a603 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -23,6 +23,7 @@
#include "timer.h"
#include "defaults.h"
#include "compat.h"
+#include "syncop.h"
#include "run.h"
#include "compat-errno.h"
#include "statedump.h"
@@ -1252,6 +1253,7 @@ glusterd_brick_connect (glusterd_volinfo_t *volinfo,
char socketpath[PATH_MAX] = {0};
dict_t *options = NULL;
struct rpc_clnt *rpc = NULL;
+ glusterd_conf_t *priv = THIS->private;
GF_ASSERT (volinfo);
GF_ASSERT (brickinfo);
@@ -1270,9 +1272,11 @@ glusterd_brick_connect (glusterd_volinfo_t *volinfo,
socketpath, 600);
if (ret)
goto out;
+ synclock_unlock (&priv->big_lock);
ret = glusterd_rpc_create (&rpc, options,
glusterd_brick_rpc_notify,
brickinfo);
+ synclock_lock (&priv->big_lock);
if (ret)
goto out;
brickinfo->rpc = rpc;
@@ -1422,10 +1426,14 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
runner_add_arg (&runner, "--mem-accounting");
runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
- if (wait)
+ if (wait) {
+ synclock_unlock (&priv->big_lock);
ret = runner_run (&runner);
- else
+ synclock_lock (&priv->big_lock);
+
+ } else {
ret = runner_run_nowait (&runner);
+ }
if (ret)
goto out;
@@ -2403,6 +2411,19 @@ out:
return in;
}
+int
+glusterd_spawn_daemons (void *opaque)
+{
+ glusterd_conf_t *conf = THIS->private;
+ gf_boolean_t start_bricks = (long) opaque;
+
+ if (start_bricks)
+ glusterd_restart_bricks (conf);
+ glusterd_restart_gsyncds (conf);
+ glusterd_restart_rebalance (conf);
+ return 0;
+}
+
void
glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
gf_boolean_t meets_quorum)
@@ -3399,6 +3420,7 @@ glusterd_nodesvc_connect (char *server, char *socketpath) {
int ret = 0;
dict_t *options = NULL;
struct rpc_clnt *rpc = NULL;
+ glusterd_conf_t *priv = THIS->private;
rpc = glusterd_nodesvc_get_rpc (server);
@@ -3412,9 +3434,11 @@ glusterd_nodesvc_connect (char *server, char *socketpath) {
socketpath, 600);
if (ret)
goto out;
+ synclock_unlock (&priv->big_lock);
ret = glusterd_rpc_create (&rpc, options,
glusterd_nodesvc_rpc_notify,
server);
+ synclock_lock (&priv->big_lock);
if (ret)
goto out;
(void) glusterd_nodesvc_set_rpc (server, rpc);
@@ -4055,13 +4079,8 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
if (volinfo->status != GLUSTERD_STATUS_STARTED)
continue;
start_nodesvcs = _gf_true;
- if (glusterd_is_volume_in_server_quorum (volinfo)) {
- //these bricks will be restarted once the quorum is met
- continue;
- }
-
list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
- glusterd_brick_start (volinfo, brickinfo, _gf_true);
+ glusterd_brick_start (volinfo, brickinfo, _gf_false);
}
}
@@ -5577,7 +5596,9 @@ glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
runner_argprintf (&runner, ":%s", master_vol->volname);
runner_add_args (&runner, slave, "--config-set", "session-owner",
uuid_str, NULL);
+ synclock_unlock (&priv->big_lock);
ret = runner_run (&runner);
+ synclock_lock (&priv->big_lock);
if (ret == -1) {
errcode = -1;
goto out;
@@ -5588,7 +5609,9 @@ glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
runner_argprintf (&runner, "%s/"GSYNC_CONF, priv->workdir);
runner_argprintf (&runner, ":%s", master_vol->volname);
runner_add_arg (&runner, slave);
+ synclock_unlock (&priv->big_lock);
ret = runner_run (&runner);
+ synclock_lock (&priv->big_lock);
if (ret == -1) {
gf_asprintf (op_errstr, GEOREP" start failed for %s %s",
master_vol->volname, slave);
@@ -6051,7 +6074,6 @@ glusterd_restart_rebalance (glusterd_conf_t *conf)
return ret;
}
-
void
glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo)
{
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 28182d6ba5b..24a65aebe6e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -362,6 +362,8 @@ glusterd_delete_brick (glusterd_volinfo_t* volinfo,
int32_t
glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo);
int
+glusterd_spawn_daemons (void *opaque);
+int
glusterd_restart_gsyncds (glusterd_conf_t *conf);
int
glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index fd4e0268a57..50b03f71e49 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -31,7 +31,7 @@
glusterd_op_stop_volume_args_get (dict, volname, flags)
int
-glusterd_handle_create_volume (rpcsvc_request_t *req)
+__glusterd_handle_create_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -192,7 +192,14 @@ out:
}
int
-glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+glusterd_handle_create_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_create_volume);
+}
+
+int
+__glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -257,9 +264,15 @@ out:
return ret;
}
+int
+glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_start_volume);
+}
int
-glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -327,7 +340,14 @@ out:
}
int
-glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_stop_volume);
+}
+
+int
+__glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,},};
@@ -397,7 +417,14 @@ out:
}
int
-glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_delete_volume);
+}
+
+int
+__glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -481,7 +508,14 @@ out:
}
int
-glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_heal_volume);
+}
+
+int
+__glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
{
int32_t ret = -1;
gf_cli_req cli_req = {{0,}};
@@ -562,6 +596,13 @@ out:
return ret;
}
+int
+glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+{
+ return glusterd_big_locked_handler (req,
+ __glusterd_handle_cli_statedump_volume);
+}
+
#ifdef HAVE_BD_XLATOR
int
glusterd_is_valid_vg (const char *name)
@@ -1852,7 +1893,9 @@ glusterd_clearlocks_unmount (glusterd_volinfo_t *volinfo, char *mntpt)
runner_add_args (&runner, "/bin/umount", "-f", NULL);
runner_argprintf (&runner, "%s", mntpt);
+ synclock_unlock (&priv->big_lock);
ret = runner_run (&runner);
+ synclock_lock (&priv->big_lock);
if (ret) {
ret = 0;
gf_log ("", GF_LOG_DEBUG,
@@ -1924,7 +1967,9 @@ glusterd_clearlocks_mount (glusterd_volinfo_t *volinfo, char **xl_opts,
}
runner_argprintf (&runner, "%s", mntpt);
+ synclock_unlock (&priv->big_lock);
ret = runner_run (&runner);
+ synclock_lock (&priv->big_lock);
if (ret) {
gf_log (THIS->name, GF_LOG_DEBUG,
"Could not start glusterfs");
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 2b55149ffd6..cb5daf4bb3d 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -869,27 +869,28 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
return -1;
}
+
static int
-glusterd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
{
- return ret;
+ return ret;
}
-static int
-glusterd_launch_synctask (xlator_t *this, synctask_fn_t fn)
+static void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
{
- glusterd_conf_t *priv = NULL;
- int ret = -1;
-
- priv = this->private;
+ xlator_t *this = NULL;
+ glusterd_conf_t *priv = NULL;
+ int ret = -1;
- ret = synctask_new (this->ctx->env, fn,
- glusterd_default_synctask_cbk, NULL, priv);
+ this = THIS;
+ priv = this->private;
- if (ret)
- gf_log (this->name, GF_LOG_CRITICAL, "Failed to create synctask"
- "for starting process");
- return ret;
+ ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+ opaque);
+ if (ret)
+ gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+ " and other volume related services");
}
/*
@@ -1082,6 +1083,7 @@ init (xlator_t *this)
conf->gfs_mgmt = &gd_brick_prog;
strncpy (conf->workdir, workdir, PATH_MAX);
+ synclock_init (&conf->big_lock);
pthread_mutex_init (&conf->xprt_lock, NULL);
INIT_LIST_HEAD (&conf->xprt_list);
@@ -1144,6 +1146,14 @@ init (xlator_t *this)
if (ret < 0)
goto out;
+ /* If there are no 'friends', this would be the best time to
+ * spawn process/bricks that may need (re)starting since last
+ * time (this) glusterd was up.*/
+
+ if (list_empty (&conf->peers)) {
+ glusterd_launch_synctask (glusterd_spawn_daemons,
+ (void*) _gf_true);
+ }
ret = glusterd_options_init (this);
if (ret < 0)
goto out;
@@ -1152,13 +1162,6 @@ init (xlator_t *this)
if (ret)
goto out;
- glusterd_launch_synctask (this,
- (synctask_fn_t) glusterd_restart_bricks);
- glusterd_launch_synctask (this,
- (synctask_fn_t) glusterd_restart_gsyncds);
- glusterd_launch_synctask (this,
- (synctask_fn_t) glusterd_restart_rebalance);
-
ret = glusterd_hooks_spawn_worker (this);
if (ret)
goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index dfdbbb3bea7..4547e080d4e 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -151,6 +151,7 @@ typedef struct {
xlator_t *xl; /* Should be set to 'THIS' before creating thread */
gf_boolean_t pending_quorum_action;
dict_t *opts;
+ synclock_t big_lock;
} glusterd_conf_t;
@@ -427,6 +428,16 @@ __glusterd_uuid()
return &priv->uuid[0];
}
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+ rpc_clnt_event_t event,
+ void *data, rpc_clnt_notify_t notify_fn);
+
+int
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+ int count, void *myframe, fop_cbk_fn_t fn);
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn);
+
int32_t
glusterd_brick_from_brickinfo (glusterd_brickinfo_t *brickinfo,
char **new_brick);