glusterd: On gaining quorum spawn_daemons in new thread

  Backport of http://review.gluster.org/7703 from master

During startup, if a glusterd has peers, it waits till quorum is
obtained to spawn bricks and other services. If peers are not present,
the daemons are started during glusterd' startup itself.

The spawning of daemons as a quorum action was done without using a
seperate thread, unlike the spawn on startup. Since, quotad was launched
using the blocking runner_run api, this leads to the thread being
blocked. The calling thread is almost always the epoll thread and this
leads to a deadlock. The runner_run call blocks the epoll thread waiting
for quotad to start, as a result glusterd cannot serve any requests. But
the startup of quotad is blocked as it cannot fetch the volfile from
glusterd.

The fix for this is to launch the spawn daemons task in a seperate
thread. This will free up the epoll thread and prevents the above
deadlock from happening.

BUG: 1105188
Change-Id: Idad1e96fbe1411dfd4b1a542fb5fa115673636c0
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.org/7995
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Niels de Vos <ndevos@redhat.com>

4 files changed, 44 insertions, 28 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index c671edf6885..2e3eb790ecf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -996,6 +996,13 @@ glusterd_friend_sm ()
         gf_boolean_t                     is_await_conn = _gf_false;
         gf_boolean_t                     quorum_action = _gf_false;
         glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
 
         while (!list_empty (&gd_friend_sm_queue)) {
                 list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue, list) {
@@ -1094,7 +1101,9 @@ out:
              * the functions spawn process(es) only if they are not started yet.
              *
              * */
-                glusterd_spawn_daemons (NULL);
+                synclock_unlock (&priv->big_lock);
+                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
+                synclock_lock (&priv->big_lock);
                 glusterd_do_quorum_action ();
         }
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 593bddfda78..36cd2ddc741 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -4015,6 +4015,9 @@ glusterd_nodesvc_start (char *server, gf_boolean_t wait)
         glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
                                               sockfpath, sizeof (sockfpath));
 
+        if (gf_is_service_running(pidfile, NULL))
+                goto connect;
+
         runinit (&runner);
 
         if (priv->valgrind) {
@@ -4062,7 +4065,7 @@ glusterd_nodesvc_start (char *server, gf_boolean_t wait)
                 }
                 synclock_lock (&priv->big_lock);
         }
-
+connect:
         if (ret == 0) {
                 glusterd_nodesvc_connect (server, sockfpath);
         }
@@ -9351,3 +9354,30 @@ glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc)
         return ret;
 }
 
+
+static int
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        glusterd_conf_t     *priv = THIS->private;
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+        int             ret   = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        synclock_lock (&priv->big_lock);
+        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+                            opaque);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+                        " and other volume related services");
+}
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 05d5c7172b2..d5497f03631 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -644,4 +644,7 @@ gd_stop_rebalance_process (glusterd_volinfo_t *volinfo);
 
 rpc_clnt_t *
 glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc);
+
+void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index d59aaa44aae..f9c38337473 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -938,32 +938,6 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
 }
 
 
-static int
-gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
-{
-        glusterd_conf_t     *priv = THIS->private;
-        synclock_unlock (&priv->big_lock);
-        return ret;
-}
-
-static void
-glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
-{
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
-        int             ret   = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        synclock_lock (&priv->big_lock);
-        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
-                            opaque);
-        if (ret)
-                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
-                        " and other volume related services");
-}
-
 int
 glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
                             void *data)