syncop: improve scaling and implement more tools

The current scaling of the syncop thread pool is not working properly
and can leave some tasks in the run queue more time than necessary
when the maximum number of threads is not reached.

This patch provides a better scaling condition to react faster to
pending work.

Condition variables and sleep in the context of a synctask have also
been implemented. Their purpose is to replace regular condition
variables and sleeps that block synctask threads and prevent other
tasks to be executed.

The new features have been applied to several places in glusterd.

Change-Id: Ic50b7c73c104f9e41f08101a357d30b95efccfbf
Fixes: #1116
Signed-off-by: Xavi Hernandez <xhernandez@redhat.com>

9 files changed, 40 insertions, 33 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 14915b3fc17..26a0eb7d20f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -5838,13 +5838,8 @@ glusterd_op_stage_validate(glusterd_op_t op, dict_t *dict, char **op_errstr,
 static void
 glusterd_wait_for_blockers(glusterd_conf_t *priv)
 {
-    uint64_t blockers = GF_ATOMIC_GET(priv->blockers);
-
-    while (blockers) {
-        synclock_unlock(&priv->big_lock);
-        sleep(1);
-        blockers = GF_ATOMIC_GET(priv->blockers);
-        synclock_lock(&priv->big_lock);
+    while (GF_ATOMIC_GET(priv->blockers)) {
+        synccond_wait(&priv->cond_blockers, &priv->big_lock);
     }
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
index d96adcae89e..a05c90d7b10 100644
--- a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
@@ -114,7 +114,7 @@ glusterd_proc_stop(glusterd_proc_t *proc, int sig, int flags)
         goto out;
 
     synclock_unlock(&conf->big_lock);
-    sleep(1);
+    synctask_sleep(1);
     synclock_lock(&conf->big_lock);
     if (gf_is_service_running(proc->pidfile, &pid)) {
         ret = kill(pid, SIGKILL);
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
index e106398e697..c5fa4f76417 100644
--- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
@@ -270,9 +270,7 @@ glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
     }
 
     while (conf->restart_shd) {
-        synclock_unlock(&conf->big_lock);
-        sleep(2);
-        synclock_lock(&conf->big_lock);
+        synccond_wait(&conf->cond_restart_shd, &conf->big_lock);
     }
     conf->restart_shd = _gf_true;
     shd_restart = _gf_true;
@@ -328,8 +326,10 @@ glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
         }
     }
 out:
-    if (shd_restart)
+    if (shd_restart) {
         conf->restart_shd = _gf_false;
+        synccond_broadcast(&conf->cond_restart_shd);
+    }
     if (volinfo)
         glusterd_volinfo_unref(volinfo);
     if (ret)
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
index 43735d33fee..810f3b7d5b5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
@@ -1961,9 +1961,7 @@ glusterd_update_snaps_synctask(void *opaque)
     synclock_lock(&conf->big_lock);
 
     while (conf->restart_bricks) {
-        synclock_unlock(&conf->big_lock);
-        sleep(2);
-        synclock_lock(&conf->big_lock);
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
     }
     conf->restart_bricks = _gf_true;
 
@@ -2070,6 +2068,7 @@ out:
     if (dict)
         dict_unref(dict);
     conf->restart_bricks = _gf_false;
+    synccond_broadcast(&conf->cond_restart_bricks);
 
     return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
index 18990fe365b..ef584c7cfad 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
@@ -634,7 +634,9 @@ my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
     conf = this->private;
     GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-    GF_ATOMIC_DEC(conf->blockers);
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
 
     STACK_DESTROY(frame->root);
 out:
@@ -722,7 +724,9 @@ out:
     if (volinfo)
         glusterd_volinfo_unref(volinfo);
 
-    GF_ATOMIC_DEC(conf->blockers);
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
     STACK_DESTROY(frame->root);
     return 0;
 }
@@ -969,7 +973,7 @@ glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int flags)
          * TBD: see if there's a better way
          */
         synclock_unlock(&conf->big_lock);
-        sleep(1);
+        synctask_sleep(1);
         synclock_lock(&conf->big_lock);
     }
     ret = -1;
@@ -1023,7 +1027,7 @@ glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig)
          * TBD: see if there's a better way
          */
         synclock_unlock(&conf->big_lock);
-        sleep(1);
+        synctask_sleep(1);
         synclock_lock(&conf->big_lock);
     }
     ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
index ce4a940c7a0..a265f2135c6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.h
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -32,7 +32,7 @@
         ret = gd_syncop_submit_request(rpc, req, stb, cookie, prog, procnum,   \
                                        cbk, (xdrproc_t)xdrproc);               \
         if (!ret)                                                              \
-            synctask_yield(stb->task);                                         \
+            synctask_yield(stb->task, NULL);                                   \
         else                                                                   \
             gf_asprintf(&stb->errstr,                                          \
                         "%s failed. Check log file"                            \
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index c947e6c926b..62fefe2de0d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -5040,22 +5040,22 @@ glusterd_import_friend_volumes_synctask(void *opaque)
      * restarted (refer glusterd_restart_bricks ())
      */
     while (conf->restart_bricks) {
-        synclock_unlock(&conf->big_lock);
-        sleep(2);
-        synclock_lock(&conf->big_lock);
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
     }
     conf->restart_bricks = _gf_true;
 
     while (i <= count) {
         ret = glusterd_import_friend_volume(peer_data, i);
         if (ret) {
-            conf->restart_bricks = _gf_false;
-            goto out;
+            break;
         }
         i++;
     }
-    glusterd_svcs_manager(NULL);
+    if (i > count) {
+        glusterd_svcs_manager(NULL);
+    }
     conf->restart_bricks = _gf_false;
+    synccond_broadcast(&conf->cond_restart_bricks);
 out:
     if (peer_data)
         dict_unref(peer_data);
@@ -5735,7 +5735,9 @@ my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
     call_frame_t *frame = v_frame;
     glusterd_conf_t *conf = frame->this->private;
 
-    GF_ATOMIC_DEC(conf->blockers);
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
 
     STACK_DESTROY(frame->root);
     return 0;
@@ -5837,7 +5839,9 @@ attach_brick_callback(struct rpc_req *req, struct iovec *iov, int count,
         }
     }
 out:
-    GF_ATOMIC_DEC(conf->blockers);
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
     STACK_DESTROY(frame->root);
     return 0;
 }
@@ -6022,7 +6026,7 @@ attach_brick(xlator_t *this, glusterd_brickinfo_t *brickinfo,
          * TBD: see if there's a better way
          */
         synclock_unlock(&conf->big_lock);
-        sleep(1);
+        synctask_sleep(1);
         synclock_lock(&conf->big_lock);
     }
 
@@ -6162,7 +6166,7 @@ find_compat_brick_in_vol(glusterd_conf_t *conf,
                          "brick %s is still"
                          " starting, waiting for 2 seconds ",
                          other_brick->path);
-            sleep(2);
+            synctask_sleep(2);
             synclock_lock(&conf->big_lock);
             retries--;
         }
@@ -6651,9 +6655,7 @@ glusterd_restart_bricks(void *opaque)
      * glusterd_compare_friend_data ())
      */
     while (conf->restart_bricks) {
-        synclock_unlock(&conf->big_lock);
-        sleep(2);
-        synclock_lock(&conf->big_lock);
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
     }
     conf->restart_bricks = _gf_true;
 
@@ -6770,6 +6772,7 @@ out:
     GF_ATOMIC_DEC(conf->blockers);
     conf->restart_done = _gf_true;
     conf->restart_bricks = _gf_false;
+    synccond_broadcast(&conf->cond_restart_bricks);
 
 return_block:
     return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index cd2c5da628b..39aea886051 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1872,6 +1872,9 @@ init(xlator_t *this)
     (void)strncpy(conf->logdir, logdir, sizeof(conf->logdir));
 
     synclock_init(&conf->big_lock, SYNC_LOCK_RECURSIVE);
+    synccond_init(&conf->cond_restart_bricks);
+    synccond_init(&conf->cond_restart_shd);
+    synccond_init(&conf->cond_blockers);
     pthread_mutex_init(&conf->xprt_lock, NULL);
     INIT_LIST_HEAD(&conf->xprt_list);
     pthread_mutex_init(&conf->import_volumes, NULL);
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index d7e4da8425a..f2651ebe43c 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -209,6 +209,9 @@ typedef struct {
     gf_boolean_t restart_done;
     dict_t *opts;
     synclock_t big_lock;
+    synccond_t cond_restart_bricks;
+    synccond_t cond_restart_shd;
+    synccond_t cond_blockers;
     rpcsvc_t *uds_rpc; /* RPCSVC for the unix domain socket */
     uint32_t base_port;
     uint32_t max_port;