cluster/dht: Make rebalance throttle option tuned by number

Current rebalance throttle options: lazy/normal/aggressive may not always be
sufficient for the purpose of throttling.  In our recent test, we observed for
certain setups, normal and aggressive modes behaved similarly consuming full
disk bandwidth. So in cases like this admin should be able to  tune it
down(or vice versa) depending on the need.

Along with old throttle configurations, thread counts are tuned based on number.
e.g. gluster v set vol-name cluster-rebal.throttle  5.

Admin can tune up/down between 0 and the number of cores available.

Note: For heterogenous servers, validation will fail on the old server if "number"
is given for throttle configuration.
The message looks something like this:
"volume set: failed: Staging failed on vm2. Error: cluster.rebal-throttle should be {lazy|normal|aggressive}"

Test: Manual test by logging active thread number after reconfiguring throttle option.
testcase: tests/basic/distribute/throttle-rebal.t

> Change-Id: I46e3cde546900307831028b344ecf601fd9b02c3
> BUG: 1438370
> Signed-off-by: Susant Palai <spalai@redhat.com>
> Reviewed-on: https://review.gluster.org/16980
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
> Signed-off-by: Susant Palai <spalai@redhat.com>

Change-Id: I46e3cde546900307831028b344ecf601fd9b02c3
BUG: 1473136
Signed-off-by: Susant Palai <spalai@redhat.com>
Reviewed-on: https://review.gluster.org/17835
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shyamsundar Ranganathan <srangana@redhat.com>

1 files changed, 87 insertions, 21 deletions

diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
index f13762e34fb..9c3eb16869c 100644
--- a/xlators/cluster/dht/src/dht-shared.c
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -400,7 +400,7 @@ dht_reconfigure (xlator_t *this, dict_t *options)
         char            *temp_str = NULL;
         gf_boolean_t     search_unhashed;
         int              ret = -1;
-        int              throttle_count = 0;
+        int              rebal_thread_count = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", this, out);
         GF_VALIDATE_OR_GOTO ("dht", options, out);
@@ -456,22 +456,54 @@ dht_reconfigure (xlator_t *this, dict_t *options)
                           conf->randomize_by_gfid,
                           options, bool, out);
 
-        GF_OPTION_RECONF ("rebal-throttle", conf->dthrottle, options,
-                          str, out);
-
         GF_OPTION_RECONF ("lock-migration", conf->lock_migration_enabled,
                           options, bool, out);
 
         if (conf->defrag) {
+                pthread_mutex_lock (&conf->defrag->dfq_mutex);
+                {
+                if (dict_get_str (options, "rebal-throttle", &temp_str) == 0) {
+                        if (!strcasecmp (temp_str, "lazy")) {
+                                conf->defrag->recon_thread_count = 1;
+                        } else if (!strcasecmp (temp_str, "normal")) {
+                                conf->defrag->recon_thread_count = 3;
+                        } else if (!strcasecmp (temp_str, "aggressive")) {
+                                conf->defrag->recon_thread_count = MAX ((MAX_REBAL_THREADS - 4), 4);
+                        } else if ((gf_string2int (temp_str, &rebal_thread_count) == 0)) {
+                                if ((rebal_thread_count > 0) && (rebal_thread_count <= MAX_REBAL_THREADS)) {
+                                        gf_msg_debug (this->name, 0, "rebal throttle count reconfigured to %d", rebal_thread_count);
+                                        conf->defrag->recon_thread_count = rebal_thread_count;
+                                } else {
+                                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                                               DHT_MSG_INVALID_OPTION,
+                                               "Invalid option: Reconfigure: "
+                                               "rebal-throttle should be "
+                                               "within range of 0 and maximum number of"
+                                               " cores available");
+                                        ret = -1;
+                                        pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+                                        goto out;
+                                }
+                        } else {
+                                gf_msg(this->name, GF_LOG_ERROR, 0,
+                                       DHT_MSG_INVALID_OPTION,
+                                       "Invalid option: Reconfigure: "
+                                       "rebal-throttle should be {lazy|normal|aggressive}"
+                                       " or a number upto number of cores available,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->dthrottle);
+                                ret = -1;
+                                pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+                                goto out;
+                        }
+                }
+                }
+                pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+        }
+
+        if (conf->defrag) {
                 conf->defrag->lock_migration_enabled =
                                         conf->lock_migration_enabled;
-
-                GF_DECIDE_DEFRAG_THROTTLE_COUNT (throttle_count, conf);
-                gf_msg ("DHT", GF_LOG_INFO, 0,
-                        DHT_MSG_REBAL_THROTTLE_INFO,
-                        "conf->dthrottle: %s, "
-                        "conf->defrag->recon_thread_count: %d",
-                         conf->dthrottle, conf->defrag->recon_thread_count);
         }
 
         if (conf->defrag) {
@@ -608,8 +640,8 @@ dht_init (xlator_t *this)
         gf_defrag_info_t                *defrag         = NULL;
         int                              cmd            = 0;
         char                            *node_uuid      = NULL;
-        int                              throttle_count = 0;
         uint32_t                         commit_hash    = 0;
+        int                              rebal_thread_count = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", this, err);
 
@@ -809,15 +841,49 @@ dht_init (xlator_t *this)
                         conf->randomize_by_gfid, bool, err);
 
         if (defrag) {
-                GF_OPTION_INIT ("rebal-throttle",
-                                 conf->dthrottle, str, err);
-
-                GF_DECIDE_DEFRAG_THROTTLE_COUNT(throttle_count, conf);
-
-                gf_msg_debug ("DHT", 0, "conf->dthrottle: %s, "
-                              "conf->defrag->recon_thread_count: %d",
-                              conf->dthrottle,
-                              conf->defrag->recon_thread_count);
+                GF_OPTION_INIT ("rebal-throttle", temp_str, str, err);
+                if (temp_str) {
+
+                        pthread_mutex_lock (&conf->defrag->dfq_mutex);
+                        {
+                        if (!strcasecmp (temp_str, "lazy")) {
+                                conf->defrag->recon_thread_count = 1;
+                        } else if (!strcasecmp (temp_str, "normal")) {
+                                conf->defrag->recon_thread_count = 2;
+                        } else if (!strcasecmp (temp_str, "aggressive")) {
+                                conf->defrag->recon_thread_count = MAX (MAX_REBAL_THREADS - 4, 4);
+                        } else if ((gf_string2int (temp_str, &rebal_thread_count) == 0)) {
+                                if ((rebal_thread_count > 0) && (rebal_thread_count <= MAX_REBAL_THREADS)) {
+                                        gf_msg (this->name, GF_LOG_INFO, 0, 0,
+                                                "rebal thread count configured to %d",
+                                                 rebal_thread_count);
+                                        conf->defrag->recon_thread_count = rebal_thread_count;
+                                } else {
+                                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                                               DHT_MSG_INVALID_OPTION,
+                                               "Invalid option: Reconfigure: "
+                                               "rebal-throttle should be "
+                                               "within range of 0 and maximum number of"
+                                               " cores available");
+                                        ret = -1;
+                                        pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+                                        goto err;
+                                }
+                        } else {
+                                gf_msg(this->name, GF_LOG_ERROR, 0,
+                                       DHT_MSG_INVALID_OPTION,
+                                       "Invalid option: Reconfigure: "
+                                       "rebal-throttle should be {lazy|normal|aggressive}"
+                                       " or a number upto number of cores available,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->dthrottle);
+                                ret = -1;
+                                pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+                                goto err;
+                        }
+                        }
+                        pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+                }
         }
 
         GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);