summaryrefslogtreecommitdiffstats
path: root/xlators/cluster
diff options
context:
space:
mode:
authorN Balachandran <nbalacha@redhat.com>2017-06-22 15:56:28 +0530
committerRaghavendra G <rgowdapp@redhat.com>2017-06-23 10:12:17 +0000
commitd66fb14a952729caf51c8328448a548c4d198082 (patch)
tree5c5189daecfc1b05f5dd0ef303b69c4972a74745 /xlators/cluster
parent0c0bc42ddfef4f05b50c3d1510e93ef3ec292a56 (diff)
cluster/dht: rebalance gets file count periodically
The rebalance used to get the file count in the beginning and not update it. This caused estimates to fail if the number changed during the rebalance. The rebalance now updates the file count periodically. Change-Id: I1667ee69e8a1d7d6bc6bc2f060fad7f989d19ed4 BUG: 1464110 Signed-off-by: N Balachandran <nbalacha@redhat.com> Reviewed-on: https://review.gluster.org/17607 Smoke: Gluster Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.org> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Diffstat (limited to 'xlators/cluster')
-rw-r--r--xlators/cluster/dht/src/dht-common.h3
-rw-r--r--xlators/cluster/dht/src/dht-rebalance.c122
-rw-r--r--xlators/cluster/dht/src/dht-shared.c1
-rw-r--r--xlators/cluster/dht/src/tier.c1
4 files changed, 97 insertions, 30 deletions
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index b63d4e18882..cf300385abd 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -550,6 +550,9 @@ struct gf_defrag_info_ {
/* lock migration flag */
gf_boolean_t lock_migration_enabled;
+ /* backpointer to make it easier to write functions for rebalance */
+ xlator_t *this;
+
};
typedef struct gf_defrag_info_ gf_defrag_info_t;
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 17d3bd95ee0..7bbd5956028 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -24,6 +24,7 @@
#define MAX_MIGRATE_QUEUE_COUNT 500
#define MIN_MIGRATE_QUEUE_COUNT 200
#define MAX_REBAL_TYPE_SIZE 16
+#define FILE_CNT_INTERVAL 600 /* 10 mins */
#ifndef MAX
#define MAX(a, b) (((a) > (b))?(a):(b))
@@ -3974,6 +3975,9 @@ gf_tier_wait_fix_lookup (gf_defrag_info_t *defrag) {
/******************Tier background Fix layout functions END********************/
+
+
+
uint64_t gf_defrag_subvol_file_cnt (xlator_t *this, loc_t *root_loc)
{
int ret = -1;
@@ -3991,23 +3995,23 @@ uint64_t gf_defrag_subvol_file_cnt (xlator_t *this, loc_t *root_loc)
}
-int gf_defrag_total_file_cnt (xlator_t *this, loc_t *root_loc)
+uint64_t
+gf_defrag_total_file_cnt (xlator_t *this, loc_t *root_loc)
{
dht_conf_t *conf = NULL;
- int ret = -1;
int i = 0;
uint64_t num_files = 0;
-
+ uint64_t total_entries = 0;
conf = this->private;
if (!conf) {
- return ret;
+ return 0;
}
for (i = 0 ; i < conf->local_subvols_cnt; i++) {
num_files = gf_defrag_subvol_file_cnt (conf->local_subvols[i],
root_loc);
- g_totalfiles += num_files;
+ total_entries += num_files;
gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvol: %s,"
"cnt = %"PRIu64, conf->local_subvols[i]->name,
num_files);
@@ -4016,14 +4020,14 @@ int gf_defrag_total_file_cnt (xlator_t *this, loc_t *root_loc)
/* FIXFIXFIX: halve the number of files to negate .glusterfs contents
We need a better way to figure this out */
- g_totalfiles = g_totalfiles/2;
- if (g_totalfiles > 20000)
- g_totalfiles += 10000;
+ total_entries = total_entries/2;
+ if (total_entries > 20000)
+ total_entries += 10000;
gf_msg (this->name, GF_LOG_INFO, 0, 0,
- "Total number of files = %"PRIu64, g_totalfiles);
+ "Total number of files = %"PRIu64, total_entries);
- return 0;
+ return total_entries;
}
@@ -4053,6 +4057,39 @@ out:
return ret;
}
+static void*
+dht_file_counter_thread (void *args)
+{
+ gf_defrag_info_t *defrag = NULL;
+ loc_t root_loc = {0,};
+
+ if (!args)
+ return NULL;
+
+ defrag = (gf_defrag_info_t *) args;
+ dht_build_root_loc (defrag->root_inode, &root_loc);
+
+ while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+
+ sleep (FILE_CNT_INTERVAL);
+ g_totalfiles = gf_defrag_total_file_cnt (defrag->this,
+ &root_loc);
+
+ if (!g_totalfiles) {
+ gf_msg ("dht", GF_LOG_ERROR, 0, 0, "Failed to get "
+ "the total number of files. Unable to estimate "
+ "time to complete rebalance.");
+ } else {
+ gf_msg_debug ("dht", 0,
+ "total number of files =%"PRIu64,
+ g_totalfiles);
+ }
+ }
+
+ return NULL;
+}
+
+
int
gf_defrag_start_crawl (void *data)
@@ -4071,11 +4108,12 @@ gf_defrag_start_crawl (void *data)
glusterfs_ctx_t *ctx = NULL;
dht_methods_t *methods = NULL;
int i = 0;
- int thread_index = 0;
- int err = 0;
- int thread_spawn_count = 0;
+ int thread_index = 0;
+ int err = 0;
+ int thread_spawn_count = 0;
pthread_t *tid = NULL;
- gf_boolean_t is_tier_detach = _gf_false;
+ pthread_t filecnt_thread;
+ gf_boolean_t is_tier_detach = _gf_false;
call_frame_t *statfs_frame = NULL;
xlator_t *old_THIS = NULL;
int j = 0;
@@ -4223,13 +4261,23 @@ gf_defrag_start_crawl (void *data)
}
}
- ret = gf_defrag_total_file_cnt (this, &loc);
- if (ret) {
+ g_totalfiles = gf_defrag_total_file_cnt (this, &loc);
+ if (!g_totalfiles) {
gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Failed to get "
"the total number of files. Unable to estimate "
"time to complete rebalance.");
}
+ ret = gf_thread_create_detached (&filecnt_thread,
+ &dht_file_counter_thread,
+ (void *)defrag);
+
+ if (ret) {
+ gf_msg (this->name, GF_LOG_ERROR, ret, 0, "Failed to "
+ "create the file counter thread ");
+ ret = 0;
+ }
+
/* Initialize global entry queue */
defrag->queue = GF_CALLOC (1, sizeof (struct dht_container),
gf_dht_mt_container_t);
@@ -4348,6 +4396,8 @@ out:
pthread_join (tid[i], NULL);
}
+
+
GF_FREE (tid);
if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
@@ -4465,13 +4515,16 @@ uint64_t
gf_defrag_get_estimates (dht_conf_t *conf)
{
gf_defrag_info_t *defrag = NULL;
- double rate_lookedup = 0;
- uint64_t dirs_processed = 0;
- uint64_t total_processed = 0;
- uint64_t tmp_count = 0;
- uint64_t time_to_complete = 0;
- struct timeval end = {0,};
- double elapsed = 0;
+ loc_t loc = {0,};
+ double rate_lookedup = 0;
+ uint64_t dirs_processed = 0;
+ uint64_t files_processed = 0;
+ uint64_t total_processed = 0;
+ uint64_t tmp_count = 0;
+ uint64_t time_to_complete = 0;
+ struct timeval end = {0,};
+ double elapsed = 0;
+
defrag = conf->defrag;
@@ -4488,26 +4541,34 @@ gf_defrag_get_estimates (dht_conf_t *conf)
*/
dirs_processed = defrag->num_dirs_processed;
+ files_processed = defrag->num_files_lookedup;
- total_processed = defrag->num_files_lookedup
- + dirs_processed;
+ total_processed = files_processed + dirs_processed;
+
+ if (total_processed > g_totalfiles) {
+ /* lookup the number of files again
+ * The problem here is that not all the newly added files
+ * might need to be processed. So this need not work
+ * in some cases
+ */
+ dht_build_root_loc (defrag->root_inode, &loc);
+ g_totalfiles = gf_defrag_total_file_cnt (defrag->this, &loc);
+ if (!g_totalfiles)
+ goto out;
+ }
/* rate at which files looked up */
rate_lookedup = (total_processed)/elapsed;
-
/* We initially sum up dirs across all local subvols because we get the
* file count from the inodes on each subvol.
* The same directories will be counted for each subvol but
- * we want that they are only counted once.
+ * we want them to be counted once.
*/
tmp_count = g_totalfiles
- (dirs_processed * (conf->local_subvols_cnt - 1));
- if (total_processed > g_totalfiles)
- g_totalfiles = total_processed + 10000;
-
if (rate_lookedup) {
time_to_complete = (tmp_count)/rate_lookedup;
@@ -4522,6 +4583,7 @@ gf_defrag_get_estimates (dht_conf_t *conf)
"rate_lookedup=%f", total_processed, tmp_count,
rate_lookedup);
+out:
return time_to_complete;
}
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
index d947ed401ec..0814db3fe18 100644
--- a/xlators/cluster/dht/src/dht-shared.c
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -705,6 +705,7 @@ dht_init (xlator_t *this)
defrag->is_exiting = 0;
conf->defrag = defrag;
+ defrag->this = this;
ret = dict_get_str (this->options, "node-uuid", &node_uuid);
if (ret) {
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index e4b910eb0e6..01b6ada3677 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -2958,6 +2958,7 @@ tier_init (xlator_t *this)
defrag->tier_conf.last_promote_qfile_index = 0;
defrag->tier_conf.is_tier = 1;
+ defrag->this = this;
ret = dict_get_int32 (this->options,
"tier-max-promote-file-size", &maxsize);