From bcc34ce05c1be76dae42838d55c15d3af5f80e48 Mon Sep 17 00:00:00 2001 From: Xavier Hernandez Date: Fri, 12 May 2017 09:23:47 +0200 Subject: cluster/ec: return all node uuids from all subvolumes EC was retuning the UUID of the brick with smaller value. This had the side effect of not evenly balancing the load between bricks on rebalance operations. This patch modifies the common functions that combine multiple subvolume values into a single result to take into account the subvolume order and, optionally, other subvolumes that could be damaged. This makes easier to add future features where brick order is important. It also makes possible to easily identify the originating brick of each answer, in case some brick will have an special meaning in the future. Change-Id: Iee0a4da710b41224a6dc8e13fa8dcddb36c73a2f BUG: 1366817 Signed-off-by: Xavier Hernandez Reviewed-on: https://review.gluster.org/17297 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Ashish Pandey Reviewed-by: Pranith Kumar Karampuri --- xlators/cluster/ec/src/ec-combine.c | 241 ++++++++++++++++++++---------------- xlators/cluster/ec/src/ec.c | 5 +- 2 files changed, 141 insertions(+), 105 deletions(-) diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c index f949dbd0c9f..e09381b9f8a 100644 --- a/xlators/cluster/ec/src/ec-combine.c +++ b/xlators/cluster/ec/src/ec-combine.c @@ -22,6 +22,8 @@ #define EC_QUOTA_PREFIX "trusted.glusterfs.quota." +#define EC_MISSING_DATA ((data_t *)1ULL) + struct _ec_dict_info; typedef struct _ec_dict_info ec_dict_info_t; @@ -270,35 +272,45 @@ ec_dict_compare (dict_t *dict1, dict_t *dict2) return 0; } -int32_t ec_dict_list(data_t ** list, int32_t * count, ec_cbk_data_t * cbk, - int32_t which, char * key) +static uint32_t +ec_dict_list(data_t **list, ec_cbk_data_t *cbk, int32_t which, char *key, + gf_boolean_t global) { - ec_cbk_data_t *ans = NULL; - dict_t *dict = NULL; - int32_t i, max; - - max = *count; - i = 0; - for (ans = cbk; ans != NULL; ans = ans->next) { - if (i >= max) { - gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, EINVAL, - EC_MSG_INVALID_DICT_NUMS, - "Unexpected number of " - "dictionaries"); - - return -EINVAL; + ec_t *ec = cbk->fop->xl->private; + ec_cbk_data_t *ans = NULL; + dict_t *dict = NULL; + data_t *data; + uint32_t count; + int32_t i; + + for (i = 0; i < ec->nodes; i++) { + /* We initialize the list with EC_MISSING_DATA if we are + * returning a global list or the current subvolume belongs + * to the group of the accepted answer. Note that if some + * subvolume is known to be down before issuing the request, + * we won't have any answer from it, so we set here the + * appropriate default value. */ + if (global || ((cbk->mask & (1ULL << i)) != 0)) { + list[i] = EC_MISSING_DATA; + } else { + list[i] = NULL; + } } - dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict; - list[i] = dict_get(dict, key); - if (list[i] != NULL) { - i++; + count = 0; + list_for_each_entry(ans, &cbk->fop->answer_list, answer_list) { + if (global || ((cbk->mask & ans->mask) != 0)) { + dict = (which == EC_COMBINE_XDATA) ? ans->xdata + : ans->dict; + data = dict_get(dict, key); + if (data != NULL) { + list[ans->idx] = data; + count++; + } + } } - } - - *count = i; - return 0; + return count; } int32_t ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post, @@ -337,23 +349,21 @@ out: return -EINVAL; } -int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk, - int32_t which, char * key, ...) +static int32_t +ec_dict_data_concat(const char *fmt, ec_cbk_data_t *cbk, int32_t which, + char *key, const char *def, gf_boolean_t global, ...) { - data_t * data[cbk->count]; - char * str = NULL, * pre = NULL, * sep, * post; - dict_t * dict; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + char *str = NULL, *pre = NULL, *sep, *post; + dict_t *dict; va_list args; - int32_t i, num, len, prelen, postlen, seplen, tmp; + int32_t i, num, len, deflen, prelen, postlen, seplen, tmp; int32_t err; - num = cbk->count; - err = ec_dict_list(data, &num, cbk, which, key); - if (err != 0) { - return err; - } + ec_dict_list(data, cbk, which, key, global); - va_start(args, key); + va_start(args, global); err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args); va_end(args); @@ -365,9 +375,29 @@ int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk, seplen = strlen(sep); postlen = strlen(post); - len = prelen + (num - 1) * seplen + postlen + 1; - for (i = 0; i < num; i++) { - len += data[i]->len - 1; + deflen = 0; + if (def != NULL) { + deflen = strlen(def); + } + + len = prelen + postlen + 1; + num = -1; + for (i = 0; i < ec->nodes; i++) { + if (data[i] == NULL) { + continue; + } + if (data[i] == EC_MISSING_DATA) { + if (def == NULL) { + continue; + } + len += deflen; + } else { + len += data[i]->len - 1; + } + if (num >= 0) { + len += seplen; + } + num++; } err = -ENOMEM; @@ -379,14 +409,25 @@ int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk, memcpy(str, pre, prelen); len = prelen; - for (i = 0; i < num; i++) { - if (i > 0) { + for (i = 0; i < ec->nodes; i++) { + if (data[i] == NULL) { + continue; + } + if (data[i] == EC_MISSING_DATA) { + if (deflen == 0) { + continue; + } + tmp = deflen; + memcpy(str + len, def, tmp); + } else { + tmp = data[i]->len - 1; + memcpy(str + len, data[i]->data, tmp); + } + len += tmp; + if (i < num) { memcpy(str + len, sep, seplen); len += seplen; } - tmp = data[i]->len - 1; - memcpy(str + len, data[i]->data, tmp); - len += tmp; } memcpy(str + len, post, postlen + 1); @@ -407,30 +448,26 @@ out: int32_t ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t *data[cbk->count]; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; dict_t *dict, *lockinfo, *tmp = NULL; char *ptr = NULL; - int32_t i, num, len; + int32_t i, len; int32_t err; - num = cbk->count; - err = ec_dict_list(data, &num, cbk, which, key); - if (err != 0) { - return err; - } + + ec_dict_list(data, cbk, which, key, _gf_false); lockinfo = dict_new(); if (lockinfo == NULL) { return -ENOMEM; } - err = dict_unserialize(data[0]->data, data[0]->len, &lockinfo); - if (err != 0) { - goto out; - } + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } - for (i = 1; i < num; i++) - { tmp = dict_new(); if (tmp == NULL) { err = -ENOMEM; @@ -517,19 +554,20 @@ int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key) int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t * data[cbk->count]; - dict_t * dict; - int32_t i, num, err; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + int32_t i; uint32_t max, tmp; - num = cbk->count; - err = ec_dict_list(data, &num, cbk, which, key); - if (err != 0) { - return err; - } + ec_dict_list(data, cbk, which, key, _gf_false); + + max = 0; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } - max = data_to_uint32(data[0]); - for (i = 1; i < num; i++) { tmp = data_to_uint32(data[i]); if (max < tmp) { max = tmp; @@ -542,19 +580,20 @@ int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key) int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t *data[cbk->count]; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; dict_t *dict; - int32_t i, num, err; + int32_t i; uint64_t max, tmp; - num = cbk->count; - err = ec_dict_list(data, &num, cbk, which, key); - if (err != 0) { - return err; - } + ec_dict_list(data, cbk, which, key, _gf_false); + + max = 0; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } - max = data_to_uint64(data[0]); - for (i = 1; i < num; i++) { tmp = data_to_uint64(data[i]); if (max < tmp) { max = tmp; @@ -567,22 +606,14 @@ int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key) int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) { - data_t *data[cbk->count]; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; dict_t *dict = NULL; - ec_t *ec = NULL; int32_t i = 0; - int32_t num = 0; - int32_t err = 0; quota_meta_t size = {0, }; quota_meta_t max_size = {0, }; - num = cbk->count; - err = ec_dict_list(data, &num, cbk, which, key); - if (err != 0) { - return err; - } - - if (num == 0) { + if (ec_dict_list(data, cbk, which, key, _gf_false) == 0) { return 0; } @@ -591,8 +622,9 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) * bricks and we can receive slightly different values. If that's the * case, we take the maximum of all received values. */ - for (i = 0; i < num; i++) { - if (quota_data_to_meta (data[i], QUOTA_SIZE_KEY, &size) < 0) { + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA) || + (quota_data_to_meta (data[i], QUOTA_SIZE_KEY, &size) < 0)) { continue; } @@ -604,7 +636,6 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) max_size.dir_count = size.dir_count; } - ec = cbk->fop->xl->private; max_size.size *= ec->fragments; dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; @@ -613,18 +644,18 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key) { - data_t * data[cbk->count]; - dict_t * dict; - int32_t i, num, err; + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + int32_t i, err; - num = cbk->count; - err = ec_dict_list(data, &num, cbk, which, key); - if (err != 0) { - return err; - } + ec_dict_list(data, cbk, which, key, _gf_false); dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; - for (i = 1; i < num; i++) { + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } err = gf_get_max_stime(cbk->fop->xl, dict, key, data[i]); if (err != 0) { gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, -err, @@ -646,12 +677,14 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value, (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) { return ec_dict_data_concat("( { })", data->cbk, data->which, - key, data->cbk->fop->xl->name); + key, NULL, _gf_false, + data->cbk->fop->xl->name); } if (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0) { - return ec_dict_data_concat("{\n}", data->cbk, data->which, key); + return ec_dict_data_concat("{\n}", data->cbk, data->which, key, NULL, + _gf_false); } if (strncmp(key, GF_XATTR_LOCKINFO_KEY, @@ -681,9 +714,9 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value, return 0; } - if (XATTR_IS_NODE_UUID(key)) - { - return ec_dict_data_uuid(data->cbk, data->which, key); + if (XATTR_IS_NODE_UUID(key)) { + return ec_dict_data_concat("{ }", data->cbk, data->which, key, + UUID0_STR, _gf_true); } if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c index e628183190e..2009faccbaf 100644 --- a/xlators/cluster/ec/src/ec.c +++ b/xlators/cluster/ec/src/ec.c @@ -863,8 +863,11 @@ ec_gf_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, NULL, ec_marker_populate_args) == 0) return 0; - if (name && (fnmatch (GF_XATTR_STIME_PATTERN, name, 0) == 0)) + if (name && + ((fnmatch (GF_XATTR_STIME_PATTERN, name, 0) == 0) || + (XATTR_IS_NODE_UUID(name)))) { minimum = EC_MINIMUM_ALL; + } ec_getxattr (frame, this, -1, minimum, default_getxattr_cbk, NULL, loc, name, xdata); -- cgit