summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKaushal M <kaushal@redhat.com>2013-12-10 11:34:06 +0530
committerVijay Bellur <vbellur@redhat.com>2013-12-16 05:02:11 -0800
commitf502e28e8b416f80bd9506ac204948681610b305 (patch)
tree71745aac6184a34818d184822c50b819c9d91a99
parent7f70a9d2b2a0c3141ccdabb79401d39c871e47a9 (diff)
glusterd: Save/restore/sync rebalance dict
A dictionary was added to store additional information of a rebalance process, like the bricks being removed in case of a rebalance started by remove-brick. This dictionary wasn't being stored/restored or synced during volume sync, leading to errors like a volume status command failing. These issues have been fixed in this patch. The rebalance dict is now stored/restored and also exported/imported during volume sync. Also, this makes sure that the rebalance dict is only create on remove-brick start. This adds a bricks decommissioned status to the information imported/exported during volume sync. Change-Id: I56fed23dc2de80a96648055fe705e9c3ffd55227 BUG: 1040809 Signed-off-by: Kaushal M <kaushal@redhat.com> Reviewed-on: http://review.gluster.org/6492 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com> Reviewed-by: Vijay Bellur <vbellur@redhat.com>
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-brick-ops.c60
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-store.c100
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c125
3 files changed, 208 insertions, 77 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index d832cdf2429..26d608a2f4c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1842,19 +1842,22 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
goto out;
}
- /* Save the list of bricks for later usage. Right now this is required
- * for displaying the task parameters with task status in volume status.
+ /* Save the list of bricks for later usage only on starting a
+ * remove-brick. Right now this is required for displaying the task
+ * parameters with task status in volume status.
*/
- bricks_dict = dict_new ();
- if (!bricks_dict) {
- ret = -1;
- goto out;
- }
- ret = dict_set_int32 (bricks_dict, "count", count);
- if (ret) {
- gf_log (this->name, GF_LOG_ERROR,
- "Failed to save remove-brick count");
- goto out;
+ if (GF_OP_CMD_START == cmd) {
+ bricks_dict = dict_new ();
+ if (!bricks_dict) {
+ ret = -1;
+ goto out;
+ }
+ ret = dict_set_int32 (bricks_dict, "count", count);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to save remove-brick count");
+ goto out;
+ }
}
while ( i <= count) {
snprintf (key, 256, "brick%d", i);
@@ -1865,20 +1868,22 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
goto out;
}
- brick_tmpstr = gf_strdup (brick);
- if (!brick_tmpstr) {
- ret = -1;
- gf_log (this->name, GF_LOG_ERROR,
- "Failed to duplicate brick name");
- goto out;
- }
- ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
- if (ret) {
- gf_log (this->name, GF_LOG_ERROR,
- "Failed to add brick to dict");
- goto out;
+ if (GF_OP_CMD_START == cmd) {
+ brick_tmpstr = gf_strdup (brick);
+ if (!brick_tmpstr) {
+ ret = -1;
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to duplicate brick name");
+ goto out;
+ }
+ ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to add brick to dict");
+ goto out;
+ }
+ brick_tmpstr = NULL;
}
- brick_tmpstr = NULL;
ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
&need_rebalance);
@@ -1886,6 +1891,9 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
goto out;
i++;
}
+ if (GF_OP_CMD_START == cmd)
+ volinfo->rebal.dict = dict_ref (bricks_dict);
+
ret = dict_get_int32 (dict, "replica-count", &replica_count);
if (!ret) {
gf_log (this->name, GF_LOG_INFO,
@@ -1910,8 +1918,6 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
}
}
}
- volinfo->rebal.dict = bricks_dict;
- bricks_dict = NULL;
ret = glusterd_create_volfiles_and_notify_services (volinfo);
if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 5b51aabaddb..2ba2548d20d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -883,6 +883,19 @@ out:
return ret;
}
+int
+_gd_store_rebalance_dict (dict_t *dict, char *key, data_t *value, void *data)
+{
+ int ret = -1;
+ int fd = 0;
+
+ fd = *(int *)data;
+
+ ret = gf_store_save_value (fd, key, value->data);
+
+ return ret;
+}
+
int32_t
glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
{
@@ -907,9 +920,14 @@ glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
if (ret)
goto out;
- if (volinfo->rebal.defrag_cmd) {
- uuid_unparse (volinfo->rebal.rebalance_id, buf);
- ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+ uuid_unparse (volinfo->rebal.rebalance_id, buf);
+ ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+ if (ret)
+ goto out;
+
+ if (volinfo->rebal.dict) {
+ dict_foreach (volinfo->rebal.dict, _gd_store_rebalance_dict,
+ &fd);
}
out:
gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -1726,17 +1744,22 @@ out:
int32_t
glusterd_store_retrieve_node_state (char *volname)
{
- int32_t ret = -1;
- glusterd_volinfo_t *volinfo = NULL;
- gf_store_iter_t *iter = NULL;
- char *key = NULL;
- char *value = NULL;
- char volpath[PATH_MAX] = {0,};
- glusterd_conf_t *priv = NULL;
- char path[PATH_MAX] = {0,};
- gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+ int32_t ret = -1;
+ glusterd_volinfo_t *volinfo = NULL;
+ gf_store_iter_t *iter = NULL;
+ char *key = NULL;
+ char *value = NULL;
+ char *dup_value = NULL;
+ char volpath[PATH_MAX] = {0,};
+ glusterd_conf_t *priv = NULL;
+ char path[PATH_MAX] = {0,};
+ gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+ dict_t *tmp_dict = NULL;
+ xlator_t *this = NULL;
- priv = THIS->private;
+ this = THIS;
+ GF_ASSERT (this);
+ priv = this->private;
ret = glusterd_volinfo_find (volname, &volinfo);
if (ret) {
@@ -1766,16 +1789,35 @@ glusterd_store_retrieve_node_state (char *volname)
if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
volinfo->rebal.defrag_cmd = atoi (value);
- }
-
- if (volinfo->rebal.defrag_cmd) {
- if (!strncmp (key, GF_REBALANCE_TID_KEY,
- strlen (GF_REBALANCE_TID_KEY)))
- uuid_parse (value, volinfo->rebal.rebalance_id);
-
- if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
- strlen (GLUSTERD_STORE_KEY_DEFRAG_OP)))
- volinfo->rebal.op = atoi (value);
+ } else if (!strncmp (key, GF_REBALANCE_TID_KEY,
+ strlen (GF_REBALANCE_TID_KEY))) {
+ uuid_parse (value, volinfo->rebal.rebalance_id);
+ } else if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+ strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) {
+ volinfo->rebal.op = atoi (value);
+ } else {
+ if (!tmp_dict) {
+ tmp_dict = dict_new ();
+ if (!tmp_dict) {
+ ret = -1;
+ goto out;
+ }
+ }
+ dup_value = gf_strdup (value);
+ if (!dup_value) {
+ ret = -1;
+ gf_log (this->name, GF_LOG_ERROR,
+ "Failed to strdup value string");
+ goto out;
+ }
+ ret = dict_set_str (tmp_dict, key, dup_value);
+ if (ret) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Error setting data in rebal "
+ "dict.");
+ goto out;
+ }
+ dup_value = NULL;
}
GF_FREE (key);
@@ -1785,9 +1827,13 @@ glusterd_store_retrieve_node_state (char *volname)
ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
}
+ if (tmp_dict)
+ volinfo->rebal.dict = dict_ref (tmp_dict);
- if (op_errno != GD_STORE_EOF)
+ if (op_errno != GD_STORE_EOF) {
+ ret = -1;
goto out;
+ }
ret = gf_store_iter_destroy (iter);
@@ -1795,6 +1841,12 @@ glusterd_store_retrieve_node_state (char *volname)
goto out;
out:
+ if (dup_value)
+ GF_FREE (dup_value);
+ if (ret && volinfo->rebal.dict)
+ dict_unref (volinfo->rebal.dict);
+ if (tmp_dict)
+ dict_unref (tmp_dict);
gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 8cf9f790f2f..f398e49dc62 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1956,20 +1956,18 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
if (ret)
goto out;
- if (volinfo->rebal.defrag_cmd) {
- rebalance_id_str = gf_strdup (uuid_utoa
- (volinfo->rebal.rebalance_id));
- if (!rebalance_id_str) {
- ret = -1;
- goto out;
- }
- memset (key, 0, sizeof (key));
- snprintf (key, 256, "volume%d.rebalance-id", count);
- ret = dict_set_dynstr (dict, key, rebalance_id_str);
- if (ret)
- goto out;
- rebalance_id_str = NULL;
+ rebalance_id_str = gf_strdup (uuid_utoa
+ (volinfo->rebal.rebalance_id));
+ if (!rebalance_id_str) {
+ ret = -1;
+ goto out;
}
+ memset (key, 0, sizeof (key));
+ snprintf (key, 256, "volume%d.rebalance-id", count);
+ ret = dict_set_dynstr (dict, key, rebalance_id_str);
+ if (ret)
+ goto out;
+ rebalance_id_str = NULL;
memset (key, 0, sizeof (key));
snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
@@ -1977,6 +1975,23 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
if (ret)
goto out;
+ if (volinfo->rebal.dict) {
+ snprintf (prefix, sizeof (prefix), "volume%d", count);
+ ctx.dict = dict;
+ ctx.prefix = prefix;
+ ctx.opt_count = 1;
+ ctx.key_name = "rebal-dict-key";
+ ctx.val_name = "rebal-dict-value";
+
+ dict_foreach (volinfo->rebal.dict, _add_dict_to_prdict, &ctx);
+ ctx.opt_count--;
+ memset (key, 0, sizeof (key));
+ snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
+ ret = dict_set_int32 (dict, key, ctx.opt_count);
+ if (ret)
+ goto out;
+ }
+
memset (key, 0, sizeof (key));
snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status);
@@ -2066,6 +2081,13 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
if (ret)
goto out;
+ memset (key, 0, sizeof (key));
+ snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned",
+ count, i);
+ ret = dict_set_int32 (dict, key, brickinfo->decommissioned);
+ if (ret)
+ goto out;
+
i++;
}
@@ -2733,6 +2755,7 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
int ret = -1;
char *hostname = NULL;
char *path = NULL;
+ int decommissioned = 0;
glusterd_brickinfo_t *new_brickinfo = NULL;
char msg[2048] = {0};
@@ -2758,12 +2781,22 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
goto out;
}
+ memset (key, 0, sizeof (key));
+ snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned",
+ vol_count, brick_count);
+ ret = dict_get_int32 (vols, key, &decommissioned);
+ if (ret) {
+ /* For backward compatibility */
+ ret = 0;
+ }
+
ret = glusterd_brickinfo_new (&new_brickinfo);
if (ret)
goto out;
strcpy (new_brickinfo->path, path);
strcpy (new_brickinfo->hostname, hostname);
+ new_brickinfo->decommissioned = decommissioned;
//peerinfo might not be added yet
(void) glusterd_resolve_brick (new_brickinfo);
ret = 0;
@@ -2908,6 +2941,43 @@ out:
return ret;
}
+int
+gd_import_friend_volume_rebal_dict (dict_t *dict, int count,
+ glusterd_volinfo_t *volinfo)
+{
+ int ret = -1;
+ char key[256] = {0,};
+ int dict_count = 0;
+ char prefix[64] = {0};
+
+ GF_ASSERT (dict);
+ GF_ASSERT (volinfo);
+
+ memset (key, 0, sizeof (key));
+ snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
+ ret = dict_get_int32 (dict, key, &dict_count);
+ if (ret) {
+ /* Older peers will not have this dict */
+ ret = 0;
+ goto out;
+ }
+
+ volinfo->rebal.dict = dict_new ();
+ if(!volinfo->rebal.dict) {
+ ret = -1;
+ goto out;
+ }
+
+ snprintf (prefix, sizeof (prefix), "volume%d", count);
+ ret = import_prdict_dict (dict, volinfo->rebal.dict, "rebal-dict-key",
+ "rebal-dict-value", dict_count, prefix);
+out:
+ if (ret && volinfo->rebal.dict)
+ dict_unref (volinfo->rebal.dict);
+ gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+ return ret;
+}
+
int32_t
glusterd_import_volinfo (dict_t *vols, int count,
glusterd_volinfo_t **volinfo)
@@ -3071,19 +3141,16 @@ glusterd_import_volinfo (dict_t *vols, int count,
goto out;
}
- if (new_volinfo->rebal.defrag_cmd) {
- memset (key, 0, sizeof (key));
- snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
- ret = dict_get_str (vols, key, &rebalance_id_str);
- if (ret) {
- /* This is not present in older glusterfs versions,
- * so don't error out
- */
- ret = 0;
- } else {
- uuid_parse (rebalance_id_str,
- new_volinfo->rebal.rebalance_id);
- }
+ memset (key, 0, sizeof (key));
+ snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
+ ret = dict_get_str (vols, key, &rebalance_id_str);
+ if (ret) {
+ /* This is not present in older glusterfs versions,
+ * so don't error out
+ */
+ ret = 0;
+ } else {
+ uuid_parse (rebalance_id_str, new_volinfo->rebal.rebalance_id);
}
memset (key, 0, sizeof (key));
@@ -3095,6 +3162,12 @@ glusterd_import_volinfo (dict_t *vols, int count,
*/
ret = 0;
}
+ ret = gd_import_friend_volume_rebal_dict (vols, count, new_volinfo);
+ if (ret) {
+ snprintf (msg, sizeof (msg), "Failed to import rebalance dict "
+ "for volume.");
+ goto out;
+ }
memset (key, 0, sizeof (key));
snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);