glusterd: Save/restore/sync rebalance dict

        Backport of http://review.gluster.org/6492

A dictionary was added to store additional information of a rebalance
process, like the bricks being removed in case of a rebalance started
by remove-brick. This dictionary wasn't being stored/restored or synced
during volume sync, leading to errors like a volume status command
failing. These issues have been fixed in this patch. The rebalance dict
is now stored/restored and also exported/imported during volume sync.

Also, this makes sure that the rebalance dict is only create on
remove-brick start. This adds a bricks decommissioned status to the
information imported/exported during volume sync.

Change-Id: I56fed23dc2de80a96648055fe705e9c3ffd55227
BUG: 1040809
Signed-off-by: Kaushal M <kaushal@redhat.com>
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/6565
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 208 insertions, 77 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index df6087659..ae17e68ed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1831,19 +1831,22 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        /* Save the list of bricks for later usage. Right now this is required
-         * for displaying the task parameters with task status in volume status.
+        /* Save the list of bricks for later usage only on starting a
+         * remove-brick. Right now this is required for displaying the task
+         * parameters with task status in volume status.
          */
-        bricks_dict = dict_new ();
-        if (!bricks_dict) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_int32 (bricks_dict, "count", count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to save remove-brick count");
-                goto out;
+        if (GF_OP_CMD_START == cmd) {
+                bricks_dict = dict_new ();
+                if (!bricks_dict) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_int32 (bricks_dict, "count", count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to save remove-brick count");
+                        goto out;
+                }
         }
         while ( i <= count) {
                 snprintf (key, 256, "brick%d", i);
@@ -1854,20 +1857,22 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
-                brick_tmpstr = gf_strdup (brick);
-                if (!brick_tmpstr) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to duplicate brick name");
-                        goto out;
-                }
-                ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add brick to dict");
-                        goto out;
+                if (GF_OP_CMD_START == cmd) {
+                        brick_tmpstr = gf_strdup (brick);
+                        if (!brick_tmpstr) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to duplicate brick name");
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add brick to dict");
+                                goto out;
+                        }
+                        brick_tmpstr = NULL;
                 }
-                brick_tmpstr = NULL;
 
                 ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
                                                         &need_rebalance);
@@ -1875,6 +1880,9 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 i++;
         }
+        if (GF_OP_CMD_START == cmd)
+                volinfo->rebal.dict = dict_ref (bricks_dict);
+
         ret = dict_get_int32 (dict, "replica-count", &replica_count);
         if (!ret) {
                 gf_log (this->name, GF_LOG_INFO,
@@ -1899,8 +1907,6 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         }
                 }
         }
-        volinfo->rebal.dict = bricks_dict;
-        bricks_dict = NULL;
 
         ret = glusterd_create_volfiles_and_notify_services (volinfo);
         if (ret) {
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 5b51aabad..2ba2548d2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -883,6 +883,19 @@ out:
         return ret;
 }
 
+int
+_gd_store_rebalance_dict (dict_t *dict, char *key, data_t *value, void *data)
+{
+        int             ret = -1;
+        int             fd = 0;
+
+        fd = *(int *)data;
+
+        ret = gf_store_save_value (fd, key, value->data);
+
+        return ret;
+}
+
 int32_t
 glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
 {
@@ -907,9 +920,14 @@ glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
         if (ret)
                 goto out;
 
-        if (volinfo->rebal.defrag_cmd) {
-                uuid_unparse (volinfo->rebal.rebalance_id, buf);
-                ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+        uuid_unparse (volinfo->rebal.rebalance_id, buf);
+        ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rebal.dict) {
+                dict_foreach (volinfo->rebal.dict, _gd_store_rebalance_dict,
+                              &fd);
         }
 out:
         gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
@@ -1726,17 +1744,22 @@ out:
 int32_t
 glusterd_store_retrieve_node_state (char   *volname)
 {
-        int32_t                   ret                   = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
-        gf_store_iter_t           *iter                 = NULL;
-        char                      *key                  = NULL;
-        char                      *value                = NULL;
-        char                      volpath[PATH_MAX]     = {0,};
-        glusterd_conf_t           *priv                 = NULL;
-        char                      path[PATH_MAX]        = {0,};
-        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        int32_t              ret               = -1;
+        glusterd_volinfo_t  *volinfo           = NULL;
+        gf_store_iter_t     *iter              = NULL;
+        char                *key               = NULL;
+        char                *value             = NULL;
+        char                *dup_value         = NULL;
+        char                 volpath[PATH_MAX] = {0,};
+        glusterd_conf_t     *priv              = NULL;
+        char                 path[PATH_MAX]    = {0,};
+        gf_store_op_errno_t  op_errno          = GD_STORE_SUCCESS;
+        dict_t              *tmp_dict          = NULL;
+        xlator_t            *this              = NULL;
 
-        priv = THIS->private;
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
 
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
@@ -1766,16 +1789,35 @@ glusterd_store_retrieve_node_state (char   *volname)
                 if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
                               strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
                         volinfo->rebal.defrag_cmd = atoi (value);
-                }
-
-                if (volinfo->rebal.defrag_cmd) {
-                        if (!strncmp (key, GF_REBALANCE_TID_KEY,
-                                       strlen (GF_REBALANCE_TID_KEY)))
-                                uuid_parse (value, volinfo->rebal.rebalance_id);
-
-                        if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
-                                      strlen (GLUSTERD_STORE_KEY_DEFRAG_OP)))
-                                volinfo->rebal.op = atoi (value);
+                } else if (!strncmp (key, GF_REBALANCE_TID_KEY,
+                                     strlen (GF_REBALANCE_TID_KEY))) {
+                        uuid_parse (value, volinfo->rebal.rebalance_id);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                                        strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) {
+                        volinfo->rebal.op = atoi (value);
+                } else {
+                        if (!tmp_dict) {
+                                tmp_dict = dict_new ();
+                                if (!tmp_dict) {
+                                        ret = -1;
+                                        goto out;
+                                }
+                        }
+                        dup_value = gf_strdup (value);
+                        if (!dup_value) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to strdup value string");
+                                goto out;
+                        }
+                        ret = dict_set_str (tmp_dict, key, dup_value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                                "Error setting data in rebal "
+                                                "dict.");
+                                goto out;
+                        }
+                        dup_value = NULL;
                 }
 
                 GF_FREE (key);
@@ -1785,9 +1827,13 @@ glusterd_store_retrieve_node_state (char   *volname)
 
                 ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
         }
+        if (tmp_dict)
+                volinfo->rebal.dict = dict_ref (tmp_dict);
 
-        if (op_errno != GD_STORE_EOF)
+        if (op_errno != GD_STORE_EOF) {
+                ret = -1;
                 goto out;
+        }
 
         ret = gf_store_iter_destroy (iter);
 
@@ -1795,6 +1841,12 @@ glusterd_store_retrieve_node_state (char   *volname)
                 goto out;
 
 out:
+        if (dup_value)
+                GF_FREE (dup_value);
+        if (ret && volinfo->rebal.dict)
+                dict_unref (volinfo->rebal.dict);
+        if (tmp_dict)
+                dict_unref (tmp_dict);
         gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
 
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 15c40f3e4..03ef2ef3b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1956,20 +1956,18 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
-        if (volinfo->rebal.defrag_cmd) {
-                rebalance_id_str = gf_strdup (uuid_utoa
-                                                (volinfo->rebal.rebalance_id));
-                if (!rebalance_id_str) {
-                        ret = -1;
-                        goto out;
-                }
-                memset (key, 0, sizeof (key));
-                snprintf (key, 256, "volume%d.rebalance-id", count);
-                ret = dict_set_dynstr (dict, key, rebalance_id_str);
-                if (ret)
-                        goto out;
-                rebalance_id_str = NULL;
+        rebalance_id_str = gf_strdup (uuid_utoa
+                        (volinfo->rebal.rebalance_id));
+        if (!rebalance_id_str) {
+                ret = -1;
+                goto out;
         }
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d.rebalance-id", count);
+        ret = dict_set_dynstr (dict, key, rebalance_id_str);
+        if (ret)
+                goto out;
+        rebalance_id_str = NULL;
 
         memset (key, 0, sizeof (key));
         snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
@@ -1977,6 +1975,23 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
         if (ret)
                 goto out;
 
+        if (volinfo->rebal.dict) {
+                snprintf (prefix, sizeof (prefix), "volume%d", count);
+                ctx.dict = dict;
+                ctx.prefix = prefix;
+                ctx.opt_count = 1;
+                ctx.key_name = "rebal-dict-key";
+                ctx.val_name = "rebal-dict-value";
+
+                dict_foreach (volinfo->rebal.dict, _add_dict_to_prdict, &ctx);
+                ctx.opt_count--;
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
+                ret = dict_set_int32 (dict, key, ctx.opt_count);
+                if (ret)
+                        goto out;
+        }
+
         memset (key, 0, sizeof (key));
         snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
         ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status);
@@ -2066,6 +2081,13 @@ glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
                 if (ret)
                         goto out;
 
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned",
+                          count, i);
+                ret = dict_set_int32 (dict, key, brickinfo->decommissioned);
+                if (ret)
+                        goto out;
+
                 i++;
         }
 
@@ -2733,6 +2755,7 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
         int                     ret = -1;
         char                    *hostname = NULL;
         char                    *path = NULL;
+        int                     decommissioned = 0;
         glusterd_brickinfo_t    *new_brickinfo = NULL;
         char                    msg[2048] = {0};
 
@@ -2758,12 +2781,22 @@ glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
                 goto out;
         }
 
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick%d.decommissioned",
+                  vol_count, brick_count);
+        ret = dict_get_int32 (vols, key, &decommissioned);
+        if (ret) {
+                /* For backward compatibility */
+                ret = 0;
+        }
+
         ret = glusterd_brickinfo_new (&new_brickinfo);
         if (ret)
                 goto out;
 
         strcpy (new_brickinfo->path, path);
         strcpy (new_brickinfo->hostname, hostname);
+        new_brickinfo->decommissioned = decommissioned;
         //peerinfo might not be added yet
         (void) glusterd_resolve_brick (new_brickinfo);
         ret = 0;
@@ -2908,6 +2941,43 @@ out:
         return ret;
 }
 
+int
+gd_import_friend_volume_rebal_dict (dict_t *dict, int count,
+                                    glusterd_volinfo_t *volinfo)
+{
+        int  ret        = -1;
+        char key[256]   = {0,};
+        int  dict_count  = 0;
+        char prefix[64] = {0};
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
+        ret = dict_get_int32 (dict, key, &dict_count);
+        if (ret) {
+                /* Older peers will not have this dict */
+                ret = 0;
+                goto out;
+        }
+
+        volinfo->rebal.dict = dict_new ();
+        if(!volinfo->rebal.dict) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (prefix, sizeof (prefix), "volume%d", count);
+        ret = import_prdict_dict (dict, volinfo->rebal.dict, "rebal-dict-key",
+                                  "rebal-dict-value", dict_count, prefix);
+out:
+        if (ret && volinfo->rebal.dict)
+                dict_unref (volinfo->rebal.dict);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
 int32_t
 glusterd_import_volinfo (dict_t *vols, int count,
                          glusterd_volinfo_t **volinfo)
@@ -3071,19 +3141,16 @@ glusterd_import_volinfo (dict_t *vols, int count,
                 goto out;
         }
 
-        if (new_volinfo->rebal.defrag_cmd) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
-                ret = dict_get_str (vols, key, &rebalance_id_str);
-                if (ret) {
-                        /* This is not present in older glusterfs versions,
-                         * so don't error out
-                         */
-                        ret = 0;
-                } else {
-                        uuid_parse (rebalance_id_str,
-                                    new_volinfo->rebal.rebalance_id);
-                }
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
+        ret = dict_get_str (vols, key, &rebalance_id_str);
+        if (ret) {
+                /* This is not present in older glusterfs versions,
+                 * so don't error out
+                 */
+                ret = 0;
+        } else {
+                uuid_parse (rebalance_id_str, new_volinfo->rebal.rebalance_id);
         }
 
         memset (key, 0, sizeof (key));
@@ -3095,6 +3162,12 @@ glusterd_import_volinfo (dict_t *vols, int count,
                  */
                 ret = 0;
         }
+        ret = gd_import_friend_volume_rebal_dict (vols, count, new_volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to import rebalance dict "
+                          "for volume.");
+                goto out;
+        }
 
         memset (key, 0, sizeof (key));
         snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);