Tier: "tier start force" command implementation

The start command doesnt restart the tier deamon if the deamon
is running at one node. hence to bring up the tierd on the nodes
where the deamon is down, the force command is implemented.
It skips the check for tierd running.

Change-Id: I0037d3e5ecfe56637d0da201a97903c435d26436
BUG: 1292112
Signed-off-by: hari gowtham <hgowtham@redhat.com>
Reviewed-on: http://review.gluster.org/12983
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Tested-by: Dan Lambright <dlambrig@redhat.com>

10 files changed, 185 insertions, 59 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index c72f99db194..9254a3e34bd 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1696,9 +1696,9 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount,
 {
         dict_t  *dict    = NULL;
         char    *volname = NULL;
-        char    *word    = NULL;
         int      ret     = -1;
         int32_t  command = GF_OP_CMD_NONE;
+        int32_t  is_force    = 0;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -1708,7 +1708,7 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount,
         if (!dict)
                 goto out;
 
-        if (wordcount != 4) {
+        if (!(wordcount == 4 || wordcount == 5)) {
                 gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
                 ret = -1;
                 goto out;
@@ -1730,11 +1730,28 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount,
                 goto out;
 
         volname = (char *)words[2];
-
-        word = (char *)words[3];
-        if (!strcmp(word, "status"))
-                command = GF_DEFRAG_CMD_STATUS_TIER;
-        else {
+        if (wordcount == 4) {
+                if (!strcmp(words[3], "status"))
+                        command = GF_DEFRAG_CMD_STATUS_TIER;
+                else if (!strcmp(words[3], "start"))
+                        command = GF_DEFRAG_CMD_START_TIER;
+                else {
+                        ret = -1;
+                        goto out;
+                }
+        } else if (wordcount == 5) {
+                if ((!strcmp (words[3], "start")) &&
+                    (!strcmp (words[4], "force"))) {
+                        command = GF_DEFRAG_CMD_START_TIER;
+                        is_force = 1;
+                        ret = dict_set_int32 (dict, "force", is_force);
+                        if (ret)
+                                goto out;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
                 ret = -1;
                 goto out;
         }
@@ -3656,7 +3673,7 @@ cli_cmd_volume_defrag_parse (const char **words, int wordcount,
                     strcmp (words[3], "status"))
                             goto out;
         } else if ((strcmp (words[3], "tier") == 0) &&
-               (strcmp (words[4], "start") == 0)) {
+                   (strcmp (words[4], "start") == 0)) {
                 volname = (char *) words[2];
                 cmd = GF_DEFRAG_CMD_START_TIER;
                 goto done;
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index bc292864cb0..16e1d1b1e98 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -2659,6 +2659,7 @@ struct cli_cmd volume_cmds[] = {
           "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
         { "volume tier <VOLNAME> status\n"
+        "volume tier <VOLNAME> start [force]"
         "volume tier <VOLNAME> attach [<replica COUNT>] <NEW-BRICK>...\n"
         "volume tier <VOLNAME> detach <start|stop|status|commit|[force]>\n",
         cli_cmd_volume_tier_cbk,
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 666d44ff831..6288b01a6af 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1775,7 +1775,6 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                  * case since unlock failures can be highlighted
                                  * event though rebalance command was successful
                                  */
-
                                  if (cmd == GF_DEFRAG_CMD_START_TIER) {
                                          snprintf (msg, sizeof (msg),
                                                   "Attach tier is successful "
diff --git a/tests/basic/tier/tierd_check.t b/tests/basic/tier/tierd_check.t
new file mode 100644
index 00000000000..ee97883688d
--- /dev/null
+++ b/tests/basic/tier/tierd_check.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../tier.rc
+
+
+# Creates a tiered volume with pure distribute hot and cold tiers
+# Both hot and cold tiers will have an equal number of bricks.
+
+function create_dist_tier_vol () {
+        mkdir $B0/cold
+        mkdir $B0/hot
+        TEST $CLI volume create $V0 $H0:$B0/cold/${V0}{1..3}
+        TEST $CLI volume set $V0 performance.quick-read off
+        TEST $CLI volume set $V0 performance.io-cache off
+        TEST $CLI volume start $V0
+        TEST $CLI volume attach-tier $V0 $H0:$B0/hot/${V0}{1..2}
+        TEST $CLI volume set $V0 cluster.tier-mode test
+}
+
+function tier_deamon_check () {
+pgrep -f "rebalance/$V0"
+echo "$?"
+}
+
+function tier_deamon_kill () {
+pkill -f "rebalance/$V0"
+echo "$?"
+}
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume status
+
+
+#Create and start a tiered volume
+create_dist_tier_vol
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_check
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_kill
+
+TEST $CLI volume tier $V0 start
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_kill
+
+TEST $CLI volume tier $V0 start force
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check
+
+TEST $CLI volume tier $V0 start force
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check
+
+cleanup
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 968167d5d5a..4c43c42750e 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -4547,7 +4547,8 @@ dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
         }
 
         if ((conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_TIER) ||
-            (conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) ||
+            (conf->defrag && conf->defrag->cmd ==
+             GF_DEFRAG_CMD_START_DETACH_TIER) ||
             (!(conf->local_subvols_cnt) || !conf->defrag)) {
                 local->call_cnt = conf->subvolume_cnt;
 
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 387c764cb17..cc323c46f0d 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -2997,8 +2997,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                  * CTR DB Lookup Heal triggered on existing
                                  * data.
                                  * */
-                                if (defrag->cmd ==
-                                        GF_DEFRAG_CMD_START_TIER) {
+                                if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
                                         gf_fix_layout_tier_attach_lookup
                                                 (this, loc, entry);
                                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 6acfc18da77..598c0555c24 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -265,7 +265,8 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
                                                    priv);
                 snprintf (logfile, PATH_MAX, "%s/%s-%s-%s.log",
                           DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname,
-                          (cmd == GF_DEFRAG_CMD_START_TIER ? "tier":"rebalance"),
+                          (cmd == GF_DEFRAG_CMD_START_TIER ?
+                           "tier":"rebalance"),
                           uuid_utoa(MY_UUID));
 
         } else {
@@ -600,6 +601,7 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
         char                    *task_id_str = NULL;
         dict_t                  *op_ctx      = NULL;
         xlator_t                *this        = 0;
+        int32_t                 is_force     = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -624,16 +626,20 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
         }
         switch (cmd) {
         case GF_DEFRAG_CMD_START_TIER:
+                ret = dict_get_int32 (dict, "force", &is_force);
+                if (ret)
+                        is_force = 0;
+
                 if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
                         gf_asprintf (op_errstr, "volume %s is not a tier "
                                      "volume.", volinfo->volname);
                         ret = -1;
                         goto out;
                 }
-                if (glusterd_is_tier_daemon_running (volinfo)) {
+                if ((!is_force) && glusterd_is_tier_daemon_running (volinfo)) {
                         ret = gf_asprintf (op_errstr, "A Tier daemon is "
-                                           "already running on volume %s",
-                                           volname);
+                                        "already running on volume %s",
+                                        volname);
                         ret = -1;
                         goto out;
                 }
@@ -792,6 +798,7 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         dict_t                  *ctx = NULL;
         xlator_t                *this = NULL;
         uint32_t                commit_hash;
+        int32_t                 is_force    = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -855,39 +862,79 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
         case GF_DEFRAG_CMD_START_TIER:
-                /* Reset defrag status to 'NOT STARTED' whenever a
-                 * remove-brick/rebalance command is issued to remove
-                 * stale information from previous run.
-                 */
-                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 
-                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Missing rebalance "
-                                "id");
-                        ret = 0;
+
+                ret = dict_get_int32 (dict, "force", &is_force);
+                if (ret)
+                        is_force = 0;
+                if (!is_force) {
+                        /* Reset defrag status to 'NOT STARTED' whenever a
+                         * remove-brick/rebalance command is issued to remove
+                         * stale information from previous run.
+                         */
+                        volinfo->rebal.defrag_status =
+                               GF_DEFRAG_STATUS_NOT_STARTED;
+
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_msg_debug (this->name, 0, "Missing rebalance"
+                                              " id");
+                                ret = 0;
+                        } else {
+                                gf_uuid_parse (task_id_str,
+                                               volinfo->rebal.rebalance_id);
+                                volinfo->rebal.op = GD_OP_REBALANCE;
+                        }
+                        if (!gd_should_i_start_rebalance (volinfo)) {
+                                /* Store the rebalance-id and rebalance command
+                                 * even if the peer isn't starting a rebalance
+                                 * process. On peers where a rebalance process
+                                 * is started, glusterd_handle_defrag_start
+                                 * performs the storing.
+                                 * Storing this is needed for having
+                                 * 'volume status' work correctly.
+                                 */
+                                glusterd_store_perform_node_state_store
+                                        (volinfo);
+                                break;
+                        }
+                        if (dict_get_uint32 (dict, "commit-hash", &commit_hash)
+                                             == 0) {
+                                volinfo->rebal.commit_hash = commit_hash;
+                        }
+                        ret = glusterd_handle_defrag_start (volinfo, msg,
+                                        sizeof (msg),
+                                        cmd, NULL, GD_OP_REBALANCE);
+                        break;
                 } else {
-                        gf_uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
-                        volinfo->rebal.op = GD_OP_REBALANCE;
-                }
-                if (!gd_should_i_start_rebalance (volinfo)) {
-                        /* Store the rebalance-id and rebalance command even if
-                         * the peer isn't starting a rebalance process. On peers
-                         * where a rebalance process is started,
-                         * glusterd_handle_defrag_start performs the storing.
-                         *
-                         * Storing this is needed for having 'volume status'
-                         * work correctly.
+                        /* Reset defrag status to 'STARTED' so that the
+                         * pid is checked and restarted accordingly.
+                         * If the pid is not running it executes the
+                         * "NOT_STARTED" case and restarts the process
                          */
-                        glusterd_store_perform_node_state_store (volinfo);
+                        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+                        volinfo->rebal.defrag_cmd = cmd;
+                        volinfo->rebal.op = GD_OP_REBALANCE;
+
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_msg_debug (this->name, 0, "Missing rebalance"
+                                              " id");
+                                ret = 0;
+                        } else {
+                                gf_uuid_parse (task_id_str,
+                                               volinfo->rebal.rebalance_id);
+                                volinfo->rebal.op = GD_OP_REBALANCE;
+                        }
+                        if (dict_get_uint32 (dict, "commit-hash", &commit_hash)
+                                             == 0) {
+                                volinfo->rebal.commit_hash = commit_hash;
+                        }
+                        ret = glusterd_restart_rebalance_for_volume (volinfo);
                         break;
                 }
-                if (dict_get_uint32 (dict, "commit-hash", &commit_hash) == 0) {
-                        volinfo->rebal.commit_hash = commit_hash;
-                }
-                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
-                                                    cmd, NULL, GD_OP_REBALANCE);
-                break;
         case GF_DEFRAG_CMD_STOP:
         case GF_DEFRAG_CMD_STOP_DETACH_TIER:
                 /* Clear task-id only on explicitly stopping rebalance.
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index c25d301def9..3a02234d691 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -5667,7 +5667,7 @@ glusterd_is_tier_daemon_running (glusterd_volinfo_t *volinfo)
                 return _gf_false;
 
         if (volinfo->rebal.defrag &&
-             volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) {
+            volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) {
                 return _gf_true;
         }
 
@@ -7050,12 +7050,12 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
         case GF_DEFRAG_STATUS_STARTED:
                 GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
                 if (gf_is_service_running (pidfile, &pid)) {
-                        glusterd_rebalance_rpc_create (volinfo, _gf_true);
+                        ret = glusterd_rebalance_rpc_create (volinfo, _gf_true);
                         break;
                 }
         case GF_DEFRAG_STATUS_NOT_STARTED:
-                glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd,
-                                              cbk, volinfo->rebal.op);
+                ret = glusterd_handle_defrag_start (volinfo, op_errstr, len,
+                                cmd, cbk, volinfo->rebal.op);
                 break;
         default:
                 gf_msg (this->name, GF_LOG_ERROR, 0,
@@ -7122,14 +7122,14 @@ out:
 }
 
 
-void
+int
 glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo)
 {
-
+        int             ret = -1;
         char          op_errstr[PATH_MAX];
 
         if (!volinfo->rebal.defrag_cmd)
-                return;
+                return -1;
         if (!gd_should_i_start_rebalance (volinfo)) {
 
                 /* Store the rebalance-id and rebalance command even if
@@ -7143,10 +7143,11 @@ glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo)
                 if (volinfo->type == GF_CLUSTER_TYPE_TIER)
                         glusterd_store_perform_node_state_store (volinfo);
 
-                return;
+                return 0;
         }
-        glusterd_volume_defrag_restart (volinfo, op_errstr, PATH_MAX,
+        ret = glusterd_volume_defrag_restart (volinfo, op_errstr, PATH_MAX,
                                 volinfo->rebal.defrag_cmd, NULL);
+        return ret;
 }
 int
 glusterd_restart_rebalance (glusterd_conf_t *conf)
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 622a6352b22..7195fe30658 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -369,7 +369,7 @@ glusterd_create_sub_tier_volinfo (glusterd_volinfo_t *volinfo,
                                    glusterd_volinfo_t **dup_volinfo,
                                    gf_boolean_t is_hot_tier,
                                    const char *new_name);
-void
+int
 glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo);
 
 void
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 35195eb2197..0af7be07300 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -627,11 +627,11 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                 STACK_DESTROY (frame->root);                            \
         } while (0)
 
-#define GLUSTERD_GET_DEFRAG_PROCESS(path, volinfo) do {                     \
-                if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER)  \
-                        snprintf (path, NAME_MAX, "tier"); \
-                else                                                        \
-                        snprintf (path, NAME_MAX, "rebalance"); \
+#define GLUSTERD_GET_DEFRAG_PROCESS(path, volinfo) do {                 \
+                if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) \
+                        snprintf (path, NAME_MAX, "tier");              \
+                else                                                    \
+                        snprintf (path, NAME_MAX, "rebalance");         \
         } while (0)
 
 #define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do {               \