From 85d34ea0cf8b687c10093ae06417e498e252e563 Mon Sep 17 00:00:00 2001 From: hari gowtham Date: Wed, 16 Dec 2015 16:18:29 +0530 Subject: Tier: "tier start force" command implementation back port of : http://review.gluster.org/#/c/12983/ The start command doesnt restart the tier deamon if the deamon is running at one node. hence to bring up the tierd on the nodes where the deamon is down, the force command is implemented. It skips the check for tierd running. >Change-Id: I0037d3e5ecfe56637d0da201a97903c435d26436 >BUG: 1292112 >Signed-off-by: hari gowtham Change-Id: Idaca442c1a41ded8bf555a6e34eed0ebb9ea4034 BUG: 1293698 Signed-off-by: hari Reviewed-on: http://review.gluster.org/13069 Tested-by: Gluster Build System Reviewed-by: Dan Lambright Tested-by: Dan Lambright --- cli/src/cli-cmd-parser.c | 33 ++++++-- cli/src/cli-cmd-volume.c | 1 + cli/src/cli-rpc-ops.c | 1 - tests/basic/tier/tierd_check.t | 61 ++++++++++++++ xlators/cluster/dht/src/dht-common.c | 3 +- xlators/cluster/dht/src/dht-rebalance.c | 3 +- xlators/mgmt/glusterd/src/glusterd-rebalance.c | 111 ++++++++++++++++++------- xlators/mgmt/glusterd/src/glusterd-utils.c | 19 +++-- xlators/mgmt/glusterd/src/glusterd-utils.h | 2 +- xlators/mgmt/glusterd/src/glusterd.h | 10 +-- 10 files changed, 185 insertions(+), 59 deletions(-) create mode 100644 tests/basic/tier/tierd_check.t diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c index 1d4d6cf89c7..68c407d5b4f 100644 --- a/cli/src/cli-cmd-parser.c +++ b/cli/src/cli-cmd-parser.c @@ -1701,9 +1701,9 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount, { dict_t *dict = NULL; char *volname = NULL; - char *word = NULL; int ret = -1; int32_t command = GF_OP_CMD_NONE; + int32_t is_force = 0; GF_ASSERT (words); GF_ASSERT (options); @@ -1713,7 +1713,7 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount, if (!dict) goto out; - if (wordcount != 4) { + if (!(wordcount == 4 || wordcount == 5)) { gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax"); ret = -1; goto out; @@ -1735,11 +1735,28 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount, goto out; volname = (char *)words[2]; - - word = (char *)words[3]; - if (!strcmp(word, "status")) - command = GF_DEFRAG_CMD_STATUS_TIER; - else { + if (wordcount == 4) { + if (!strcmp(words[3], "status")) + command = GF_DEFRAG_CMD_STATUS_TIER; + else if (!strcmp(words[3], "start")) + command = GF_DEFRAG_CMD_START_TIER; + else { + ret = -1; + goto out; + } + } else if (wordcount == 5) { + if ((!strcmp (words[3], "start")) && + (!strcmp (words[4], "force"))) { + command = GF_DEFRAG_CMD_START_TIER; + is_force = 1; + ret = dict_set_int32 (dict, "force", is_force); + if (ret) + goto out; + } else { + ret = -1; + goto out; + } + } else { ret = -1; goto out; } @@ -3661,7 +3678,7 @@ cli_cmd_volume_defrag_parse (const char **words, int wordcount, strcmp (words[3], "status")) goto out; } else if ((strcmp (words[3], "tier") == 0) && - (strcmp (words[4], "start") == 0)) { + (strcmp (words[4], "start") == 0)) { volname = (char *) words[2]; cmd = GF_DEFRAG_CMD_START_TIER; goto done; diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c index 92edc356705..3707e130183 100644 --- a/cli/src/cli-cmd-volume.c +++ b/cli/src/cli-cmd-volume.c @@ -2662,6 +2662,7 @@ struct cli_cmd volume_cmds[] = { "rename volume to "},*/ { "volume tier status\n" + "volume tier start [force]" "volume tier attach [] ...\n" "volume tier detach \n", cli_cmd_volume_tier_cbk, diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c index 6dcf0f37e3f..b7e0aedbcbb 100644 --- a/cli/src/cli-rpc-ops.c +++ b/cli/src/cli-rpc-ops.c @@ -1780,7 +1780,6 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov, * case since unlock failures can be highlighted * event though rebalance command was successful */ - if (cmd == GF_DEFRAG_CMD_START_TIER) { snprintf (msg, sizeof (msg), "Attach tier is successful " diff --git a/tests/basic/tier/tierd_check.t b/tests/basic/tier/tierd_check.t new file mode 100644 index 00000000000..ee97883688d --- /dev/null +++ b/tests/basic/tier/tierd_check.t @@ -0,0 +1,61 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../tier.rc + + +# Creates a tiered volume with pure distribute hot and cold tiers +# Both hot and cold tiers will have an equal number of bricks. + +function create_dist_tier_vol () { + mkdir $B0/cold + mkdir $B0/hot + TEST $CLI volume create $V0 $H0:$B0/cold/${V0}{1..3} + TEST $CLI volume set $V0 performance.quick-read off + TEST $CLI volume set $V0 performance.io-cache off + TEST $CLI volume start $V0 + TEST $CLI volume attach-tier $V0 $H0:$B0/hot/${V0}{1..2} + TEST $CLI volume set $V0 cluster.tier-mode test +} + +function tier_deamon_check () { +pgrep -f "rebalance/$V0" +echo "$?" +} + +function tier_deamon_kill () { +pkill -f "rebalance/$V0" +echo "$?" +} + +cleanup; + +#Basic checks +TEST glusterd +TEST pidof glusterd +TEST $CLI volume status + + +#Create and start a tiered volume +create_dist_tier_vol + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_check + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_kill + +TEST $CLI volume tier $V0 start + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_kill + +TEST $CLI volume tier $V0 start force + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check + +TEST $CLI volume tier $V0 start force + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check + +cleanup diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index bc7390b8b40..6e3320f9931 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -4532,7 +4532,8 @@ dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, } if ((conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_TIER) || - (conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) || + (conf->defrag && conf->defrag->cmd == + GF_DEFRAG_CMD_START_DETACH_TIER) || (!(conf->local_subvols_cnt) || !conf->defrag)) { local->call_cnt = conf->subvolume_cnt; diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c index 6375286f636..239ac26fe24 100644 --- a/xlators/cluster/dht/src/dht-rebalance.c +++ b/xlators/cluster/dht/src/dht-rebalance.c @@ -2929,8 +2929,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, * CTR DB Lookup Heal triggered on existing * data. * */ - if (defrag->cmd == - GF_DEFRAG_CMD_START_TIER) { + if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) { gf_fix_layout_tier_attach_lookup (this, loc, entry); } diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c index ea443aaa05e..5f49e4b8752 100644 --- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c +++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c @@ -270,7 +270,8 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr, priv); snprintf (logfile, PATH_MAX, "%s/%s-%s-%s.log", DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname, - (cmd == GF_DEFRAG_CMD_START_TIER ? "tier":"rebalance"), + (cmd == GF_DEFRAG_CMD_START_TIER ? + "tier":"rebalance"), uuid_utoa(MY_UUID)); } else { @@ -605,6 +606,7 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr) char *task_id_str = NULL; dict_t *op_ctx = NULL; xlator_t *this = 0; + int32_t is_force = 0; this = THIS; GF_ASSERT (this); @@ -629,16 +631,20 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr) } switch (cmd) { case GF_DEFRAG_CMD_START_TIER: + ret = dict_get_int32 (dict, "force", &is_force); + if (ret) + is_force = 0; + if (volinfo->type != GF_CLUSTER_TYPE_TIER) { gf_asprintf (op_errstr, "volume %s is not a tier " "volume.", volinfo->volname); ret = -1; goto out; } - if (glusterd_is_tier_daemon_running (volinfo)) { + if ((!is_force) && glusterd_is_tier_daemon_running (volinfo)) { ret = gf_asprintf (op_errstr, "A Tier daemon is " - "already running on volume %s", - volname); + "already running on volume %s", + volname); ret = -1; goto out; } @@ -797,6 +803,7 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict) dict_t *ctx = NULL; xlator_t *this = NULL; uint32_t commit_hash; + int32_t is_force = 0; this = THIS; GF_ASSERT (this); @@ -860,39 +867,79 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict) case GF_DEFRAG_CMD_START_LAYOUT_FIX: case GF_DEFRAG_CMD_START_FORCE: case GF_DEFRAG_CMD_START_TIER: - /* Reset defrag status to 'NOT STARTED' whenever a - * remove-brick/rebalance command is issued to remove - * stale information from previous run. - */ - volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED; - ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str); - if (ret) { - gf_msg_debug (this->name, 0, "Missing rebalance " - "id"); - ret = 0; + + ret = dict_get_int32 (dict, "force", &is_force); + if (ret) + is_force = 0; + if (!is_force) { + /* Reset defrag status to 'NOT STARTED' whenever a + * remove-brick/rebalance command is issued to remove + * stale information from previous run. + */ + volinfo->rebal.defrag_status = + GF_DEFRAG_STATUS_NOT_STARTED; + + ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, + &task_id_str); + if (ret) { + gf_msg_debug (this->name, 0, "Missing rebalance" + " id"); + ret = 0; + } else { + gf_uuid_parse (task_id_str, + volinfo->rebal.rebalance_id); + volinfo->rebal.op = GD_OP_REBALANCE; + } + if (!gd_should_i_start_rebalance (volinfo)) { + /* Store the rebalance-id and rebalance command + * even if the peer isn't starting a rebalance + * process. On peers where a rebalance process + * is started, glusterd_handle_defrag_start + * performs the storing. + * Storing this is needed for having + * 'volume status' work correctly. + */ + glusterd_store_perform_node_state_store + (volinfo); + break; + } + if (dict_get_uint32 (dict, "commit-hash", &commit_hash) + == 0) { + volinfo->rebal.commit_hash = commit_hash; + } + ret = glusterd_handle_defrag_start (volinfo, msg, + sizeof (msg), + cmd, NULL, GD_OP_REBALANCE); + break; } else { - gf_uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ; - volinfo->rebal.op = GD_OP_REBALANCE; - } - if (!gd_should_i_start_rebalance (volinfo)) { - /* Store the rebalance-id and rebalance command even if - * the peer isn't starting a rebalance process. On peers - * where a rebalance process is started, - * glusterd_handle_defrag_start performs the storing. - * - * Storing this is needed for having 'volume status' - * work correctly. + /* Reset defrag status to 'STARTED' so that the + * pid is checked and restarted accordingly. + * If the pid is not running it executes the + * "NOT_STARTED" case and restarts the process */ - glusterd_store_perform_node_state_store (volinfo); + volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED; + volinfo->rebal.defrag_cmd = cmd; + volinfo->rebal.op = GD_OP_REBALANCE; + + ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, + &task_id_str); + if (ret) { + gf_msg_debug (this->name, 0, "Missing rebalance" + " id"); + ret = 0; + } else { + gf_uuid_parse (task_id_str, + volinfo->rebal.rebalance_id); + volinfo->rebal.op = GD_OP_REBALANCE; + } + if (dict_get_uint32 (dict, "commit-hash", &commit_hash) + == 0) { + volinfo->rebal.commit_hash = commit_hash; + } + ret = glusterd_restart_rebalance_for_volume (volinfo); break; } - if (dict_get_uint32 (dict, "commit-hash", &commit_hash) == 0) { - volinfo->rebal.commit_hash = commit_hash; - } - ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg), - cmd, NULL, GD_OP_REBALANCE); - break; case GF_DEFRAG_CMD_STOP: case GF_DEFRAG_CMD_STOP_DETACH_TIER: /* Clear task-id only on explicitly stopping rebalance. diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index c9457e89b45..6fc0dcdc603 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -5647,7 +5647,7 @@ glusterd_is_tier_daemon_running (glusterd_volinfo_t *volinfo) return _gf_false; if (volinfo->rebal.defrag && - volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) { + volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) { return _gf_true; } @@ -7030,12 +7030,12 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr, case GF_DEFRAG_STATUS_STARTED: GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv); if (gf_is_service_running (pidfile, &pid)) { - glusterd_rebalance_rpc_create (volinfo, _gf_true); + ret = glusterd_rebalance_rpc_create (volinfo, _gf_true); break; } case GF_DEFRAG_STATUS_NOT_STARTED: - glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd, - cbk, volinfo->rebal.op); + ret = glusterd_handle_defrag_start (volinfo, op_errstr, len, + cmd, cbk, volinfo->rebal.op); break; default: gf_msg (this->name, GF_LOG_ERROR, 0, @@ -7102,14 +7102,14 @@ out: } -void +int glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo) { - + int ret = -1; char op_errstr[PATH_MAX]; if (!volinfo->rebal.defrag_cmd) - return; + return -1; if (!gd_should_i_start_rebalance (volinfo)) { /* Store the rebalance-id and rebalance command even if @@ -7123,10 +7123,11 @@ glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo) if (volinfo->type == GF_CLUSTER_TYPE_TIER) glusterd_store_perform_node_state_store (volinfo); - return; + return 0; } - glusterd_volume_defrag_restart (volinfo, op_errstr, PATH_MAX, + ret = glusterd_volume_defrag_restart (volinfo, op_errstr, PATH_MAX, volinfo->rebal.defrag_cmd, NULL); + return ret; } int glusterd_restart_rebalance (glusterd_conf_t *conf) diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 99225c674eb..e912f64ae23 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -374,7 +374,7 @@ glusterd_create_sub_tier_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_t **dup_volinfo, gf_boolean_t is_hot_tier, const char *new_name); -void +int glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo); void diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 1e573f8c5ee..ceb472284b2 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -632,11 +632,11 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args); STACK_DESTROY (frame->root); \ } while (0) -#define GLUSTERD_GET_DEFRAG_PROCESS(path, volinfo) do { \ - if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) \ - snprintf (path, NAME_MAX, "tier"); \ - else \ - snprintf (path, NAME_MAX, "rebalance"); \ +#define GLUSTERD_GET_DEFRAG_PROCESS(path, volinfo) do { \ + if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) \ + snprintf (path, NAME_MAX, "tier"); \ + else \ + snprintf (path, NAME_MAX, "rebalance"); \ } while (0) #define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do { \ -- cgit