From 8beaf169e39b262416e2274a028292379d39b310 Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Fri, 9 Jan 2015 14:43:22 +0000 Subject: cluster/afr: split-brain resolution CLI Extend the AFR heal command to include automated split-brain resolution. This patch [3/3] is the final patch for afr automated split-brain resolution implementation. "gluster volume heal [full | statistics [heal-count [replica ]] |info [healed | heal-failed | split-brain]| split-brain {bigger-file |source-brick []}]" The new additions being: 1.gluster volume heal split-brain bigger-file Locates the replica containing the FILE, selects bigger-file as source and completes heal. 2.gluster volume heal split-brain source-brick Selects present in as source and completes heal. 3.gluster volume heal split-brain Selects all split-brained files in as source and completes heal. Note: can be either the full file name as seen from the root of the volume (or) the gfid-string representation of the file, which sometimes gets displayed in the heal info command's output. Entry/gfid split-brain resolution is not supported. Example can be found in the test case. Change-Id: I4649733922d406f14f28ee9033a5cb627b9538b3 BUG: 1136769 Signed-off-by: Ravishankar N Reviewed-on: http://review.gluster.org/9377 Reviewed-by: Pranith Kumar Karampuri Tested-by: Pranith Kumar Karampuri Tested-by: Gluster Build System --- cli/src/cli-cmd-parser.c | 91 ++++++++++++++++++++++++++++++++++++++++++------ cli/src/cli-cmd-volume.c | 79 +++++++++++++++++++++++++++++++---------- cli/src/cli-rpc-ops.c | 6 ++++ 3 files changed, 147 insertions(+), 29 deletions(-) (limited to 'cli/src') diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c index 28888ba656d..53b14d27708 100644 --- a/cli/src/cli-cmd-parser.c +++ b/cli/src/cli-cmd-parser.c @@ -2929,6 +2929,43 @@ out: return ret; } +static int +set_hostname_path_in_dict (const char *token, dict_t *dict, int heal_op) +{ + char *hostname = NULL; + char *path = NULL; + int ret = 0; + + ret = extract_hostname_path_from_token (token, &hostname, &path); + if (ret) + goto out; + + switch (heal_op) { + case GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK: + ret = dict_set_dynstr (dict, "heal-source-hostname", + hostname); + if (ret) + goto out; + ret = dict_set_dynstr (dict, "heal-source-brickpath", + path); + break; + case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA: + ret = dict_set_dynstr (dict, "per-replica-cmd-hostname", + hostname); + if (ret) + goto out; + ret = dict_set_dynstr (dict, "per-replica-cmd-path", + path); + break; + default: + ret = -1; + break; + } + +out: + return ret; + +} int cli_cmd_volume_heal_options_parse (const char **words, int wordcount, @@ -2936,8 +2973,6 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount, { int ret = 0; dict_t *dict = NULL; - char *hostname = NULL; - char *path = NULL; dict = dict_new (); if (!dict) @@ -3008,6 +3043,35 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount, ret = -1; goto out; } + if (wordcount == 6) { + if (strcmp (words[3], "split-brain")) { + ret = -1; + goto out; + } + if (!strcmp (words[4], "bigger-file")) { + ret = dict_set_int32 (dict, "heal-op", + GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE); + if (ret) + goto out; + ret = dict_set_str (dict, "file", (char *)words[5]); + if (ret) + goto out; + goto done; + } + if (!strcmp (words[4], "source-brick")) { + ret = dict_set_int32 (dict, "heal-op", + GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK); + if (ret) + goto out; + ret = set_hostname_path_in_dict (words[5], dict, + GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK); + if (ret) + goto out; + goto done; + } + ret = -1; + goto out; + } if (wordcount == 7) { if (!strcmp (words[3], "statistics") && !strcmp (words[4], "heal-count") @@ -3017,21 +3081,26 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount, GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA); if (ret) goto out; - ret = extract_hostname_path_from_token (words[6], - &hostname, &path); + ret = set_hostname_path_in_dict (words[6], dict, + GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA); if (ret) goto out; - ret = dict_set_dynstr (dict, "per-replica-cmd-hostname", - hostname); + goto done; + + } + if (!strcmp (words[3], "split-brain") && + !strcmp (words[4], "source-brick")) { + ret = dict_set_int32 (dict, "heal-op", + GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK); + ret = set_hostname_path_in_dict (words[5], dict, + GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK); if (ret) goto out; - ret = dict_set_dynstr (dict, "per-replica-cmd-path", - path); + ret = dict_set_str (dict, "file", + (char *) words[6]); if (ret) goto out; - else - goto done; - + goto done; } } ret = -1; diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c index 238c8673d75..501b5776dec 100644 --- a/cli/src/cli-cmd-volume.c +++ b/cli/src/cli-cmd-volume.c @@ -1879,6 +1879,60 @@ cli_print_brick_status (cli_volume_status_t *status) return 0; } +#define NEEDS_GLFS_HEAL(op) ((op == GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) || \ + (op == GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK) || \ + (op == GF_AFR_OP_INDEX_SUMMARY)) + +int +cli_launch_glfs_heal (int heal_op, dict_t *options) +{ + char buff[PATH_MAX] = {0}; + runner_t runner = {0}; + char *filename = NULL; + char *hostname = NULL; + char *path = NULL; + char *volname = NULL; + char *out = NULL; + int ret = 0; + + runinit (&runner); + ret = dict_get_str (options, "volname", &volname); + runner_add_args (&runner, SBIN_DIR"/glfsheal", volname, NULL); + runner_redir (&runner, STDOUT_FILENO, RUN_PIPE); + + switch (heal_op) { + case GF_AFR_OP_INDEX_SUMMARY: + break; + case GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE: + ret = dict_get_str (options, "file", &filename); + runner_add_args (&runner, "bigger-file", filename, NULL); + break; + case GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK: + ret = dict_get_str (options, "heal-source-hostname", + &hostname); + ret = dict_get_str (options, "heal-source-brickpath", + &path); + runner_add_args (&runner, "source-brick", NULL); + runner_argprintf (&runner, "%s:%s", hostname, path); + if (dict_get_str (options, "file", &filename) == 0) + runner_argprintf (&runner, filename); + break; + default: + ret = -1; + } + ret = runner_start (&runner); + if (ret == -1) + goto out; + while ((out = fgets (buff, sizeof(buff), + runner_chio (&runner, STDOUT_FILENO)))) { + printf ("%s", out); + } + ret = runner_end (&runner); + ret = WEXITSTATUS (ret); + +out: + return ret; +} int cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word, const char **words, int wordcount) @@ -1892,9 +1946,6 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word, xlator_t *this = NULL; cli_local_t *local = NULL; int heal_op = 0; - runner_t runner = {0}; - char buff[PATH_MAX] = {0}; - char *out = NULL; this = THIS; frame = create_frame (this, this->ctx->pool); @@ -1916,21 +1967,10 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word, ret = dict_get_int32 (options, "heal-op", &heal_op); if (ret < 0) goto out; - - if (heal_op == GF_AFR_OP_INDEX_SUMMARY) { - runinit (&runner); - runner_add_args (&runner, SBIN_DIR"/glfsheal", words[2], NULL); - runner_redir (&runner, STDOUT_FILENO, RUN_PIPE); - ret = runner_start (&runner); + if (NEEDS_GLFS_HEAL (heal_op)) { + ret = cli_launch_glfs_heal (heal_op, options); if (ret == -1) goto out; - while ((out = fgets(buff, sizeof(buff), - runner_chio (&runner, STDOUT_FILENO)))) { - printf ("%s", out); - } - - ret = runner_end (&runner); - ret = WEXITSTATUS (ret); } else { proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME]; @@ -1946,7 +1986,7 @@ out: if (ret) { cli_cmd_sent_status_get (&sent); if ((sent == 0) && (parse_error == 0)) - cli_out ("Volume heal failed"); + cli_out ("Volume heal failed."); } CLI_STACK_DESTROY (frame); @@ -2316,7 +2356,10 @@ struct cli_cmd volume_cmds[] = { cli_cmd_volume_status_cbk, "display status of all or specified volume(s)/brick"}, - { "volume heal [{full | statistics {heal-count {replica }} |info {healed | heal-failed | split-brain}}]", + { "volume heal [full | statistics [heal-count "\ + "[replica ]] |info [healed | heal-failed | "\ + "split-brain]| split-brain {bigger-file |source-brick "\ + " []}]", cli_cmd_volume_heal_cbk, "self-heal commands on volume specified by "}, diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c index 1d8cf23ff42..72ffaf4129a 100644 --- a/cli/src/cli-rpc-ops.c +++ b/cli/src/cli-rpc-ops.c @@ -7358,6 +7358,12 @@ gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov, case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA: heal_op_str = "count of entries to be healed per replica"; break; + /* The below 2 cases are never hit; they're coded only to make + * compiler warnings go away.*/ + case GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE: + case GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK: + break; + case GF_AFR_OP_INVALID: heal_op_str = "invalid heal op"; break; -- cgit