From 626da0e0c8d9240dc78bc596ee6592dc8875e14d Mon Sep 17 00:00:00 2001 From: Gaurav Kumar Garg Date: Thu, 23 Apr 2015 18:44:30 +0530 Subject: features/bitrot: Per volume bitrot translator Currently whatever bitrot/scrubber tunable value user set for one volume that value is considering for all other volumes also. Each volume should act on their respective bitrot/scrubber tunable value. For handling bitrot/scrubber tunable value independently with respect to all the volume bitrot and scrubber translator should run seperatly for each volume. Change-Id: I1d9379508afe6cfd2f78e3ebf29c829c362d84a9 BUG: 1218048 Signed-off-by: Gaurav Kumar Garg Reviewed-on: http://review.gluster.org/10352 (cherry picked from commit f81deb95db417eeededf7442a30304a880cc8169) Reviewed-on: http://review.gluster.org/10516 Tested-by: NetBSD Build System Tested-by: Gluster Build System Reviewed-by: Kaushal M --- xlators/features/bit-rot/src/bitd/bit-rot.c | 20 ++ xlators/mgmt/glusterd/src/glusterd-bitd-svc.c | 18 +- xlators/mgmt/glusterd/src/glusterd-bitrot.c | 22 ++ xlators/mgmt/glusterd/src/glusterd-scrub-svc.c | 19 +- xlators/mgmt/glusterd/src/glusterd-volgen.c | 409 +++++++++++++++++------- xlators/mgmt/glusterd/src/glusterd-volume-set.c | 15 +- 6 files changed, 349 insertions(+), 154 deletions(-) (limited to 'xlators') diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c index 1202ca260eb..4ace55fcfe1 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot.c @@ -1494,10 +1494,30 @@ struct volume_options options[] = { .description = "default time duration for which an object waits " "before it is signed", }, + { .key = {"brick-count"}, + .type = GF_OPTION_TYPE_STR, + .description = "Total number of bricks for the current node for " + "all volumes in the trusted storage pool.", + }, { .key = {"scrubber"}, .type = GF_OPTION_TYPE_BOOL, .default_value = "false", .description = "option to run as a scrubber", }, + { .key = {"scrub-throttle"}, + .type = GF_OPTION_TYPE_STR, + .description = "Scrub-throttle value is a measure of how fast " + "or slow the scrubber scrubs the filesystem for " + "volume ", + }, + { .key = {"scrub-freq"}, + .type = GF_OPTION_TYPE_STR, + .description = "Scrub frequency for volume ", + }, + { .key = {"scrub-state"}, + .type = GF_OPTION_TYPE_STR, + .description = "Pause/Resume scrub. Upon resume, scrubber " + "continues from where it left off.", + }, { .key = {NULL} }, }; diff --git a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c index 8bd6d8a7276..4748b5ef68d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c @@ -30,7 +30,6 @@ glusterd_bitdsvc_create_volfile () char filepath[PATH_MAX] = {0,}; int ret = -1; glusterd_conf_t *conf = NULL; - dict_t *mod_dict = NULL; xlator_t *this = NULL; this = THIS; @@ -38,29 +37,16 @@ glusterd_bitdsvc_create_volfile () GF_ASSERT (conf); - mod_dict = dict_new (); - if (!mod_dict) { - gf_log (this->name, GF_LOG_ERROR, "failed to allocate new " - "dict"); - goto out; - } - - ret = dict_set_uint32 (mod_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) - goto free_dict; - glusterd_svc_build_volfile_path (bitd_svc_name, conf->workdir, filepath, sizeof (filepath)); ret = glusterd_create_global_volfile (build_bitd_graph, - filepath, mod_dict); + filepath, NULL); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Failed to create volfile"); - goto free_dict; + goto out; } -free_dict: - dict_unref (mod_dict); out: gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret); diff --git a/xlators/mgmt/glusterd/src/glusterd-bitrot.c b/xlators/mgmt/glusterd/src/glusterd-bitrot.c index ae654a0aa87..5474f30626a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-bitrot.c +++ b/xlators/mgmt/glusterd/src/glusterd-bitrot.c @@ -25,6 +25,7 @@ #include "syscall.h" #include "byte-order.h" #include "compat-errno.h" +#include "glusterd-scrub-svc.h" #include #include @@ -154,6 +155,13 @@ glusterd_bitrot_scrub_throttle (glusterd_volinfo_t *volinfo, dict_t *dict, goto out; } + ret = glusterd_scrubsvc_reconfigure (); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to reconfigure scrub " + "services"); + goto out; + } + out: return ret; } @@ -185,6 +193,13 @@ glusterd_bitrot_scrub_freq (glusterd_volinfo_t *volinfo, dict_t *dict, goto out; } + ret = glusterd_scrubsvc_reconfigure (); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to reconfigure scrub " + "services"); + goto out; + } + out: return ret; } @@ -221,6 +236,13 @@ glusterd_bitrot_scrub (glusterd_volinfo_t *volinfo, dict_t *dict, goto out; } + ret = glusterd_scrubsvc_reconfigure (); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to reconfigure scrub " + "services"); + goto out; + } + out: return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c index 9b72053e89c..987acb60955 100644 --- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c +++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c @@ -32,37 +32,22 @@ glusterd_scrubsvc_create_volfile () char filepath[PATH_MAX] = {0,}; int ret = -1; glusterd_conf_t *conf = NULL; - dict_t *mod_dict = NULL; xlator_t *this = NULL; this = THIS; conf = this->private; GF_ASSERT (conf); - - mod_dict = dict_new (); - if (!mod_dict) { - gf_log (this->name, GF_LOG_ERROR, "failed to allocate new " - "dict"); - goto out; - } - - ret = dict_set_uint32 (mod_dict, "trusted-client", GF_CLIENT_TRUSTED); - if (ret) - goto free_dict; - glusterd_svc_build_volfile_path (scrub_svc_name, conf->workdir, filepath, sizeof (filepath)); ret = glusterd_create_global_volfile (build_scrub_graph, - filepath, mod_dict); + filepath, NULL); if (ret) { gf_log (this->name, GF_LOG_ERROR, "Failed to create volfile"); - goto free_dict; + goto out; } -free_dict: - dict_unref (mod_dict); out: gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret); diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index fec13562216..1b9225128c6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -3716,6 +3716,58 @@ gd_get_matching_option (char **options, char *option) return *options; } +static int +scrubber_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme, + void *param) +{ + xlator_t *xl = NULL; + char *scrub_option = NULL; + int ret = 0; + glusterd_volinfo_t *volinfo = NULL; + + volinfo = param; + + xl = first_of (graph); + + if (!strcmp (vme->option, "scrub-throttle")) { + ret = gf_asprintf (&scrub_option, "scrub-throttle"); + if (ret != -1) { + ret = xlator_set_option (xl, scrub_option, vme->value); + GF_FREE (scrub_option); + } + + if (ret) + return -1; + } + + if (!strcmp (vme->option, "scrub-frequency")) { + ret = gf_asprintf (&scrub_option, "scrub-freq"); + if (ret != -1) { + ret = xlator_set_option (xl, scrub_option, vme->value); + GF_FREE (scrub_option); + } + + if (ret) + return -1; + } + + if (!strcmp (vme->option, "scrubber")) { + if (!strcmp (vme->value, "pause")) { + ret = gf_asprintf (&scrub_option, "scrub-state"); + if (ret != -1) { + ret = xlator_set_option (xl, scrub_option, + vme->value); + GF_FREE (scrub_option); + } + + if (ret) + return -1; + } + } + + return ret; +} + static int shd_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme, void *param) @@ -4784,25 +4836,96 @@ glusterd_snapdsvc_generate_volfile (volgen_graph_t *graph, return ret; } -int -build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict) +static int +prepare_bitrot_scrub_volume_options (glusterd_volinfo_t *volinfo, + dict_t *mod_dict, dict_t *set_dict) +{ + int ret = 0; + + + ret = dict_set_uint32 (set_dict, "trusted-client", GF_CLIENT_TRUSTED); + if (ret) + goto out; + + dict_copy (volinfo->dict, set_dict); + if (mod_dict) + dict_copy (mod_dict, set_dict); + +out: + return ret; +} + +static int +build_bitd_clusters (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + dict_t *set_dict, unsigned int numbricks) +{ + int ret = -1; + int clusters = 0; + xlator_t *xl = NULL; + xlator_t *trav = NULL; + xlator_t *this = NULL; + char transt[16] = {0,}; + char *brick_hint = NULL; + int brick_count = 0; + glusterd_brickinfo_t *brickinfo = NULL; + char *bitrot_args[] = {"features/bit-rot", + "%s-bit-rot-%d"}; + + this = THIS; + GF_ASSERT (this); + + get_transport_type (volinfo, set_dict, transt, _gf_false); + if (!strncmp (transt, "tcp,rdma", strlen ("tcp,rdma"))) + strncpy (transt, "tcp", strlen ("tcp")); + + cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { + if (!glusterd_is_local_brick (this, volinfo, brickinfo)) + continue; + + xl = volgen_graph_build_client (graph, volinfo, + brickinfo->hostname, + brickinfo->path, + brickinfo->brick_id, + transt, set_dict); + if (!xl) { + ret = -1; + goto out; + } + brick_count++; + } + + + ret = volgen_link_bricks_from_list_tail (graph, volinfo, bitrot_args[0], + bitrot_args[1], brick_count, + brick_count); + clusters = ret; + + xl = first_of (graph); + + ret = gf_asprintf (&brick_hint, "%d", numbricks); + if (ret < 0) + goto out; + + ret = xlator_set_option (xl, "brick-count", brick_hint); + if (ret) + goto out; + + ret = clusters; + +out: + return ret; +} + +static int +build_bitd_volume_graph (volgen_graph_t *graph, + glusterd_volinfo_t *volinfo, dict_t *mod_dict, + dict_t *set_dict, unsigned int numbricks) { volgen_graph_t cgraph = {0}; - glusterd_volinfo_t *voliter = NULL; xlator_t *this = NULL; glusterd_conf_t *priv = NULL; - dict_t *set_dict = NULL; int ret = 0; - xlator_t *bitd_xl = NULL; - xlator_t *xl = NULL; - xlator_t *trav = NULL; - xlator_t *txl = NULL; - char *skey = NULL; - char transt[16] = {0,}; - glusterd_brickinfo_t *brickinfo = NULL; - char *br_args[] = {"features/bit-rot", - "bit-rot"}; - int32_t count = 0; + int clusters = -1; this = THIS; GF_ASSERT (this); @@ -4810,93 +4933,154 @@ build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict) priv = this->private; GF_ASSERT (priv); + ret = prepare_bitrot_scrub_volume_options (volinfo, mod_dict, set_dict); + if (ret) + goto out; + + clusters = build_bitd_clusters (&cgraph, volinfo, set_dict, numbricks); + if (clusters < 0) { + ret = -1; + goto out; + } + + ret = volgen_graph_merge_sub (graph, &cgraph, clusters); + if (ret) + goto out; + + ret = graph_set_generic_options (this, graph, set_dict, "Bitrot"); +out: + return ret; +} + +int +build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict) +{ + glusterd_volinfo_t *voliter = NULL; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + dict_t *set_dict = NULL; + int ret = 0; + gf_boolean_t valid_config = _gf_false; + xlator_t *iostxl = NULL; + glusterd_brickinfo_t *brickinfo = NULL; + unsigned int numbricks = 0; + + this = THIS; + GF_ASSERT (this); + priv = this->private; + GF_ASSERT (priv); + set_dict = dict_new (); if (!set_dict) { - ret = -ENOMEM; + ret = -1; goto out; } - if (mod_dict) - dict_copy (mod_dict, set_dict); + iostxl = volgen_graph_add_as (graph, "debug/io-stats", "bitd"); + if (!iostxl) { + ret = -1; + goto out; + } + /* TODO: do way with this extra loop _if possible_ */ cds_list_for_each_entry (voliter, &priv->volumes, vol_list) { if (voliter->status != GLUSTERD_STATUS_STARTED) continue; - if (!glusterd_is_bitrot_enabled (voliter)) continue; - memset (transt, '\0', 16); - - get_transport_type (voliter, set_dict, transt, _gf_false); - if (!strcmp (transt, "tcp,rdma")) - strcpy (transt, "tcp"); - - - cds_list_for_each_entry (brickinfo, &voliter->bricks, - brick_list) { + cds_list_for_each_entry (brickinfo, + &voliter->bricks, brick_list) { if (!glusterd_is_local_brick (this, voliter, brickinfo)) continue; - xl = volgen_graph_build_client (graph, voliter, - brickinfo->hostname, - brickinfo->path, - brickinfo->brick_id, transt, - set_dict); - if (!xl) { - ret = -1; - goto out; - } - - count++; + numbricks++; } } - bitd_xl = volgen_graph_add_nolink (graph, br_args[0], br_args[1]); - if (!bitd_xl) { - ret = -1; - goto out; - } + cds_list_for_each_entry (voliter, &priv->volumes, vol_list) { + if (voliter->status != GLUSTERD_STATUS_STARTED) + continue; - txl = first_of (graph); - for (trav = txl; count; trav = trav->next) - count--; + if (!glusterd_is_bitrot_enabled (voliter)) + continue; - for (; trav != txl; trav = trav->prev) { - ret = volgen_xlator_link (bitd_xl, trav); + ret = build_bitd_volume_graph (graph, voliter, + mod_dict, set_dict, numbricks); + ret = dict_reset (set_dict); if (ret) goto out; } - - ret = 0; - out: if (set_dict) dict_unref (set_dict); - gf_log(this->name, GF_LOG_DEBUG, "Returning %d", ret); + return ret; +} + +static int +build_scrub_clusters (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + dict_t *set_dict) +{ + int ret = -1; + int clusters = 0; + xlator_t *xl = NULL; + xlator_t *trav = NULL; + xlator_t *this = NULL; + char transt[16] = {0,}; + int brick_count = 0; + glusterd_brickinfo_t *brickinfo = NULL; + char *scrub_args[] = {"features/bit-rot", + "%s-bit-rot-%d"}; + this = THIS; + GF_ASSERT (this); + + get_transport_type (volinfo, set_dict, transt, _gf_false); + if (!strncmp (transt, "tcp,rdma", strlen ("tcp,rdma"))) + strncpy (transt, "tcp", strlen ("tcp")); + + cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { + if (!glusterd_is_local_brick (this, volinfo, brickinfo)) + continue; + + xl = volgen_graph_build_client (graph, volinfo, + brickinfo->hostname, + brickinfo->path, + brickinfo->brick_id, + transt, set_dict); + if (!xl) { + ret = -1; + goto out; + } + brick_count++; + } + + ret = volgen_link_bricks_from_list_tail (graph, volinfo, scrub_args[0], + scrub_args[1], brick_count, + brick_count); + clusters = ret; + + xl = first_of (graph); + + ret = xlator_set_option (xl, "scrubber", "true"); + if (ret) + goto out; + + ret = clusters; + +out: return ret; } -int -build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict) +static int +build_scrub_volume_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, + dict_t *mod_dict, dict_t *set_dict) { volgen_graph_t cgraph = {0}; - glusterd_volinfo_t *voliter = NULL; xlator_t *this = NULL; glusterd_conf_t *priv = NULL; - dict_t *set_dict = NULL; int ret = 0; - xlator_t *bitd_xl = NULL; - xlator_t *xl = NULL; - xlator_t *trav = NULL; - xlator_t *txl = NULL; - char *skey = NULL; - char transt[16] = {0,}; - glusterd_brickinfo_t *brickinfo = NULL; - char *br_args[] = {"features/bit-rot", - "bit-rot"}; - int32_t count = 0; + int clusters = -1; this = THIS; GF_ASSERT (this); @@ -4904,75 +5088,76 @@ build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict) priv = this->private; GF_ASSERT (priv); - set_dict = dict_new (); - if (!set_dict) { - ret = -ENOMEM; + ret = prepare_bitrot_scrub_volume_options (volinfo, mod_dict, set_dict); + if (ret) goto out; - } - if (mod_dict) - dict_copy (mod_dict, set_dict); - - cds_list_for_each_entry (voliter, &priv->volumes, vol_list) { - if (voliter->status != GLUSTERD_STATUS_STARTED) - continue; + clusters = build_scrub_clusters (&cgraph, volinfo, set_dict); + if (clusters < 0) { + ret = -1; + goto out; + } - if (!glusterd_is_bitrot_enabled (voliter)) - continue; + ret = volgen_graph_set_options_generic (&cgraph, set_dict, + volinfo, + scrubber_option_handler); + if (ret) + goto out; - memset (transt, '\0', 16); + ret = volgen_graph_merge_sub (graph, &cgraph, clusters); + if (ret) + goto out; - get_transport_type (voliter, set_dict, transt, _gf_false); - if (!strcmp (transt, "tcp,rdma")) - strcpy (transt, "tcp"); + ret = graph_set_generic_options (this, graph, set_dict, "Scrubber"); +out: + return ret; +} +int +build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict) +{ + glusterd_volinfo_t *voliter = NULL; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + dict_t *set_dict = NULL; + int ret = 0; + gf_boolean_t valid_config = _gf_false; + xlator_t *iostxl = NULL; - cds_list_for_each_entry (brickinfo, &voliter->bricks, - brick_list) { - if (!glusterd_is_local_brick (this, voliter, brickinfo)) - continue; - xl = volgen_graph_build_client (graph, voliter, - brickinfo->hostname, - brickinfo->path, - brickinfo->brick_id, transt, - set_dict); - if (!xl) { - ret = -1; - goto out; - } + this = THIS; + GF_ASSERT (this); + priv = this->private; + GF_ASSERT (priv); - count++; - } + set_dict = dict_new (); + if (!set_dict) { + ret = -1; + goto out; } - bitd_xl = volgen_graph_add_nolink (graph, br_args[0], br_args[1]); - if (!bitd_xl) { + iostxl = volgen_graph_add_as (graph, "debug/io-stats", "scrub"); + if (!iostxl) { ret = -1; goto out; } - ret = xlator_set_option (bitd_xl, "scrubber", "true"); - if (ret) - goto out; + cds_list_for_each_entry (voliter, &priv->volumes, vol_list) { + if (voliter->status != GLUSTERD_STATUS_STARTED) + continue; - txl = first_of (graph); - for (trav = txl; count; trav = trav->next) - count--; + if (!glusterd_is_bitrot_enabled (voliter)) + continue; - for (; trav != txl; trav = trav->prev) { - ret = volgen_xlator_link (bitd_xl, trav); + ret = build_scrub_volume_graph (graph, voliter, mod_dict, + set_dict); + ret = dict_reset (set_dict); if (ret) goto out; } - - ret = 0; - out: if (set_dict) dict_unref (set_dict); - gf_log(this->name, GF_LOG_DEBUG, "Returning %d", ret); - return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index c2af51d0f65..fd1ba3e9b49 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -1268,7 +1268,7 @@ struct volopt_map_entry glusterd_volopt_map[] = { { .key = VKEY_FEATURES_BITROT, .voltype = "features/bitrot", .option = "bitrot", - .value = "off", + .value = "disable", .type = NO_DOC, .flags = OPT_FLAG_FORCE, .op_version = GD_OP_VERSION_3_7_0 @@ -1779,26 +1779,23 @@ struct volopt_map_entry glusterd_volopt_map[] = { .op_version = GD_OP_VERSION_3_7_0, .flags = OPT_FLAG_CLIENT_OPT }, - { .key = "features.bitrot", - .voltype = "features/bitrot-stub", - .value = "disable", - .op_version = GD_OP_VERSION_3_7_0, - .type = NO_DOC, - }, { .key = "features.scrub-throttle", .voltype = "features/bitrot", - .value = "normal", + .value = "lazy", + .option = "scrub-throttle", .op_version = GD_OP_VERSION_3_7_0, .type = NO_DOC, }, { .key = "features.scrub-freq", .voltype = "features/bitrot", - .value = "monthly", + .value = "biweekly", + .option = "scrub-frequency", .op_version = GD_OP_VERSION_3_7_0, .type = NO_DOC, }, { .key = "features.scrub", .voltype = "features/bitrot", + .option = "scrubber", .op_version = GD_OP_VERSION_3_7_0, .type = NO_DOC, }, -- cgit