From 3e18f093974c85ac92a4c48f0cd13aa9ff9c5cac Mon Sep 17 00:00:00 2001 From: vmallika Date: Wed, 18 Mar 2015 23:17:23 +0530 Subject: features/quota : Introducing inode quota ========================================================================== Inode quota ========================================================================== = Currently, the only way to retrieve the number of files/objects in a = = directory or volume is to do a crawl of the entire directory/volume. = = This is expensive and is not scalable. = = = = The proposed mechanism will provide an easier alternative to determine = = the count of files/objects in a directory or volume. = = = = The new mechanism proposes to store count of objects/files as part of = = an extended attribute of a directory. Each directory's extended = = attribute value will indicate the number of files/objects present = = in a tree with the directory being considered as the root of the tree. = = = = The count value can be accessed by performing a getxattr(). = = Cluster translators like afr, dht and stripe will perform aggregation = = of count values from various bricks when getxattr() happens on the key = = associated with file/object count. = A new interface is introduced: ------------------------------ limit-objects : limit the number of inodes at directory level list-objects : list the directories where the limit is set remove-objects : remove the limit from the directory ========================================================================== CLI COMMAND: gluster volume quota limit-objects [] * is a hard-limit for number of objects limitation for path "" If hard-limit is exceeded, creation of file/directory is no longer permitted. * is a soft-limit for number of objects creation for path "" If soft-limit is exceeded, a warning is issued for each creation. CLI COMMAND: gluster volume quota remove-objects [path] ========================================================================== CLI COMMAND: gluster volume quota list-objects [path] ... Sample output: ------------------ Path Hard-limit Soft-limit Used Available Soft-limit exceeded? Hard-limit exceeded? ------------------------------------------------------------------------ -------------------------------------- /dir 10 80% 10 0 Yes Yes ========================================================================== [root@snapshot-28 dir]# ls a b file11 file12 file13 file14 file15 file16 file17 [root@snapshot-28 dir]# touch a1 touch: cannot touch `a1': Disk quota exceeded * Nine files are created in directory "dir" and directory is included in * the count too. Hence the limit "10" is reached and further file creation fails ========================================================================== Note: We have also done some re-factoring in cli for volume name validation. New function cli_validate_volname is created ========================================================================== Change-Id: I1823497de4f790a2a20ebb1770293472ea33ee2b BUG: 1190108 Signed-off-by: Sachin Pandit Signed-off-by: vmallika Reviewed-on: http://review.gluster.org/9769 Tested-by: Gluster Build System Reviewed-by: Vijay Bellur --- cli/src/cli-cmd-parser.c | 218 ++++++++++++++--------- cli/src/cli-cmd-volume.c | 65 ++++--- cli/src/cli-rpc-ops.c | 445 +++++++++++++++++++++++------------------------ cli/src/cli-xml-output.c | 67 +++++++ cli/src/cli.c | 32 ++++ cli/src/cli.h | 13 ++ 6 files changed, 509 insertions(+), 331 deletions(-) (limited to 'cli/src') diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c index c7fdafb32b6..83a9fbd7e7d 100644 --- a/cli/src/cli-cmd-parser.c +++ b/cli/src/cli-cmd-parser.c @@ -368,6 +368,55 @@ out: return ret; } +int32_t +cli_validate_volname (const char *volname) +{ + int32_t ret = -1; + int32_t i = -1; + static const char * const invalid_volnames[] = { + "volume", "type", "subvolumes", "option", + "end-volume", "all", "volume_not_in_ring", + "description", "force", + "snap-max-hard-limit", + "snap-max-soft-limit", "auto-delete", + "activate-on-create", NULL}; + + if (volname[0] == '-') + goto out; + + for (i = 0; invalid_volnames[i]; i++) { + if (!strcmp (volname, invalid_volnames[i])) { + cli_err ("\"%s\" cannot be the name of a volume.", + volname); + goto out; + } + } + + if (strchr (volname, '/')) + goto out; + + if (strlen (volname) > GD_VOLUME_NAME_MAX) { + cli_err("Volume name exceeds %d characters.", + GD_VOLUME_NAME_MAX); + goto out; + } + + for (i = 0; i < strlen (volname); i++) { + if (!isalnum (volname[i]) && (volname[i] != '_') && + (volname[i] != '-')) { + cli_err ("Volume name should not contain \"%c\"" + " character.\nVolume names can only" + "contain alphanumeric, '-' and '_' " + "characters.", volname[i]); + goto out; + } + } + + ret = 0; +out: + return ret; +} + int32_t cli_cmd_volume_create_parse (struct cli_state *state, const char **words, int wordcount, dict_t **options) @@ -379,7 +428,6 @@ cli_cmd_volume_create_parse (struct cli_state *state, const char **words, int count = 1; int sub_count = 1; int brick_index = 0; - int i = 0; char *trans_type = NULL; int32_t index = 0; char *bricks = NULL; @@ -387,12 +435,6 @@ cli_cmd_volume_create_parse (struct cli_state *state, const char **words, char *opwords[] = { "replica", "stripe", "transport", "disperse", "redundancy", "disperse-data", NULL }; - char *invalid_volnames[] = {"volume", "type", "subvolumes", "option", - "end-volume", "all", "volume_not_in_ring", - "description", "force", - "snap-max-hard-limit", - "snap-max-soft-limit", "auto-delete", - "activate-on-create", NULL}; char *w = NULL; char *ptr = NULL; int op_count = 0; @@ -420,37 +462,8 @@ cli_cmd_volume_create_parse (struct cli_state *state, const char **words, GF_ASSERT (volname); /* Validate the volume name here itself */ - { - if (volname[0] == '-') - goto out; - - for (i = 0; invalid_volnames[i]; i++) { - if (!strcmp (volname, invalid_volnames[i])) { - cli_err ("\"%s\" cannot be the name of a volume.", - volname); - goto out; - } - } - - if (strchr (volname, '/')) - goto out; - - if (strlen (volname) > GD_VOLUME_NAME_MAX) { - cli_err("Volume name exceeds %d characters.", - GD_VOLUME_NAME_MAX); - goto out; - } - - for (i = 0; i < strlen (volname); i++) - if (!isalnum (volname[i]) && (volname[i] != '_') && - (volname[i] != '-')) { - cli_err ("Volume name should not contain \"%c\"" - " character.\nVolume names can only" - "contain alphanumeric, '-' and '_' " - "characters.",volname[i]); - goto out; - } - } + if (cli_validate_volname (volname) < 0) + goto out; if (wordcount < 4) { ret = -1; @@ -853,24 +866,29 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) dict_t *dict = NULL; char *volname = NULL; int ret = -1; - int i = 0; + int i = -1; char key[20] = {0, }; uint64_t value = 0; gf_quota_type type = GF_QUOTA_OPTION_TYPE_NONE; char *opwords[] = { "enable", "disable", "limit-usage", "remove", "list", "alert-time", "soft-timeout", "hard-timeout", - "default-soft-limit", NULL}; + "default-soft-limit", "limit-objects", + "list-objects", "remove-objects", NULL}; char *w = NULL; uint32_t time = 0; double percent = 0; + char *end_ptr = NULL; + int64_t limit = 0; GF_ASSERT (words); GF_ASSERT (options); dict = dict_new (); - if (!dict) + if (!dict) { + gf_log ("cli", GF_LOG_ERROR, "dict_new failed"); goto out; + } if (wordcount < 4) goto out; @@ -882,34 +900,8 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) } /* Validate the volume name here itself */ - { - if (volname[0] == '-') - goto out; - - if (!strcmp (volname, "all")) { - cli_err ("\"all\" cannot be the name of a volume."); - goto out; - } - - if (strchr (volname, '/')) - goto out; - - if (strlen (volname) > GD_VOLUME_NAME_MAX) { - cli_err("Volname can not exceed %d characters.", - GD_VOLUME_NAME_MAX); - goto out; - } - - for (i = 0; i < strlen (volname); i++) - if (!isalnum (volname[i]) && (volname[i] != '_') && - (volname[i] != '-')) { - cli_err ("Volume name should not contain \"%c\"" - " character.\nVolume names can only" - "contain alphanumeric, '-' and '_' " - "characters.",volname[i]); - goto out; - } - } + if (cli_validate_volname (volname) < 0) + goto out; ret = dict_set_str (dict, "volname", volname); if (ret < 0) @@ -945,14 +937,19 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) } if (strcmp (w, "limit-usage") == 0) { + type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE; + } else if (strcmp (w, "limit-objects") == 0) { + type = GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS; + } + + if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE || + type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) { if (wordcount < 6 || wordcount > 7) { ret = -1; goto out; } - type = GF_QUOTA_OPTION_TYPE_LIMIT_USAGE; - if (words[4][0] != '/') { cli_err ("Please enter absolute path"); ret = -1; @@ -968,13 +965,26 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) goto out; } - ret = gf_string2bytesize_uint64 (words[5], &value); - if (ret != 0) { - if (errno == ERANGE) - cli_err ("Value too large: %s", words[5]); - else - cli_err ("Please enter a correct value"); - goto out; + if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) { + ret = gf_string2bytesize_uint64 (words[5], &value); + if (ret != 0) { + if (errno == ERANGE) + cli_err ("Value too large: %s", + words[5]); + else + cli_err ("Please enter a correct " + "value"); + goto out; + } + } else { + errno = 0; + limit = strtol (words[5], &end_ptr, 10); + if (errno == ERANGE || errno == EINVAL || limit <= 0) { + ret = -1; + cli_err ("Please enter an interger value in " + "the range 1 - %"PRId64, INT64_MAX); + goto out; + } } ret = dict_set_str (dict, "hard-limit", (char *) words[5]); @@ -997,6 +1007,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) goto set_type; } + if (strcmp (w, "remove") == 0) { if (wordcount != 5) { ret = -1; @@ -1017,6 +1028,26 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) goto set_type; } + if (strcmp (w, "remove-objects") == 0) { + if (wordcount != 5) { + ret = -1; + goto out; + } + + type = GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS; + + if (words[4][0] != '/') { + cli_err ("Please enter absolute path"); + ret = -1; + goto out; + } + + ret = dict_set_str (dict, "path", (char *) words[4]); + if (ret < 0) + goto out; + goto set_type; + } + if (strcmp (w, "list") == 0) { if (wordcount < 4) { ret = -1; @@ -1041,6 +1072,35 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) goto set_type; } + if (strcmp (w, "list-objects") == 0) { + if (wordcount < 4) { + ret = -1; + goto out; + } + + type = GF_QUOTA_OPTION_TYPE_LIST_OBJECTS; + + i = 4; + while (i < wordcount) { + snprintf (key, 20, "path%d", i-4); + + ret = dict_set_str (dict, key, (char *) words[i++]); + if (ret < 0) { + gf_log ("cli", GF_LOG_ERROR, "Failed to set " + "quota patch in request dictionary"); + goto out; + } + } + + ret = dict_set_int32 (dict, "count", i - 4); + if (ret < 0) { + gf_log ("cli", GF_LOG_ERROR, "Failed to set quota " + "limit count in request dictionary"); + goto out; + } + + goto set_type; + } if (strcmp (w, "alert-time") == 0) { if (wordcount != 5) { @@ -1061,6 +1121,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) goto out; goto set_type; } + if (strcmp (w, "soft-timeout") == 0) { if (wordcount != 5) { ret = -1; @@ -1080,6 +1141,7 @@ cli_cmd_quota_parse (const char **words, int wordcount, dict_t **options) goto out; goto set_type; } + if (strcmp (w, "hard-timeout") == 0) { if(wordcount != 5) { ret = -1; diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c index dc223990741..5632a9798bb 100644 --- a/cli/src/cli-cmd-volume.c +++ b/cli/src/cli-cmd-volume.c @@ -900,40 +900,30 @@ cli_stage_quota_op (char *volname, int op_code) int ret = -1; switch (op_code) { - case GF_QUOTA_OPTION_TYPE_ENABLE: - case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE: - case GF_QUOTA_OPTION_TYPE_REMOVE: - case GF_QUOTA_OPTION_TYPE_LIST: - ret = gf_cli_create_auxiliary_mount (volname); - if (ret) { - cli_err ("quota: Could not start quota " - "auxiliary mount"); - goto out; - } - ret = 0; - break; + case GF_QUOTA_OPTION_TYPE_ENABLE: + case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE: + case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS: + case GF_QUOTA_OPTION_TYPE_REMOVE: + case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS: + case GF_QUOTA_OPTION_TYPE_LIST: + ret = gf_cli_create_auxiliary_mount (volname); + if (ret) { + cli_err ("quota: Could not start quota " + "auxiliary mount"); + goto out; + } + ret = 0; + break; - default: - ret = 0; - break; + default: + ret = 0; + break; } out: return ret; } -static void -print_quota_list_header (void) -{ - //Header - cli_out (" Path Hard-limit " - "Soft-limit Used Available Soft-limit exceeded?" - " Hard-limit exceeded?"); - cli_out ("-----------------------------------------------------" - "-----------------------------------------------------" - "-----------------"); -} - int cli_get_soft_limit (dict_t *options, const char **words, dict_t *xdata) { @@ -1082,6 +1072,7 @@ cli_cmd_quota_handle_list_all (const char **words, dict_t *options) char quota_conf_file[PATH_MAX] = {0}; gf_boolean_t xml_err_flag = _gf_false; char err_str[NAME_MAX] = {0,}; + int32_t type = 0; xdata = dict_new (); if (!xdata) { @@ -1095,6 +1086,18 @@ cli_cmd_quota_handle_list_all (const char **words, dict_t *options) goto out; } + ret = dict_get_int32 (options, "type", &type); + if (ret) { + gf_log ("cli", GF_LOG_ERROR, "Failed to get quota option type"); + goto out; + } + + ret = dict_set_int32 (xdata, "type", type); + if (ret) { + gf_log ("cli", GF_LOG_ERROR, "Failed to set type in xdata"); + goto out; + } + ret = cli_get_soft_limit (options, words, xdata); if (ret) { gf_log ("cli", GF_LOG_ERROR, "Failed to fetch default " @@ -1164,7 +1167,7 @@ cli_cmd_quota_handle_list_all (const char **words, dict_t *options) proc = &cli_quotad_clnt.proctable[GF_AGGREGATOR_GETLIMIT]; if (!(global_state->mode & GLUSTER_MODE_XML)) { - print_quota_list_header (); + print_quota_list_header (type); } else { ret = cli_xml_output_vol_quota_limit_list_begin (local, 0, 0, NULL); @@ -1336,6 +1339,7 @@ cli_cmd_quota_cbk (struct cli_state *state, struct cli_cmd_word *word, goto out; break; case GF_QUOTA_OPTION_TYPE_LIST: + case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS: if (wordcount != 4) break; ret = cli_cmd_quota_handle_list_all (words, options); @@ -2437,8 +2441,11 @@ struct cli_cmd volume_cmds[] = { cli_cmd_volume_profile_cbk, "volume profile operations"}, - { "volume quota {enable|disable|list [ ...]|remove | default-soft-limit } |\n" + { "volume quota {enable|disable|list [ ...]| " + "list-objects [ ...] | remove | remove-objects | " + "default-soft-limit } |\n" "volume quota {limit-usage []} |\n" + "volume quota {limit-objects []} |\n" "volume quota {alert-time|soft-timeout|hard-timeout} {