From 04ede2e1635d664ea18e266544bffc569ae69bb8 Mon Sep 17 00:00:00 2001 From: Sanoj Unnikrishnan Date: Fri, 23 Feb 2018 15:26:20 +0530 Subject: Quota: heal directory on newly added bricks when quota limit is reached Problem: if a lookup is done on a newly added brick for a path on which limit has been reached, the lookup fails to heal the directory tree due to quota. Solution: Tag the lookup as an internal fop and ignore it in quota. Since marking internal fop does not usually give enough contextual information. Introducing new flags to pass the contextual info. Adding dict_check_flag and dict_set_flag to aid flag operations. A flag is a single bit in a bit array (currently limited to 256 bits). Change-Id: Ifb6a68bcaffedd425dd0f01f7db24edd5394c095 fixes: bz#1505355 BUG: 1505355 Signed-off-by: Sanoj Unnikrishnan --- libglusterfs/src/dict.c | 190 ++++++++++++++++++++++++++++ libglusterfs/src/dict.h | 11 +- libglusterfs/src/glusterfs.h | 27 ++++ libglusterfs/src/libglusterfs.sym | 3 + xlators/cluster/dht/src/dht-selfheal.c | 17 ++- xlators/features/quota/src/quota-messages.h | 4 +- xlators/features/quota/src/quota.c | 31 ++++- xlators/storage/posix/src/posix-helpers.c | 4 + 8 files changed, 281 insertions(+), 6 deletions(-) diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c index d1a64c4a3a2..0f9632160cc 100644 --- a/libglusterfs/src/dict.c +++ b/libglusterfs/src/dict.c @@ -2022,6 +2022,196 @@ err: return ret; } +/* + * dict_check_flag can be used to check a one bit flag in an array of flags + * The flag argument indicates the bit position (within the array of bits). + * Currently limited to max of 256 flags for a key. + * return value, + * 1 : flag is set + * 0 : flag is not set + * <0: Error + */ +int +dict_check_flag (dict_t *this, char *key, int flag) +{ + data_t *data = NULL; + int ret = -ENOENT; + + ret = dict_get_with_ref (this, key, &data); + if (ret < 0) { + return ret; + } + + if (BIT_VALUE((unsigned char *)(data->data), flag)) + ret = 1; + else + ret = 0; + + data_unref(data); + return ret; +} + +/* + * _dict_modify_flag can be used to set/clear a bit flag in an array of flags + * flag: indicates the bit position. limited to max of DICT_MAX_FLAGS. + * op: Indicates operation DICT_FLAG_SET / DICT_FLAG_CLEAR + */ +static int +_dict_modify_flag (dict_t *this, char *key, int flag, int op) +{ + data_t *data = NULL; + int ret = 0; + uint32_t hash = 0; + data_pair_t *pair = NULL; + char *ptr = NULL; + int hashval = 0; + + if (!this || !key) { + gf_msg_callingfn ("dict", GF_LOG_WARNING, EINVAL, + LG_MSG_INVALID_ARG, + "dict OR key (%s) is NULL", key); + ret = -EINVAL; + goto err; + } + + /* + * Using a size of 32 bytes to support max of 256 + * flags in a single key. This should be suffcient. + */ + GF_ASSERT(flag >= 0 && flag < DICT_MAX_FLAGS); + + hash = SuperFastHash (key, strlen (key)); + LOCK (&this->lock); + { + pair = dict_lookup_common (this, key, hash); + + if (pair) { + data = pair->value; + if (op == DICT_FLAG_SET) + BIT_SET((unsigned char *)(data->data), flag); + else + BIT_CLEAR((unsigned char *)(data->data), flag); + ret = 0; + } else { + ptr = GF_CALLOC(1, DICT_MAX_FLAGS / 8, + gf_common_mt_char); + if (!ptr) { + gf_msg("dict", GF_LOG_ERROR, ENOMEM, + LG_MSG_NO_MEMORY, + "unable to allocate flag bit array"); + ret = -ENOMEM; + goto err; + } + + data = data_from_dynptr(ptr, DICT_MAX_FLAGS / 8); + + if (!data) { + gf_msg("dict", GF_LOG_ERROR, ENOMEM, + LG_MSG_NO_MEMORY, + "unable to allocate data"); + GF_FREE(ptr); + ret = -ENOMEM; + goto err; + } + + if (op == DICT_FLAG_SET) + BIT_SET((unsigned char *)(data->data), flag); + else + BIT_CLEAR((unsigned char *)(data->data), flag); + + if (this->free_pair_in_use) { + pair = mem_get0 (THIS->ctx->dict_pair_pool); + if (!pair) { + gf_msg("dict", GF_LOG_ERROR, ENOMEM, + LG_MSG_NO_MEMORY, + "unable to allocate dict pair"); + ret = -ENOMEM; + goto err; + } + } else { + pair = &this->free_pair; + this->free_pair_in_use = _gf_true; + } + + pair->key = (char *)GF_CALLOC(1, strlen (key) + 1, + gf_common_mt_char); + if (!pair->key) { + gf_msg("dict", GF_LOG_ERROR, ENOMEM, + LG_MSG_NO_MEMORY, + "unable to allocate dict pair"); + ret = -ENOMEM; + goto err; + } + strcpy (pair->key, key); + pair->key_hash = hash; + pair->value = data_ref (data); + + hashval = hash % this->hash_size; + pair->hash_next = this->members[hashval]; + this->members[hashval] = pair; + + pair->next = this->members_list; + pair->prev = NULL; + if (this->members_list) + this->members_list->prev = pair; + this->members_list = pair; + this->count++; + + + if (this->max_count < this->count) + this->max_count = this->count; + } + } + + UNLOCK (&this->lock); + return 0; + +err: + UNLOCK (&this->lock); + if (pair) { + if (pair->key) + free(pair->key); + + if (pair == &this->free_pair) { + this->free_pair_in_use = _gf_false; + } else { + mem_put (pair); + } + } + + if (data) + data_destroy(data); + + + gf_msg("dict", GF_LOG_ERROR, EINVAL, + LG_MSG_DICT_SET_FAILED, + "unable to set key (%s) in dict ", key); + + return ret; +} + +/* + * Todo: + * Add below primitives as needed: + * dict_check_flags(this, key, flag...): variadic function to check + * multiple flags at a time. + * dict_set_flags(this, key, flag...): set multiple flags + * dict_clear_flags(this, key, flag...): reset multiple flags + */ + +int +dict_set_flag (dict_t *this, char *key, int flag) +{ + return _dict_modify_flag (this, key, flag, DICT_FLAG_SET); +} + +int +dict_clear_flag (dict_t *this, char *key, int flag) +{ + return _dict_modify_flag (this, key, flag, DICT_FLAG_CLEAR); +} + + int dict_get_double (dict_t *this, char *key, double *val) { diff --git a/libglusterfs/src/dict.h b/libglusterfs/src/dict.h index e481330d6b5..d0b05172c2e 100644 --- a/libglusterfs/src/dict.h +++ b/libglusterfs/src/dict.h @@ -60,10 +60,13 @@ typedef struct _data_pair data_pair_t; \ } while (0) -#define DICT_KEY_VALUE_MAX_SIZE 1048576 - #define dict_foreach_inline(d, c) for (c = d->members_list; c; c = c->next) +#define DICT_KEY_VALUE_MAX_SIZE 1048576 +#define DICT_MAX_FLAGS 256 +#define DICT_FLAG_SET 1 +#define DICT_FLAG_CLEAR 0 + struct _data { unsigned char is_static:1; unsigned char is_const:1; @@ -227,6 +230,10 @@ GF_MUST_CHECK int dict_set_uint32 (dict_t *this, char *key, uint32_t val); GF_MUST_CHECK int dict_get_uint64 (dict_t *this, char *key, uint64_t *val); GF_MUST_CHECK int dict_set_uint64 (dict_t *this, char *key, uint64_t val); +GF_MUST_CHECK int dict_check_flag (dict_t *this, char *key, int flag); +GF_MUST_CHECK int dict_set_flag (dict_t *this, char *key, int flag); +GF_MUST_CHECK int dict_clear_flag (dict_t *this, char *key, int flag); + GF_MUST_CHECK int dict_get_double (dict_t *this, char *key, double *val); GF_MUST_CHECK int dict_set_double (dict_t *this, char *key, double val); diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h index 28666de3da4..84e33449dad 100644 --- a/libglusterfs/src/glusterfs.h +++ b/libglusterfs/src/glusterfs.h @@ -173,6 +173,33 @@ #define GLUSTERFS_VERSION_XCHG_KEY "glusterfs.version.xchg" #define GLUSTERFS_INTERNAL_FOP_KEY "glusterfs-internal-fop" + +/* GlusterFS Internal FOP Indicator flags + * (To pass information on the context in which a paritcular + * fop is performed between translators) + * The presence of a particular flag must be treated as an + * indicator of the context, however the flag is added only in + * a scenario where there is a need for such context across translators. + * So it cannot be an absolute information on context. + */ +#define GF_INTERNAL_CTX_KEY "glusterfs.internal-ctx" + +/* + * Always append entries to end of the enum, do not delete entries. + * Currently dict_set_flag allows to set upto 256 flag, if the enum + * needs to grow beyond this dict_set_flag has to be changed accordingly + */ +enum gf_internal_fop_indicator { + GF_DHT_HEAL_DIR /* Index 0 in bit array*/ +}; + +/* Todo: + * Add GF_FOP_LINK_FILE 0x2ULL + * address GLUSTERFS_MARKER_DONT_ACCOUNT_KEY and + * GLUSTERFS_INTERNAL_FOP_KEY with this flag + */ + + #define DHT_CHANGELOG_RENAME_OP_KEY "changelog.rename-op" #define ZR_FILE_CONTENT_STR "glusterfs.file." diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym index 6340bc8a3a2..1d21cfa8465 100644 --- a/libglusterfs/src/libglusterfs.sym +++ b/libglusterfs/src/libglusterfs.sym @@ -411,6 +411,9 @@ dict_set_static_ptr dict_set_str dict_set_uint32 dict_set_uint64 +dict_set_flag +dict_clear_flag +dict_check_flag dict_unref dict_unserialize drop_token diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c index d14020fabd0..4762f25067f 100644 --- a/xlators/cluster/dht/src/dht-selfheal.c +++ b/xlators/cluster/dht/src/dht-selfheal.c @@ -1411,10 +1411,25 @@ dht_selfheal_dir_mkdir_lookup_done (call_frame_t *frame, xlator_t *this) dht_dir_set_heal_xattr (this, local, dict, local->xattr, NULL, NULL); - if (!dict) + if (!dict) { gf_msg (this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED, "dict is NULL, need to make sure gfids are same"); + dict = dict_new (); + if (!dict) + return -1; + } + ret = dict_set_flag (dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + DHT_MSG_DICT_SET_FAILED, + "Failed to set dictionary value for" + " key = %s at path: %s", + GF_INTERNAL_CTX_KEY, loc->path); + /* We can still continue. As heal can still happen + * unless quota limits have reached for the dir. + */ + } cnt = layout->cnt; for (i = 0; i < cnt; i++) { diff --git a/xlators/features/quota/src/quota-messages.h b/xlators/features/quota/src/quota-messages.h index 4292c5b4dcc..85f5abf7b29 100644 --- a/xlators/features/quota/src/quota-messages.h +++ b/xlators/features/quota/src/quota-messages.h @@ -46,7 +46,9 @@ GLFS_MSGID(QUOTA, Q_MSG_INODE_CTX_GET_FAILED, Q_MSG_INODE_CTX_SET_FAILED, Q_MSG_LOOKUP_FAILED, - Q_MSG_RPC_SUBMIT_FAILED + Q_MSG_RPC_SUBMIT_FAILED, + Q_MSG_ENFORCEMENT_SKIPPED, + Q_MSG_INTERNAL_FOP_KEY_MISSING ); #endif /* !_QUOTA_MESSAGES_H_ */ diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c index 068d2e8c1f1..6ef2e20cf61 100644 --- a/xlators/features/quota/src/quota.c +++ b/xlators/features/quota/src/quota.c @@ -1591,6 +1591,28 @@ out: return ret; } +/* + * return _gf_true if enforcement is needed and _gf_false otherwise + */ +gf_boolean_t +should_quota_enforce (xlator_t *this, dict_t *dict, glusterfs_fop_t fop) +{ + int ret = 0; + + ret = dict_check_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR); + + if (fop == GF_FOP_MKDIR && ret == DICT_FLAG_SET) { + return _gf_false; + } else if (ret == -ENOENT) { + gf_msg (this->name, GF_LOG_DEBUG, EINVAL, + Q_MSG_INTERNAL_FOP_KEY_MISSING, + "No internal fop context present"); + goto out; + } +out: + return _gf_true; +} + int32_t quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, int32_t op_errno, inode_t *inode, @@ -1965,7 +1987,6 @@ unwind: return 0; } - int32_t quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata) @@ -1976,9 +1997,15 @@ quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, call_stub_t *stub = NULL; priv = this->private; - WIND_IF_QUOTAOFF (priv->is_quota_on, off); + if (!should_quota_enforce(this, xdata, GF_FOP_MKDIR)) { + gf_msg (this->name, GF_LOG_DEBUG, 0, + Q_MSG_ENFORCEMENT_SKIPPED, + "Enforcement has been skipped(internal fop)."); + goto off; + } + local = quota_local_new (); if (local == NULL) { op_errno = ENOMEM; diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index 816fb3587d2..6d7d8c512db 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -1208,6 +1208,10 @@ posix_handle_pair (xlator_t *this, const char *real_path, } else if (!strncmp(key, POSIX_ACL_ACCESS_XATTR, strlen(key)) && stbuf && IS_DHT_LINKFILE_MODE (stbuf)) { goto out; + } else if (!strncmp(key, GF_INTERNAL_CTX_KEY, strlen(key))) { + /* ignore this key value pair */ + ret = 0; + goto out; } else { sys_ret = sys_lsetxattr (real_path, key, value->data, value->len, flags); -- cgit