From e00d6ba4ad0a0d92907363b56e241b4f515b4dca Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Mon, 7 May 2018 17:18:06 +0530 Subject: cluster/dht: Leverage MDS subvol for dht_removexattr also Problem: In a distributed volume situation can be arise when custom extended attributed are not removed from all bricks after stop/start or added newly brick. Solution: To resolve the same use MDS subvol for remove xattr also BUG: 1575587 Change-Id: I7701e0d3833e3064274cb269f26061bff9b71f50 fixes: bz#1575587 Signed-off-by: Mohit Agrawal --- xlators/cluster/dht/src/dht-common.c | 250 ++++++++++++++++++++++++++--------- 1 file changed, 190 insertions(+), 60 deletions(-) (limited to 'xlators/cluster/dht/src') diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index 9165164578f..53ff58efa8b 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -3793,8 +3793,16 @@ unlock: this_call_cnt = dht_frame_return (frame); if (is_last_call (this_call_cnt)) { - DHT_STACK_UNWIND (setxattr, frame, local->op_ret, - local->op_errno, NULL); + if ((local->fop == GF_FOP_SETXATTR) || + (local->fop == GF_FOP_FSETXATTR)) { + DHT_STACK_UNWIND (setxattr, frame, local->op_ret, + local->op_errno, NULL); + } + if ((local->fop == GF_FOP_REMOVEXATTR) || + (local->fop == GF_FOP_FREMOVEXATTR)) { + DHT_STACK_UNWIND (removexattr, frame, local->op_ret, + local->op_errno, NULL); + } } return 0; @@ -3841,11 +3849,22 @@ dht_common_mds_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this, "subvolume %s returned -1", prev->this->name); - if (local->fop == GF_FOP_SETXATTR) { + if (local->fop == GF_FOP_SETXATTR) { DHT_STACK_UNWIND (setxattr, frame, 0, op_errno, local->xdata); - } else { + } + + if (local->fop == GF_FOP_FSETXATTR) { DHT_STACK_UNWIND (fsetxattr, frame, 0, op_errno, local->xdata); - } + } + + if (local->fop == GF_FOP_REMOVEXATTR) { + DHT_STACK_UNWIND (removexattr, frame, 0, op_errno, NULL); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + DHT_STACK_UNWIND (fremovexattr, frame, 0, op_errno, NULL); + } + return 0; } @@ -3901,7 +3920,8 @@ dht_setxattr_non_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this, ret = -1; goto out; } - if (local->fop == GF_FOP_SETXATTR) { + if ((local->fop == GF_FOP_SETXATTR) || + (local->fop == GF_FOP_REMOVEXATTR)) { STACK_WIND (frame, dht_common_mds_xattrop_cbk, local->mds_subvol, local->mds_subvol->fops->xattrop, @@ -3915,20 +3935,42 @@ dht_setxattr_non_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this, xattrop, NULL); } } else { - if (local->fop == GF_FOP_SETXATTR) + if (local->fop == GF_FOP_SETXATTR) { DHT_STACK_UNWIND (setxattr, frame, 0, 0, local->xdata); - else + } + + if (local->fop == GF_FOP_FSETXATTR) { DHT_STACK_UNWIND (fsetxattr, frame, 0, 0, local->xdata); + } + + if (local->fop == GF_FOP_REMOVEXATTR) { + DHT_STACK_UNWIND (removexattr, frame, 0, 0, NULL); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + DHT_STACK_UNWIND (fremovexattr, frame, 0, 0, NULL); + } } } out: if (xattrop) dict_unref (xattrop); if (ret) { - if (local->fop == GF_FOP_SETXATTR) + if (local->fop == GF_FOP_SETXATTR) { DHT_STACK_UNWIND (setxattr, frame, 0, 0, local->xdata); - else + } + + if (local->fop == GF_FOP_FSETXATTR) { DHT_STACK_UNWIND (fsetxattr, frame, 0, 0, local->xdata); + } + + if (local->fop == GF_FOP_REMOVEXATTR) { + DHT_STACK_UNWIND (removexattr, frame, 0, 0, NULL); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + DHT_STACK_UNWIND (fremovexattr, frame, 0, 0, NULL); + } } return 0; } @@ -3971,13 +4013,31 @@ dht_setxattr_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this, conf->subvolumes[i]->fops->setxattr, &local->loc, local->xattr, local->flags, local->xattr_req); - } else { + } + + if (local->fop == GF_FOP_FSETXATTR) { STACK_WIND (frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i], conf->subvolumes[i]->fops->fsetxattr, local->fd, local->xattr, local->flags, local->xattr_req); } + + if (local->fop == GF_FOP_REMOVEXATTR) { + STACK_WIND (frame, dht_setxattr_non_mds_cbk, + conf->subvolumes[i], + conf->subvolumes[i]->fops->removexattr, + &local->loc, local->key, + local->xattr_req); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + STACK_WIND (frame, dht_setxattr_non_mds_cbk, + conf->subvolumes[i], + conf->subvolumes[i]->fops->fremovexattr, + local->fd, local->key, + local->xattr_req); + } } return 0; @@ -3985,11 +4045,23 @@ out: if (local->fop == GF_FOP_SETXATTR) { DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno, xdata); - } else { + } + + if (local->fop == GF_FOP_FSETXATTR) { DHT_STACK_UNWIND (fsetxattr, frame, local->op_ret, local->op_errno, xdata); } + if (local->fop == GF_FOP_REMOVEXATTR) { + DHT_STACK_UNWIND (removexattr, frame, local->op_ret, + local->op_errno, NULL); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + DHT_STACK_UNWIND (fremovexattr, frame, local->op_ret, + local->op_errno, NULL); + } + return 0; } @@ -4018,21 +4090,55 @@ dht_xattrop_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->mds_subvol->fops->setxattr, &local->loc, local->xattr, local->flags, local->xattr_req); - } else { + } + + if (local->fop == GF_FOP_FSETXATTR) { STACK_WIND (frame, dht_setxattr_mds_cbk, local->mds_subvol, local->mds_subvol->fops->fsetxattr, local->fd, local->xattr, local->flags, local->xattr_req); } + + if (local->fop == GF_FOP_REMOVEXATTR) { + STACK_WIND (frame, dht_setxattr_mds_cbk, + local->mds_subvol, + local->mds_subvol->fops->removexattr, + &local->loc, local->key, + local->xattr_req); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + STACK_WIND (frame, dht_setxattr_mds_cbk, + local->mds_subvol, + local->mds_subvol->fops->fremovexattr, + local->fd, local->key, + local->xattr_req); + } + + return 0; out: - if (local->fop == GF_FOP_SETXATTR) + if (local->fop == GF_FOP_SETXATTR) { DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno, xdata); - else + } + + if (local->fop == GF_FOP_FSETXATTR) { DHT_STACK_UNWIND (fsetxattr, frame, local->op_ret, local->op_errno, xdata); + } + + if (local->fop == GF_FOP_REMOVEXATTR) { + DHT_STACK_UNWIND (removexattr, frame, local->op_ret, + local->op_errno, NULL); + } + + if (local->fop == GF_FOP_FREMOVEXATTR) { + DHT_STACK_UNWIND (fremovexattr, frame, local->op_ret, + local->op_errno, NULL); + } + return 0; } @@ -5299,12 +5405,12 @@ dht_is_user_xattr (dict_t *this, char *key, data_t *value, void *data) } -/* Common code to wind a (f)setxattr call to set xattr on directory +/* Common code to wind a (f)(set|remove)xattr call to set xattr on directory */ int -dht_dir_common_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, - fd_t *fd, dict_t *xattr, int flags, dict_t *xdata, - int *op_errno) +dht_dir_common_set_remove_xattr (call_frame_t *frame, xlator_t *this, loc_t *loc, + fd_t *fd, dict_t *xattr, int flags, dict_t *xdata, + int *op_errno) { dict_t *xattrop = NULL; @@ -5327,18 +5433,30 @@ dht_dir_common_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, if (local->gfid) gf_uuid_unparse(local->gfid, gfid_local); - /* Check if any user xattr present in xattr - */ - dict_foreach_fnmatch (xattr, "user*", dht_is_user_xattr, - &uxattr_key_found); + if ((local->fop == GF_FOP_SETXATTR) || + (local->fop == GF_FOP_FSETXATTR)) { + /* Check if any user xattr present in xattr + */ + dict_foreach_fnmatch (xattr, "user*", dht_is_user_xattr, + &uxattr_key_found); - /* Check if any custom key xattr present in dict xattr - and start index from 1 because user xattr already - checked in previous line - */ - for (i = 1; xattrs_to_heal[i]; i++) - if (dict_get (xattr, xattrs_to_heal[i])) - uxattr_key_found = _gf_true; + /* Check if any custom key xattr present in dict xattr + and start index from 1 because user xattr already + checked in previous line + */ + for (i = 1; xattrs_to_heal[i]; i++) + if (dict_get (xattr, xattrs_to_heal[i])) + uxattr_key_found = _gf_true; + } + + if ((local->fop == GF_FOP_REMOVEXATTR) || + (local->fop == GF_FOP_FREMOVEXATTR)) { + /* Check if any custom key xattr present in local->key + */ + for (i = 0; xattrs_to_heal[i]; i++) + if (strstr (local->key, xattrs_to_heal[i])) + uxattr_key_found = _gf_true; + } /* If there is no custom key xattr present or gfid is root or call_cnt is 1 then wind a (f)setxattr call on all subvols @@ -5346,16 +5464,34 @@ dht_dir_common_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, if (!uxattr_key_found || __is_root_gfid (local->gfid) || call_cnt == 1) { for (i = 0; i < conf->subvolume_cnt; i++) { travvol = conf->subvolumes[i]; - if (fd) { - STACK_WIND_COOKIE (frame, dht_err_cbk, - travvol, travvol, - travvol->fops->fsetxattr, - fd, xattr, flags, xdata); - } else { - STACK_WIND_COOKIE (frame, dht_err_cbk, - travvol, travvol, - travvol->fops->setxattr, - loc, xattr, flags, xdata); + if ((local->fop == GF_FOP_SETXATTR) || + (local->fop == GF_FOP_FSETXATTR)) { + if (fd) { + STACK_WIND_COOKIE (frame, dht_err_cbk, + travvol, travvol, + travvol->fops->fsetxattr, + fd, xattr, flags, xdata); + } else { + STACK_WIND_COOKIE (frame, dht_err_cbk, + travvol, travvol, + travvol->fops->setxattr, + loc, xattr, flags, xdata); + } + } + + if ((local->fop == GF_FOP_REMOVEXATTR) || + (local->fop == GF_FOP_FREMOVEXATTR)) { + if (fd) { + STACK_WIND_COOKIE (frame, dht_err_cbk, + travvol, travvol, + travvol->fops->fremovexattr, + fd, local->key, local->xattr_req); + } else { + STACK_WIND_COOKIE (frame, dht_err_cbk, + travvol, travvol, + travvol->fops->removexattr, + loc, local->key, local->xattr_req); + } } } @@ -5503,8 +5639,8 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, if (IA_ISDIR (fd->inode->ia_type)) { local->hashed_subvol = NULL; - ret = dht_dir_common_setxattr (frame, this, NULL, fd, - xattr, flags, xdata, &op_errno); + ret = dht_dir_common_set_remove_xattr (frame, this, NULL, fd, + xattr, flags, xdata, &op_errno); if (ret) goto err; } else { @@ -5912,8 +6048,8 @@ dht_setxattr (call_frame_t *frame, xlator_t *this, if (IA_ISDIR (loc->inode->ia_type)) { local->hashed_subvol = NULL; - ret = dht_dir_common_setxattr (frame, this, loc, NULL, - xattr, flags, xdata, &op_errno); + ret = dht_dir_common_set_remove_xattr (frame, this, loc, NULL, + xattr, flags, xdata, &op_errno); if (ret) goto err; } else { @@ -6114,7 +6250,6 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, dht_layout_t *layout = NULL; int call_cnt = 0; dht_conf_t *conf = NULL; - int i; int ret = 0; VALIDATE_OR_GOTO (this, err); @@ -6161,13 +6296,11 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, } if (IA_ISDIR (loc->inode->ia_type)) { - for (i = 0; i < call_cnt; i++) { - STACK_WIND_COOKIE (frame, dht_removexattr_cbk, - layout->list[i].xlator, - layout->list[i].xlator, - layout->list[i].xlator->fops->removexattr, - loc, key, local->xattr_req); - } + local->hashed_subvol = NULL; + ret = dht_dir_common_set_remove_xattr (frame, this, loc, NULL, + NULL, 0, local->xattr_req, &op_errno); + if (ret) + goto err; } else { @@ -6206,7 +6339,6 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this, dht_conf_t *conf = 0; int ret = 0; - int i; VALIDATE_OR_GOTO (this, err); VALIDATE_OR_GOTO (this->private, err); @@ -6246,13 +6378,11 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this, local->key = gf_strdup (key); if (IA_ISDIR (fd->inode->ia_type)) { - for (i = 0; i < call_cnt; i++) { - STACK_WIND_COOKIE (frame, dht_removexattr_cbk, - layout->list[i].xlator, - layout->list[i].xlator, - layout->list[i].xlator->fops->fremovexattr, - fd, key, local->xattr_req); - } + local->hashed_subvol = NULL; + ret = dht_dir_common_set_remove_xattr (frame, this, NULL, fd, + NULL, 0, local->xattr_req, &op_errno); + if (ret) + goto err; } else { -- cgit