From d60ca8e96bbc16b13f8f3456f30ebeb16d0d1e47 Mon Sep 17 00:00:00 2001 From: Krutika Dhananjay Date: Thu, 20 Apr 2017 10:08:02 +0530 Subject: cluster/dht: Pass the req dict instead of NULL in dht_attr2() This bug was causing VMs to pause during rebalance. When qemu winds down a STAT, shard fills the trusted.glusterfs.shard.file-size attribute in the req dict which DHT doesn't wind its STAT fop with upon detecting the file has undergone migration. As a result shard doesn't find the value to this key in the unwind path, causing it to fail the STAT with EINVAL. Also, the same bug exists in other fops too, which is also fixed in this patch. Change-Id: Id7823fd932b4e5a9b8779ebb2b612a399c0ef5f0 BUG: 1440051 Signed-off-by: Krutika Dhananjay Reviewed-on: https://review.gluster.org/17085 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Raghavendra G --- xlators/cluster/dht/src/dht-common.c | 53 ++++++++++++------------------- xlators/cluster/dht/src/dht-inode-read.c | 39 ++++++++++++++++------- xlators/cluster/dht/src/dht-inode-write.c | 33 +++++++++++++------ 3 files changed, 71 insertions(+), 54 deletions(-) (limited to 'xlators/cluster') diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index e3af5a7fe8b..836a009c362 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -3834,6 +3834,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, goto err; } + local->xattr_req = xdata ? dict_ref (xdata) : dict_new (); local->call_cnt = call_cnt = layout->cnt; if (IA_ISDIR (fd->inode->ia_type)) { @@ -3842,7 +3843,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, layout->list[i].xlator, layout->list[i].xlator, layout->list[i].xlator->fops->fsetxattr, - fd, xattr, flags, NULL); + fd, xattr, flags, xdata); } } else { @@ -3851,9 +3852,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, local->rebalance.xattr = dict_ref (xattr); local->rebalance.flags = flags; - xdata = xdata ? dict_ref (xdata) : dict_new (); - if (xdata) - ret = dict_set_int8 (xdata, DHT_IATT_IN_XDATA_KEY, 1); + ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1); if (ret) { gf_msg_debug (this->name, 0, "Failed to set dictionary key %s for fd=%p", @@ -3862,11 +3861,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this, STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol, subvol, subvol->fops->fsetxattr, fd, xattr, - flags, xdata); - - if (xdata) - dict_unref (xdata); - + flags, local->xattr_req); } return 0; @@ -3962,12 +3957,12 @@ dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol, subvol, subvol->fops->setxattr, &local->loc, local->rebalance.xattr, - local->rebalance.flags, NULL); + local->rebalance.flags, local->xattr_req); } else { STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol, subvol, subvol->fops->fsetxattr, local->fd, local->rebalance.xattr, - local->rebalance.flags, NULL); + local->rebalance.flags, local->xattr_req); } return 0; @@ -4251,6 +4246,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this, if (tmp) { return dht_nuke_dir (frame, this, loc, tmp); } + local->xattr_req = xdata ? dict_ref (xdata) : dict_new (); if (IA_ISDIR (loc->inode->ia_type)) { @@ -4268,16 +4264,11 @@ dht_setxattr (call_frame_t *frame, xlator_t *this, local->rebalance.flags = flags; local->call_cnt = 1; - xdata = xdata ? dict_ref (xdata) : dict_new (); - if (xdata) - ret = dict_set_int8 (xdata, DHT_IATT_IN_XDATA_KEY, 1); + ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1); STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol, subvol, subvol->fops->setxattr, loc, xattr, - flags, xdata); - - if (xdata) - dict_unref (xdata); + flags, local->xattr_req); } return 0; @@ -4396,11 +4387,11 @@ dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, if (local->fop == GF_FOP_REMOVEXATTR) { STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol, subvol, subvol->fops->removexattr, - &local->loc, local->key, NULL); + &local->loc, local->key, local->xattr_req); } else { STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol, subvol, subvol->fops->fremovexattr, - local->fd, local->key, NULL); + local->fd, local->key, local->xattr_req); } return 0; @@ -4437,8 +4428,6 @@ dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, unlock: UNLOCK (&frame->lock); - - this_call_cnt = dht_frame_return (frame); if (is_last_call (this_call_cnt)) { DHT_STACK_UNWIND (removexattr, frame, local->op_ret, @@ -4494,6 +4483,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, op_errno = EINVAL; goto err; } + local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new (); local->call_cnt = call_cnt = layout->cnt; local->key = gf_strdup (key); @@ -4504,15 +4494,13 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, layout->list[i].xlator, layout->list[i].xlator, layout->list[i].xlator->fops->removexattr, - loc, key, NULL); + loc, key, local->xattr_req); } } else { local->call_cnt = 1; - xdata = xdata ? dict_ref (xdata) : dict_new (); - if (xdata) - ret = dict_set_int8 (xdata, DHT_IATT_IN_XDATA_KEY, 1); + ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1); if (ret) { gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED, "Failed to " @@ -4522,10 +4510,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this, STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol, subvol, subvol->fops->removexattr, loc, key, - xdata); - - if (xdata) - dict_unref (xdata); + local->xattr_req); } return 0; @@ -6561,7 +6546,7 @@ dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) local->call_cnt = 2; STACK_WIND (frame, dht_link_cbk, subvol, subvol->fops->link, - &local->loc, &local->loc2, NULL); + &local->loc, &local->loc2, local->xattr_req); return 0; err: @@ -6588,7 +6573,7 @@ dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this, srcvol = local->linkfile.srcvol; STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link, - &local->loc, &local->loc2, xdata); + &local->loc, &local->loc2, local->xattr_req); return 0; @@ -6597,7 +6582,7 @@ err: dht_set_fixed_dir_stat (preparent); dht_set_fixed_dir_stat (postparent); DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent, - postparent, NULL); + postparent, xdata); return 0; } @@ -6648,6 +6633,8 @@ dht_link (call_frame_t *frame, xlator_t *this, op_errno = ENOMEM; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); if (hashed_subvol != cached_subvol) { gf_uuid_copy (local->gfid, oldloc->inode->gfid); diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c index 58168dec81b..c53662d6740 100644 --- a/xlators/cluster/dht/src/dht-inode-read.c +++ b/xlators/cluster/dht/src/dht-inode-read.c @@ -76,7 +76,7 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) if (we_are_not_migrating (ret)) { /* This DHT layer is not migrating the file */ DHT_STACK_UNWIND (open, frame, -1, local->op_errno, - NULL, NULL); + NULL, local->rebalance.xdata); return 0; } @@ -88,7 +88,7 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_open_cbk, subvol, subvol, subvol->fops->open, &local->loc, - local->rebalance.flags, local->fd, NULL); + local->rebalance.flags, local->fd, local->xattr_req); return 0; out: @@ -122,6 +122,8 @@ dht_open (call_frame_t *frame, xlator_t *this, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); local->rebalance.flags = flags; local->call_cnt = 1; @@ -239,10 +241,12 @@ dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) if (local->fop == GF_FOP_FSTAT) { STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol, subvol, - subvol->fops->fstat, local->fd, NULL); + subvol->fops->fstat, local->fd, + local->xattr_req); } else { STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol, subvol, - subvol->fops->stat, &local->loc, NULL); + subvol->fops->stat, &local->loc, + local->xattr_req); } return 0; @@ -325,6 +329,8 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); if (IA_ISREG (loc->inode->ia_type)) { local->call_cnt = 1; @@ -384,6 +390,8 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); if (IA_ISREG (fd->inode->ia_type)) { local->call_cnt = 1; @@ -504,7 +512,7 @@ dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd, local->rebalance.size, local->rebalance.offset, - local->rebalance.flags, NULL); + local->rebalance.flags, local->xattr_req); return 0; @@ -539,6 +547,8 @@ dht_readv (call_frame_t *frame, xlator_t *this, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); local->rebalance.offset = off; local->rebalance.size = size; @@ -635,7 +645,7 @@ dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_access_cbk, subvol, subvol, subvol->fops->access, &local->loc, - local->rebalance.flags, NULL); + local->rebalance.flags, local->xattr_req); return 0; @@ -674,6 +684,8 @@ dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); STACK_WIND_COOKIE (frame, dht_access_cbk, subvol, subvol, subvol->fops->access, loc, mask, xdata); @@ -708,9 +720,6 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, local->op_ret = op_ret; local->op_errno = op_errno; - if (xdata) - local->rebalance.xdata = dict_ref (xdata); - /* If context is set, then send flush() it to the destination */ dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol); if (subvol && dht_fd_open_on_dst (this, local->fd, subvol)) { @@ -751,7 +760,7 @@ dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND (frame, dht_flush_cbk, subvol, subvol->fops->flush, local->fd, - local->rebalance.xdata); + local->xattr_req); return 0; @@ -785,6 +794,8 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); local->call_cnt = 1; @@ -908,7 +919,7 @@ dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync, local->fd, - local->rebalance.flags, NULL); + local->rebalance.flags, local->xattr_req); return 0; @@ -935,6 +946,8 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync, goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); local->call_cnt = 1; local->rebalance.flags = datasync; @@ -1025,7 +1038,7 @@ dht_lk2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd, local->rebalance.lock_cmd, &local->rebalance.flock, - local->rebalance.xdata); + local->xattr_req); return 0; @@ -1060,6 +1073,8 @@ dht_lk (call_frame_t *frame, xlator_t *this, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); local->rebalance.flock = *flock; local->rebalance.lock_cmd = cmd; diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c index 93755b9703a..825822a27e6 100644 --- a/xlators/cluster/dht/src/dht-inode-write.c +++ b/xlators/cluster/dht/src/dht-inode-write.c @@ -144,7 +144,7 @@ dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) local->fd, local->rebalance.vector, local->rebalance.count, local->rebalance.offset, local->rebalance.flags, - local->rebalance.iobref, NULL); + local->rebalance.iobref, local->xattr_req); return 0; @@ -181,7 +181,8 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, op_errno = EINVAL; goto err; } - + if (xdata) + local->xattr_req = dict_ref (xdata); local->rebalance.vector = iov_dup (vector, count); local->rebalance.offset = off; @@ -324,11 +325,11 @@ dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) if (local->fop == GF_FOP_TRUNCATE) { STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol, subvol->fops->truncate, &local->loc, - local->rebalance.offset, NULL); + local->rebalance.offset, local->xattr_req); } else { STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol, subvol->fops->ftruncate, local->fd, - local->rebalance.offset, NULL); + local->rebalance.offset, local->xattr_req); } return 0; @@ -367,6 +368,8 @@ dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol, subvol->fops->truncate, loc, offset, xdata); @@ -407,6 +410,8 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol, subvol->fops->ftruncate, fd, offset, xdata); @@ -533,7 +538,7 @@ dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol, subvol->fops->fallocate, local->fd, local->rebalance.flags, local->rebalance.offset, - local->rebalance.size, NULL); + local->rebalance.size, local->xattr_req); return 0; @@ -572,6 +577,8 @@ dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol, subvol->fops->fallocate, fd, mode, offset, len, @@ -699,7 +706,7 @@ dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol, subvol->fops->discard, local->fd, local->rebalance.offset, local->rebalance.size, - NULL); + local->xattr_req); return 0; @@ -737,6 +744,8 @@ dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol, subvol->fops->discard, fd, offset, len, xdata); @@ -863,7 +872,7 @@ dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol, subvol->fops->zerofill, local->fd, local->rebalance.offset, - local->rebalance.size, NULL); + local->rebalance.size, local->xattr_req); return 0; @@ -902,6 +911,8 @@ dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol, subvol->fops->zerofill, fd, offset, len, xdata); @@ -1006,12 +1017,12 @@ dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret) STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol, subvol, subvol->fops->setattr, &local->loc, &local->rebalance.stbuf, local->rebalance.flags, - NULL); + local->xattr_req); } else { STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol, subvol, subvol->fops->fsetattr, local->fd, &local->rebalance.stbuf, local->rebalance.flags, - NULL); + local->xattr_req); } return 0; @@ -1104,6 +1115,8 @@ dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); if (IA_ISREG (loc->inode->ia_type)) { /* in the regular file _cbk(), we need to check for @@ -1176,6 +1189,8 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf, op_errno = EINVAL; goto err; } + if (xdata) + local->xattr_req = dict_ref (xdata); if (IA_ISREG (fd->inode->ia_type)) { /* in the regular file _cbk(), we need to check for -- cgit