cluster/dht: Pass the req dict instead of NULL in dht_attr2()

        Backport of: https://review.gluster.org/17085

This bug was causing VMs to pause during rebalance. When qemu winds
down a STAT, shard fills the trusted.glusterfs.shard.file-size attribute
in the req dict which DHT doesn't wind its STAT fop with upon detecting
the file has undergone migration. As a result shard doesn't find the
value to this key in the unwind path, causing it to fail the STAT
with EINVAL.

Also, the same bug exists in other fops too, which is also fixed in
this patch.

Change-Id: I56273b1a65347dabd38bc6bdd12d618f68287a00
BUG: 1440635
Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-on: https://review.gluster.org/17121
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

3 files changed, 74 insertions, 57 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index a50d6fe5628..a55922f0a74 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -3823,6 +3823,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
+        local->xattr_req = xdata ? dict_ref (xdata) : dict_new ();
         local->call_cnt = call_cnt = layout->cnt;
 
         if (IA_ISDIR (fd->inode->ia_type)) {
@@ -3830,7 +3831,7 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, dht_err_cbk,
                                     layout->list[i].xlator,
                                     layout->list[i].xlator->fops->fsetxattr,
-                                    fd, xattr, flags, NULL);
+                                    fd, xattr, flags, xdata);
                 }
 
         } else {
@@ -3839,10 +3840,8 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                 local->rebalance.xattr = dict_ref (xattr);
                 local->rebalance.flags = flags;
 
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                        DHT_IATT_IN_XDATA_KEY, "yes");
+                ret = dict_set_dynstr_with_alloc (local->xattr_req,
+                                                  DHT_IATT_IN_XDATA_KEY, "yes");
                 if (ret) {
                         gf_msg_debug (this->name, 0,
                                       "Failed to set dictionary key %s for fd=%p",
@@ -3850,11 +3849,8 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                 }
 
                 STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
-                    subvol->fops->fsetxattr, fd, xattr, flags, xdata);
-
-                if (xdata)
-                        dict_unref (xdata);
-
+                            subvol->fops->fsetxattr, fd, xattr, flags,
+                            local->xattr_req);
         }
         return 0;
 
@@ -3950,12 +3946,12 @@ dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
                 STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
                             subvol->fops->setxattr, &local->loc,
                             local->rebalance.xattr, local->rebalance.flags,
-                            NULL);
+                            local->xattr_req);
         } else {
                 STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
                             subvol->fops->fsetxattr, local->fd,
                             local->rebalance.xattr, local->rebalance.flags,
-                            NULL);
+                            local->xattr_req);
         }
 
         return 0;
@@ -4239,6 +4235,7 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         if (tmp) {
                 return dht_nuke_dir (frame, this, loc, tmp);
         }
+        local->xattr_req = xdata ? dict_ref (xdata) : dict_new ();
 
         if (IA_ISDIR (loc->inode->ia_type)) {
 
@@ -4255,17 +4252,12 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 local->rebalance.flags = flags;
                 local->call_cnt = 1;
 
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                              DHT_IATT_IN_XDATA_KEY, "yes");
+                ret = dict_set_dynstr_with_alloc (local->xattr_req,
+                                                  DHT_IATT_IN_XDATA_KEY, "yes");
 
                 STACK_WIND (frame, dht_file_setxattr_cbk,
                             subvol, subvol->fops->setxattr,
-                            loc, xattr, flags, xdata);
-
-                if (xdata)
-                        dict_unref (xdata);
+                            loc, xattr, flags, local->xattr_req);
         }
 
         return 0;
@@ -4384,11 +4376,11 @@ dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
         if (local->fop == GF_FOP_REMOVEXATTR) {
                 STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
                             subvol->fops->removexattr, &local->loc,
-                            local->key, NULL);
+                            local->key, local->xattr_req);
         } else {
                 STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
                             subvol->fops->fremovexattr, local->fd,
-                            local->key, NULL);
+                            local->key, local->xattr_req);
         }
 
         return 0;
@@ -4425,8 +4417,6 @@ dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
-
-
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
                 DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
@@ -4482,6 +4472,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
                 op_errno = EINVAL;
                 goto err;
         }
+        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
 
         local->call_cnt = call_cnt = layout->cnt;
         local->key = gf_strdup (key);
@@ -4491,16 +4482,15 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
                         STACK_WIND (frame, dht_removexattr_cbk,
                                     layout->list[i].xlator,
                                     layout->list[i].xlator->fops->removexattr,
-                                    loc, key, NULL);
+                                    loc, key, local->xattr_req);
                 }
 
         } else {
 
                 local->call_cnt = 1;
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                 DHT_IATT_IN_XDATA_KEY, "yes");
+
+                ret = dict_set_dynstr_with_alloc (local->xattr_req,
+                                                  DHT_IATT_IN_XDATA_KEY, "yes");
                 if (ret) {
                         gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
                                 DHT_MSG_DICT_SET_FAILED, "Failed to "
@@ -4510,10 +4500,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
 
                 STACK_WIND (frame, dht_file_removexattr_cbk,
                             subvol, subvol->fops->removexattr,
-                            loc, key, xdata);
-
-                if (xdata)
-                        dict_unref (xdata);
+                            loc, key, local->xattr_req);
         }
 
         return 0;
@@ -6511,7 +6498,7 @@ dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
         local->call_cnt = 2;
 
         STACK_WIND (frame, dht_link_cbk, subvol, subvol->fops->link,
-                    &local->loc, &local->loc2, NULL);
+                    &local->loc, &local->loc2, local->xattr_req);
 
         return 0;
 err:
@@ -6538,7 +6525,7 @@ dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         srcvol = local->linkfile.srcvol;
 
         STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2, xdata);
+                    &local->loc, &local->loc2, local->xattr_req);
 
         return 0;
 
@@ -6547,7 +6534,7 @@ err:
         dht_set_fixed_dir_stat (preparent);
         dht_set_fixed_dir_stat (postparent);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent, NULL);
+                          postparent, xdata);
 
         return 0;
 }
@@ -6598,6 +6585,8 @@ dht_link (call_frame_t *frame, xlator_t *this,
                 op_errno = ENOMEM;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         if (hashed_subvol != cached_subvol) {
                 gf_uuid_copy (local->gfid, oldloc->inode->gfid);
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 8abf0d59b88..298eca711b4 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -76,7 +76,7 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
         if (we_are_not_migrating (ret)) {
                 /* This DHT layer is not migrating the file */
                 DHT_STACK_UNWIND (open, frame, -1, local->op_errno,
-                                  NULL, NULL);
+                                  NULL, local->rebalance.xdata);
                 return 0;
 
         }
@@ -88,7 +88,7 @@ dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
         STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
                     &local->loc, local->rebalance.flags, local->fd,
-                    NULL);
+                    local->xattr_req);
         return 0;
 
 out:
@@ -122,6 +122,8 @@ dht_open (call_frame_t *frame, xlator_t *this,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         local->rebalance.flags = flags;
         local->call_cnt = 1;
@@ -239,10 +241,10 @@ dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
         if (local->fop == GF_FOP_FSTAT) {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd, NULL);
+                            subvol->fops->fstat, local->fd, local->xattr_req);
         } else {
                 STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc, NULL);
+                            subvol->fops->stat, &local->loc, local->xattr_req);
         }
 
         return 0;
@@ -325,6 +327,8 @@ dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         if (IA_ISREG (loc->inode->ia_type)) {
                 local->call_cnt = 1;
@@ -385,6 +389,8 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         if (IA_ISREG (fd->inode->ia_type)) {
                 local->call_cnt = 1;
@@ -506,7 +512,7 @@ dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
         STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
                     local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags, NULL);
+                    local->rebalance.flags, local->xattr_req);
 
         return 0;
 
@@ -541,6 +547,8 @@ dht_readv (call_frame_t *frame, xlator_t *this,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         local->rebalance.offset = off;
         local->rebalance.size   = size;
@@ -635,7 +643,7 @@ dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
         local->call_cnt = 2;
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags, NULL);
+                    &local->loc, local->rebalance.flags, local->xattr_req);
 
         return 0;
 
@@ -674,6 +682,8 @@ dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
                     loc, mask, xdata);
@@ -708,9 +718,6 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->op_ret = op_ret;
         local->op_errno = op_errno;
 
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
-
         /* If context is set, then send flush() it to the destination */
         dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol);
         if (subvol && dht_fd_open_on_dst (this, local->fd, subvol)) {
@@ -751,7 +758,7 @@ dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
         STACK_WIND (frame, dht_flush_cbk,
                     subvol, subvol->fops->flush, local->fd,
-                    local->rebalance.xdata);
+                    local->xattr_req);
 
         return 0;
 
@@ -785,6 +792,8 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         local->call_cnt = 1;
 
@@ -907,7 +916,7 @@ dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
         local->call_cnt = 2; /* This is the second attempt */
 
         STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags, NULL);
+                    local->fd, local->rebalance.flags, local->xattr_req);
 
         return 0;
 
@@ -934,6 +943,8 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
 
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         local->call_cnt = 1;
         local->rebalance.flags = datasync;
@@ -1024,7 +1035,7 @@ dht_lk2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
         STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
                     local->rebalance.lock_cmd, &local->rebalance.flock,
-                    local->rebalance.xdata);
+                    local->xattr_req);
 
         return 0;
 
@@ -1059,6 +1070,8 @@ dht_lk (call_frame_t *frame, xlator_t *this,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         local->rebalance.flock = *flock;
         local->rebalance.lock_cmd = cmd;
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 112685b659e..364b66c942e 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -143,7 +143,7 @@ dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
                     subvol, subvol->fops->writev,
                     local->fd, local->rebalance.vector, local->rebalance.count,
                     local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref, NULL);
+                    local->rebalance.iobref, local->xattr_req);
 
         return 0;
 
@@ -180,7 +180,8 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 op_errno = EINVAL;
                 goto err;
         }
-
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         local->rebalance.vector = iov_dup (vector, count);
         local->rebalance.offset = off;
@@ -323,11 +324,11 @@ dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
         if (local->fop == GF_FOP_TRUNCATE) {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset, NULL);
+                            local->rebalance.offset, local->xattr_req);
         } else {
                 STACK_WIND (frame, dht_truncate_cbk, subvol,
                             subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset, NULL);
+                            local->rebalance.offset, local->xattr_req);
         }
 
         return 0;
@@ -366,6 +367,8 @@ dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         STACK_WIND (frame, dht_truncate_cbk,
                     subvol, subvol->fops->truncate,
@@ -407,6 +410,8 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         STACK_WIND (frame, dht_truncate_cbk,
                     subvol, subvol->fops->ftruncate,
@@ -533,7 +538,7 @@ dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
 	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
 		   local->fd, local->rebalance.flags, local->rebalance.offset,
-		   local->rebalance.size, NULL);
+		   local->rebalance.size, local->xattr_req);
 
         return 0;
 
@@ -572,6 +577,8 @@ dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         STACK_WIND (frame, dht_fallocate_cbk,
                     subvol, subvol->fops->fallocate,
@@ -698,7 +705,7 @@ dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
 	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
 		   local->fd, local->rebalance.offset, local->rebalance.size,
-		   NULL);
+		   local->xattr_req);
 
         return 0;
 
@@ -736,6 +743,8 @@ dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
                     fd, offset, len, xdata);
@@ -861,7 +870,7 @@ dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 
         STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
                    local->fd, local->rebalance.offset, local->rebalance.size,
-                   NULL);
+                   local->xattr_req);
 
         return 0;
 
@@ -900,6 +909,8 @@ dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
                     fd, offset, len, xdata);
@@ -1004,12 +1015,12 @@ dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->setattr, &local->loc,
                             &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
+                            local->xattr_req);
         } else {
                 STACK_WIND (frame, dht_file_setattr_cbk, subvol,
                             subvol->fops->fsetattr, local->fd,
                             &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
+                            local->xattr_req);
         }
 
         return 0;
@@ -1102,6 +1113,8 @@ dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         if (IA_ISREG (loc->inode->ia_type)) {
                 /* in the regular file _cbk(), we need to check for
@@ -1173,6 +1186,8 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
                 op_errno = EINVAL;
                 goto err;
         }
+        if (xdata)
+                local->xattr_req = dict_ref (xdata);
 
         if (IA_ISREG (fd->inode->ia_type)) {
                 /* in the regular file _cbk(), we need to check for