cluster/dht: Fix dht_setxattr to follow files under migration

If a file is under migration, then any xattrs created on it
are lost post migration of the file. This is because
the xattrs are set only on the cached subvol of the source
and as the source is under migration, it becomes a linkto file
post migration.

Change-Id: Ib8e233b519cf954e7723c6e26b38fa8f9b8c85c0
BUG: 1225839
Signed-off-by: Nithya Balachandran <nbalacha@redhat.com>
Reviewed-on: http://review.gluster.org/10968
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>

6 files changed, 603 insertions, 31 deletions

diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 9ed02e5400f..6f20185f80b 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -230,8 +230,9 @@
 #define DHT_LINKFILE_STR "linkto"
 #define DHT_COMMITHASH_STR "commithash"
 
-#define DHT_SKIP_NON_LINKTO_UNLINK "unlink-only-if-dht-linkto-file"
-#define DHT_SKIP_OPEN_FD_UNLINK "dont-unlink-for-open-fd"
+#define DHT_SKIP_NON_LINKTO_UNLINK  "unlink-only-if-dht-linkto-file"
+#define DHT_SKIP_OPEN_FD_UNLINK     "dont-unlink-for-open-fd"
+#define DHT_IATT_IN_XDATA_KEY       "dht-get-iatt-in-xattr"
 
 /*CTR requires inode dentry link count from posix*/
 #define CTR_RESPONSE_LINK_COUNT_XDATA "ctr_response_link_count"
diff --git a/tests/bugs/distribute/bug-1193636.c b/tests/bugs/distribute/bug-1193636.c
new file mode 100644
index 00000000000..eae90783f8e
--- /dev/null
+++ b/tests/bugs/distribute/bug-1193636.c
@@ -0,0 +1,70 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <attr/xattr.h>
+#include <fcntl.h>
+#include <string.h>
+
+
+#define MY_XATTR_NAME   "user.ftest"
+#define MY_XATTR_VAL    "ftestval"
+
+
+void usage (void)
+{
+        printf ("Usage : bug-1193636 <filename> <xattr_name> <op>\n");
+        printf ("   op : 0 - set, 1 - remove\n");
+}
+
+
+int main (int argc, char **argv)
+{
+        int fd;
+        int err = 0;
+        char *xattr_name = NULL;
+        int op = 0;
+
+        if (argc != 4) {
+                usage ();
+                exit (1);
+        }
+
+        op = atoi (argv[3]);
+
+        if ((op != 0) && (op != 1)) {
+                printf ("Invalid operation specified.\n");
+                usage ();
+                exit (1);
+        }
+
+        xattr_name = argv[2];
+
+        fd = open(argv[1], O_RDWR);
+        if (fd == -1) {
+                printf ("Failed to open file %s\n", argv[1]);
+                exit (1);
+        }
+
+        if (!op) {
+                err = fsetxattr (fd, xattr_name, MY_XATTR_VAL,
+                                 strlen (MY_XATTR_VAL) + 1, XATTR_CREATE);
+
+                if (err) {
+                        printf ("Failed to set xattr %s: %m\n", xattr_name);
+                        exit (1);
+                }
+
+        } else {
+                err = fremovexattr (fd, xattr_name);
+
+                if (err) {
+                        printf ("Failed to remove xattr %s: %m\n", xattr_name);
+                        exit (1);
+                }
+        }
+
+        close (fd);
+
+        return 0;
+}
+
diff --git a/tests/bugs/distribute/bug-1193636.t b/tests/bugs/distribute/bug-1193636.t
new file mode 100644
index 00000000000..ccde02edc70
--- /dev/null
+++ b/tests/bugs/distribute/bug-1193636.t
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+
+checksticky () {
+        i=0;
+        while [ ! -k $1 ]; do
+                sleep 1
+                i=$((i+1));
+                if [[ $i == 10 ]]; then
+                        return $i
+                fi
+                echo "Waiting... $i"
+        done
+        echo "done ...got out @ $i"
+        return 0
+}
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info
+
+#Create a distributed volume
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1..3};
+TEST $CLI volume start $V0
+
+# Mount FUSE
+TEST glusterfs -s $H0 --volfile-id $V0 $M0
+
+TEST mkdir $M0/dir1
+
+# Create a large file (1GB), so that rebalance takes time
+dd if=/dev/zero of=$M0/dir1/FILE2 bs=64k count=10240
+
+# Rename the file to create a linkto, for rebalance to
+# act on the file
+TEST mv $M0/dir1/FILE2 $M0/dir1/FILE1
+
+build_tester $(dirname $0)/bug-1193636.c
+
+TEST $CLI volume rebalance $V0 start force
+
+TEST checksticky $B0/${V0}3/dir1/FILE1
+
+TEST setfattr -n "user.test1" -v "test1" $M0/dir1/FILE1
+TEST setfattr -n "user.test2" -v "test1" $M0/dir1/FILE1
+TEST setfattr -n "user.test3" -v "test1" $M0/dir1/FILE1
+
+TEST $(dirname $0)/bug-1193636 $M0/dir1/FILE1 user.fsetx 0
+TEST $(dirname $0)/bug-1193636 $M0/dir1/FILE1 user.fremx 0
+
+TEST getfattr -n "user.fremx" $M0/dir1/FILE1
+TEST setfattr -x "user.test2" $M0/dir1/FILE1
+
+
+TEST $(dirname $0)/bug-1193636 $M0/dir1/FILE1 user.fremx 1
+
+EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" rebalance_status_field $V0
+
+TEST getfattr -n "user.fsetx" $M0/dir1/FILE1
+TEST getfattr -n "user.test1" $M0/dir1/FILE1
+TEST ! getfattr -n "user.test2" $M0/dir1/FILE1
+TEST ! getfattr -n "user.fremx" $M0/dir1/FILE1
+TEST getfattr -n "user.test3" $M0/dir1/FILE1
+
+
+cleanup;
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index d41dbffba5d..3a461cbf3a8 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -32,6 +32,12 @@
 int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame);
 
 int
+dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
+
+int
+dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame);
+
+int
 dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)
 {
         int              ret            = -1;
@@ -96,6 +102,8 @@ out:
         return ret;
 }
 
+
+
 int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
@@ -3265,6 +3273,81 @@ err:
 }
 
 int
+dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+        int           ret     = -1;
+        dht_local_t  *local   = NULL;
+        call_frame_t *prev    = NULL;
+        struct iatt  *stbuf   = NULL;
+        inode_t      *inode   = NULL;
+        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+
+        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
+                gf_msg_debug (this->name, op_errno,
+                              "subvolume %s returned -1.",
+                              prev->this->name);
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+
+        if ((!op_ret) && !stbuf) {
+                goto out;
+        }
+
+        local->op_ret = 0;
+
+        local->rebalance.target_op_fn = dht_setxattr2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Phase 1 of migration */
+        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+                ret = dht_inode_ctx_get_mig_info (this, inode,
+                                                  &subvol1, &subvol2);
+                if (!dht_mig_info_is_invalid (local->cached_subvol,
+                                              subvol1, subvol2)) {
+                        dht_setxattr2 (this, subvol2, frame);
+                        return 0;
+                }
+
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        if (local->rebalance.xdata)
+                dict_unref (local->rebalance.xdata);
+
+        if (local->fop == GF_FOP_SETXATTR) {
+                DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
+        } else {
+                DHT_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        }
+
+        return 0;
+}
+
+
+
+int
 dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
@@ -3272,6 +3355,10 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
         dht_conf_t   *conf     = NULL;
+        dht_layout_t *layout   = NULL;
+        int           ret      = -1;
+        int           call_cnt = 0;
+        int           i        = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3299,11 +3386,47 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        local->call_cnt = 1;
+        layout = local->layout;
+        if (!layout) {
+                gf_msg_debug (this->name, 0,
+                              "no layout for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        if (IA_ISDIR (fd->inode->ia_type)) {
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_err_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->fsetxattr,
+                                    fd, xattr, flags, NULL);
+                }
+
+        } else {
 
-        STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags, NULL);
+                local->call_cnt = 1;
+                local->rebalance.xdata = dict_ref (xattr);
+                local->rebalance.flags = flags;
 
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                        DHT_IATT_IN_XDATA_KEY, "yes");
+                if (ret) {
+                        gf_msg_debug (this->name, 0,
+                              "Failed to set dictionary key %s for fd=%p",
+                               DHT_IATT_IN_XDATA_KEY, fd);
+                }
+
+                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
+                    subvol->fops->fsetxattr, fd, xattr, flags, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
+
+        }
         return 0;
 
 err:
@@ -3324,6 +3447,7 @@ dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
+
 int
 dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int op_ret, int op_errno, dict_t *xattr,
@@ -3365,6 +3489,40 @@ out:
 
 }
 
+
+int
+dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+{
+        dht_local_t *local    = NULL;
+        int          op_errno = EINVAL;
+
+        if (!frame || !frame->local || !subvol)
+                goto err;
+
+        local = frame->local;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_SETXATTR) {
+                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
+                            subvol->fops->setxattr, &local->loc,
+                            local->rebalance.xdata, local->rebalance.flags,
+                            NULL);
+        } else {
+                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
+                            subvol->fops->fsetxattr, local->fd,
+                            local->rebalance.xdata, local->rebalance.flags,
+                            NULL);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
 int
 dht_setxattr (call_frame_t *frame, xlator_t *this,
               loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
@@ -3587,11 +3745,32 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_err_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setxattr,
+        if (IA_ISDIR (loc->inode->ia_type)) {
+
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_err_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->setxattr,
+                                    loc, xattr, flags, xdata);
+                }
+
+        } else {
+
+                local->rebalance.xdata = dict_ref (xattr);
+                local->rebalance.flags = flags;
+                local->call_cnt = 1;
+
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                              DHT_IATT_IN_XDATA_KEY, "yes");
+
+                STACK_WIND (frame, dht_file_setxattr_cbk,
+                            subvol, subvol->fops->setxattr,
                             loc, xattr, flags, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
         }
 
         return 0;
@@ -3604,6 +3783,108 @@ err:
 }
 
 
+
+
+int
+dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+        int           ret     = -1;
+        dht_local_t  *local   = NULL;
+        call_frame_t *prev    = NULL;
+        struct iatt  *stbuf   = NULL;
+        inode_t      *inode   = NULL;
+        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+
+        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
+                gf_msg_debug (this->name, op_errno,
+                              "subvolume %s returned -1",
+                              prev->this->name);
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+
+        if ((!op_ret) && !stbuf) {
+                goto out;
+        }
+
+        local->op_ret = 0;
+
+        local->rebalance.target_op_fn = dht_removexattr2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Phase 1 of migration */
+        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get_mig_info (this, inode,
+                                                  &subvol1, &subvol2);
+                if (!dht_mig_info_is_invalid (local->cached_subvol,
+                                              subvol1, subvol2)) {
+                        dht_removexattr2 (this, subvol2, frame);
+                        return 0;
+                }
+
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+                DHT_STACK_UNWIND (removexattr, frame, op_ret, op_errno, NULL);
+        } else {
+                DHT_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        }
+        return 0;
+
+}
+
+int
+dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame)
+{
+        dht_local_t *local    = NULL;
+        int          op_errno = EINVAL;
+
+        if (!frame || !frame->local || !subvol)
+                goto err;
+
+        local = frame->local;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+                STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
+                            subvol->fops->removexattr, &local->loc,
+                            local->key, NULL);
+        } else {
+                STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
+                            subvol->fops->fremovexattr, local->fd,
+                            local->key, NULL);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
 int
 dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int op_ret, int op_errno, dict_t *xdata)
@@ -3630,6 +3911,8 @@ dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
+
+
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
                 DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
@@ -3651,6 +3934,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
         int           call_cnt = 0;
         dht_conf_t   *conf = NULL;
         int i;
+        int           ret = 0;
 
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (this->private, err);
@@ -3688,11 +3972,33 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = call_cnt = layout->cnt;
         local->key = gf_strdup (key);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->removexattr,
-                            loc, key, NULL);
+        if (IA_ISDIR (loc->inode->ia_type)) {
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_removexattr_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->removexattr,
+                                    loc, key, NULL);
+                }
+
+        } else {
+
+                local->call_cnt = 1;
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                 DHT_IATT_IN_XDATA_KEY, "yes");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set dictionary key %s for %s",
+                                DHT_IATT_IN_XDATA_KEY, loc->path);
+                }
+
+                STACK_WIND (frame, dht_file_removexattr_cbk,
+                            subvol, subvol->fops->removexattr,
+                            loc, key, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
         }
 
         return 0;
@@ -3714,6 +4020,7 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
         dht_conf_t   *conf = 0;
+        int           ret = 0;
 
         int i;
 
@@ -3753,11 +4060,33 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = call_cnt = layout->cnt;
         local->key = gf_strdup (key);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fremovexattr,
-                            fd, key, NULL);
+        if (IA_ISDIR (fd->inode->ia_type)) {
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_removexattr_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->fremovexattr,
+                                    fd, key, NULL);
+                }
+
+        } else {
+
+                local->call_cnt = 1;
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                 DHT_IATT_IN_XDATA_KEY, "yes");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set dictionary key %s for fd=%p",
+                                DHT_IATT_IN_XDATA_KEY, fd);
+                }
+
+                STACK_WIND (frame, dht_file_removexattr_cbk,
+                            subvol, subvol->fops->fremovexattr,
+                            fd, key, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
         }
 
         return 0;
diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c
index b787d9be303..3fc8ab5c015 100644
--- a/xlators/cluster/ec/src/ec-combine.c
+++ b/xlators/cluster/ec/src/ec-combine.c
@@ -260,6 +260,7 @@ ec_value_ignore (char *key)
             (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
             (strncmp(key, GF_XATTR_CLRLK_CMD,
                      strlen (GF_XATTR_CLRLK_CMD)) == 0) ||
+            (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
             (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) ||
             (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
             (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index fbbeacd28b1..71b83c2e0ac 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -3239,6 +3239,31 @@ map_xattr_flags(int flags)
 }
 #endif
 
+static
+int32_t posix_set_iatt_in_dict (dict_t *dict, struct iatt *in_stbuf)
+{
+        int ret             = -1;
+        struct iatt *stbuf  = NULL;
+        int32_t len         = sizeof(struct iatt);
+
+        if (!dict || !in_stbuf)
+                return ret;
+
+        stbuf = GF_CALLOC (1, len, gf_common_mt_char);
+        if (!stbuf)
+                return ret;
+
+        memcpy (stbuf, in_stbuf, len);
+
+        ret = dict_set_bin (dict, DHT_IATT_IN_XDATA_KEY, stbuf, len);
+        if (ret)
+                GF_FREE (stbuf);
+
+        return ret;
+}
+
+
+
 int32_t
 posix_setxattr (call_frame_t *frame, xlator_t *this,
                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
@@ -3246,7 +3271,9 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         int32_t       op_ret                  = -1;
         int32_t       op_errno                = 0;
         char *        real_path               = NULL;
-
+        struct iatt   stbuf                   = {0};
+        int32_t       ret                     = 0;
+        dict_t       *xattr                   = NULL;
         posix_xattr_filler_t filler = {0,};
 
         DECLARE_OLD_FS_ID_VAR;
@@ -3265,6 +3292,7 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         }
 
         op_ret = -1;
+
         dict_del (dict, GFID_XATTR_KEY);
         dict_del (dict, GF_XATTR_VOL_ID_KEY);
 
@@ -3280,12 +3308,31 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         if (op_ret < 0) {
                 op_errno = -op_ret;
                 op_ret = -1;
+                goto out;
         }
 
+/*
+ * FIXFIX: Send the stbuf info in the xdata for now
+ * This is used by DHT to redirect FOPs if the file is being migrated
+ * Ignore errors for now
+ */
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_pstat(this, loc->gfid, real_path, &stbuf);
+                if (ret)
+                        goto out;
+
+                xattr = dict_new();
+                if (!xattr)
+                        goto out;
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref(xattr);
 
         return 0;
 }
@@ -4313,13 +4360,14 @@ int32_t
 posix_fsetxattr (call_frame_t *frame, xlator_t *this,
                  fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
 {
-        int32_t            op_ret       = -1;
-        int32_t            op_errno     = 0;
-        struct posix_fd *  pfd          = NULL;
-        int                _fd          = -1;
-        int                ret          = -1;
-
-        posix_xattr_filler_t filler = {0,};
+        int32_t            op_ret         = -1;
+        int32_t            op_errno       = 0;
+        struct posix_fd   *pfd            = NULL;
+        int                _fd            = -1;
+        int                ret            = -1;
+        struct  iatt       stbuf          = {0,};
+        dict_t            *xattr          = NULL;
+        posix_xattr_filler_t filler       = {0,};
 
         DECLARE_OLD_FS_ID_VAR;
         SET_FS_ID (frame->root->uid, frame->root->gid);
@@ -4366,10 +4414,28 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_fdstat (this, pfd->fd, &stbuf);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "fsetxattr (fstat) failed on fd=%p: %s",
+                                fd, strerror (op_errno));
+                        goto out;
+                }
+
+                xattr = dict_new ();
+                if (!xattr)
+                        goto out;
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
+
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
 
         return 0;
 }
@@ -4429,7 +4495,10 @@ posix_removexattr (call_frame_t *frame, xlator_t *this,
 {
         int32_t op_ret    = -1;
         int32_t op_errno  = 0;
+        int32_t ret    = -1;
         char *  real_path = NULL;
+        struct iatt   stbuf         = {0};
+        dict_t  *xattr    = NULL;
         posix_xattr_filler_t filler = {0,};
 
         DECLARE_OLD_FS_ID_VAR;
@@ -4485,12 +4554,26 @@ posix_removexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_pstat(this, loc->gfid, real_path, &stbuf);
+                if (ret)
+                        goto out;
+                xattr = dict_new();
+                if (!xattr)
+                        goto out;
+
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
         op_ret = 0;
 
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 }
 
@@ -4501,6 +4584,8 @@ posix_fremovexattr (call_frame_t *frame, xlator_t *this,
         int32_t           op_ret   = -1;
         int32_t           op_errno = 0;
         struct posix_fd * pfd      = NULL;
+        struct iatt       stbuf    = {0,};
+        dict_t           *xattr    = NULL;
         int               _fd      = -1;
         int               ret      = -1;
 
@@ -4541,12 +4626,26 @@ posix_fremovexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_fdstat (this, pfd->fd, &stbuf);
+                if (ret)
+                        goto out;
+                xattr = dict_new();
+                if (!xattr)
+                        goto out;
+
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
         op_ret = 0;
 
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 }