cluster/dht: Fix dht_setxattr to follow files under migration

If a file is under migration, any xattrs created on it
are lost post migration of the file. This is because
the xattrs are set only on the cached subvol of the source
and as the source is under migration, it becomes a linkto file
post migration.

Change-Id: Ib8e233b519cf954e7723c6e26b38fa8f9b8c85c0
BUG: 1193636
Signed-off-by: Nithya Balachandran <nbalacha@redhat.com>
Reviewed-on: http://review.gluster.org/10212
Tested-by: NetBSD Build System
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>

3 files changed, 486 insertions, 29 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 72ca77ea5ab..092ddebb8f5 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -32,6 +32,13 @@
 int dht_link2 (xlator_t *this, call_frame_t *frame, int op_ret);
 
 int
+dht_removexattr2 (xlator_t *this, call_frame_t *frame, int op_ret);
+
+int
+dht_setxattr2 (xlator_t *this, call_frame_t *frame, int op_ret);
+
+
+int
 dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)
 {
         int              ret            = -1;
@@ -96,6 +103,8 @@ out:
         return ret;
 }
 
+
+
 int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
@@ -3265,6 +3274,78 @@ err:
 }
 
 int
+dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+        int           ret  = -1;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        struct iatt  *stbuf = NULL;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = NULL;
+
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+
+        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
+                gf_msg_debug (this->name, op_errno,
+                              "subvolume %s returned -1.",
+                              prev->this->name);
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+
+        if ((!op_ret) && !stbuf) {
+                goto out;
+        }
+
+        local->op_ret = 0;
+
+        local->rebalance.target_op_fn = dht_setxattr2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Phase 1 of migration */
+        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
+                        dht_setxattr2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        if (local->rebalance.xdata)
+                dict_unref (local->rebalance.xdata);
+
+        if (local->fop == GF_FOP_SETXATTR) {
+                DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
+        } else {
+                DHT_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        }
+
+        return 0;
+}
+
+
+
+int
 dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
@@ -3272,6 +3353,10 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
         dht_conf_t   *conf     = NULL;
+        dht_layout_t *layout   = NULL;
+        int           ret      = -1;
+        int           call_cnt = 0;
+        int           i        = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3299,11 +3384,47 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        local->call_cnt = 1;
+        layout = local->layout;
+        if (!layout) {
+                gf_msg_debug (this->name, 0,
+                              "no layout for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        if (IA_ISDIR (fd->inode->ia_type)) {
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_err_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->fsetxattr,
+                                    fd, xattr, flags, NULL);
+                }
+
+        } else {
+
+                local->call_cnt = 1;
+                local->rebalance.xdata = dict_ref (xattr);
+                local->rebalance.flags = flags;
+
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                        DHT_IATT_IN_XDATA_KEY, "yes");
+                if (ret) {
+                        gf_msg_debug (this->name, 0,
+                              "Failed to set dictionary key %s for fd=%p",
+                               DHT_IATT_IN_XDATA_KEY, fd);
+                }
 
-        STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags, NULL);
+                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
+                    subvol->fops->fsetxattr, fd, xattr, flags, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
 
+        }
         return 0;
 
 err:
@@ -3324,6 +3445,7 @@ dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
+
 int
 dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                            int op_ret, int op_errno, dict_t *xattr,
@@ -3365,6 +3487,55 @@ out:
 
 }
 
+
+int
+dht_setxattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+        int          op_errno = EINVAL;
+        inode_t      *inode = NULL;
+
+        local = frame->local;
+
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
+
+        /* In phase2, dht_migration_complete_check_task will
+         * reset inode_ctx_reset1 and update local->cached_subvol
+         * with the dst subvol.
+         */
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        if (!subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_SETXATTR) {
+                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
+                            subvol->fops->setxattr, &local->loc,
+                            local->rebalance.xdata, local->rebalance.flags,
+                            NULL);
+        } else {
+                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
+                            subvol->fops->fsetxattr, local->fd,
+                            local->rebalance.xdata, local->rebalance.flags,
+                            NULL);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (setxattr, frame, local->op_ret, op_errno, NULL);
+        return 0;
+}
+
+
 int
 dht_setxattr (call_frame_t *frame, xlator_t *this,
               loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
@@ -3588,11 +3759,32 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 goto err;
         }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_err_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setxattr,
+        if (IA_ISDIR (loc->inode->ia_type)) {
+
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_err_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->setxattr,
+                                    loc, xattr, flags, xdata);
+                }
+
+        } else {
+
+                local->rebalance.xdata = dict_ref (xattr);
+                local->rebalance.flags = flags;
+                local->call_cnt = 1;
+
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                              DHT_IATT_IN_XDATA_KEY, "yes");
+
+                STACK_WIND (frame, dht_file_setxattr_cbk,
+                            subvol, subvol->fops->setxattr,
                             loc, xattr, flags, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
         }
 
         return 0;
@@ -3605,6 +3797,123 @@ err:
 }
 
 
+
+
+int
+dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+        int           ret  = -1;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        struct iatt  *stbuf = NULL;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = NULL;
+
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+
+        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
+                gf_msg_debug (this->name, op_errno,
+                              "subvolume %s returned -1",
+                              prev->this->name);
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+
+        if ((!op_ret) && !stbuf) {
+                goto out;
+        }
+
+        local->op_ret = 0;
+
+        local->rebalance.target_op_fn = dht_removexattr2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Phase 1 of migration */
+        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
+                        dht_removexattr2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+                DHT_STACK_UNWIND (removexattr, frame, op_ret, op_errno, NULL);
+        } else {
+                DHT_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        }
+        return 0;
+
+}
+
+int
+dht_removexattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+        int          op_errno = EINVAL;
+        inode_t      *inode = NULL;
+
+        local = frame->local;
+
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
+
+        /* In phase2, dht_migration_complete_check_task will
+         * reset inode_ctx_reset1 and update local->cached_subvol
+         * with the dst subvol.
+         */
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        if (!subvol) {
+                op_errno = EINVAL;
+                goto err;
+
+        }
+
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+                STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
+                            subvol->fops->removexattr, &local->loc,
+                            local->key, NULL);
+        } else {
+                STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
+                            subvol->fops->fremovexattr, local->fd,
+                            local->key, NULL);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (removexattr, frame, local->op_ret, op_errno, NULL);
+        return 0;
+}
+
+
 int
 dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int op_ret, int op_errno, dict_t *xdata)
@@ -3631,6 +3940,8 @@ dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
+
+
         this_call_cnt = dht_frame_return (frame);
         if (is_last_call (this_call_cnt)) {
                 DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
@@ -3652,6 +3963,7 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
         int           call_cnt = 0;
         dht_conf_t   *conf = NULL;
         int i;
+        int           ret = 0;
 
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (this->private, err);
@@ -3689,11 +4001,33 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = call_cnt = layout->cnt;
         local->key = gf_strdup (key);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->removexattr,
-                            loc, key, NULL);
+        if (IA_ISDIR (loc->inode->ia_type)) {
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_removexattr_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->removexattr,
+                                    loc, key, NULL);
+                }
+
+        } else {
+
+                local->call_cnt = 1;
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                 DHT_IATT_IN_XDATA_KEY, "yes");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set dictionary key %s for %s",
+                                DHT_IATT_IN_XDATA_KEY, loc->path);
+                }
+
+                STACK_WIND (frame, dht_file_removexattr_cbk,
+                            subvol, subvol->fops->removexattr,
+                            loc, key, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
         }
 
         return 0;
@@ -3715,6 +4049,7 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
         dht_conf_t   *conf = 0;
+        int           ret = 0;
 
         int i;
 
@@ -3754,11 +4089,33 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
         local->call_cnt = call_cnt = layout->cnt;
         local->key = gf_strdup (key);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fremovexattr,
-                            fd, key, NULL);
+        if (IA_ISDIR (fd->inode->ia_type)) {
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_removexattr_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->fremovexattr,
+                                    fd, key, NULL);
+                }
+
+        } else {
+
+                local->call_cnt = 1;
+                xdata = xdata ? dict_ref (xdata) : dict_new ();
+                if (xdata)
+                        ret = dict_set_dynstr_with_alloc (xdata,
+                                 DHT_IATT_IN_XDATA_KEY, "yes");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set dictionary key %s for fd=%p",
+                                DHT_IATT_IN_XDATA_KEY, fd);
+                }
+
+                STACK_WIND (frame, dht_file_removexattr_cbk,
+                            subvol, subvol->fops->fremovexattr,
+                            fd, key, xdata);
+
+                if (xdata)
+                        dict_unref (xdata);
         }
 
         return 0;
diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c
index 4617a0430f1..5842a4dd0a8 100644
--- a/xlators/cluster/ec/src/ec-combine.c
+++ b/xlators/cluster/ec/src/ec-combine.c
@@ -191,6 +191,7 @@ ec_value_ignore (char *key)
             (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
             (strncmp(key, GF_XATTR_CLRLK_CMD,
                      strlen (GF_XATTR_CLRLK_CMD)) == 0) ||
+            (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
             (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) ||
             (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
             (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index b235ff4a794..d38964143ba 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -3239,6 +3239,31 @@ map_xattr_flags(int flags)
 }
 #endif
 
+static
+int32_t posix_set_iatt_in_dict (dict_t *dict, struct iatt *in_stbuf)
+{
+        int ret             = -1;
+        struct iatt *stbuf  = NULL;
+        int32_t len         = sizeof(struct iatt);
+
+        if (!dict || !in_stbuf)
+                return ret;
+
+        stbuf = GF_CALLOC (1, len, gf_common_mt_char);
+        if (!stbuf)
+                return ret;
+
+        memcpy (stbuf, in_stbuf, len);
+
+        ret = dict_set_bin (dict, DHT_IATT_IN_XDATA_KEY, stbuf, len);
+        if (ret)
+                GF_FREE (stbuf);
+
+        return ret;
+}
+
+
+
 int32_t
 posix_setxattr (call_frame_t *frame, xlator_t *this,
                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
@@ -3246,7 +3271,9 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         int32_t       op_ret                  = -1;
         int32_t       op_errno                = 0;
         char *        real_path               = NULL;
-
+        struct iatt   stbuf                   = {0};
+        int32_t       ret                     = 0;
+        dict_t       *xattr                   = NULL;
         posix_xattr_filler_t filler = {0,};
 
         DECLARE_OLD_FS_ID_VAR;
@@ -3265,6 +3292,7 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         }
 
         op_ret = -1;
+
         dict_del (dict, GFID_XATTR_KEY);
         dict_del (dict, GF_XATTR_VOL_ID_KEY);
 
@@ -3280,12 +3308,31 @@ posix_setxattr (call_frame_t *frame, xlator_t *this,
         if (op_ret < 0) {
                 op_errno = -op_ret;
                 op_ret = -1;
+                goto out;
         }
 
+/*
+ * FIXFIX: Send the stbuf info in the xdata for now
+ * This is used by DHT to redirect FOPs if the file is being migrated
+ * Ignore errors for now
+ */
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_pstat(this, loc->gfid, real_path, &stbuf);
+                if (ret)
+                        goto out;
+
+                xattr = dict_new();
+                if (!xattr)
+                        goto out;
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref(xattr);
 
         return 0;
 }
@@ -4313,13 +4360,14 @@ int32_t
 posix_fsetxattr (call_frame_t *frame, xlator_t *this,
                  fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
 {
-        int32_t            op_ret       = -1;
-        int32_t            op_errno     = 0;
-        struct posix_fd *  pfd          = NULL;
-        int                _fd          = -1;
-        int                ret          = -1;
-
-        posix_xattr_filler_t filler = {0,};
+        int32_t            op_ret         = -1;
+        int32_t            op_errno       = 0;
+        struct posix_fd   *pfd            = NULL;
+        int                _fd            = -1;
+        int                ret            = -1;
+        struct  iatt       stbuf          = {0,};
+        dict_t            *xattr          = NULL;
+        posix_xattr_filler_t filler       = {0,};
 
         DECLARE_OLD_FS_ID_VAR;
         SET_FS_ID (frame->root->uid, frame->root->gid);
@@ -4366,10 +4414,28 @@ posix_fsetxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_fdstat (this, pfd->fd, &stbuf);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "fsetxattr (fstat) failed on fd=%p: %s",
+                                fd, strerror (op_errno));
+                        goto out;
+                }
+
+                xattr = dict_new ();
+                if (!xattr)
+                        goto out;
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
+
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
 
         return 0;
 }
@@ -4429,7 +4495,10 @@ posix_removexattr (call_frame_t *frame, xlator_t *this,
 {
         int32_t op_ret    = -1;
         int32_t op_errno  = 0;
+        int32_t ret    = -1;
         char *  real_path = NULL;
+        struct iatt   stbuf         = {0};
+        dict_t  *xattr    = NULL;
         posix_xattr_filler_t filler = {0,};
 
         DECLARE_OLD_FS_ID_VAR;
@@ -4485,12 +4554,26 @@ posix_removexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_pstat(this, loc->gfid, real_path, &stbuf);
+                if (ret)
+                        goto out;
+                xattr = dict_new();
+                if (!xattr)
+                        goto out;
+
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
         op_ret = 0;
 
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 }
 
@@ -4501,6 +4584,8 @@ posix_fremovexattr (call_frame_t *frame, xlator_t *this,
         int32_t           op_ret   = -1;
         int32_t           op_errno = 0;
         struct posix_fd * pfd      = NULL;
+        struct iatt       stbuf    = {0,};
+        dict_t           *xattr    = NULL;
         int               _fd      = -1;
         int               ret      = -1;
 
@@ -4541,12 +4626,26 @@ posix_fremovexattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
+        if (dict_get (xdata, DHT_IATT_IN_XDATA_KEY)) {
+                ret = posix_fdstat (this, pfd->fd, &stbuf);
+                if (ret)
+                        goto out;
+                xattr = dict_new();
+                if (!xattr)
+                        goto out;
+
+                ret = posix_set_iatt_in_dict (xattr, &stbuf);
+        }
         op_ret = 0;
 
 out:
         SET_TO_OLD_FS_ID ();
 
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, NULL);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xattr);
+
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 }