Merge remote-tracking branch 'origin/release-3.8' into merge-3.8

Change-Id: Ib336c2ada491c2d2fcbbbe6865f9eb975a405b36

18 files changed, 518 insertions, 214 deletions

diff --git a/doc/release-notes/3.8.10.md b/doc/release-notes/3.8.10.md
new file mode 100644
index 00000000000..d67b8ca7494
--- /dev/null
+++ b/doc/release-notes/3.8.10.md
@@ -0,0 +1,57 @@
+# Release notes for Gluster 3.8.10
+
+This is a bugfix release. The [Release Notes for 3.8.0](3.8.0.md),
+[3.8.1](3.8.1.md), [3.8.2](3.8.2.md), [3.8.3](3.8.3.md), [3.8.4](3.8.4.md),
+[3.8.5](3.8.5.md), [3.8.6](3.8.6.md), [3.8.7](3.8.7.md), [3.8.8](3.8.8.md) and
+[3.8.9](3.8.9.md) contain a listing of all the new features that were added and
+bugs fixed in the GlusterFS 3.8 stable release.
+
+
+## Improved configuration with additional 'virt' options
+
+This release includes 5 more options to group virt (for VM workloads) for
+optimal performance.
+
+Updating to the glusterfs version containing this patch won't automatically set
+these newer options on already existing volumes that have group virt
+configured. The changes take effect only when post-upgrade
+
+    # gluster volume-set <VOL> group virt
+
+is performed.
+
+For already existing volumes the users may execute the following five commands,
+if not already set:
+
+    # gluster volume set <VOL> performance.low-prio-threads 32
+    # gluster volume set <VOL> cluster.locking-scheme granular
+    # gluster volume set <VOL> features.shard on
+    # gluster volume set <VOL> cluster.shd-max-threads 8
+    # gluster volume set <VOL> cluster.shd-wait-qlength 10000
+    # gluster volume set <VOL> user.cifs off
+
+It is most likely that `features.shard` would already have been set on the
+volume even before the upgrade, in which case the third `volume set` command
+above may be skipped.
+
+
+## Bugs addressed
+
+A total of 18 patches have been merged, addressing 16 bugs:
+
+- [#1387878](https://bugzilla.redhat.com/1387878): Rebalance after add bricks corrupts files
+- [#1412994](https://bugzilla.redhat.com/1412994): Memory leak on mount/fuse when setxattr fails
+- [#1420993](https://bugzilla.redhat.com/1420993): Modified volume options not synced once offline nodes comes up.
+- [#1422352](https://bugzilla.redhat.com/1422352): glustershd process crashed on systemic setup
+- [#1422394](https://bugzilla.redhat.com/1422394): Gluster NFS server crashing in __mnt3svc_umountall
+- [#1422811](https://bugzilla.redhat.com/1422811): [Geo-rep] Recreating geo-rep session with same slave after deleting with reset-sync-time fails to sync
+- [#1424915](https://bugzilla.redhat.com/1424915): dht_setxattr returns EINVAL when a file is deleted during the FOP
+- [#1424934](https://bugzilla.redhat.com/1424934): Include few more options in virt file
+- [#1424974](https://bugzilla.redhat.com/1424974): remove-brick status shows 0 rebalanced files
+- [#1425112](https://bugzilla.redhat.com/1425112): [Ganesha] : Unable to bring up a Ganesha HA cluster on RHEL 6.9.
+- [#1425307](https://bugzilla.redhat.com/1425307): Fix statvfs for FreeBSD in Python
+- [#1427390](https://bugzilla.redhat.com/1427390): systemic testing: seeing lot of ping time outs  which would lead to splitbrains
+- [#1427419](https://bugzilla.redhat.com/1427419): Warning messages throwing when EC volume offline brick comes up are difficult to understand for end user.
+- [#1428743](https://bugzilla.redhat.com/1428743): Fix crash in dht resulting from tests/features/nuke.t
+- [#1429312](https://bugzilla.redhat.com/1429312): Prevent reverse heal from happening
+- [#1429405](https://bugzilla.redhat.com/1429405): Restore atime/mtime for symlinks and other non-regular files.
diff --git a/extras/group-virt.example b/extras/group-virt.example
index 4fe3760be2c..6361f1b72bc 100644
--- a/extras/group-virt.example
+++ b/extras/group-virt.example
@@ -2,9 +2,14 @@ performance.quick-read=off
 performance.read-ahead=off
 performance.io-cache=off
 performance.stat-prefetch=off
-cluster.eager-lock=enable
+performance.low-prio-threads=32
 network.remote-dio=enable
+cluster.eager-lock=enable
 cluster.quorum-type=auto
 cluster.server-quorum-type=server
-features.shard=on
 cluster.data-self-heal-algorithm=full
+cluster.locking-scheme=granular
+cluster.shd-max-threads=8
+cluster.shd-wait-qlength=10000
+features.shard=on
+user.cifs=off
diff --git a/extras/rebalance.py b/extras/rebalance.py
index 80c614c5dfe..9579e5616ad 100755
--- a/extras/rebalance.py
+++ b/extras/rebalance.py
@@ -11,6 +11,7 @@ import subprocess
 import sys
 import tempfile
 import volfilter
+import platform
 
 # It's just more convenient to have named fields.
 class Brick:
@@ -218,12 +219,19 @@ if __name__ == "__main__":
         total = 0
         for b in bricks:
                 info = os.statvfs(b.path)
+                # On FreeBSD f_bsize (info[0]) contains the optimal I/O size,
+                # not the block size as it's found on Linux. In this case we
+                # use f_frsize (info[1]).
+                if platform.system() == 'FreeBSD':
+                        bsize = info[1]
+                else:
+                        bsize = info[0]
                 # We want a standard unit even if different bricks use
                 # different block sizes.  The size is chosen to avoid overflows
                 # for very large bricks with very small block sizes, but also
                 # accommodate filesystems which use very large block sizes to
                 # cheat on benchmarks.
-                blocksper100mb = 104857600 / info[0]
+                blocksper100mb = 104857600 / bsize
                 if options.free_space:
                         size = info[3] / blocksper100mb
                 else:
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index bc3a3ee5ca1..549c3c1ba71 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -26,6 +26,8 @@ afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 	local = frame->local;
 
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 	syncbarrier_wake (&local->barrier);
 
 	return 0;
@@ -39,6 +41,7 @@ afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
 	afr_private_t *priv = NULL;
 	afr_local_t *local = NULL;
 	loc_t loc = {0, };
+        int ret = 0;
 
 	priv = this->private;
 	local = frame->local;
@@ -46,15 +49,20 @@ afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
 	loc.inode = inode_ref (inode);
 	gf_uuid_copy (loc.gfid, inode->gfid);
 
+        local->op_ret = 0;
+
 	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
 		    priv->children[subvol]->fops->xattrop, &loc,
 		    GF_XATTROP_ADD_ARRAY, xattr, xdata);
 
 	syncbarrier_wait (&local->barrier, 1);
+        if (local->op_ret < 0)
+                ret = -local->op_errno;
 
         loc_wipe (&loc);
+        local->op_ret = 0;
 
-	return 0;
+	return ret;
 }
 
 int
@@ -78,6 +86,49 @@ afr_check_stale_error (struct afr_reply *replies, afr_private_t *priv)
                 return -op_errno;
 }
 
+int
+afr_sh_generic_fop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+          int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+          dict_t *xdata)
+{
+        int i = (long) cookie;
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->replies[i].valid = 1;
+        local->replies[i].op_ret = op_ret;
+        local->replies[i].op_errno = op_errno;
+        if (pre)
+                local->replies[i].prestat = *pre;
+        if (post)
+                local->replies[i].poststat = *post;
+        if (xdata)
+                local->replies[i].xdata = dict_ref (xdata);
+
+        syncbarrier_wake (&local->barrier);
+
+        return 0;
+}
+
+int
+afr_selfheal_restore_time (call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, unsigned char *healed_sinks,
+                           struct afr_reply *replies)
+{
+        loc_t loc = {0, };
+
+        loc.inode = inode_ref (inode);
+        gf_uuid_copy (loc.gfid, inode->gfid);
+
+        AFR_ONLIST (healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
+                    &replies[source].poststat,
+                    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
+
+        loc_wipe (&loc);
+
+        return 0;
+}
 
 dict_t *
 afr_selfheal_output_xattr (xlator_t *this, gf_boolean_t is_full_crawl,
@@ -1937,8 +1988,10 @@ afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
         changelog = afr_mark_pending_changelog (priv, newentry, xattr,
                                             replies[source].poststat.ia_type);
 
-        if (!changelog)
+        if (!changelog) {
+                ret = -ENOMEM;
                 goto out;
+        }
 
         /* Pre-compute how many sources we have, if we made it in here
          * without any sources defined, we are doing a conservative
@@ -1958,7 +2011,8 @@ afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
                 if (!sources[i] && source_count) {
                         continue;
                 }
-                afr_selfheal_post_op (frame, this, inode, i, xattr, NULL);
+                ret |= afr_selfheal_post_op (frame, this, inode, i, xattr,
+                                             NULL);
             }
 out:
         if (changelog)
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 9c12e433097..98ee8059640 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -47,33 +47,6 @@ __checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	return 0;
 }
 
-
-static int
-attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	  int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
-	  dict_t *xdata)
-{
-	int i = (long) cookie;
-	afr_local_t *local = NULL;
-
-	local = frame->local;
-
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (pre)
-		local->replies[i].prestat = *pre;
-	if (post)
-		local->replies[i].poststat = *post;
-	if (xdata)
-		local->replies[i].xdata = dict_ref (xdata);
-
-	syncbarrier_wake (&local->barrier);
-
-	return 0;
-}
-
-
 static gf_boolean_t
 __afr_can_skip_data_block_heal (call_frame_t *frame, xlator_t *this, fd_t *fd,
                                 int source, unsigned char *healed_sinks,
@@ -307,7 +280,8 @@ afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (!priv->ensure_durability)
                 return 0;
 
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, fsync, fd, 0, NULL);
+	AFR_ONLIST (healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0,
+                    NULL);
 
 	for (i = 0; i < priv->child_count; i++)
 		if (healed_sinks[i] && local->replies[i].op_ret != 0)
@@ -318,27 +292,6 @@ afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	return 0;
 }
 
-
-static int
-afr_selfheal_data_restore_time (call_frame_t *frame, xlator_t *this,
-				inode_t *inode, int source,
-				unsigned char *healed_sinks,
-				struct afr_reply *replies)
-{
-	loc_t loc = {0, };
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, setattr, &loc,
-		    &replies[source].poststat,
-		    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
-
-	loc_wipe (&loc);
-
-	return 0;
-}
-
 static int
 afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
                              int source, struct afr_reply *replies)
@@ -443,7 +396,8 @@ __afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
                 healed_sinks[ARBITER_BRICK_INDEX] = 0;
         }
 
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, ftruncate, fd, size, NULL);
+	AFR_ONLIST (healed_sinks, frame, afr_sh_generic_fop_cbk, ftruncate, fd,
+                    size, NULL);
 
 	for (i = 0; i < priv->child_count; i++)
 		if (healed_sinks[i] && local->replies[i].op_ret == -1)
@@ -780,7 +734,7 @@ unlock:
 	if (ret)
                 goto out;
 restore_time:
-	afr_selfheal_data_restore_time (frame, this, fd->inode, source,
+	afr_selfheal_restore_time (frame, this, fd->inode, source,
 					healed_sinks, locked_replies);
 
         if (!is_arbiter_the_only_sink) {
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 12a5940e7cd..09f26690588 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -75,26 +75,29 @@ afr_selfheal_entry_delete (xlator_t *this, inode_t *dir, const char *name,
 
 
 int
-afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
+afr_selfheal_recreate_entry (call_frame_t *frame, int dst, int source,
+                             unsigned char *sources, inode_t *dir,
                              const char *name, inode_t *inode,
-                             struct afr_reply *replies,
-                             unsigned char *newentry)
+                             struct afr_reply *replies)
 {
 	int ret = 0;
 	loc_t loc = {0,};
 	loc_t srcloc = {0,};
+        xlator_t *this = frame->this;
 	afr_private_t *priv = NULL;
 	dict_t *xdata = NULL;
 	struct iatt *iatt = NULL;
 	char *linkname = NULL;
 	mode_t mode = 0;
 	struct iatt newent = {0,};
-	priv = this->private;
+        unsigned char *newentry = NULL;
 
+        priv = this->private;
 	xdata = dict_new();
 	if (!xdata)
 		return -ENOMEM;
 
+        newentry = alloca0 (priv->child_count);
 	loc.parent = inode_ref (dir);
 	gf_uuid_copy (loc.pargfid, dir->gfid);
 	loc.name = name;
@@ -113,6 +116,15 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
 
 	srcloc.inode = inode_ref (inode);
 	gf_uuid_copy (srcloc.gfid, iatt->ia_gfid);
+        if (iatt->ia_type != IA_IFDIR)
+                ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
+        if (iatt->ia_type == IA_IFDIR || ret == -ENOENT || ret == -ESTALE) {
+                newentry[dst] = 1;
+                ret = afr_selfheal_newentry_mark (frame, this, inode, source,
+                                                  replies, sources, newentry);
+                if (ret)
+                        goto out;
+        }
 
 	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
 
@@ -120,12 +132,9 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
 	case IA_IFDIR:
 		ret = syncop_mkdir (priv->children[dst], &loc, mode, 0,
                                     xdata, NULL);
-                if (ret == 0)
-                        newentry[dst] = 1;
 		break;
 	case IA_IFLNK:
-		ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
-		if (ret == 0) {
+		if (!newentry[dst]) {
 			ret = syncop_link (priv->children[dst], &srcloc, &loc,
 					   &newent, NULL, NULL);
 		} else {
@@ -135,8 +144,6 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
 				goto out;
 			ret = syncop_symlink (priv->children[dst], &loc,
                                               linkname, NULL, xdata, NULL);
-                        if (ret == 0)
-                                newentry[dst] = 1;
                 }
 		break;
 	default:
@@ -146,10 +153,6 @@ afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
 		ret = syncop_mknod (priv->children[dst], &loc, mode,
                     makedev (ia_major(iatt->ia_rdev), ia_minor (iatt->ia_rdev)),
                     &newent, xdata, NULL);
-		if (ret == 0 && newent.ia_nlink == 1) {
-			/* New entry created. Mark @dst pending on all sources */
-                        newentry[dst] = 1;
-		}
 		break;
 	}
 
@@ -172,12 +175,9 @@ __afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	int ret = 0;
 	afr_private_t *priv = NULL;
 	int i = 0;
-	unsigned char *newentry = NULL;
 
 	priv = this->private;
 
-	newentry = alloca0 (priv->child_count);
-
 	if (!replies[source].valid)
 		return -EIO;
 
@@ -200,17 +200,15 @@ __afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
 					   replies[source].poststat.ia_gfid))
 				continue;
 
-			ret = afr_selfheal_recreate_entry (this, i, source,
-							   fd->inode, name, inode,
-							   replies, newentry);
+			ret = afr_selfheal_recreate_entry (frame, i, source,
+                                                           sources, fd->inode,
+                                                           name, inode,
+							   replies);
                 }
 		if (ret < 0)
 			break;
 	}
 
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
-					    sources, newentry);
 	return ret;
 }
 
@@ -278,13 +276,10 @@ __afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
         int             ret       = 0;
         int             i         = 0;
         int             source    = -1;
-        unsigned char  *newentry  = NULL;
         afr_private_t  *priv      = NULL;
 
 	priv = this->private;
 
-	newentry = alloca0 (priv->child_count);
-
 	for (i = 0; i < priv->child_count; i++) {
 		if (replies[i].valid && replies[i].op_ret == 0) {
 			source = i;
@@ -319,14 +314,11 @@ __afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		if (replies[i].op_errno != ENOENT)
 			continue;
 
-		ret = afr_selfheal_recreate_entry (this, i, source, fd->inode,
-                                                   name, inode, replies,
-                                                   newentry);
+		ret = afr_selfheal_recreate_entry (frame, i, source, sources,
+                                                   fd->inode, name, inode,
+                                                   replies);
 	}
 
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
-					    sources, newentry);
 	return ret;
 }
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index a67717bf9d4..d5d95ef8852 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -427,6 +427,13 @@ afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
                 if (ret)
                         goto unlock;
 
+                /* Restore atime/mtime for files that don't need data heal as
+                 * restoring timestamps happens only as a part of data-heal.
+                 */
+                if (!IA_ISREG (locked_replies[source].poststat.ia_type))
+                        afr_selfheal_restore_time (frame, this, inode, source,
+                                                  healed_sinks, locked_replies);
+
                 ret = afr_selfheal_undo_pending (frame, this, inode, sources,
                                                  sinks, healed_sinks,
                                                  undid_pending,
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
index 3445ecccf9c..b28ce4170f1 100644
--- a/xlators/cluster/afr/src/afr-self-heal-name.c
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -109,12 +109,10 @@ __afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this,
 	int i = 0;
 	afr_private_t *priv = NULL;
         int ret = 0;
-        unsigned char *newentry = NULL;
         unsigned char *sources = NULL;
 
         priv = this->private;
 
-	newentry = alloca0 (priv->child_count);
 	sources = alloca0 (priv->child_count);
 
 	gf_uuid_copy (parent->gfid, pargfid);
@@ -128,15 +126,17 @@ __afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this,
                         sources[i] = 1;
 			continue;
                 }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i])
+                        continue;
 
-		ret |= afr_selfheal_recreate_entry (this, i, gfid_idx, parent,
-                                                    bname, inode, replies,
-                                                    newentry);
+		ret |= afr_selfheal_recreate_entry (frame, i, gfid_idx, sources,
+                                                    parent, bname, inode,
+                                                    replies);
 	}
 
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, gfid_idx, replies,
-					    sources, newentry);
 	return ret;
 }
 
@@ -457,8 +457,7 @@ __afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
         }
 
 	ret = __afr_selfheal_name_impunge (frame, this, parent, pargfid,
-                                           bname, inode,
-                                           replies, gfid_idx);
+                                           bname, inode, replies, gfid_idx);
         if (ret == -EIO)
                 ret = -1;
 
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index a33905033cc..0a3d6482ca3 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -166,6 +166,15 @@ afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
 			    afr_transaction_type type, int *dirty, int **matrix);
 
 int
+afr_sh_generic_fop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *pre,
+                        struct iatt *post, dict_t *xdata);
+
+int
+afr_selfheal_restore_time (call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, unsigned char *healed_sinks,
+                           struct afr_reply *replies);
+int
 afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
 			   unsigned char *sources, unsigned char *sinks,
 			   unsigned char *healed_sinks,
@@ -174,10 +183,10 @@ afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
                            unsigned char *locked_on);
 
 int
-afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
-                             const char *name, inode_t *inode,
-                             struct afr_reply *replies,
-                             unsigned char *newentry);
+afr_selfheal_recreate_entry (call_frame_t *frame, int dst, int source,
+                             unsigned char *sources,
+                             inode_t *dir, const char *name, inode_t *inode,
+                             struct afr_reply *replies);
 
 int
 afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index a97d03bb055..81f7a8a40e6 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -3929,6 +3929,7 @@ dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
                 goto err;
 
         local = frame->local;
+        op_errno = local->op_errno;
 
         if (we_are_not_migrating (ret)) {
                 /* This dht xlator is not migrating the file. Unwind and
@@ -3943,7 +3944,6 @@ dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
         if (subvol == NULL)
                 goto err;
 
-        op_errno = local->op_errno;
 
         local->call_cnt = 2; /* This is the second attempt */
 
@@ -3968,6 +3968,15 @@ err:
 }
 
 int
+dht_nuke_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+int
 dht_nuke_dir (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
 {
         if (!IA_ISDIR(loc->inode->ia_type)) {
@@ -3998,7 +4007,7 @@ dht_nuke_dir (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
          * obscure the fact that we came in via this path instead of a genuine
          * rmdir.  That makes debugging just a tiny bit easier.
          */
-        STACK_WIND (frame, default_rmdir_cbk, this, this->fops->rmdir,
+        STACK_WIND (frame, dht_nuke_dir_cbk, this, this->fops->rmdir,
                     loc, 1, NULL);
 
         return 0;
@@ -4351,10 +4360,11 @@ dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
         dht_local_t *local    = NULL;
         int          op_errno = EINVAL;
 
-        if (!frame || !frame->local || !subvol)
+        if (!frame || !frame->local)
                 goto err;
 
         local = frame->local;
+        op_errno = local->op_errno;
 
         local->call_cnt = 2; /* This is the second attempt */
 
@@ -4369,6 +4379,9 @@ dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
                 return 0;
         }
 
+        if (subvol == NULL)
+                goto err;
+
         if (local->fop == GF_FOP_REMOVEXATTR) {
                 STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
                             subvol->fops->removexattr, &local->loc,
diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c
index 2b4cec406a9..8a5d0eb74e8 100644
--- a/xlators/cluster/ec/src/ec-common.c
+++ b/xlators/cluster/ec/src/ec-common.c
@@ -93,6 +93,7 @@ void ec_check_status(ec_fop_data_t * fop)
 {
     ec_t * ec = fop->xl->private;
     int32_t partial = 0;
+    char str1[32], str2[32], str3[32], str4[32], str5[32];
 
     if (!ec_fop_needs_heal(fop)) {
         return;
@@ -109,11 +110,15 @@ void ec_check_status(ec_fop_data_t * fop)
 
     gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
             EC_MSG_OP_FAIL_ON_SUBVOLS,
-            "Operation failed on some "
-            "subvolumes (up=%lX, mask=%lX, "
-            "remaining=%lX, good=%lX, bad=%lX)",
-            ec->xl_up, fop->mask, fop->remaining, fop->good,
-            ec->xl_up & ~(fop->remaining | fop->good));
+            "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, "
+            "remaining=%s, good=%s, bad=%s)",
+            gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes,
+            ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+            ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+            ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes),
+            ec_bin(str4, sizeof(str4), fop->good, ec->nodes),
+            ec_bin(str5, sizeof(str5),
+            ec->xl_up & ~(fop->remaining | fop->good), ec->nodes));
 
     if (fop->use_fd)
     {
diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c
index 934aaaf133e..39d648bb34d 100644
--- a/xlators/features/shard/src/shard.c
+++ b/xlators/features/shard/src/shard.c
@@ -178,6 +178,63 @@ shard_inode_ctx_set (inode_t *inode, xlator_t *this, struct iatt *stbuf,
 }
 
 int
+__shard_inode_ctx_set_refreshed_flag (inode_t *inode, xlator_t *this)
+{
+        int                 ret = -1;
+        shard_inode_ctx_t  *ctx = NULL;
+
+        ret = __shard_inode_ctx_get (inode, this, &ctx);
+        if (ret)
+                return ret;
+
+        ctx->refreshed = _gf_true;
+        return 0;
+}
+
+int
+shard_inode_ctx_set_refreshed_flag (inode_t *inode, xlator_t *this)
+{
+        int ret = -1;
+
+        LOCK (&inode->lock);
+        {
+                ret = __shard_inode_ctx_set_refreshed_flag (inode, this);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+gf_boolean_t
+__shard_inode_ctx_needs_lookup (inode_t *inode, xlator_t *this)
+{
+        int                 ret = -1;
+        shard_inode_ctx_t  *ctx = NULL;
+
+        ret = __shard_inode_ctx_get (inode, this, &ctx);
+        /* If inode ctx get fails, better to err on the side of caution and
+         * try again? Unless the failure is due to mem-allocation.
+         */
+        if (ret)
+                return _gf_true;
+
+        return !ctx->refreshed;
+}
+
+gf_boolean_t
+shard_inode_ctx_needs_lookup (inode_t *inode, xlator_t *this)
+{
+        gf_boolean_t flag = _gf_false;
+
+        LOCK (&inode->lock);
+        {
+                flag = __shard_inode_ctx_needs_lookup (inode, this);
+        }
+        UNLOCK (&inode->lock);
+
+        return flag;
+}
+int
 __shard_inode_ctx_invalidate (inode_t *inode, xlator_t *this, struct iatt *stbuf)
 {
         int                 ret = -1;
@@ -549,19 +606,20 @@ shard_common_inode_write_success_unwind (glusterfs_fop_t fop,
 
 int
 shard_common_resolve_shards (call_frame_t *frame, xlator_t *this,
-                             inode_t *res_inode,
                              shard_post_resolve_fop_handler_t post_res_handler)
 {
         int                   i              = -1;
         uint32_t              shard_idx_iter = 0;
         char                  path[PATH_MAX] = {0,};
         inode_t              *inode          = NULL;
+        inode_t              *res_inode      = NULL;
         shard_priv_t         *priv           = NULL;
         shard_local_t        *local          = NULL;
 
         priv = this->private;
         local = frame->local;
         shard_idx_iter = local->first_block;
+        res_inode = local->resolver_base_inode;
 
         if (local->op_ret < 0)
                 goto out;
@@ -755,7 +813,7 @@ out:
 
 }
 
-static void
+static inode_t *
 shard_link_dot_shard_inode (shard_local_t *local, inode_t *inode,
                             struct iatt *buf)
 {
@@ -764,10 +822,72 @@ shard_link_dot_shard_inode (shard_local_t *local, inode_t *inode,
 
         priv = THIS->private;
 
-        linked_inode = inode_link (inode, local->dot_shard_loc.parent,
-                                   local->dot_shard_loc.name, buf);
+        linked_inode = inode_link (inode, inode->table->root, ".shard", buf);
         inode_lookup (linked_inode);
         priv->dot_shard_inode = linked_inode;
+        return linked_inode;
+}
+
+
+int
+shard_refresh_dot_shard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, inode_t *inode,
+                             struct iatt *buf, dict_t *xdata,
+                             struct iatt *postparent)
+{
+        shard_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret) {
+                local->op_ret = op_ret;
+                local->op_errno = op_errno;
+                goto out;
+        }
+
+        /* To-Do: Fix refcount increment per call to
+         * shard_link_dot_shard_inode().
+         */
+        shard_link_dot_shard_inode (local, inode, buf);
+        shard_inode_ctx_set_refreshed_flag (inode, this);
+out:
+        shard_common_resolve_shards (frame, this, local->post_res_handler);
+        return 0;
+}
+
+int
+shard_refresh_dot_shard (call_frame_t *frame, xlator_t *this)
+{
+        loc_t          loc       = {0,};
+        inode_t       *inode     = NULL;
+        shard_priv_t  *priv      = NULL;
+        shard_local_t *local     = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+        inode = inode_find (this->itable, priv->dot_shard_gfid);
+
+        if (!shard_inode_ctx_needs_lookup (inode, this)) {
+                local->op_ret = 0;
+                goto out;
+        }
+
+        /* Plain assignment because the ref is already taken above through
+         * call to inode_find()
+         */
+        loc.inode = inode;
+        gf_uuid_copy (loc.gfid, priv->dot_shard_gfid);
+
+        STACK_WIND (frame, shard_refresh_dot_shard_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &loc, NULL);
+        loc_wipe (&loc);
+
+        return 0;
+
+out:
+        shard_common_resolve_shards (frame, this, local->post_res_handler);
+        return 0;
 }
 
 int
@@ -776,7 +896,8 @@ shard_lookup_dot_shard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                             struct iatt *buf, dict_t *xdata,
                             struct iatt *postparent)
 {
-        shard_local_t *local = NULL;
+        inode_t       *link_inode = NULL;
+        shard_local_t *local      = NULL;
 
         local = frame->local;
 
@@ -796,11 +917,14 @@ shard_lookup_dot_shard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
         }
 
-        shard_link_dot_shard_inode (local, inode, buf);
-        shard_common_resolve_shards (frame, this,
-                                     (local->fop == GF_FOP_RENAME) ?
-                                     local->loc2.inode : local->loc.inode,
-                                     local->post_res_handler);
+        link_inode = shard_link_dot_shard_inode (local, inode, buf);
+        if (link_inode != inode) {
+                shard_refresh_dot_shard (frame, this);
+        } else {
+                shard_inode_ctx_set_refreshed_flag (link_inode, this);
+                shard_common_resolve_shards (frame, this,
+                                             local->post_res_handler);
+        }
         return 0;
 
 unwind:
@@ -1813,6 +1937,8 @@ shard_truncate_begin (call_frame_t *frame, xlator_t *this)
                                                local->block_size);
 
         local->num_blocks = local->last_block - local->first_block + 1;
+        local->resolver_base_inode = (local->fop == GF_FOP_TRUNCATE) ?
+                                     local->loc.inode : local->fd->inode;
 
         if ((local->first_block == 0) && (local->num_blocks == 1)) {
                 if (local->fop == GF_FOP_TRUNCATE)
@@ -1843,11 +1969,8 @@ shard_truncate_begin (call_frame_t *frame, xlator_t *this)
                 shard_lookup_dot_shard (frame, this,
                                         shard_post_resolve_truncate_handler);
         } else {
-                shard_common_resolve_shards (frame, this,
-                                             (local->fop == GF_FOP_TRUNCATE) ?
-                                              local->loc.inode :
-                                              local->fd->inode,
-                                           shard_post_resolve_truncate_handler);
+                local->post_res_handler = shard_post_resolve_truncate_handler;
+                shard_refresh_dot_shard (frame, this);
         }
         return 0;
 
@@ -2158,10 +2281,7 @@ shard_post_lookup_shards_unlink_handler (call_frame_t *frame, xlator_t *this)
         local->op_ret = 0;
         local->op_errno = 0;
 
-        shard_unlink_shards_do (frame, this,
-                                (local->fop == GF_FOP_RENAME)
-                                             ? local->loc2.inode
-                                             : local->loc.inode);
+        shard_unlink_shards_do (frame, this, local->resolver_base_inode);
         return 0;
 }
 
@@ -2208,14 +2328,10 @@ shard_post_resolve_unlink_handler (call_frame_t *frame, xlator_t *this)
 
         if (!local->call_count)
                 shard_unlink_shards_do (frame, this,
-                                        (local->fop == GF_FOP_RENAME)
-                                                     ? local->loc2.inode
-                                                     : local->loc.inode);
+                                        local->resolver_base_inode);
         else
                 shard_common_lookup_shards (frame, this,
-                                            (local->fop == GF_FOP_RENAME)
-                                                         ? local->loc2.inode
-                                                         : local->loc.inode,
+                                            local->resolver_base_inode,
                                        shard_post_lookup_shards_unlink_handler);
         return 0;
 }
@@ -2252,6 +2368,7 @@ shard_unlink_base_file_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->last_block = get_highest_block (0, local->prebuf.ia_size,
                                                local->block_size);
         local->num_blocks = local->last_block - local->first_block + 1;
+        local->resolver_base_inode = local->loc.inode;
 
         /* num_blocks = 1 implies that the file has not crossed its
          * shard block size. So unlink boils down to unlinking just the
@@ -2282,8 +2399,8 @@ shard_unlink_base_file_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 shard_lookup_dot_shard (frame, this,
                                         shard_post_resolve_unlink_handler);
         } else {
-                shard_common_resolve_shards (frame, this, local->loc.inode,
-                                             shard_post_resolve_unlink_handler);
+                local->post_res_handler = shard_post_resolve_unlink_handler;
+                shard_refresh_dot_shard (frame, this);
         }
 
         return 0;
@@ -2605,6 +2722,7 @@ shard_rename_unlink_dst_shards_do (call_frame_t *frame, xlator_t *this)
         local->last_block = get_highest_block (0, local->postbuf.ia_size,
                                                local->dst_block_size);
         local->num_blocks = local->last_block - local->first_block + 1;
+        local->resolver_base_inode = local->loc2.inode;
 
         if ((local->xattr_rsp) &&
             (!dict_get_uint32 (local->xattr_rsp, GET_LINK_COUNT, &link_count))
@@ -2632,8 +2750,8 @@ shard_rename_unlink_dst_shards_do (call_frame_t *frame, xlator_t *this)
                 shard_lookup_dot_shard (frame, this,
                                         shard_post_resolve_unlink_handler);
         } else {
-                shard_common_resolve_shards (frame, this, local->loc2.inode,
-                                             shard_post_resolve_unlink_handler);
+                local->post_res_handler = shard_post_resolve_unlink_handler;
+                shard_refresh_dot_shard (frame, this);
         }
 
         return 0;
@@ -3361,6 +3479,7 @@ shard_post_lookup_readv_handler (call_frame_t *frame, xlator_t *this)
                                                local->block_size);
 
         local->num_blocks = local->last_block - local->first_block + 1;
+        local->resolver_base_inode = local->loc.inode;
 
         local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
                                        gf_shard_mt_inode_list);
@@ -3395,8 +3514,8 @@ shard_post_lookup_readv_handler (call_frame_t *frame, xlator_t *this)
                 shard_lookup_dot_shard (frame, this,
                                         shard_post_resolve_readv_handler);
         } else {
-                shard_common_resolve_shards (frame, this, local->loc.inode,
-                                             shard_post_resolve_readv_handler);
+                local->post_res_handler = shard_post_resolve_readv_handler;
+                shard_refresh_dot_shard (frame, this);
         }
         return 0;
 
@@ -3856,7 +3975,8 @@ shard_mkdir_dot_shard_cbk (call_frame_t *frame, void *cookie,
                                        struct iatt *buf, struct iatt *preparent,
                                        struct iatt *postparent, dict_t *xdata)
 {
-        shard_local_t *local = NULL;
+        inode_t       *link_inode = NULL;
+        shard_local_t *local      = NULL;
 
         local = frame->local;
 
@@ -3876,11 +3996,17 @@ shard_mkdir_dot_shard_cbk (call_frame_t *frame, void *cookie,
                 }
         }
 
-        shard_link_dot_shard_inode (local, inode, buf);
-
+        link_inode = shard_link_dot_shard_inode (local, inode, buf);
+        if (link_inode != inode) {
+                shard_refresh_dot_shard (frame, this);
+        } else {
+                shard_inode_ctx_set_refreshed_flag (link_inode, this);
+                shard_common_resolve_shards (frame, this,
+                                             local->post_res_handler);
+        }
+        return 0;
 unwind:
-        shard_common_resolve_shards (frame, this, local->loc.inode,
-                                     local->post_res_handler);
+        shard_common_resolve_shards (frame, this, local->post_res_handler);
         return 0;
 }
 
@@ -4623,6 +4749,7 @@ shard_common_inode_write_begin (call_frame_t *frame, xlator_t *this,
         local->last_block = get_highest_block (offset, local->total_size,
                                                local->block_size);
         local->num_blocks = local->last_block - local->first_block + 1;
+        local->resolver_base_inode = local->fd->inode;
         local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
                                        gf_shard_mt_inode_list);
         if (!local->inode_list)
@@ -4641,12 +4768,13 @@ shard_common_inode_write_begin (call_frame_t *frame, xlator_t *this,
         local->dot_shard_loc.inode = inode_find (this->itable,
                                                  priv->dot_shard_gfid);
 
-        if (!local->dot_shard_loc.inode)
+        if (!local->dot_shard_loc.inode) {
                 shard_mkdir_dot_shard (frame, this,
                                  shard_common_inode_write_post_resolve_handler);
-        else
-                shard_common_resolve_shards (frame, this, local->loc.inode,
-                                 shard_common_inode_write_post_resolve_handler);
+        } else {
+                local->post_res_handler = shard_common_inode_write_post_resolve_handler;
+                shard_refresh_dot_shard (frame, this);
+        }
 
         return 0;
 out:
diff --git a/xlators/features/shard/src/shard.h b/xlators/features/shard/src/shard.h
index 8303a2ca030..09232a45f24 100644
--- a/xlators/features/shard/src/shard.h
+++ b/xlators/features/shard/src/shard.h
@@ -254,6 +254,7 @@ typedef struct shard_local {
                 fop_inodelk_cbk_t inodelk_cbk;
                 shard_lock_t *shard_lock;
         } lock;
+        inode_t *resolver_base_inode;
 } shard_local_t;
 
 typedef struct shard_inode_ctx {
@@ -267,6 +268,7 @@ typedef struct shard_inode_ctx {
         struct list_head ilist;
         uuid_t base_gfid;
         int block_num;
+        gf_boolean_t refreshed;
 } shard_inode_ctx_t;
 
 #endif /* __SHARD_H__ */
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 525a6a6fbbc..0e1ba0b4dba 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -3213,6 +3213,7 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_state_t *state = NULL;
         char         *dict_value = NULL;
         int32_t       ret = -1;
+        int32_t       op_errno = 0;
         char *newkey = NULL;
 
         priv = this->private;
@@ -3225,57 +3226,49 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         "%"PRIu64": SETXATTR %s/%"PRIu64" (%s):"
                         "refusing positioned setxattr",
                         finh->unique, state->loc.path, finh->nodeid, name);
-                send_fuse_err (this, finh, EINVAL);
-                FREE (finh);
-                return;
+                op_errno = EINVAL;
+                goto done;
         }
 #endif
 
         if (fuse_ignore_xattr_set (priv, name)) {
-                (void) send_fuse_err (this, finh, 0);
-                return;
+                goto done;
         }
 
         if (!priv->acl) {
                 if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
                     (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
-                        send_fuse_err (this, finh, EOPNOTSUPP);
-                        GF_FREE (finh);
-                        return;
+                        op_errno = EOPNOTSUPP;
+                        goto done;
                 }
         }
 
         ret = fuse_check_selinux_cap_xattr (priv, name);
         if (ret) {
-                send_fuse_err (this, finh, EOPNOTSUPP);
-                GF_FREE (finh);
-                return;
+                op_errno = EOPNOTSUPP;
+                goto done;
         }
 
         /* Check if the command is for changing the log
            level of process or specific xlator */
         ret = is_gf_log_command (this, name, value);
         if (ret >= 0) {
-                send_fuse_err (this, finh, ret);
-                GF_FREE (finh);
-                return;
+                op_errno = ret;
+                goto done;
         }
 
         if (!strcmp ("inode-invalidate", name)) {
                 gf_log ("fuse", GF_LOG_TRACE,
                         "got request to invalidate %"PRIu64, finh->nodeid);
-                send_fuse_err (this, finh, 0);
 #if FUSE_KERNEL_MINOR_VERSION >= 11
                 fuse_invalidate_entry (this, finh->nodeid);
 #endif
-                GF_FREE (finh);
-                return;
+                goto done;
         }
 
         if (!strcmp (GFID_XATTR_KEY, name) || !strcmp (GF_XATTR_VOL_ID_KEY, name)) {
-                send_fuse_err (this, finh, EPERM);
-                GF_FREE (finh);
-                return;
+                op_errno = EPERM;
+                goto done;
         }
 
         state->size = fsi->size;
@@ -3287,17 +3280,14 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
                         "%"PRIu64": SETXATTR dict allocation failed",
                         finh->unique);
-
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                return;
+                op_errno = ENOMEM;
+                goto done;
         }
 
         ret = fuse_flip_xattr_ns (priv, name, &newkey);
         if (ret) {
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                return;
+                op_errno = ENOMEM;
+                goto done;
         }
 
         if (fsi->size > 0) {
@@ -3320,6 +3310,10 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         fuse_resolve_and_resume (state, fuse_setxattr_resume);
 
         return;
+
+done:
+        send_fuse_err(this, finh, op_errno);
+        free_fuse_state(state);
 }
 
 
diff --git a/xlators/storage/posix/src/posix-handle.c b/xlators/storage/posix/src/posix-handle.c
index ddafb0d9b04..d3f48f859bf 100644
--- a/xlators/storage/posix/src/posix-handle.c
+++ b/xlators/storage/posix/src/posix-handle.c
@@ -940,6 +940,7 @@ posix_create_link_if_gfid_exists (xlator_t *this, uuid_t gfid, char *real_path,
         inode_t               *inode       = NULL;
         struct stat            stbuf       = {0,};
         struct posix_private  *priv        = NULL;
+        posix_inode_ctx_t     *ctx         = NULL;
 
         priv = this->private;
 
@@ -961,12 +962,14 @@ posix_create_link_if_gfid_exists (xlator_t *this, uuid_t gfid, char *real_path,
 
                 LOCK (&inode->lock);
                 {
-                        ret = __inode_ctx_get0 (inode, this, &ctx_int);
+                        ret = __posix_inode_ctx_get_all (inode, this, &ctx);
                         if (ret)
                                 goto unlock;
 
-                        if (ctx_int != GF_UNLINK_TRUE)
+                        if (ctx->unlink_flag != GF_UNLINK_TRUE) {
+                                ret = -1;
                                 goto unlock;
+                        }
 
                         POSIX_GET_FILE_UNLINK_PATH (priv->base_path, gfid,
                                                     unlink_path);
@@ -985,7 +988,8 @@ posix_create_link_if_gfid_exists (xlator_t *this, uuid_t gfid, char *real_path,
                                 goto unlock;
                         }
                         ctx_int = GF_UNLINK_FALSE;
-                        ret = __inode_ctx_set0 (inode, this, &ctx_int);
+                        ret = __posix_inode_ctx_set_unlink_flag (inode, this,
+                                                                 ctx_int);
                 }
 unlock:
                 UNLOCK (&inode->lock);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 1ebff47879d..8694383241e 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -2194,39 +2194,88 @@ posix_fdget_objectsignature (int fd, dict_t *xattr)
         return -EINVAL;
 }
 
+posix_inode_ctx_t *
+__posix_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        int                 ret      = -1;
+        uint64_t            ctx_uint = 0;
+        posix_inode_ctx_t  *ctx_p    = NULL;
+
+        ret = __inode_ctx_get (inode, this, &ctx_uint);
+        if (ret == 0) {
+                return (posix_inode_ctx_t *)ctx_uint;
+        }
+
+        ctx_p = GF_CALLOC (1, sizeof (*ctx_p), gf_posix_mt_inode_ctx_t);
+        if (!ctx_p)
+                return NULL;
+
+        pthread_mutex_init (&ctx_p->xattrop_lock, NULL);
+
+        ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);
+        if (ret < 0) {
+                pthread_mutex_destroy (&ctx_p->xattrop_lock);
+                GF_FREE (ctx_p);
+                return NULL;
+        }
+
+        return ctx_p;
+}
 
 int
-posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx)
+__posix_inode_ctx_set_unlink_flag (inode_t *inode, xlator_t *this, uint64_t ctx)
 {
-        int             ret     = -1;
-        uint64_t        ctx_int = 0;
+        posix_inode_ctx_t  *ctx_p    = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        ctx_p = __posix_inode_ctx_get (inode, this);
+        if (ctx_p == NULL)
+                return -1;
 
-        ret = inode_ctx_get (inode, this, &ctx_int);
+        ctx_p->unlink_flag = ctx;
 
-        if (ret)
-                return ret;
+        return 0;
+}
 
-        if (ctx)
-                *ctx = ctx_int;
+int
+posix_inode_ctx_set_unlink_flag (inode_t *inode, xlator_t *this, uint64_t ctx)
+{
+        int             ret = -1;
+
+        LOCK(&inode->lock);
+        {
+                ret = __posix_inode_ctx_set_unlink_flag (inode, this, ctx);
+        }
+        UNLOCK(&inode->lock);
 
-out:
         return ret;
 }
 
+int
+__posix_inode_ctx_get_all (inode_t *inode, xlator_t *this,
+                           posix_inode_ctx_t **ctx)
+{
+        posix_inode_ctx_t  *ctx_p    = NULL;
+
+        ctx_p = __posix_inode_ctx_get (inode, this);
+        if (ctx_p == NULL)
+                return -1;
+
+        *ctx = ctx_p;
+
+        return 0;
+}
 
 int
-posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx)
+posix_inode_ctx_get_all (inode_t *inode, xlator_t *this,
+                         posix_inode_ctx_t **ctx)
 {
-        int             ret = -1;
+        int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+        LOCK(&inode->lock);
+        {
+                ret = __posix_inode_ctx_get_all (inode, this, ctx);
+        }
+        UNLOCK(&inode->lock);
 
-        ret = inode_ctx_set (inode, this, &ctx);
-out:
         return ret;
 }
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 49665884a7e..3cbb947d6b2 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -107,19 +107,21 @@ out:
 int
 posix_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t tmp_cache = 0;
-        int ret = 0;
-        char *unlink_path = NULL;
-        struct posix_private    *priv_posix = NULL;
+        int                     ret         = 0;
+        char                   *unlink_path = NULL;
+        uint64_t                ctx_uint    = 0;
+        posix_inode_ctx_t      *ctx         = NULL;
+        struct posix_private   *priv_posix  = NULL;
 
         priv_posix = (struct posix_private *) this->private;
 
-        ret = inode_ctx_del (inode, this, &tmp_cache);
-        if (ret < 0) {
-                ret = 0;
-                goto out;
-        }
-        if (tmp_cache == GF_UNLINK_TRUE) {
+        ret = inode_ctx_del (inode, this, &ctx_uint);
+        if (!ctx_uint)
+                return 0;
+
+        ctx = (posix_inode_ctx_t *)ctx_uint;
+
+        if (ctx->unlink_flag == GF_UNLINK_TRUE) {
                 POSIX_GET_FILE_UNLINK_PATH(priv_posix->base_path,
                                            inode->gfid, unlink_path);
                 if (!unlink_path) {
@@ -133,6 +135,8 @@ posix_forget (xlator_t *this, inode_t *inode)
                 ret = sys_unlink(unlink_path);
         }
 out:
+        pthread_mutex_destroy (&ctx->xattrop_lock);
+        GF_FREE (ctx);
         return ret;
 }
 
@@ -1778,7 +1782,7 @@ posix_add_unlink_to_ctx (inode_t *inode, xlator_t *this, char *unlink_path)
         }
 
         ctx = GF_UNLINK_TRUE;
-        ret = posix_inode_ctx_set (inode, this, ctx);
+        ret = posix_inode_ctx_set_unlink_flag (inode, this, ctx);
         if (ret < 0) {
                 goto out;
         }
@@ -5513,6 +5517,7 @@ _posix_handle_xattr_keyvalue_pair (dict_t *d, char *k, data_t *v,
         inode_t              *inode    = NULL;
         xlator_t             *this     = NULL;
         posix_xattr_filler_t *filler   = NULL;
+        posix_inode_ctx_t    *ctx      = NULL;
 
         filler = tmp;
 
@@ -5535,8 +5540,13 @@ _posix_handle_xattr_keyvalue_pair (dict_t *d, char *k, data_t *v,
                 }
         }
 #endif
+        op_ret = posix_inode_ctx_get_all (inode, this, &ctx);
+        if (op_ret < 0) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-        LOCK (&inode->lock);
+        pthread_mutex_lock (&ctx->xattrop_lock);
         {
                 if (filler->real_path) {
                         size = sys_lgetxattr (filler->real_path, k,
@@ -5645,7 +5655,7 @@ _posix_handle_xattr_keyvalue_pair (dict_t *d, char *k, data_t *v,
                 op_errno = errno;
         }
 unlock:
-        UNLOCK (&inode->lock);
+        pthread_mutex_unlock (&ctx->xattrop_lock);
 
         if (op_ret == -1)
                 goto out;
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index ef4bc66ecbc..2249b238e8b 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -197,6 +197,11 @@ typedef struct {
         int32_t     op_errno;
 } posix_xattr_filler_t;
 
+typedef struct {
+        uint64_t unlink_flag;
+        pthread_mutex_t xattrop_lock;
+} posix_inode_ctx_t;
+
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)
 
 #define POSIX_BASE_PATH_LEN(this) (((struct posix_private *)this->private)->base_path_length)
@@ -224,8 +229,17 @@ typedef struct {
 
 
 /* Helper functions */
-int posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx);
-int posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx);
+int posix_inode_ctx_set_unlink_flag (inode_t *inode, xlator_t *this,
+                                     uint64_t ctx);
+
+int posix_inode_ctx_get_all (inode_t *inode, xlator_t *this,
+                             posix_inode_ctx_t **ctx);
+
+int __posix_inode_ctx_set_unlink_flag (inode_t *inode, xlator_t *this,
+                                       uint64_t ctx);
+
+int __posix_inode_ctx_get_all (inode_t *inode, xlator_t *this,
+                               posix_inode_ctx_t **ctx);
 
 int posix_gfid_set (xlator_t *this, const char *path, loc_t *loc,
                     dict_t *xattr_req);