cluster/afr: Make [f]xattrop metadata transaction

        Backport of http://review.gluster.com/11809

Problem:
When xlators above afr do [f]xattrop when one of the bricks is down, after the
brick comes backup, the metadata is not healed because [f]xattrop is not
considered a transaction.

Fix:
Treat [f]xattrop as transaction so that changes done by xlators above afr are
marked for heal when some of the bricks were down at the time of [f]xattrop.

BUG: 1248890
Change-Id: Ibe69aa0ca6be9b4b4134dc2879b306e2e9c4cde8
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/11810
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>

4 files changed, 234 insertions, 183 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index dd111ba6564..2eb3946ab7c 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1226,6 +1226,11 @@ afr_replies_wipe (struct afr_reply *replies, int count)
                         dict_unref (replies[i].xdata);
                         replies[i].xdata = NULL;
                 }
+
+                if (replies[i].xattr) {
+                        dict_unref (replies[i].xattr);
+                        replies[i].xattr = NULL;
+                }
         }
 }
 
@@ -1278,6 +1283,9 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->xattr_req)
                 dict_unref (local->xattr_req);
 
+        if (local->xattr_rsp)
+                dict_unref (local->xattr_rsp);
+
         if (local->dict)
                 dict_unref (local->dict);
 
@@ -1355,10 +1363,6 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
                 if (local->cont.xattrop.xattr)
                         dict_unref (local->cont.xattrop.xattr);
         }
-        { /* fxattrop */
-                if (local->cont.fxattrop.xattr)
-                        dict_unref (local->cont.fxattrop.xattr);
-        }
         { /* symlink */
                 GF_FREE (local->cont.symlink.linkpath);
         }
@@ -2910,166 +2914,6 @@ out:
 
 /* }}} */
 
-/* {{{ xattrop */
-
-int32_t
-afr_xattrop_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.xattrop.xattr)
-                                local->cont.xattrop.xattr = dict_ref (xattr);
-
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-
-                        local->op_ret = 0;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                local->cont.xattrop.xattr, local->xdata_rsp);
-
-        return 0;
-}
-
-
-int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_xattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
-
-/* }}} */
-
-/* {{{ fxattrop */
-
-int32_t
-afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.fxattrop.xattr)
-                                local->cont.fxattrop.xattr = dict_ref (xattr);
-
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                        local->op_ret = 0;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.fxattrop.xattr, local->xdata_rsp);
-
-        return 0;
-}
-
-
-int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-                goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fxattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
-
-/* }}} */
-
 int32_t
 afr_unlock_partial_inodelk_cbk (call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index be83781f3f1..387d0d6cb7b 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -97,6 +97,12 @@ __afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 				local->xdata_rsp =
 					dict_ref (local->replies[i].xdata);
 			}
+			if (local->replies[i].xattr) {
+				if (local->xattr_rsp)
+					dict_unref (local->xattr_rsp);
+				local->xattr_rsp =
+					dict_ref (local->replies[i].xattr);
+			}
 		}
 	}
 
@@ -107,7 +113,8 @@ __afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 static void
 __afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
 			int op_ret, int op_errno,
-			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+			struct iatt *prebuf, struct iatt *postbuf,
+			dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
 
@@ -122,6 +129,8 @@ __afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
 			local->replies[child_index].prestat = *prebuf;
 		if (postbuf)
 			local->replies[child_index].poststat = *postbuf;
+		if (xattr)
+			local->replies[child_index].xattr = dict_ref (xattr);
 		if (xdata)
 			local->replies[child_index].xdata = dict_ref (xdata);
 	} else {
@@ -135,7 +144,7 @@ __afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
 static int
 __afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int child_index = (long) cookie;
@@ -146,7 +155,8 @@ __afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&frame->lock);
         {
                 __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xdata);
+					op_errno, prebuf, postbuf, xattr,
+					xdata);
         }
         UNLOCK (&frame->lock);
 
@@ -255,7 +265,7 @@ afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         LOCK (&frame->lock);
         {
                 __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xdata);
+					op_errno, prebuf, postbuf, NULL, xdata);
 		if (op_ret == -1 || !xdata)
 			goto unlock;
 
@@ -509,7 +519,7 @@ afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		local->stable_write = _gf_false;
 
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -628,7 +638,7 @@ afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		local->stable_write = _gf_false;
 
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -740,7 +750,7 @@ afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, xdata);
+				      preop, postop, NULL, xdata);
 }
 
 
@@ -845,7 +855,7 @@ afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, xdata);
+				      preop, postop, NULL, xdata);
 }
 
 
@@ -952,7 +962,7 @@ afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1499,7 +1509,7 @@ afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1613,7 +1623,7 @@ afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1721,7 +1731,7 @@ afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+				      NULL, NULL, NULL, xdata);
 }
 
 
@@ -1831,7 +1841,7 @@ afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         struct iatt *postbuf, dict_t *xdata)
 {
         return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -1945,7 +1955,7 @@ afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *postbuf, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -2055,7 +2065,7 @@ afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *postbuf, dict_t *xdata)
 {
 	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+				      prebuf, postbuf, NULL, xdata);
 }
 
 
@@ -2137,3 +2147,192 @@ out:
 }
 
 /* }}} */
+
+int32_t
+afr_xattrop_wind_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int32_t op_ret, int32_t op_errno,
+                      dict_t *xattr, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xattr, xdata);
+}
+
+int
+afr_xattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_xattrop_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->xattrop,
+			   &local->loc, local->cont.xattrop.optype,
+			   local->cont.xattrop.xattr, local->xdata_req);
+        return 0;
+}
+
+int
+afr_xattrop_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
+
+        local = frame->local;
+
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (xattrop, main_frame, local->op_ret, local->op_errno,
+			  local->xattr_rsp, local->xdata_rsp);
+        return 0;
+}
+
+int32_t
+afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.xattrop.xattr = dict_ref (xattr);
+        local->cont.xattrop.optype = optype;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        local->transaction.wind   = afr_xattrop_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_xattrop_unwind;
+
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+	local->op = GF_FOP_XATTROP;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
+
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int32_t
+afr_fxattrop_wind_cbk (call_frame_t *frame, void *cookie,
+                       xlator_t *this, int32_t op_ret, int32_t op_errno,
+                       dict_t *xattr, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xattr, xdata);
+}
+
+int
+afr_fxattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_fxattrop_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fxattrop,
+			   local->fd, local->cont.xattrop.optype,
+			   local->cont.xattrop.xattr, local->xdata_req);
+        return 0;
+}
+
+int
+afr_fxattrop_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
+
+        local = frame->local;
+
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (fxattrop, main_frame, local->op_ret, local->op_errno,
+			  local->xattr_rsp, local->xdata_rsp);
+        return 0;
+}
+
+int32_t
+afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.xattrop.xattr = dict_ref (xattr);
+        local->cont.xattrop.optype = optype;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        local->transaction.wind   = afr_fxattrop_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fxattrop_unwind;
+
+	local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FXATTROP;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this,
+                               AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
+
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 7b1fc552880..e174cc2d610 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -79,4 +79,12 @@ afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
 int
 afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              off_t len, dict_t *xdata);
+
+int32_t
+afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int32_t
+afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 2c225cd6462..008839a7312 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -262,6 +262,7 @@ struct afr_reply {
 	int	valid;
 	int32_t	op_ret;
 	int32_t	op_errno;
+	dict_t *xattr;/*For xattrop*/
 	dict_t *xdata;
 	struct iatt poststat;
 	struct iatt postparent;
@@ -568,12 +569,9 @@ typedef struct _afr_local {
 
                 struct {
                         dict_t *xattr;
+                        gf_xattrop_flags_t optype;
                 } xattrop;
 
-                struct {
-                        dict_t *xattr;
-                } fxattrop;
-
                 /* dir write */
 
                 struct {
@@ -734,6 +732,8 @@ typedef struct _afr_local {
         dict_t         *xdata_req;
         dict_t         *xdata_rsp;
 
+        dict_t         *xattr_rsp; /*for [f]xattrop*/
+
         mode_t          umask;
         int             xflag;
         gf_boolean_t    do_discovery;