cluster/afr: Fixing the flaws in arbiter becoming source patch

Problem:
Setting the write_subvol value to read_subvol in case of metadata
transaction during pre-op (commit 19f9bcff4aada589d4321356c2670ed283f02c03)
might lead to the original problem of arbiter becoming source.

Scenario:
1) All bricks are up and good
2) 2 writes w1 and w2 are in progress in parallel
3) ctx->read_subvol is good for all the subvolumes
4) w1 succeeds on brick0 and fails on brick1, yet to do post-op on
   the disk
5) read/lookup comes on the same file and refreshes read_subvols back
   to all good
6) metadata transaction happens which makes ctx->write_subvol to be
   assigned with ctx->read_subvol which is all good
7) w2 succeeds on brick1 and fails on brick0 and this will update the
   brick in reverse order leading to arbiter becoming source

Fix:
Instead of setting the ctx->write_subvol to ctx->read_subvol in the
pre-op statge, if there is a metadata transaction, check in the
function __afr_set_in_flight_sb_status() if it is a data/metadata
transaction. Use the value of ctx->write_subvol if it is a data
transactions and ctx->read_subvol value for other transactions.

With this patch we assign the value of ctx->write_subvol in the
afr_transaction_perform_fop() with the on disk value, instead of
assigning it in the afr_changelog_pre_op() with the in memory value.

Change-Id: Id2025a7e965f0578af35b1abaac793b019c43cc4
BUG: 1482064
Signed-off-by: karthik-us <ksubrahm@redhat.com>

1 files changed, 43 insertions, 14 deletions

diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 8078f4403c0..89ec6ee4d61 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -510,6 +510,7 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 {
         afr_local_t *local = NULL;
         int op_errno = ENOMEM;
+        int ret = -1;
 
 	local = AFR_FRAME_INIT (frame, op_errno);
 	if (!local)
@@ -532,7 +533,9 @@ afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		goto out;
 
         local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_OPEN_FD_COUNT, 4)) {
 		op_errno = ENOMEM;
@@ -657,7 +660,9 @@ afr_truncate (call_frame_t *frame, xlator_t *this,
         local->transaction.unwind = afr_truncate_unwind;
 
         loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+        ret = afr_set_inode_local (this, local, loc->inode);
+        if (ret)
+                goto out;
 
         local->op = GF_FOP_TRUNCATE;
 
@@ -771,7 +776,9 @@ afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
 		goto out;
 
         local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
         local->op = GF_FOP_FTRUNCATE;
 
@@ -889,7 +896,9 @@ afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
         local->transaction.unwind = afr_setattr_unwind;
 
         loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+        ret = afr_set_inode_local (this, local, loc->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_SETATTR;
 
@@ -994,7 +1003,9 @@ afr_fsetattr (call_frame_t *frame, xlator_t *this,
         local->transaction.unwind = afr_fsetattr_unwind;
 
         local->fd                 = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_FSETATTR;
 
@@ -1636,7 +1647,9 @@ afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
         local->transaction.unwind = afr_setxattr_unwind;
 
         loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+        ret = afr_set_inode_local (this, local, loc->inode);
+        if (ret)
+                goto out;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
@@ -1748,7 +1761,9 @@ afr_fsetxattr (call_frame_t *frame, xlator_t *this,
         local->transaction.unwind = afr_fsetxattr_unwind;
 
         local->fd                 = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_FSETXATTR;
 
@@ -1861,7 +1876,9 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
         local->transaction.unwind = afr_removexattr_unwind;
 
         loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+        ret = afr_set_inode_local (this, local, loc->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_REMOVEXATTR;
 
@@ -1968,7 +1985,9 @@ afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         local->transaction.unwind = afr_fremovexattr_unwind;
 
         local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_FREMOVEXATTR;
 
@@ -2063,7 +2082,9 @@ afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
         local->cont.fallocate.len = len;
 
         local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	if (xdata)
 		local->xdata_req = dict_copy_with_ref (xdata, NULL);
@@ -2175,7 +2196,9 @@ afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         local->cont.discard.len = len;
 
         local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	if (xdata)
 		local->xdata_req = dict_copy_with_ref (xdata, NULL);
@@ -2284,7 +2307,9 @@ afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         local->cont.zerofill.len = len;
 
         local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	if (xdata)
 		local->xdata_req = dict_copy_with_ref (xdata, NULL);
@@ -2396,7 +2421,9 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         local->transaction.unwind = afr_xattrop_unwind;
 
         loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+        ret = afr_set_inode_local (this, local, loc->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_XATTROP;
 
@@ -2490,7 +2517,9 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
         local->transaction.unwind = afr_fxattrop_unwind;
 
 	local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+        ret = afr_set_inode_local (this, local, fd->inode);
+        if (ret)
+                goto out;
 
 	local->op = GF_FOP_FXATTROP;