afr: send the correct iatt values in fsync cbk

Problem:
afr unwinds the fsync fop with an iatt buffer from one of its children
on whom fsync was successful. But that child might not be a valid read
subvolume for that inode because of pending heals or because it happens
to be the arbiter brick etc. Thus we end up sending the wrong iatt to
mdcache which will in turn serve it to the application on a subsequent
stat call as reported in the BZ.

Fix:
Pick a child on whom the fsync was successful *and* that is readable as
indicated in the inode context.

> Reviewed-on: https://review.gluster.org/17227
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Smoke: Gluster Build System <jenkins@build.gluster.org>
(cherry picked from commit 1a8fa910ccba7aa941f673302c1ddbd7bd818e39)

Change-Id: Ie8647289219cebe02dde4727e19a729b3353ebcf
BUG: 1449941
RCA'ed-by: Miklós Fokin <miklos.fokin@appeartv.com>
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: https://review.gluster.org/17248
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: jiffin tony Thottan <jthottan@redhat.com>

1 files changed, 43 insertions, 25 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index c324c02c008..5ebe86bb368 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -3130,47 +3130,65 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int i = 0;
         int call_count = -1;
         int child_index = (long) cookie;
 	int read_subvol = 0;
 	call_stub_t *stub = NULL;
 
         local = frame->local;
-
-	read_subvol = afr_data_subvol_get (local->inode, this, NULL, NULL,
-                                           NULL, NULL);
+        priv = this->private;
 
         LOCK (&frame->lock);
         {
+                local->replies[child_index].valid = 1;
+                local->replies[child_index].op_ret = op_ret;
+                local->replies[child_index].op_errno = op_errno;
                 if (op_ret == 0) {
-                        if (local->op_ret == -1) {
-				local->op_ret = 0;
-
-                                local->cont.inode_wfop.prebuf  = *prebuf;
-                                local->cont.inode_wfop.postbuf = *postbuf;
-
-				if (xdata)
-					local->xdata_rsp = dict_ref (xdata);
-                        }
-
-                        if (child_index == read_subvol) {
-                                local->cont.inode_wfop.prebuf  = *prebuf;
-                                local->cont.inode_wfop.postbuf = *postbuf;
-				if (xdata) {
-					if (local->xdata_rsp)
-						dict_unref (local->xdata_rsp);
-					local->xdata_rsp = dict_ref (xdata);
-				}
-                        }
-                } else {
-			local->op_errno = op_errno;
-		}
+                        if (prebuf)
+                                local->replies[child_index].prestat = *prebuf;
+                        if (postbuf)
+                                local->replies[child_index].poststat = *postbuf;
+                        if (xdata)
+                                local->replies[child_index].xdata =
+                                        dict_ref (xdata);
+                }
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
+                local->op_ret = -1;
+                local->op_errno = afr_final_errno (local, priv);
+	        read_subvol = afr_data_subvol_get (local->inode, this, NULL,
+                                                   local->readable, NULL, NULL);
+                /* Pick a reply that is valid and readable, with a preference
+                 * given to read_subvol. */
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->replies[i].valid)
+                                continue;
+                        if (local->replies[i].op_ret != 0)
+                                continue;
+                        if (!local->readable[i])
+                                continue;
+                        local->op_ret = local->replies[i].op_ret;
+                        local->op_errno = local->replies[i].op_errno;
+                        local->cont.inode_wfop.prebuf =
+                                local->replies[i].prestat;
+                        local->cont.inode_wfop.postbuf =
+                                local->replies[i].poststat;
+                        if (local->replies[i].xdata) {
+                                if (local->xdata_rsp)
+                                        dict_unref (local->xdata_rsp);
+                                local->xdata_rsp =
+                                        dict_ref (local->replies[i].xdata);
+                        }
+                        if (i == read_subvol)
+                                break;
+                }
+
 		/* Make a stub out of the frame, and register it
 		   with the waking up post-op. When the call-stub resumes,
 		   we are guaranteed that there was no post-op pending