cluster/afr: AFR2 discovery should always do entry heal flow

Summary:
- Fixes case where when a brick is completely wiped, the AFR2 discovery
  mechanism would potentially (1/R chance where R is your replication
  factor) pin a NFSd or client to the wiped brick.  This would in turn
  prevent the client from seeing the contents of the (degraded)
  subvolume.
- The fix proposed in this patch is to force the entry-self
  heal code path when the discovery process happens.  And furthermore,
  forcing a conservative merge in the case where no brick is found to be
  degraded.
- This also restores the property of our 3.4.x builds where-by bricks
  automagically rebuild via the SHDs without having to run any sort of
  "full heal".  SHDs are given enough signal via this patch to figure
  out what they need to heal.

Test Plan:
Run "prove -v tests/bugs/fb8149516.t"
Output: https://phabricator.fb.com/P19989638

Prove test showing failed run on v3.6.3-fb_10 without the patch -> https://phabricator.fb.com/P19989643

Reviewers: dph, moox, sshreyas

Reviewed By: sshreyas

FB-commit-id: 3d6f171

Change-Id: I7e0dec82c160a2981837d3f07e3aa6f6a701703f
Signed-off-by: Kevin Vigor <kvigor@fb.com>
Reviewed-on: https://review.gluster.org/16862
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shreyas Siravara <sshreyas@fb.com>

1 files changed, 12 insertions, 6 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index ebadba99a05..cfea53208c8 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1085,7 +1085,8 @@ refresh_done:
 int
 afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *heal_frame = NULL;
+	afr_private_t *priv = NULL;
+    call_frame_t *heal_frame = NULL;
 	afr_local_t *local = NULL;
         gf_boolean_t start_heal = _gf_false;
         afr_local_t *heal_local = NULL;
@@ -1094,13 +1095,15 @@ afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
 	int err = 0;
 
 	local = frame->local;
+    priv = this->private;
 
 	ret = afr_replies_interpret (frame, this, local->refreshinode,
                                      &start_heal);
 
 	err = afr_inode_refresh_err (frame, this);
 
-	if (ret && afr_selfheal_enabled (this) && start_heal) {
+	if (priv->did_discovery == _gf_false ||
+        (afr_selfheal_enabled (this) && start_heal)) {
                 heal_frame = copy_frame (frame);
                 if (!heal_frame)
                         goto refresh_done;
@@ -2580,6 +2583,8 @@ unwind:
                         local->op_errno = ENOTCONN;
         }
 
+    priv->did_discovery = _gf_true;
+
 	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
 			  local->inode, &local->replies[read_subvol].poststat,
 			  local->replies[read_subvol].xdata,
@@ -2612,7 +2617,7 @@ afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			local->replies[child_index].xdata = dict_ref (xdata);
 	}
 
-        if (local->do_discovery && (op_ret == 0))
+        if (local->do_local_discovery && (op_ret == 0))
                 afr_attempt_local_discovery (this, child_index);
 
         if (xdata) {
@@ -2717,12 +2722,12 @@ afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req
 		if (!priv->root_inode)
 			priv->root_inode = inode_ref (loc->inode);
 
-		if (priv->choose_local && !priv->did_discovery) {
+		if (priv->choose_local && !priv->did_local_discovery) {
 			/* Logic to detect which subvolumes of AFR are
 			   local, in order to prefer them for reads
 			*/
-			local->do_discovery = _gf_true;
-                        priv->did_discovery = _gf_true;
+			local->do_local_discovery = _gf_true;
+                        priv->did_local_discovery = _gf_true;
                 }
 	}
 
@@ -4951,6 +4956,7 @@ afr_notify (xlator_t *this, int32_t event,
          * that we could end up issuing N lookups to the first subvolume, and
          * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
          */
+        priv->did_local_discovery = _gf_false;
         priv->did_discovery = _gf_false;
         latency_samples = child_xlator->client_latency.count;