cluster/ta: Notify the clients only if there are pending heals

Problem:
In case of thin arbiter, before index healer starts crawling the
indices at every heal-timeout interval, even if there is nothing to
be healed it will send an upcall notification to all the clients to
release any AFR_TA_DOM_NOTIFY locks that they hold. SHD will wait
for the upcall to return before proceeding with the heal even though
there is nothing to be healed. This will also invalidates the cached
information about the bricks states on the clients which leads to
extra calls on TA from clients for the next reads & writes if needed.
This will impact the IO performance.

Fix:
- Before sending the upcall to the clients, check for any pending heals
on TA without taking  any locks.
- If there is nothing marked bad on TA, then continue with the index
crawl to heal any dirty markings present on the files due to any post-op
failure.
- If there is a brick marked as bad on TA, then take the
AFR_TA_DOM_NOTIFY lock on TA from SHD, get the state on TA and
continue with the current healing process.

Change-Id: Ieb477bc6cb18bbdfd4e7a0453c5ed79b574ec9d6
fixes: bz#1724184
Signed-off-by: karthik-us <ksubrahm@redhat.com>

4 files changed, 69 insertions, 22 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 6863bd02c50..bce0af5791a 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -7121,3 +7121,24 @@ afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
         local->op_ret = -1;
     }
 }
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
+{
+    int *pending = NULL;
+    int ret = 0;
+    int i = 0;
+
+    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
+    if (ret == 0) {
+        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+            /* Not doing a ntoh32(pending) as we just want to check
+             * if it is non-zero or not. */
+            if (pending[i]) {
+                return _gf_true;
+            }
+        }
+    }
+
+    return _gf_false;
+}
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
index 7e258049005..772b59f9a2f 100644
--- a/xlators/cluster/afr/src/afr-read-txn.c
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -30,27 +30,6 @@ afr_pending_read_decrement(afr_private_t *priv, int child_index)
     GF_ATOMIC_DEC(priv->pending_reads[child_index]);
 }
 
-static gf_boolean_t
-afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
-{
-    int *pending = NULL;
-    int ret = 0;
-    int i = 0;
-
-    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
-    if (ret == 0) {
-        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
-            /* Not doing a ntoh32(pending) as we just want to check
-             * if it is non-zero or not. */
-            if (pending[i]) {
-                return _gf_true;
-            }
-        }
-    }
-
-    return _gf_false;
-}
-
 void
 afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index e476ac62ec4..9c94835714f 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -801,6 +801,48 @@ afr_bricks_available_for_heal(afr_private_t *priv)
     return _gf_true;
 }
 
+static gf_boolean_t
+afr_shd_ta_needs_heal(xlator_t *this, struct subvol_healer *healer)
+{
+    dict_t *xdata = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+    gf_boolean_t need_heal = _gf_false;
+
+    priv = this->private;
+
+    ret = afr_shd_fill_ta_loc(this, &loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        healer->rerun = 1;
+        goto out;
+    }
+
+    if (_afr_shd_ta_get_xattrs(this, &loc, &xdata)) {
+        healer->rerun = 1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (afr_ta_dict_contains_pending_xattr(xdata, priv, i)) {
+            need_heal = _gf_true;
+            break;
+        }
+    }
+
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+
+    return need_heal;
+}
+
 void *
 afr_shd_index_healer(void *data)
 {
@@ -827,7 +869,8 @@ afr_shd_index_healer(void *data)
         priv->local[healer->subvol] = healer->local;
 
         if (priv->thin_arbiter_count) {
-            afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
+            if (afr_shd_ta_needs_heal(this, healer))
+                afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
         }
 
         do {
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index db83b395e02..c066099d5d3 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1347,4 +1347,8 @@ afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this);
 
 void
 afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv,
+                                   int child);
 #endif /* __AFR_H__ */