afr: Add throttled background client-side heals

If a heal is needed after inode refresh (lookup, read_txn), launch it in
the background instead of blocking the fop (that triggered refresh) until the
heal happens.

afr_replies_interpret() is modified such that the heal is
launched only if atleast one sink brick is up.

Max. no of heals that can happen in parallel is configurable via the
'background-self-heal-count' volume option. Any number greater than that
is put in a wait queue whose length is configurable via
'heal-wait-queue-leng' volume option. If the wait queue is also full,
further heals will be ignored.

Default values:  background-self-heal-count=8, heal-wait-queue-leng=128

Change-Id: I1d4a52814cdfd43d90591b6d2ad7b6219937ce70
BUG: 1297172
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/13207
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Tested-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

1 files changed, 110 insertions, 0 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 49c6bd0cc98..74e1a444069 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -15,6 +15,9 @@
 #include "protocol-common.h"
 #include "afr-messages.h"
 
+void
+afr_heal_synctask (xlator_t *this, afr_local_t *local);
+
 int
 afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
@@ -1422,3 +1425,110 @@ afr_selfheal (xlator_t *this, uuid_t gfid)
 
 	return ret;
 }
+
+afr_local_t*
+__afr_dequeue_heals (afr_private_t *priv)
+{
+        afr_local_t *local = NULL;
+
+        if (list_empty (&priv->heal_waiting))
+                goto none;
+        if ((priv->background_self_heal_count > 0) &&
+            (priv->healers >= priv->background_self_heal_count))
+                goto none;
+
+        local = list_entry (priv->heal_waiting.next, afr_local_t, healer);
+        priv->heal_waiters--;
+        GF_ASSERT (priv->heal_waiters >= 0);
+        list_del_init(&local->healer);
+        list_add(&local->healer, &priv->healing);
+        priv->healers++;
+        return local;
+none:
+        gf_msg_debug (THIS->name, 0, "Nothing dequeued. "
+                      "Num healers: %d, Num Waiters: %d",
+                      priv->healers, priv->heal_waiters);
+        return NULL;
+}
+
+int
+afr_refresh_selfheal_wrap (void *opaque)
+{
+        call_frame_t *heal_frame = opaque;
+        afr_local_t *local = heal_frame->local;
+        int ret = 0;
+
+        ret = afr_selfheal (heal_frame->this, local->refreshinode->gfid);
+        return ret;
+}
+
+int
+afr_refresh_heal_done (int ret, call_frame_t *frame, void *opaque)
+{
+        call_frame_t *heal_frame = opaque;
+        xlator_t *this = heal_frame->this;
+        afr_private_t *priv = this->private;
+        afr_local_t *local = heal_frame->local;
+
+        LOCK (&priv->lock);
+        {
+                list_del_init(&local->healer);
+                priv->healers--;
+                GF_ASSERT (priv->healers >= 0);
+                local = __afr_dequeue_heals (priv);
+        }
+        UNLOCK (&priv->lock);
+
+        if (heal_frame)
+                AFR_STACK_DESTROY (heal_frame);
+
+        if (local)
+                afr_heal_synctask (this, local);
+        return 0;
+
+}
+
+void
+afr_heal_synctask (xlator_t *this, afr_local_t *local)
+{
+        int ret = 0;
+        call_frame_t *heal_frame = NULL;
+
+        heal_frame = local->heal_frame;
+        ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
+                            afr_refresh_heal_done, heal_frame, heal_frame);
+        if (ret < 0)
+                /* Heal not launched. Will be queued when the next inode
+                 * refresh happens and shd hasn't healed it yet. */
+                afr_refresh_heal_done (ret, heal_frame, heal_frame);
+}
+
+void
+afr_throttled_selfheal (call_frame_t *frame, xlator_t *this)
+{
+        gf_boolean_t can_heal = _gf_true;
+        afr_private_t *priv = this->private;
+        afr_local_t *local = frame->local;
+
+        LOCK (&priv->lock);
+        {
+                if ((priv->background_self_heal_count > 0) &&
+                    (priv->heal_wait_qlen + priv->background_self_heal_count) >
+                    (priv->heal_waiters + priv->healers)) {
+                        list_add_tail(&local->healer, &priv->heal_waiting);
+                        priv->heal_waiters++;
+                        local = __afr_dequeue_heals (priv);
+                } else {
+                        can_heal = _gf_false;
+                }
+        }
+        UNLOCK (&priv->lock);
+
+        if (can_heal) {
+                if (local)
+                        afr_heal_synctask (this, local);
+                else
+                        gf_msg_debug (this->name, 0, "Max number of heals are "
+                                      "pending, background self-heal rejected.");
+        }
+}