afr: Add throttled background client-side heals

Backport of: http://review.gluster.org/13207

If a heal is needed after inode refresh (lookup, read_txn), launch it in
the background instead of blocking the fop (that triggered refresh)
until the heal happens.

afr_replies_interpret() is modified such that the heal is
launched only if atleast one sink brick is up.

Max. no of heals that can happen in parallel is configurable via the
'background-self-heal-count' volume option. Any number greater than that
is put in a wait queue whose length is configurable via
'heal-wait-queue-leng' volume option. If the wait queue is also full,
further heals will be ignored.

Default values:  background-self-heal-count=8, heal-wait-queue-leng=128

Change-Id: I9a134b2c29d66b70b7b1278811bd504963aabacc
BUG: 1313312
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/13564
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

1 files changed, 22 insertions, 6 deletions

diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 52f9c513a9e..9915344c1f5 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -74,8 +74,17 @@ typedef struct _afr_private {
         unsigned int data_self_heal_window_size;  /* max number of pipelined
                                                      read/writes */
 
-        unsigned int background_self_heal_count;
-        unsigned int background_self_heals_started;
+        struct list_head heal_waiting; /*queue for files that need heal*/
+        uint32_t  heal_wait_qlen; /*configurable queue length for heal_waiting*/
+        int32_t  heal_waiters; /* No. of elements currently in wait queue.*/
+
+        struct list_head healing;/* queue for files that are undergoing
+                                    background heal*/
+        uint32_t  background_self_heal_count;/*configurable queue length for
+                                               healing queue*/
+        int32_t  healers;/* No. of elements currently undergoing background
+                          heal*/
+
         gf_boolean_t metadata_self_heal;   /* on/off */
         gf_boolean_t entry_self_heal;      /* on/off */
 
@@ -127,12 +136,14 @@ typedef struct _afr_private {
 
 	afr_self_heald_t       shd;
 
-	/* pump dependencies */
-	void                   *pump_private;
-	gf_boolean_t           use_afr_in_pump;
         gf_boolean_t           consistent_metadata;
         uint64_t               spb_choice_timeout;
         gf_boolean_t           need_heal;
+
+	/* pump dependencies */
+	void                   *pump_private;
+	gf_boolean_t           use_afr_in_pump;
+
 } afr_private_t;
 
 
@@ -740,6 +751,10 @@ typedef struct _afr_local {
         int             xflag;
         gf_boolean_t    do_discovery;
 	struct afr_reply *replies;
+
+        /* For  client side background heals. */
+        struct list_head healer;
+        call_frame_t *heal_frame;
 } afr_local_t;
 
 
@@ -891,7 +906,8 @@ int
 afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
 
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       gf_boolean_t *start_heal);
 
 void
 afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv);