glusterd/rebalance: initialize defrag variable after glusterd restart

During reblance restart after glusterd restarted, we are not
connecting to rebalance process from glusterd, because the
defrag variable in volinfo will be null.

Initializing the variable will connect the rpc

Back port of>
>Change-Id: Id820cad6a3634a9fc976427fbe1c45844d3d4b9b
>BUG: 1303028
>Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
>Reviewed-on: http://review.gluster.org/13319
>Smoke: Gluster Build System <jenkins@build.gluster.com>
>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>Reviewed-by: Dan Lambright <dlambrig@redhat.com>
>CentOS-regression: Gluster Build System <jenkins@build.gluster.com>

(cherry picked from commit a67331f3f79e827ffa4f7a547f6898e12407bbf9)

Change-Id: Ieec82a798da937002e09fb9325c93678a5eefca8
BUG: 1311041
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/13494
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

6 files changed, 124 insertions, 3 deletions

diff --git a/tests/basic/tier/tier.t b/tests/basic/tier/tier.t
index c7c95690e70..cfd8e775f7f 100755
--- a/tests/basic/tier/tier.t
+++ b/tests/basic/tier/tier.t
@@ -13,6 +13,10 @@ MIGRATION_TIMEOUT=10
 DEMOTE_FREQ=4
 PROMOTE_FREQ=12
 
+function detach_start {
+        $CLI volume tier $1 detach start
+        echo $?;
+}
 
 function file_on_slow_tier {
     found=0
@@ -191,8 +195,7 @@ EXPECT "0" file_on_slow_tier d1/data.txt $md5data
 EXPECT "0" file_on_slow_tier d1/data2.txt $md5data2
 EXPECT "0" file_on_slow_tier "./d1/$SPACE_FILE" $md5space
 
-TEST $CLI volume tier $V0 detach start
-
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" detach_start $V0
 EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field "$V0 $H0:$B0/${V0}${CACHE_BRICK_FIRST}"
 
 TEST $CLI volume tier $V0 detach commit
diff --git a/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
new file mode 100644
index 00000000000..a2a10ee5ef6
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../tier.rc
+
+
+# Creates a tiered volume with pure distribute hot and cold tiers
+# Both hot and cold tiers will have an equal number of bricks.
+
+function create_dist_tier_vol () {
+        mkdir $B0/cold
+        mkdir $B0/hot
+        TEST $CLI volume create $V0 $H0:$B0/cold/${V0}{1..3}
+        TEST $CLI volume set $V0 performance.quick-read off
+        TEST $CLI volume set $V0 performance.io-cache off
+        TEST $CLI volume start $V0
+        TEST $CLI volume attach-tier $V0 $H0:$B0/hot/${V0}{1..2}
+        TEST $CLI volume set $V0 cluster.tier-mode test
+}
+
+function non_zero_check () {
+if [ "$1" -ne 0 ]
+then
+        echo "0"
+else
+        echo "1"
+fi
+}
+
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume status
+
+
+#Create and start a tiered volume
+create_dist_tier_vol
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_check
+sleep 2   #wait for some time to run tier daemon
+time_before_restarting=$(rebalance_run_time $V0);
+
+#checking for elapsed time after sleeping for two seconds.
+EXPECT "0" non_zero_check $time_before_restarting;
+
+#Difference of elapsed time should be positive
+
+kill -9 $(pidof glusterd);
+TEST glusterd;
+sleep 2;
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check;
+time1=$(rebalance_run_time $V0);
+EXPECT "0" non_zero_check $time1;
+sleep 4;
+time2=$(rebalance_run_time $V0);
+EXPECT "0" non_zero_check $time2;
+diff=`expr $time2 - $time1`
+EXPECT "0" non_zero_check $diff;
diff --git a/tests/tier.rc b/tests/tier.rc
index 32ee00cf4fe..19234c521a0 100644
--- a/tests/tier.rc
+++ b/tests/tier.rc
@@ -113,3 +113,14 @@ function sleep_until_mid_cycle {
     mod=$(( $mod + $1 / 2 ))
     sleep $mod
 }
+
+function tier_deamon_check () {
+    pgrep -f "rebalance/$V0"
+    echo "$?"
+}
+
+function rebalance_run_time () {
+    local time=$($CLI volume rebalance $1  status | awk '{print $9}' | sed -n 3p);
+    time=$(printf "%.0f\n" "$time");
+    echo $time
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 2ceab6d6d24..356cc909e17 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -331,6 +331,40 @@ out:
         return ret;
 }
 
+int
+glusterd_rebalance_defrag_init (glusterd_volinfo_t *volinfo,
+                                defrag_cbk_fn_t cbk)
+
+{
+        glusterd_defrag_info_t  *defrag         = NULL;
+        int                      ret            = -1;
+
+        if (!volinfo->rebal.defrag) {
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
+        } else {
+                /*
+                 * if defrag variable is already initialized,
+                 * we skip the initialization.
+                 */
+                ret = 0;
+                goto out;
+        }
+
+        if (!volinfo->rebal.defrag)
+                goto out;
+        defrag = volinfo->rebal.defrag;
+
+        defrag->cmd = volinfo->rebal.defrag_cmd;
+        LOCK_INIT (&defrag->lock);
+        if (cbk)
+                defrag->cbk_fn = cbk;
+        ret = 0;
+out:
+        return ret;
+
+}
 
 int
 glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 11fe0481b77..a4d6ec592d9 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7133,6 +7133,15 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
         case GF_DEFRAG_STATUS_STARTED:
                 GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
                 if (gf_is_service_running (pidfile, &pid)) {
+                        ret = glusterd_rebalance_defrag_init (volinfo, cbk);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_REBALANCE_START_FAIL,
+                                        "Failed to initialize  defrag."
+                                        "Not starting rebalance process for "
+                                        "%s.", volinfo->volname);
+                                goto out;
+                        }
                         ret = glusterd_rebalance_rpc_create (volinfo, _gf_true);
                         break;
                 }
@@ -7148,7 +7157,7 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
                         volinfo->rebal.defrag_status, volinfo->volname);
                 break;
         }
-
+out:
         return ret;
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 3b7c82faab2..c872cd2339c 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1004,6 +1004,9 @@ int
 glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
                                gf_boolean_t reconnect);
 
+int glusterd_rebalance_defrag_init (glusterd_volinfo_t *volinfo,
+                                    defrag_cbk_fn_t cbk);
+
 int glusterd_handle_cli_heal_volume (rpcsvc_request_t *req);
 
 int glusterd_handle_cli_list_volume (rpcsvc_request_t *req);