glusterd/rebalance: initialize defrag variable after glusterd restart

During reblance restart after glusterd restarted, we are not
connecting to rebalance process from glusterd, because the
defrag variable in volinfo will be null.

Initializing the variable will connect the rpc

Change-Id: Id820cad6a3634a9fc976427fbe1c45844d3d4b9b
BUG: 1303028
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/13319
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>

6 files changed, 124 insertions, 3 deletions

diff --git a/tests/basic/tier/tier.t b/tests/basic/tier/tier.t
index c7c95690e70..cfd8e775f7f 100755
--- a/tests/basic/tier/tier.t
+++ b/tests/basic/tier/tier.t
@@ -13,6 +13,10 @@ MIGRATION_TIMEOUT=10
 DEMOTE_FREQ=4
 PROMOTE_FREQ=12
 
+function detach_start {
+        $CLI volume tier $1 detach start
+        echo $?;
+}
 
 function file_on_slow_tier {
     found=0
@@ -191,8 +195,7 @@ EXPECT "0" file_on_slow_tier d1/data.txt $md5data
 EXPECT "0" file_on_slow_tier d1/data2.txt $md5data2
 EXPECT "0" file_on_slow_tier "./d1/$SPACE_FILE" $md5space
 
-TEST $CLI volume tier $V0 detach start
-
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" detach_start $V0
 EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field "$V0 $H0:$B0/${V0}${CACHE_BRICK_FIRST}"
 
 TEST $CLI volume tier $V0 detach commit
diff --git a/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
new file mode 100644
index 00000000000..a2a10ee5ef6
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../tier.rc
+
+
+# Creates a tiered volume with pure distribute hot and cold tiers
+# Both hot and cold tiers will have an equal number of bricks.
+
+function create_dist_tier_vol () {
+        mkdir $B0/cold
+        mkdir $B0/hot
+        TEST $CLI volume create $V0 $H0:$B0/cold/${V0}{1..3}
+        TEST $CLI volume set $V0 performance.quick-read off
+        TEST $CLI volume set $V0 performance.io-cache off
+        TEST $CLI volume start $V0
+        TEST $CLI volume attach-tier $V0 $H0:$B0/hot/${V0}{1..2}
+        TEST $CLI volume set $V0 cluster.tier-mode test
+}
+
+function non_zero_check () {
+if [ "$1" -ne 0 ]
+then
+        echo "0"
+else
+        echo "1"
+fi
+}
+
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume status
+
+
+#Create and start a tiered volume
+create_dist_tier_vol
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_check
+sleep 2   #wait for some time to run tier daemon
+time_before_restarting=$(rebalance_run_time $V0);
+
+#checking for elapsed time after sleeping for two seconds.
+EXPECT "0" non_zero_check $time_before_restarting;
+
+#Difference of elapsed time should be positive
+
+kill -9 $(pidof glusterd);
+TEST glusterd;
+sleep 2;
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check;
+time1=$(rebalance_run_time $V0);
+EXPECT "0" non_zero_check $time1;
+sleep 4;
+time2=$(rebalance_run_time $V0);
+EXPECT "0" non_zero_check $time2;
+diff=`expr $time2 - $time1`
+EXPECT "0" non_zero_check $diff;
diff --git a/tests/tier.rc b/tests/tier.rc
index 32ee00cf4fe..19234c521a0 100644
--- a/tests/tier.rc
+++ b/tests/tier.rc
@@ -113,3 +113,14 @@ function sleep_until_mid_cycle {
     mod=$(( $mod + $1 / 2 ))
     sleep $mod
 }
+
+function tier_deamon_check () {
+    pgrep -f "rebalance/$V0"
+    echo "$?"
+}
+
+function rebalance_run_time () {
+    local time=$($CLI volume rebalance $1  status | awk '{print $9}' | sed -n 3p);
+    time=$(printf "%.0f\n" "$time");
+    echo $time
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 5faa6b4a1b3..815b0aa6ae6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -326,6 +326,40 @@ out:
         return ret;
 }
 
+int
+glusterd_rebalance_defrag_init (glusterd_volinfo_t *volinfo,
+                                defrag_cbk_fn_t cbk)
+
+{
+        glusterd_defrag_info_t  *defrag         = NULL;
+        int                      ret            = -1;
+
+        if (!volinfo->rebal.defrag) {
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
+        } else {
+                /*
+                 * if defrag variable is already initialized,
+                 * we skip the initialization.
+                 */
+                ret = 0;
+                goto out;
+        }
+
+        if (!volinfo->rebal.defrag)
+                goto out;
+        defrag = volinfo->rebal.defrag;
+
+        defrag->cmd = volinfo->rebal.defrag_cmd;
+        LOCK_INIT (&defrag->lock);
+        if (cbk)
+                defrag->cbk_fn = cbk;
+        ret = 0;
+out:
+        return ret;
+
+}
 
 int
 glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 49b88ed4b13..f1b1a3c0c59 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7136,6 +7136,15 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
         case GF_DEFRAG_STATUS_STARTED:
                 GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
                 if (gf_is_service_running (pidfile, &pid)) {
+                        ret = glusterd_rebalance_defrag_init (volinfo, cbk);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_REBALANCE_START_FAIL,
+                                        "Failed to initialize  defrag."
+                                        "Not starting rebalance process for "
+                                        "%s.", volinfo->volname);
+                                goto out;
+                        }
                         ret = glusterd_rebalance_rpc_create (volinfo, _gf_true);
                         break;
                 }
@@ -7151,7 +7160,7 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
                         volinfo->rebal.defrag_status, volinfo->volname);
                 break;
         }
-
+out:
         return ret;
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 0a313918856..a710a08954a 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -999,6 +999,9 @@ int
 glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
                                gf_boolean_t reconnect);
 
+int glusterd_rebalance_defrag_init (glusterd_volinfo_t *volinfo,
+                                    defrag_cbk_fn_t cbk);
+
 int glusterd_handle_cli_heal_volume (rpcsvc_request_t *req);
 
 int glusterd_handle_cli_list_volume (rpcsvc_request_t *req);