glusterd: Fix volume restart issue upon glusterd restart

http://review.gluster.org/#/c/14758/ introduces a check in
glusterd_restart_bricks that makes sure that if server quorum is
enabled and if the glusterd instance has been restarted, the bricks
do not get started. This prevents bricks which have been brought
down purposely, say for maintainence, from getting started
upon a glusterd restart. However this change introduced regression
for a situation that involves multiple volumes. The bricks from
the first volume get started, but then for the subsequent volumes
the bricks do not get started. This patch fixes that by setting
the value of conf->restart_done to _gf_true only after bricks are
started correctly for all volumes.

> Reviewed-on: http://review.gluster.org/15183
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

(cherry picked from commit dd8d93f24a320805f1f67760b2d3266555acf674)

Change-Id: I2c685b43207df2a583ca890ec54dcccf109d22c3
BUG: 1366813
Signed-off-by: Samikshan Bairagya <samikshan@gmail.com>
Reviewed-on: http://review.gluster.org/15186
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

3 files changed, 42 insertions, 2 deletions

diff --git a/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t b/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t
index e5951e0a039..7f2f3cc66ca 100644
--- a/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t
+++ b/tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Test case for quorum validation in glusterd for syncop framework
+# Test case to check if bricks are down when quorum is not met
 
 . $(dirname $0)/../../include.rc
 . $(dirname $0)/../../volume.rc
diff --git a/tests/bugs/glusterd/bug-1367478-volume-start-validation-after-glusterd-restart.t b/tests/bugs/glusterd/bug-1367478-volume-start-validation-after-glusterd-restart.t
new file mode 100644
index 00000000000..4329c66474f
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1367478-volume-start-validation-after-glusterd-restart.t
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Test case to check for successful startup of volume bricks on glusterd restart
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+cleanup;
+
+TEST launch_cluster 2
+
+TEST $CLI_1 peer probe $H2;
+EXPECT_WITHIN $PROBE_TIMEOUT 1 peer_count
+
+# Lets create the volume and set quorum type as a server
+TEST $CLI_1 volume create $V0 replica 2 $H1:$B1/${V0}1 $H2:$B2/${V0}2
+TEST $CLI_1 volume create $V1 replica 2 $H1:$B1/${V1}1 $H2:$B2/${V1}2
+
+# Start the volume
+TEST $CLI_1 volume start $V0
+TEST $CLI_1 volume start $V1
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V0 $H1 $B1/${V0}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V0 $H2 $B2/${V0}2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V1 $H1 $B1/${V1}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V1 $H2 $B2/${V1}2
+
+# Restart 2nd glusterd
+TEST kill_glusterd 2
+TEST $glusterd_2
+
+# Check if all bricks are up
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V0 $H1 $B1/${V0}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V0 $H2 $B2/${V0}2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V1 $H1 $B1/${V1}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status_1 $V1 $H2 $B2/${V1}2
+
+cleanup;
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index e11585df927..58a316caaa6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -4943,7 +4943,6 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
                                 glusterd_brick_start (volinfo, brickinfo,
                                                      _gf_false);
                         }
-                        conf->restart_done = _gf_true;
                 }
         }
 
@@ -4979,6 +4978,7 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
         }
 
 out:
+        conf->restart_done = _gf_true;
         return ret;
 }