tests: Fix races in open-behind.t

Problems:
1) flush-behind is on by default, so just because write completes doesn't mean
   it will be on the disk, it could still be in write-behind's cache. This
   leads to failure where if you write from one mount and expect it to be there
   on the other mount, sometimes it won't be there.
2) Sometimes the graph switch is not completing by the time we issue read which
   is leading to opens not being sent on brick leading to failures.

Fixes:
1) Disable flush-behind
2) Add new functions to check the new graph is there and connected to bricks
   before 'cat' is executed.

BUG: 1379511
Change-Id: I0faed684e0dc70cfd2258ce6fdaed655ee915ae6
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/15575
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

3 files changed, 18 insertions, 3 deletions

diff --git a/tests/include.rc b/tests/include.rc
index b015fac1197..74c279cb8e1 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -67,6 +67,7 @@ esac
 
 DEBUG=${DEBUG:=0}             # turn on debugging?
 
+PROCESS_DOWN_TIMEOUT=5
 PROCESS_UP_TIMEOUT=20
 NFS_EXPORT_TIMEOUT=20
 CHILD_UP_TIMEOUT=20
diff --git a/tests/performance/open-behind.t b/tests/performance/open-behind.t
index 9ef10ad5a09..002a98a8352 100755
--- a/tests/performance/open-behind.t
+++ b/tests/performance/open-behind.t
@@ -10,6 +10,7 @@ TEST pidof glusterd
 TEST $CLI volume info 2>/dev/null;
 
 TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+TEST $CLI volume set $V0 performance.flush-behind off;
 
 TEST $CLI volume start $V0;
 
@@ -40,8 +41,11 @@ EXPECT "$D0" cat $M1/$F0;
 TEST $CLI volume stop $V0;
 sleep 1;
 TEST $CLI volume start $V0;
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-1
 
-sleep 2;
 cat $M1/$F0 >/dev/null;
 
 string=$(gluster volume top $V0 open | grep -w "$F0");
@@ -51,6 +55,11 @@ EXPECT "" echo $string;
 TEST $CLI volume set $V0 performance.open-behind off;
 
 EXPECT_WITHIN $GRAPH_SWITCH_TIMEOUT "2" num_graphs $M0;
+EXPECT_WITHIN $GRAPH_SWITCH_TIMEOUT "2" num_graphs $M1;
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-1
 
 D1="hello-this-is-a-test-message1";
 F1="test-file1";
@@ -60,8 +69,7 @@ EXPECT "$D1" cat $M1/$F1;
 
 EXPECT "$D0" cat $M1/$F0;
 
-gluster volume top $V0 open | grep -w "$F0" >/dev/null 2>&1
+$CLI volume top $V0 open | grep -w "$F0" >/dev/null 2>&1
 TEST [ $? -eq 0 ];
 
 cleanup;
-#G_TESTDEF_TEST_STATUS_NETBSD7=BAD_TEST,BUG=1300253
diff --git a/tests/volume.rc b/tests/volume.rc
index aa614c50489..f9725b2cba1 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -150,6 +150,12 @@ function afr_child_up_status_meta {
         grep "child_up\[$child\]" $mnt/.meta/graphs/active/$repl/private | awk '{print $3}'
 }
 
+function client_connected_status_meta {
+        local mnt=$1
+        local client=$2
+        grep "connected" $mnt/.meta/graphs/active/$client/private | awk '{print $3}'
+}
+
 function afr_child_up_status {
         local vol=$1
         #brick_id is (brick-num in volume info - 1)