From cd072b61841c19ec942871e3f06519d2a938814b Mon Sep 17 00:00:00 2001 From: Pranith Kumar K Date: Tue, 27 Sep 2016 07:51:48 +0530 Subject: tests: Fix races in open-behind.t Problems: 1) flush-behind is on by default, so just because write completes doesn't mean it will be on the disk, it could still be in write-behind's cache. This leads to failure where if you write from one mount and expect it to be there on the other mount, sometimes it won't be there. 2) Sometimes the graph switch is not completing by the time we issue read which is leading to opens not being sent on brick leading to failures. Fixes: 1) Disable flush-behind 2) Add new functions to check the new graph is there and connected to bricks before 'cat' is executed. BUG: 1379511 Change-Id: I0faed684e0dc70cfd2258ce6fdaed655ee915ae6 Signed-off-by: Pranith Kumar K Reviewed-on: http://review.gluster.org/15575 Smoke: Gluster Build System Reviewed-by: Raghavendra G NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System --- tests/include.rc | 1 + tests/performance/open-behind.t | 14 +++++++++++--- tests/volume.rc | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/include.rc b/tests/include.rc index b015fac1197..74c279cb8e1 100644 --- a/tests/include.rc +++ b/tests/include.rc @@ -67,6 +67,7 @@ esac DEBUG=${DEBUG:=0} # turn on debugging? +PROCESS_DOWN_TIMEOUT=5 PROCESS_UP_TIMEOUT=20 NFS_EXPORT_TIMEOUT=20 CHILD_UP_TIMEOUT=20 diff --git a/tests/performance/open-behind.t b/tests/performance/open-behind.t index 9ef10ad5a09..002a98a8352 100755 --- a/tests/performance/open-behind.t +++ b/tests/performance/open-behind.t @@ -10,6 +10,7 @@ TEST pidof glusterd TEST $CLI volume info 2>/dev/null; TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2}; +TEST $CLI volume set $V0 performance.flush-behind off; TEST $CLI volume start $V0; @@ -40,8 +41,11 @@ EXPECT "$D0" cat $M1/$F0; TEST $CLI volume stop $V0; sleep 1; TEST $CLI volume start $V0; +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-1 -sleep 2; cat $M1/$F0 >/dev/null; string=$(gluster volume top $V0 open | grep -w "$F0"); @@ -51,6 +55,11 @@ EXPECT "" echo $string; TEST $CLI volume set $V0 performance.open-behind off; EXPECT_WITHIN $GRAPH_SWITCH_TIMEOUT "2" num_graphs $M0; +EXPECT_WITHIN $GRAPH_SWITCH_TIMEOUT "2" num_graphs $M1; +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M0 ${V0}-client-1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "^1$" client_connected_status_meta $M1 ${V0}-client-1 D1="hello-this-is-a-test-message1"; F1="test-file1"; @@ -60,8 +69,7 @@ EXPECT "$D1" cat $M1/$F1; EXPECT "$D0" cat $M1/$F0; -gluster volume top $V0 open | grep -w "$F0" >/dev/null 2>&1 +$CLI volume top $V0 open | grep -w "$F0" >/dev/null 2>&1 TEST [ $? -eq 0 ]; cleanup; -#G_TESTDEF_TEST_STATUS_NETBSD7=BAD_TEST,BUG=1300253 diff --git a/tests/volume.rc b/tests/volume.rc index aa614c50489..f9725b2cba1 100644 --- a/tests/volume.rc +++ b/tests/volume.rc @@ -150,6 +150,12 @@ function afr_child_up_status_meta { grep "child_up\[$child\]" $mnt/.meta/graphs/active/$repl/private | awk '{print $3}' } +function client_connected_status_meta { + local mnt=$1 + local client=$2 + grep "connected" $mnt/.meta/graphs/active/$client/private | awk '{print $3}' +} + function afr_child_up_status { local vol=$1 #brick_id is (brick-num in volume info - 1) -- cgit