tests: arbiter.t fix

Backport of http://review.gluster.org/10833

Wait for AFR's children to be up in glustershd process before attempting heal.

Also, grep (version 2.21) is detecting statedump files as binary, causing tests
to succeed incorrectly. Hence adding the -a switch to force it to treat it as a
text file. Thanks to Vijay Bellur for identifying the issue
(http://lists.gnu.org/archive/html/bug-grep/2015-05/msg00000.html) and the
workaround.

Change-Id: Ie3d9591ffaf44baa0cd8c2baa327aed24378e3df
BUG: 1225077
BUG:
Reviewed-on: http://review.gluster.org/10833
Tested-by: Niels de Vos <ndevos@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
(cherry picked from commit b51ee5f8d1f80d66effffc06c1e49099c04014a4)
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/10923
Tested-by: NetBSD Build System
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 9 insertions, 4 deletions

diff --git a/run-tests.sh b/run-tests.sh
index 4b026490c4f..2d1191ae0d7 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -199,7 +199,6 @@ function is_bad_test ()
               ./tests/bugs/replicate/bug-976800.t       \
               ./tests/bugs/replicate/bug-1015990.t      \
               ./tests/bugs/quota/bug-1038598.t  \
-              ./tests/basic/afr/arbiter.t       \
               ./tests/basic/afr/read-subvol-entry.t     \
               ./tests/basic/ec/quota.t  \
               ./tests/bugs/distribute/bug-1161156.t     \
diff --git a/tests/basic/afr/arbiter.t b/tests/basic/afr/arbiter.t
index a9d485cd7b4..8a983fb0577 100644
--- a/tests/basic/afr/arbiter.t
+++ b/tests/basic/afr/arbiter.t
@@ -37,7 +37,8 @@ TEST setfattr -n user.name -v value2  $M0/file
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
 TEST kill_brick $V0 $H0 $B0/${V0}1
-TEST `echo "B2 is down, B3 is the only source, writes will fail" >> $M0/file`
+echo "B2 is down, B3 is the only source, writes will fail" >> $M0/file
+EXPECT_NOT "0" echo $?
 TEST ! cat $M0/file
 # Metadata I/O should still succeed.
 TEST getfattr -n user.name $M0/file
@@ -46,12 +47,17 @@ TEST setfattr -n user.name -v value3 $M0/file
 #shd should not data self-heal from arbiter to the sinks.
 TEST $CLI volume set $V0 cluster.self-heal-daemon on
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2
 TEST $CLI volume heal $V0
 EXPECT_WITHIN $HEAL_TIMEOUT '1' echo $(count_sh_entries $B0/$V0"1")
 EXPECT_WITHIN $HEAL_TIMEOUT '1' echo $(count_sh_entries $B0/$V0"2")
 
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
+EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
 TEST $CLI volume heal $V0
 EXPECT 0 afr_get_pending_heal_count $V0
diff --git a/tests/volume.rc b/tests/volume.rc
index b71fea4d503..bc34b608891 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -87,7 +87,7 @@ function cleanup_mount_statedump {
 function snap_client_connected_status {
          local vol=$1
          local fpath=$(generate_mount_statedump $vol)
-         up=$(grep -A2 xlator.protocol.client.$vol-snapd-client.priv $fpath | tail -1 | cut -f 2 -d'=')
+         up=$(grep -a -A2 xlator.protocol.client.$vol-snapd-client.priv $fpath | tail -1 | cut -f 2 -d'=')
          rm -f $fpath
          echo "$up"
 }
@@ -98,7 +98,7 @@ function _afr_child_up_status {
         local brick_id=$2
         local gen_state_dump=$3
         local fpath=$($gen_state_dump $vol)
-        up=$(grep -B1 trusted.afr.$vol-client-$brick_id $fpath | head -1 | cut -f2 -d'=')
+        up=$(grep -a -B1 trusted.afr.$vol-client-$brick_id $fpath | head -1 | cut -f2 -d'=')
         rm -f $fpath
         echo "$up"
 }