tests: fix afr-lock-heal-* failure

When brick-mux is enabled:

i)brick statedumps seem to be listing the same lock information multiple times.
While that is getting fixed, make changes to the .ts to check for unique values.

ii)detecting a brick as online via brick_up_status() seems to be taking
longer time when delaygen is enabled. Hence bump up PROCESS_UP_TIMEOUT to
90 for afr-lock-heal-advanced.t

Updates: #1042
Change-Id: Ife76008f7a99dd1f1fe5791a32577366baaab4b3
Signed-off-by: Ravishankar N <ravishankar@redhat.com>

2 files changed, 26 insertions, 12 deletions

diff --git a/tests/basic/fencing/afr-lock-heal-advanced.t b/tests/basic/fencing/afr-lock-heal-advanced.t
index 8a7a208db29..8a5b5989b5e 100644
--- a/tests/basic/fencing/afr-lock-heal-advanced.t
+++ b/tests/basic/fencing/afr-lock-heal-advanced.t
@@ -4,6 +4,7 @@
 . $(dirname $0)/../../volume.rc
 
 cleanup;
+PROCESS_UP_TIMEOUT=90
 
 function is_gfapi_program_alive()
 {
@@ -19,9 +20,14 @@ function is_gfapi_program_alive()
 
 function get_active_lock_count {
     brick=$1
+    i1=$2
+    i2=$3
+    pattern="ACTIVE.*client-${brick: -1}"
+
     sdump=$(generate_brick_statedump $V0 $H0 $brick)
-    lock_count="$(grep ACTIVE $sdump| wc -l)"
-    echo "$lock_count"
+    lock_count1="$(egrep "$i1" $sdump -A3| egrep "$pattern"|uniq|wc -l)"
+    lock_count2="$(egrep "$i2" $sdump -A3| egrep "$pattern"|uniq|wc -l)"
+    echo "$((lock_count1+lock_count2))"
 }
 
 TEST glusterd
@@ -49,6 +55,11 @@ TEST [ $client_pid ]
 TEST sleep 5 # By now, the client would  have opened an fd on FILE1 and FILE2 and waiting for a SIGUSR1.
 EXPECT "Y" is_gfapi_program_alive $client_pid
 
+gfid_str1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE1))
+inode1="FILE1|gfid:$gfid_str1"
+gfid_str2=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE2))
+inode2="FILE2|gfid:$gfid_str2"
+
 # Kill brick-3 and let client-1 take lock on both files.
 TEST kill_brick $V0 $H0 $B0/${V0}2
 TEST kill -SIGUSR1 $client_pid
@@ -56,15 +67,15 @@ TEST kill -SIGUSR1 $client_pid
 EXPECT "Y" is_gfapi_program_alive $client_pid
 
 # Check lock is present on brick-1 and brick-2
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1 $inode1 $inode2
 
 # Restart brick-3 and check that the lock has healed on it.
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
 TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd.
 
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2 $inode1 $inode2
 
 #------------------------------------------------------------------------------
 # Kill same brick before heal completes the first time and check it completes the second time.
@@ -80,7 +91,7 @@ TEST kill_brick $V0 $H0 $B0/${V0}0
 TEST $CLI volume reset $V0 delay-gen
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2
 
 #------------------------------------------------------------------------------
 # Kill 2 bricks and bring it back. The fds must be marked bad.
diff --git a/tests/basic/fencing/afr-lock-heal-basic.t b/tests/basic/fencing/afr-lock-heal-basic.t
index 5ac05c7aec6..c5d7d6fe8fd 100644
--- a/tests/basic/fencing/afr-lock-heal-basic.t
+++ b/tests/basic/fencing/afr-lock-heal-basic.t
@@ -45,6 +45,9 @@ TEST sleep 5 # By now, the 2 clients would  have opened an fd on FILE and waitin
 EXPECT "Y" is_gfapi_program_alive $client1_pid
 EXPECT "Y" is_gfapi_program_alive $client2_pid
 
+gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE))
+inode="FILE|gfid:$gfid_str"
+
 # Kill brick-3 and let client-1 take lock on the file.
 TEST kill_brick $V0 $H0 $B0/${V0}2
 TEST kill -SIGUSR1 $client1_pid
@@ -54,8 +57,8 @@ EXPECT "Y" is_gfapi_program_alive $client1_pid
 # Check lock is present on brick-1 and brick-2
 b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0)
 b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1)
-c1_lock_on_b1="$(grep ACTIVE $b1_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
-c1_lock_on_b2="$(grep ACTIVE $b2_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
+c1_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
+c1_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
 TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b2" ]
 
 # Restart brick-3 and check that the lock has healed on it.
@@ -64,7 +67,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2
 TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. Also wait for lock heal.
 
 b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2)
-c1_lock_on_b3="$(grep ACTIVE $b3_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
+c1_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
 TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b3" ]
 
 # Kill brick-1 and let client-2 preempt the lock on bricks 2 and 3.
@@ -82,9 +85,9 @@ TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fai
 b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0)
 b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1)
 b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2)
-c2_lock_on_b1="$(grep ACTIVE $b1_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
-c2_lock_on_b2="$(grep ACTIVE $b2_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
-c2_lock_on_b3="$(grep ACTIVE $b3_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
+c2_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
+c2_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
+c2_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')"
 TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b2" ]
 TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b3" ]
 TEST [ "$c2_lock_on_b1" != "$c1_lock_on_b1" ]