From 2e3e7a2309368376855ffac28620ad56f2c6b7b5 Mon Sep 17 00:00:00 2001 From: Ravishankar N Date: Fri, 21 Feb 2020 18:13:57 +0530 Subject: tests: fix afr-lock-heal-* failure When brick-mux is enabled: i)brick statedumps seem to be listing the same lock information multiple times. While that is getting fixed, make changes to the .ts to check for unique values. ii)detecting a brick as online via brick_up_status() seems to be taking longer time when delaygen is enabled. Hence bump up PROCESS_UP_TIMEOUT to 90 for afr-lock-heal-advanced.t Updates: #1042 Change-Id: Ife76008f7a99dd1f1fe5791a32577366baaab4b3 Signed-off-by: Ravishankar N --- tests/basic/fencing/afr-lock-heal-advanced.t | 23 +++++++++++++++++------ tests/basic/fencing/afr-lock-heal-basic.t | 15 +++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/tests/basic/fencing/afr-lock-heal-advanced.t b/tests/basic/fencing/afr-lock-heal-advanced.t index 8a7a208db29..8a5b5989b5e 100644 --- a/tests/basic/fencing/afr-lock-heal-advanced.t +++ b/tests/basic/fencing/afr-lock-heal-advanced.t @@ -4,6 +4,7 @@ . $(dirname $0)/../../volume.rc cleanup; +PROCESS_UP_TIMEOUT=90 function is_gfapi_program_alive() { @@ -19,9 +20,14 @@ function is_gfapi_program_alive() function get_active_lock_count { brick=$1 + i1=$2 + i2=$3 + pattern="ACTIVE.*client-${brick: -1}" + sdump=$(generate_brick_statedump $V0 $H0 $brick) - lock_count="$(grep ACTIVE $sdump| wc -l)" - echo "$lock_count" + lock_count1="$(egrep "$i1" $sdump -A3| egrep "$pattern"|uniq|wc -l)" + lock_count2="$(egrep "$i2" $sdump -A3| egrep "$pattern"|uniq|wc -l)" + echo "$((lock_count1+lock_count2))" } TEST glusterd @@ -49,6 +55,11 @@ TEST [ $client_pid ] TEST sleep 5 # By now, the client would have opened an fd on FILE1 and FILE2 and waiting for a SIGUSR1. EXPECT "Y" is_gfapi_program_alive $client_pid +gfid_str1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE1)) +inode1="FILE1|gfid:$gfid_str1" +gfid_str2=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE2)) +inode2="FILE2|gfid:$gfid_str2" + # Kill brick-3 and let client-1 take lock on both files. TEST kill_brick $V0 $H0 $B0/${V0}2 TEST kill -SIGUSR1 $client_pid @@ -56,15 +67,15 @@ TEST kill -SIGUSR1 $client_pid EXPECT "Y" is_gfapi_program_alive $client_pid # Check lock is present on brick-1 and brick-2 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1 $inode1 $inode2 # Restart brick-3 and check that the lock has healed on it. TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2 $inode1 $inode2 #------------------------------------------------------------------------------ # Kill same brick before heal completes the first time and check it completes the second time. @@ -80,7 +91,7 @@ TEST kill_brick $V0 $H0 $B0/${V0}0 TEST $CLI volume reset $V0 delay-gen TEST $CLI volume start $V0 force EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2 #------------------------------------------------------------------------------ # Kill 2 bricks and bring it back. The fds must be marked bad. diff --git a/tests/basic/fencing/afr-lock-heal-basic.t b/tests/basic/fencing/afr-lock-heal-basic.t index 5ac05c7aec6..c5d7d6fe8fd 100644 --- a/tests/basic/fencing/afr-lock-heal-basic.t +++ b/tests/basic/fencing/afr-lock-heal-basic.t @@ -45,6 +45,9 @@ TEST sleep 5 # By now, the 2 clients would have opened an fd on FILE and waitin EXPECT "Y" is_gfapi_program_alive $client1_pid EXPECT "Y" is_gfapi_program_alive $client2_pid +gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE)) +inode="FILE|gfid:$gfid_str" + # Kill brick-3 and let client-1 take lock on the file. TEST kill_brick $V0 $H0 $B0/${V0}2 TEST kill -SIGUSR1 $client1_pid @@ -54,8 +57,8 @@ EXPECT "Y" is_gfapi_program_alive $client1_pid # Check lock is present on brick-1 and brick-2 b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0) b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1) -c1_lock_on_b1="$(grep ACTIVE $b1_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" -c1_lock_on_b2="$(grep ACTIVE $b2_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c1_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c1_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b2" ] # Restart brick-3 and check that the lock has healed on it. @@ -64,7 +67,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. Also wait for lock heal. b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2) -c1_lock_on_b3="$(grep ACTIVE $b3_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c1_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b3" ] # Kill brick-1 and let client-2 preempt the lock on bricks 2 and 3. @@ -82,9 +85,9 @@ TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fai b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0) b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1) b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2) -c2_lock_on_b1="$(grep ACTIVE $b1_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" -c2_lock_on_b2="$(grep ACTIVE $b2_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" -c2_lock_on_b3="$(grep ACTIVE $b3_sdump| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c2_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c2_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +c2_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b2" ] TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b3" ] TEST [ "$c2_lock_on_b1" != "$c1_lock_on_b1" ] -- cgit