From a97ad9b69bb17f2351c59512fa9c6cb25d82b4da Mon Sep 17 00:00:00 2001 From: Xavier Hernandez Date: Thu, 18 Sep 2014 18:42:34 +0200 Subject: test/ec: Fix spurious failures caused by self-heal The sha1sum of a file may update the access time of that file. If this happens while a brick is down, as it is forced in the test, that brick doesn't get the update, getting out of sync. When the brick is restarted, self-heal repairs the file, but the test shouldn't access brick contents until self-heal finishes. If this is combined with a kill of another brick before self-heal has finished repairing the file, the volume could become inaccessible. Since the purpose of these tests is only to check ec functionality (there is another test that checks self-heal), the test that corrupts the file has been removed. Additional checks to validate the state of the volume have been added to avoid some timing issues. BUG: 1144108 Change-Id: Ibd9288de519914663998a1fbc4321ec92ed6082c Signed-off-by: Xavier Hernandez Reviewed-on: http://review.gluster.org/8892 Reviewed-by: Emmanuel Dreyfus Tested-by: Emmanuel Dreyfus Tested-by: Gluster Build System Reviewed-by: Dan Lambright Reviewed-by: Vijay Bellur --- tests/basic/ec/self-heal.t | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'tests/basic/ec/self-heal.t') diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t index 524c11d2220..a40193f7447 100644 --- a/tests/basic/ec/self-heal.t +++ b/tests/basic/ec/self-heal.t @@ -17,8 +17,11 @@ TESTS_EXPECTED_IN_LOOP=85 TEST glusterd TEST pidof glusterd TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +EXPECT "Created" volinfo_field $V0 'Status' TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status' TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024 @@ -40,7 +43,7 @@ for idx1 in {0..5}; do TEST chmod 666 ${brick[$idx1]}/test sleep 1 EXPECT "-rw-r--r--" stat -c "%A" test - EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test + EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test done for idx1 in {0..4}; do @@ -50,8 +53,8 @@ for idx1 in {0..4}; do TEST chmod 600 ${brick[$idx2]}/test sleep 1 EXPECT "-rw-r--r--" stat -c "%A" test - EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test - EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test + EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test + EXPECT_WITHIN $HEAL_TIMEOUT "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test fi done done @@ -67,8 +70,8 @@ sleep 1 EXPECT "1048576" stat -c "%s" test TEST ! getfattr -n user.test test -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test +EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[0]}/test +EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[1]}/test TEST ! getfattr -n user.test ${brick[0]}/test TEST ! getfattr -n user.test ${brick[1]}/test EXPECT "-rw-r--r--" stat -c "%A" ${brick[0]}/test @@ -92,31 +95,27 @@ TEST [ -f test4 ] EXPECT "2" stat -c "%h" test2 EXPECT "2" stat -c "%h" test4 -cd -TEST umount $M0 -TEST $CLI volume stop $V0 force -TEST $CLI volume start $V0 -TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; -cd $M0 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 EXPECT "1048576" stat -c "%s" test2 EXPECT "-rwxrwxrwx" stat -c "%A" test2 -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test2 -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test2 +EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[0]}/test2 +EXPECT_WITHIN $HEAL_TIMEOUT "262144" stat -c "%s" ${brick[1]}/test2 EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[0]}/test2 EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[1]}/test2 TEST ls -al dir1 -EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi" -EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi" +EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi" +EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi" TEST [ -h test3 ] -EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi" -EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi" +EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi" +EXPECT_WITHIN $HEAL_TIMEOUT "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi" EXPECT "2" stat -c "%h" test4 -EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[0]}/test4 -EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[1]}/test4 +EXPECT_WITHIN $HEAL_TIMEOUT "3" stat -c "%h" ${brick[0]}/test4 +EXPECT_WITHIN $HEAL_TIMEOUT "3" stat -c "%h" ${brick[1]}/test4 TEST rm -rf $tmp -- cgit