diff options
Diffstat (limited to 'tests/basic/ec/self-heal.t')
| -rw-r--r-- | tests/basic/ec/self-heal.t | 244 |
1 files changed, 178 insertions, 66 deletions
diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t index 99cfd9420aa..6329bb60248 100644 --- a/tests/basic/ec/self-heal.t +++ b/tests/basic/ec/self-heal.t @@ -1,5 +1,7 @@ #!/bin/bash +SCRIPT_TIMEOUT=300 + . $(dirname $0)/../../include.rc . $(dirname $0)/../../volume.rc @@ -7,40 +9,164 @@ cleanup -tmp=`mktemp -d` +function check_mount_dir +{ + getfattr -d -m. -e hex $M0 2>&1 > /dev/null + for i in {1..20}; do + ls -l $M0/ | grep "dir1" + if [ $? -ne 0 ]; then + return 1 + fi + done + + return 0 +} + +function check_size +{ + cat $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%s" $i/$1` + if [ "$res" != "$2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_mode +{ + getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%A" $i/$1` + if [ "$res" != "$2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_date +{ + getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%Y" $i/$1` + if [ "$res" != "$2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_xattr +{ + getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + getfattr -n $2 $i/$1 2>/dev/null + if [ $? -eq 0 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_dir +{ + getfattr -m. -d $M0/dir1 2>&1 > /dev/null + for i in "${brick[@]}"; do + if [ ! -d $i/dir1 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_soft_link +{ + getfattr -d -m. -e hex $M0/test3 2>&1 > /dev/null + for i in "${brick[@]}"; do + if [ ! -h $i/test3 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_hard_link +{ + getfattr -d -m. -e hex $M0/test4 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%h" $i/test4` + if [ "$res" != "3" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +tmp=`mktemp -d -t ${0##*/}.XXXXXX` if [ ! -d $tmp ]; then exit 1 fi -TESTS_EXPECTED_IN_LOOP=85 +TESTS_EXPECTED_IN_LOOP=194 TEST glusterd TEST pidof glusterd TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume set $V0 client-log-level DEBUG +#Write-behind has a bug where lookup can race over write which leads to size mismatch on the mount after a 'cp' +TEST $CLI volume set $V0 performance.write-behind off +#md-cache can return stale stat due to default timeout being 1 sec +TEST $CLI volume set $V0 performance.stat-prefetch off +EXPECT "Created" volinfo_field $V0 'Status' TEST $CLI volume start $V0 -TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status' +#direct-io-mode is to make sure 'cat' leads to READ fop which triggers heal +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 --direct-io-mode=yes $M0; +# Wait until all 6 childs have been recognized by the ec xlator +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024 cs=$(sha1sum $tmp/test | awk '{ print $1 }') -TEST df -h +TEST df -h $M0 TEST stat $M0 for idx in {0..5}; do brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx) done -cd $M0 -TEST cp $tmp/test test -TEST chmod 644 test -EXPECT "-rw-r--r--" stat -c "%A" test +TEST cp $tmp/test $M0/test +TEST chmod 644 $M0/test +TEST touch -d "@946681200" $M0/test +EXPECT "-rw-r--r--" stat -c "%A" $M0/test +EXPECT "946681200" stat -c "%Y" $M0/test for idx1 in {0..5}; do TEST chmod 666 ${brick[$idx1]}/test - sleep 1 - EXPECT "-rw-r--r--" stat -c "%A" test - EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test + TEST truncate -s 0 ${brick[$idx1]}/test + TEST setfattr -n user.test -v "test1" ${brick[$idx1]}/test + EXPECT "-rw-r--r--" stat -c "%A" $M0/test + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test "262144" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test "-rw-r--r--" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_date test "946681200" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_xattr test "user.test" done for idx1 in {0..4}; do @@ -48,76 +174,62 @@ for idx1 in {0..4}; do if [ $idx1 -ne $idx2 ]; then TEST chmod 666 ${brick[$idx1]}/test TEST chmod 600 ${brick[$idx2]}/test - sleep 1 - EXPECT "-rw-r--r--" stat -c "%A" test - EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test - EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test + TEST truncate -s 0 ${brick[$idx1]}/test + TEST truncate -s 2097152 ${brick[$idx2]}/test + TEST setfattr -n user.test -v "test1" ${brick[$idx1]}/test + TEST setfattr -n user.test -v "test2" ${brick[$idx2]}/test + EXPECT "-rw-r--r--" stat -c "%A" $M0/test + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test "262144" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test "-rw-r--r--" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_date test "946681200" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_xattr test "user.test" fi done done -TEST truncate -s 0 ${brick[0]}/test -TEST truncate -s 2097152 ${brick[1]}/test -TEST setfattr -n user.test -v "test1" ${brick[0]}/test -TEST setfattr -n user.test -v "test2" ${brick[1]}/test -TEST chmod 600 ${brick[0]}/test -TEST chmod 666 ${brick[1]}/test -sleep 1 - -EXPECT "1048576" stat -c "%s" test -TEST ! getfattr -n user.test test - -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test -TEST ! getfattr -n user.test ${brick[0]}/test -TEST ! getfattr -n user.test ${brick[1]}/test -EXPECT "-rw-r--r--" stat -c "%A" ${brick[0]}/test -EXPECT "-rw-r--r--" stat -c "%A" ${brick[1]}/test +sleep 2 TEST kill_brick $V0 $H0 $B0/${V0}0 TEST kill_brick $V0 $H0 $B0/${V0}1 -TEST cp $tmp/test test2 -EXPECT "1048576" stat -c "%s" test2 -TEST chmod 777 test2 -EXPECT "-rwxrwxrwx" stat -c "%A" test2 +TEST cp $tmp/test $M0/test2 +EXPECT "1048576" stat -c "%s" $M0/test2 +TEST chmod 777 $M0/test2 +EXPECT "-rwxrwxrwx" stat -c "%A" $M0/test2 -TEST mkdir dir1 -TEST ls -al dir1 +TEST mkdir $M0/dir1 +TEST ls -al $M0/dir1 -TEST ln -s test2 test3 -TEST [ -h test3 ] +TEST ln -s test2 $M0/test3 +TEST [ -h $M0/test3 ] -TEST ln test2 test4 -TEST [ -f test4 ] -EXPECT "2" stat -c "%h" test2 -EXPECT "2" stat -c "%h" test4 +TEST ln $M0/test2 $M0/test4 +TEST [ -f $M0/test4 ] +EXPECT "2" stat -c "%h" $M0/test2 +EXPECT "2" stat -c "%h" $M0/test4 -cd -TEST umount $M0 -TEST $CLI volume stop $V0 force -TEST $CLI volume start $V0 -TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 -cd $M0 +sleep 2 + +TEST $CLI volume start $V0 force +# Wait until the killed bricks have been started and recognized by the ec +# xlator +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +TEST check_mount_dir -EXPECT "1048576" stat -c "%s" test2 -EXPECT "-rwxrwxrwx" stat -c "%A" test2 -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test2 -EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test2 -EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[0]}/test2 -EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[1]}/test2 +EXPECT "1048576" stat -c "%s" $M0/test2 +EXPECT "-rwxrwxrwx" stat -c "%A" $M0/test2 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test2 "262144" +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test2 "-rwxrwxrwx" -TEST ls -al dir1 -EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi" -EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi" +TEST ls -al $M0/dir1 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_dir -TEST [ -h test3 ] -EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi" -EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi" +TEST [ -h $M0/test3 ] +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_soft_link -EXPECT "2" stat -c "%h" test4 -EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[0]}/test4 -EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[1]}/test4 +EXPECT "2" stat -c "%h" $M0/test4 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_hard_link -rm -rf $tmp +TEST rm -rf $tmp cleanup |