From 1392da3e237d8ea080573909015916e3544a6d2c Mon Sep 17 00:00:00 2001 From: Xavier Hernandez Date: Thu, 15 May 2014 10:35:14 +0200 Subject: cli/glusterd: Added support for dispersed volumes Two new options have been added to the 'create' command of the cli interface: disperse [] redundancy Both are optional. A dispersed volume is created by specifying, at least, one of them. If 'disperse' is missing or it's present but '' does not, the number of bricks enumerated in the command line is taken as the disperse count. If 'redundancy' is missing, the lowest optimal value is assumed. A configuration is considered optimal (for most workloads) when the disperse count - redundancy count is a power of 2. If the resulting redundancy is 1, the volume is created normally, but if it's greater than 1, a warning is shown to the user and he/she must answer yes/no to continue volume creation. If there isn't any optimal value for the given number of bricks, a warning is also shown and, if the user accepts, a redundancy of 1 is used. If 'redundancy' is specified and the resulting volume is not optimal, another warning is shown to the user. A distributed-disperse volume can be created using a number of bricks multiple of the disperse count. Change-Id: Iab93efbe78e905cdb91f54f3741599f7ea6645e4 BUG: 1118629 Signed-off-by: Xavier Hernandez Reviewed-on: http://review.gluster.org/7782 Tested-by: Gluster Build System Reviewed-by: Jeff Darcy Reviewed-by: Vijay Bellur --- tests/basic/ec/ec-12-4.t | 14 +++ tests/basic/ec/ec-3-1.t | 14 +++ tests/basic/ec/ec-4-1.t | 14 +++ tests/basic/ec/ec-5-1.t | 14 +++ tests/basic/ec/ec-5-2.t | 14 +++ tests/basic/ec/ec-6-2.t | 14 +++ tests/basic/ec/ec-7-3.t | 14 +++ tests/basic/ec/ec-common | 143 ++++++++++++++++++++++++++++ tests/basic/ec/ec.t | 233 +++++++++++++++++++++++++++++++++++++++++++++ tests/basic/ec/self-heal.t | 123 ++++++++++++++++++++++++ 10 files changed, 597 insertions(+) create mode 100644 tests/basic/ec/ec-12-4.t create mode 100644 tests/basic/ec/ec-3-1.t create mode 100644 tests/basic/ec/ec-4-1.t create mode 100644 tests/basic/ec/ec-5-1.t create mode 100644 tests/basic/ec/ec-5-2.t create mode 100644 tests/basic/ec/ec-6-2.t create mode 100644 tests/basic/ec/ec-7-3.t create mode 100644 tests/basic/ec/ec-common create mode 100644 tests/basic/ec/ec.t create mode 100644 tests/basic/ec/self-heal.t (limited to 'tests/basic/ec') diff --git a/tests/basic/ec/ec-12-4.t b/tests/basic/ec/ec-12-4.t new file mode 100644 index 00000000000..9ab47018617 --- /dev/null +++ b/tests/basic/ec/ec-12-4.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=12 +REDUNDANCY=4 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=634 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-3-1.t b/tests/basic/ec/ec-3-1.t new file mode 100644 index 00000000000..5769c202289 --- /dev/null +++ b/tests/basic/ec/ec-3-1.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=3 +REDUNDANCY=1 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=238 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-4-1.t b/tests/basic/ec/ec-4-1.t new file mode 100644 index 00000000000..d34e1fb4e95 --- /dev/null +++ b/tests/basic/ec/ec-4-1.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=4 +REDUNDANCY=1 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=282 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-5-1.t b/tests/basic/ec/ec-5-1.t new file mode 100644 index 00000000000..61d1cb6ce48 --- /dev/null +++ b/tests/basic/ec/ec-5-1.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=5 +REDUNDANCY=1 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=326 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-5-2.t b/tests/basic/ec/ec-5-2.t new file mode 100644 index 00000000000..4dc1c186f02 --- /dev/null +++ b/tests/basic/ec/ec-5-2.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=5 +REDUNDANCY=2 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=326 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-6-2.t b/tests/basic/ec/ec-6-2.t new file mode 100644 index 00000000000..23ec84e60e9 --- /dev/null +++ b/tests/basic/ec/ec-6-2.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=6 +REDUNDANCY=2 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=370 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-7-3.t b/tests/basic/ec/ec-7-3.t new file mode 100644 index 00000000000..4ebba2a1de3 --- /dev/null +++ b/tests/basic/ec/ec-7-3.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=7 +REDUNDANCY=3 + +# This must be equal to 44 * $DISPERSE + 106 +TESTS_EXPECTED_IN_LOOP=414 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-common b/tests/basic/ec/ec-common new file mode 100644 index 00000000000..95f53f250bc --- /dev/null +++ b/tests/basic/ec/ec-common @@ -0,0 +1,143 @@ + +SIZE_LIST="1048576 1000 12345 0" + +LAST_BRICK=$(($DISPERSE - 1)) + +function fragment_size +{ + local fragments=$(($DISPERSE - $REDUNDANCY)) + local block_size=$((128 * $fragments)) + local size=$(($1 + $block_size - 1)) + + echo $((( $size - ( $size ) % $block_size ) / $fragments)) +} + +cleanup + +tmp=`mktemp -d` +if [ ! -d $tmp ]; then + exit 1 +fi + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy $REDUNDANCY $H0:$B0/${V0}{0..$LAST_BRICK} +TEST $CLI volume start $V0 +TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 + +TEST dd if=/dev/urandom of=$tmp/small bs=1024 count=1 +TEST dd if=/dev/urandom of=$tmp/big bs=1024 count=4096 + +cs_small=$(sha1sum $tmp/small | awk '{ print $1 }') +cs_big=$(sha1sum $tmp/big | awk '{ print $1 }') +cp $tmp/small $tmp/small1 +for size in $SIZE_LIST; do + truncate -s $size $tmp/small1 + eval cs_small_truncate[$size]=$(sha1sum $tmp/small1 | awk '{ print $1 }') +done +cp $tmp/big $tmp/big1 +for size in $SIZE_LIST; do + truncate -s $size $tmp/big1 + eval cs_big_truncate[$size]=$(sha1sum $tmp/big1 | awk '{ print $1 }') +done + +TEST df -h +TEST stat $M0 + +for idx in `seq 0 $LAST_BRICK`; do + brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx) +done + +cd $M0 +EXPECT "2" echo $(ls -a1 | wc -l) +TEST mkdir dir1 +TEST [ -d dir1 ] +TEST touch file1 +TEST [ -f file1 ] + +for dir in . dir1; do + TEST cp $tmp/small $dir/small + TEST [ -f $dir/small ] + fsize=$(fragment_size 1024) + EXPECT "1024" stat -c "%s" $dir/small + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/small + done + + EXPECT "$cs_small" echo $(sha1sum $dir/small | awk '{ print $1 }') + + TEST cp $tmp/big $dir/big + TEST [ -f $dir/big ] + fsize=$(fragment_size 4194304) + EXPECT "4194304" stat -c "%s" $dir/big + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/big + done + + EXPECT "$cs_big" echo $(sha1sum $dir/big | awk '{ print $1 }') + + for idx in `seq 0 $LAST_BRICK`; do + TEST kill_brick $V0 $H0 $B0/$V0$idx + + EXPECT "1024" stat -c "%s" $dir/small + EXPECT "4194304" stat -c "%s" $dir/big + EXPECT "$cs_small" echo $(sha1sum $dir/small | awk '{ print $1 }') + EXPECT "$cs_big" echo $(sha1sum $dir/big | awk '{ print $1 }') + + cd + TEST umount $M0 + TEST $CLI volume stop $V0 force + TEST $CLI volume start $V0 + TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 + cd $M0 + done + + for size in $SIZE_LIST; do + TEST truncate -s $size $dir/small + TEST [ -f $dir/small ] + fsize=$(fragment_size $size) + EXPECT "$size" stat -c "%s" $dir/small + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/small + done + + EXPECT "${cs_small_truncate[$size]}" echo $(sha1sum $dir/small | awk '{ print $1 }') + + TEST truncate -s $size $dir/big + TEST [ -f $dir/big ] + EXPECT "$size" stat -c "%s" $dir/big + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/big + done + + EXPECT "${cs_big_truncate[$size]}" echo $(sha1sum $dir/big | awk '{ print $1 }') + done + + TEST rm -f $dir/small + TEST [ ! -e $dir/small ] + for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/$dir/small ] + done + + TEST rm -f $dir/big + TEST [ ! -e $dir/big ] + for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/$dir/big ] + done +done + +TEST rmdir dir1 +TEST [ ! -e dir1 ] +for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/dir1 ] +done + +TEST rm -f file1 +TEST [ ! -e file1 ] +for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/file1 ] +done + +rm -rf $tmp + +cleanup diff --git a/tests/basic/ec/ec.t b/tests/basic/ec/ec.t new file mode 100644 index 00000000000..e81de0d97bd --- /dev/null +++ b/tests/basic/ec/ec.t @@ -0,0 +1,233 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +TEST_USER=test-ec-user +TEST_UID=27341 + +function my_getfattr { + getfattr --only-values -e text $* 2> /dev/null +} + +function get_rep_count { + v=$(my_getfattr -n trusted.nsr.rep-count $1) + #echo $v > /dev/tty + echo $v +} + +function create_file { + dd if=/dev/urandom of=$1 bs=4k count=$2 conv=sync 2> /dev/null +} + +function setup_perm_file { + mkdir $1/perm_dir || return 1 + chown ${TEST_USER} $1/perm_dir || return 1 + su ${TEST_USER} -c "touch $1/perm_dir/perm_file" || return 1 + return 0 +} + +# Functions to check repair for specific operation types. + +function check_create_write { + for b in $*; do + cmp $tmpdir/create-write $b/create-write || return 1 + done + return 0 +} + +function check_truncate { + truncate --size=8192 $tmpdir/truncate + for b in $*; do + cmp $tmpdir/truncate $b/truncate || return 1 + done + return 0 +} + +function check_hard_link { + for b in $*; do + inum1=$(ls -i $b/hard-link-1 | cut -d' ' -f1) + inum2=$(ls -i $b/hard-link-2 | cut -d' ' -f1) + [ "$inum1" = "$inum2" ] || return 1 + done + echo "Y" + return 0 +} + +function check_soft_link { + for b in $*; do + [ "$(readlink $b/soft-link)" = "soft-link-tgt" ] || return 1 + done + echo "Y" + return 0 +} + +function check_unlink { + for b in $*; do + [ ! -e $b/unlink ] || return 1 + done + echo "Y" + return 0 +} + +function check_mkdir { + for b in $*; do + [ -d $b/mkdir ] || return 1 + done + echo "Y" + return 0 +} + +function check_rmdir { + for b in $*; do + [ ! -e $b/rmdir ] || return 1 + done + echo "Y" + return 0 +} + +function check_setxattr { + for b in $*; do + v=$(my_getfattr -n user.foo $b/setxattr) + [ "$v" = "ash_nazg_durbatuluk" ] || return 1 + done + echo "Y" + return 0 +} + +function check_removexattr { + for b in $*; do + my_getfattr -n user.bar $b/removexattr 2> /dev/null + [ $? = 0 ] && return 1 + done + echo "Y" + return 0 +} + +function check_perm_file { + b1=$1 + shift 1 + ftext=$(stat -c "%u %g %a" $b1/perm_dir/perm_file) + #echo "first u/g/a = $ftext" > /dev/tty + for b in $*; do + btext=$(stat -c "%u %g %a" $b/perm_dir/perm_file) + #echo " next u/a/a = $btext" > /dev/tty + if [ x"$btext" != x"$ftext" ]; then + return 1 + fi + done + echo "Y" + return 0 +} + +cleanup + +TEST useradd -o -M -u ${TEST_UID} ${TEST_USER} +trap "userdel --force ${TEST_USER}" EXIT + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info + +TEST mkdir -p $B0/${V0}{0,1,2,3,4,5,6,7,8,9} +TEST $CLI volume create $V0 disperse 10 redundancy 2 $H0:$B0/${V0}{0,1,2,3,4,5,6,7,8,9} + +EXPECT "$V0" volinfo_field $V0 'Volume Name' +EXPECT 'Created' volinfo_field $V0 'Status' +EXPECT '10' brick_count $V0 + +TEST $CLI volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +# Mount FUSE with caching disabled +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 + +# Create local files for comparisons etc. +tmpdir=$(mktemp -d) +trap "rm -rf $tmpdir" EXIT +TEST create_file $tmpdir/create-write 10 +TEST create_file $tmpdir/truncate 10 + +# Prepare files and directories we'll need later. +TEST cp $tmpdir/truncate $M0/ +TEST touch $M0/hard-link-1 +TEST touch $M0/unlink +TEST mkdir $M0/rmdir +TEST touch $M0/setxattr +TEST touch $M0/removexattr +TEST setfattr -n user.bar -v "ash_nazg_gimbatul" $M0/removexattr + +# Kill a couple of bricks and allow some time for things to settle. +TEST kill_brick $V0 $H0 $B0/${V0}3 +TEST kill_brick $V0 $H0 $B0/${V0}8 +sleep 10 + +# Test create+write +TEST cp $tmpdir/create-write $M0/ +# Test truncate +TEST truncate --size=8192 $M0/truncate +# Test hard link +TEST ln $M0/hard-link-1 $M0/hard-link-2 +# Test soft link +TEST ln -s soft-link-tgt $M0/soft-link +# Test unlink +TEST rm $M0/unlink +# Test rmdir +TEST rmdir $M0/rmdir +# Test mkdir +TEST mkdir $M0/mkdir +# Test setxattr +TEST setfattr -n user.foo -v "ash_nazg_durbatuluk" $M0/setxattr +# Test removexattr +TEST setfattr -x user.bar $M0/removexattr +# Test uid/gid behavior +TEST setup_perm_file $M0 + +# Unmount/remount so that create/write and truncate don't see cached data. +TEST umount $M0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 + +# Test create/write and truncate *before* the bricks are brought back. +TEST check_create_write $M0 +TEST check_truncate $M0 + +# Restart the bricks and allow repair to occur. +TEST $CLI volume start $V0 force +sleep 10 + +# Unmount/remount again, same reason as before. +TEST umount $M0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 + +# Make sure everything is as it should be. Most tests check for consistency +# between the bricks and the front end. This is not valid for disperse, so we +# check the mountpoint state instead. + +TEST check_create_write $M0 +TEST check_truncate $M0 + +TEST stat $M0/hard-link-1 +TEST stat $M0/hard-link-2 +TEST stat $M0/soft-link +TEST ! stat $M0/unlink +TEST ! stat $M0/rmdir +TEST stat $M0/mkdir +TEST stat $M0/setxattr +TEST stat $M0/removexattr +TEST stat $M0/perm_dir +TEST stat $M0/perm_dir/perm_file + +EXPECT_WITHIN 5 "Y" check_hard_link $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_soft_link $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_unlink $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_rmdir $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_mkdir $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_setxattr $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_removexattr $B0/${V0}{0..9} +EXPECT_WITHIN 5 "Y" check_perm_file $B0/${V0}{0..9} + +rm -rf $tmpdir +userdel --force ${TEST_USER} + +cleanup + diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t new file mode 100644 index 00000000000..99cfd9420aa --- /dev/null +++ b/tests/basic/ec/self-heal.t @@ -0,0 +1,123 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks self-healing feature of dispersed volumes + +cleanup + +tmp=`mktemp -d` +if [ ! -d $tmp ]; then + exit 1 +fi + +TESTS_EXPECTED_IN_LOOP=85 + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 + +TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024 + +cs=$(sha1sum $tmp/test | awk '{ print $1 }') + +TEST df -h +TEST stat $M0 + +for idx in {0..5}; do + brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx) +done + +cd $M0 +TEST cp $tmp/test test +TEST chmod 644 test +EXPECT "-rw-r--r--" stat -c "%A" test + +for idx1 in {0..5}; do + TEST chmod 666 ${brick[$idx1]}/test + sleep 1 + EXPECT "-rw-r--r--" stat -c "%A" test + EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test +done + +for idx1 in {0..4}; do + for idx2 in `seq $(($idx1 + 1)) 5`; do + if [ $idx1 -ne $idx2 ]; then + TEST chmod 666 ${brick[$idx1]}/test + TEST chmod 600 ${brick[$idx2]}/test + sleep 1 + EXPECT "-rw-r--r--" stat -c "%A" test + EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx1]}/test + EXPECT_WITHIN 5 "-rw-r--r--" stat -c "%A" ${brick[$idx2]}/test + fi + done +done + +TEST truncate -s 0 ${brick[0]}/test +TEST truncate -s 2097152 ${brick[1]}/test +TEST setfattr -n user.test -v "test1" ${brick[0]}/test +TEST setfattr -n user.test -v "test2" ${brick[1]}/test +TEST chmod 600 ${brick[0]}/test +TEST chmod 666 ${brick[1]}/test +sleep 1 + +EXPECT "1048576" stat -c "%s" test +TEST ! getfattr -n user.test test + +EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test +EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test +TEST ! getfattr -n user.test ${brick[0]}/test +TEST ! getfattr -n user.test ${brick[1]}/test +EXPECT "-rw-r--r--" stat -c "%A" ${brick[0]}/test +EXPECT "-rw-r--r--" stat -c "%A" ${brick[1]}/test + +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST cp $tmp/test test2 +EXPECT "1048576" stat -c "%s" test2 +TEST chmod 777 test2 +EXPECT "-rwxrwxrwx" stat -c "%A" test2 + +TEST mkdir dir1 +TEST ls -al dir1 + +TEST ln -s test2 test3 +TEST [ -h test3 ] + +TEST ln test2 test4 +TEST [ -f test4 ] +EXPECT "2" stat -c "%h" test2 +EXPECT "2" stat -c "%h" test4 + +cd +TEST umount $M0 +TEST $CLI volume stop $V0 force +TEST $CLI volume start $V0 +TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0 +cd $M0 + +EXPECT "1048576" stat -c "%s" test2 +EXPECT "-rwxrwxrwx" stat -c "%A" test2 +EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[0]}/test2 +EXPECT_WITHIN 5 "262144" stat -c "%s" ${brick[1]}/test2 +EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[0]}/test2 +EXPECT "-rwxrwxrwx" stat -c "%A" ${brick[1]}/test2 + +TEST ls -al dir1 +EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[0]}/dir1 ]; then echo 1; fi" +EXPECT_WITHIN 5 "1" eval "if [ -d ${brick[1]}/dir1 ]; then echo 1; fi" + +TEST [ -h test3 ] +EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[0]}/test3 ]; then echo 1; fi" +EXPECT_WITHIN 5 "1" eval "if [ -h ${brick[1]}/test3 ]; then echo 1; fi" + +EXPECT "2" stat -c "%h" test4 +EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[0]}/test4 +EXPECT_WITHIN 5 "3" stat -c "%h" ${brick[1]}/test4 + +rm -rf $tmp + +cleanup -- cgit