diff options
Diffstat (limited to 'tests/basic/ec')
43 files changed, 3555 insertions, 0 deletions
diff --git a/tests/basic/ec/dht-rename.t b/tests/basic/ec/dht-rename.t new file mode 100644 index 00000000000..81b41ff4c78 --- /dev/null +++ b/tests/basic/ec/dht-rename.t @@ -0,0 +1,19 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks notify part of ec + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +TEST touch $M0/1 +TEST mv $M0/1 $M0/10 +cleanup diff --git a/tests/basic/ec/ec-1468261.t b/tests/basic/ec/ec-1468261.t new file mode 100644 index 00000000000..77d704cf880 --- /dev/null +++ b/tests/basic/ec/ec-1468261.t @@ -0,0 +1,95 @@ +#!/bin/bash +# +# This test case verifies handling node down scenario with optimistic +# changelog enabled on EC volume. +### + +SCRIPT_TIMEOUT=300 + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup + +#cleate and start volume +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume set $V0 disperse.optimistic-change-log on +TEST $CLI volume set $V0 disperse.other-eager-lock on +TEST $CLI volume start $V0 + +#Mount the volume +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#Verify that all is good +TEST mkdir $M0/test_dir +TEST touch $M0/test_dir/file +sleep 2 +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^$" get_hex_xattr trusted.ec.dirty $B0/${V0}0/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^$" get_hex_xattr trusted.ec.dirty $B0/${V0}1/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^$" get_hex_xattr trusted.ec.dirty $B0/${V0}2/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^$" get_hex_xattr trusted.ec.dirty $B0/${V0}3/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^$" get_hex_xattr trusted.ec.dirty $B0/${V0}4/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^$" get_hex_xattr trusted.ec.dirty $B0/${V0}5/test_dir + +#Kill two bricks and touch a file +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 +TEST touch $M0/test_dir/new_file +sleep 2 + +#Dirty should be set on up bricks +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^00000000000000010000000000000001$" get_hex_xattr trusted.ec.dirty $B0/${V0}2/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^00000000000000010000000000000001$" get_hex_xattr trusted.ec.dirty $B0/${V0}3/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^00000000000000010000000000000001$" get_hex_xattr trusted.ec.dirty $B0/${V0}4/test_dir +EXPECT_WITHIN $IO_WAIT_TIMEOUT "^00000000000000010000000000000001$" get_hex_xattr trusted.ec.dirty $B0/${V0}5/test_dir + +#Bring up the down bricks +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#remove mount point contents +TEST rm -rf $M0"/*" 2>/dev/null + +# unmount and remount the volume +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST glusterfs -s $H0 --volfile-id $V0 $M0; + +#Create a tar file +TEST mkdir /tmp/test_dir +seq 1 3000 | xargs -n 1 -P 20 -I {} dd if=/dev/urandom of=/tmp/test_dir/file-{} bs=10K count=1 +tar -cf /tmp/test_dir.tar /tmp/test_dir/ 2>/dev/null +rm -rf /tmp/test_dir/ + +#Untar the tar file +tar -C $M0 -xf /tmp/test_dir.tar 2>/dev/null& + +#Kill 1st and 2nd brick +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#Stop untaring +TEST kill %1 +rm -f /tmp/test_dir.tar + +#Bring up the down bricks +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#Wait for heal to complete +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +#Kill 3rd and 4th brick +TEST kill_brick $V0 $H0 $B0/${V0}3 +TEST kill_brick $V0 $H0 $B0/${V0}4 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#remove mount point contents +#this will fail if things are wrong +TEST rm -rf $M0"/*" 2>/dev/null + +cleanup diff --git a/tests/basic/ec/ec-3-1.t b/tests/basic/ec/ec-3-1.t new file mode 100644 index 00000000000..511ca6420a2 --- /dev/null +++ b/tests/basic/ec/ec-3-1.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=3 +REDUNDANCY=1 + +# This must be equal to 36 * $DISPERSE + 109 +TESTS_EXPECTED_IN_LOOP=217 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-4-1.t b/tests/basic/ec/ec-4-1.t new file mode 100644 index 00000000000..3f0d0c72e44 --- /dev/null +++ b/tests/basic/ec/ec-4-1.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=4 +REDUNDANCY=1 + +# This must be equal to 36 * $DISPERSE + 109 +TESTS_EXPECTED_IN_LOOP=253 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-5-2.t b/tests/basic/ec/ec-5-2.t new file mode 100644 index 00000000000..6d9e91b0f58 --- /dev/null +++ b/tests/basic/ec/ec-5-2.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=5 +REDUNDANCY=2 + +# This must be equal to 36 * $DISPERSE + 109 +TESTS_EXPECTED_IN_LOOP=289 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-6-2.t b/tests/basic/ec/ec-6-2.t new file mode 100644 index 00000000000..b4451f905a1 --- /dev/null +++ b/tests/basic/ec/ec-6-2.t @@ -0,0 +1,14 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks basic dispersed volume functionality and cli interface + +DISPERSE=6 +REDUNDANCY=2 + +# This must be equal to 36 * $DISPERSE + 109 +TESTS_EXPECTED_IN_LOOP=325 + +. $(dirname $0)/ec-common diff --git a/tests/basic/ec/ec-anonymous-fd.t b/tests/basic/ec/ec-anonymous-fd.t new file mode 100644 index 00000000000..a61628182ef --- /dev/null +++ b/tests/basic/ec/ec-anonymous-fd.t @@ -0,0 +1,42 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../fileio.rc + +cleanup +function num_entries { + ls -l $1 | wc -l +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +fd1=`fd_available` +TEST fd_open $fd1 'w' $M0/file1 +TEST fd_write $fd1 testing +TEST cat $M0/file1 +TEST rm -rf $M0/file1 +TEST fd_write $fd1 testing +TEST fd_write $fd1 testing +TEST fd_write $fd1 testing + +EXPECT "^2$" num_entries $B0/${V0}0/.glusterfs/unlink/ +EXPECT "^2$" num_entries $B0/${V0}1/.glusterfs/unlink/ +EXPECT "^2$" num_entries $B0/${V0}2/.glusterfs/unlink/ +EXPECT "^2$" num_entries $B0/${V0}3/.glusterfs/unlink/ +EXPECT "^2$" num_entries $B0/${V0}4/.glusterfs/unlink/ +EXPECT "^2$" num_entries $B0/${V0}5/.glusterfs/unlink/ +TEST fd_close $fd1; +EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}0/.glusterfs/unlink/ +EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}1/.glusterfs/unlink/ +EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}2/.glusterfs/unlink/ +EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}3/.glusterfs/unlink/ +EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}4/.glusterfs/unlink/ +EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}5/.glusterfs/unlink/ + +cleanup; diff --git a/tests/basic/ec/ec-background-heals.t b/tests/basic/ec/ec-background-heals.t new file mode 100644 index 00000000000..29778a4f818 --- /dev/null +++ b/tests/basic/ec/ec-background-heals.t @@ -0,0 +1,105 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks background heals option + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume heal $V0 disable +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume set $V0 disperse.eager-lock off +TEST $CLI volume set $V0 disperse.other-eager-lock off +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +TEST touch $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" count_sh_entries $B0/${V0}0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" count_sh_entries $B0/${V0}1 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" count_sh_entries $B0/${V0}2 + +TEST kill_brick $V0 $H0 $B0/${V0}2 +echo abc > $M0/a +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +#Accessing file shouldn't heal the file +EXPECT "abc" cat $M0/a +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +TEST $CLI volume set $V0 disperse.background-heals 1 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "128" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +#Accessing file should heal the file now +EXPECT "abc" cat $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +#Test above test cases with reset instead of setting background-heals to 1 +TEST $CLI volume set $V0 disperse.heal-wait-qlength 1024 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1024" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +TEST $CLI volume set $V0 disperse.background-heals 0 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +TEST $CLI volume set $V0 disperse.heal-wait-qlength 200 #Changing qlength shouldn't affect anything now +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +TEST kill_brick $V0 $H0 $B0/${V0}2 +echo abc > $M0/a +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +#Accessing file shouldn't heal the file +EXPECT "abc" cat $M0/a +sleep 3 +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +TEST $CLI volume reset $V0 disperse.background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "8" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "200" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +#Accessing file should heal the file now +EXPECT "abc" cat $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +#Test that disabling background-heals still drains the queue +TEST $CLI volume set $V0 disperse.background-heals 1 +TEST touch $M0/{a,b,c,d} +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "200" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +TEST truncate -s 1GB $M0/a +echo abc > $M0/b +echo abc > $M0/c +echo abc > $M0/d +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST chown root:root $M0/{a,b,c,d} +TEST $CLI volume set $V0 disperse.background-heals 0 +EXPECT_NOT "0" mount_get_option_value $M0 $V0-disperse-0 heal-waiters +TEST truncate -s 0 $M0/a # This completes the heal fast ;-) +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +#Test that background heals get rejected on meeting background-qlen limit +TEST $CLI volume set $V0 disperse.background-heals 1 +TEST $CLI volume set $V0 disperse.heal-wait-qlength 0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength +TEST truncate -s 1GB $M0/a +echo abc > $M0/b +echo abc > $M0/c +echo abc > $M0/d +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST chown root:root $M0/{a,b,c,d} +EXPECT "0" mount_get_option_value $M0 $V0-disperse-0 heal-waiters +cleanup diff --git a/tests/basic/ec/ec-badfd.c b/tests/basic/ec/ec-badfd.c new file mode 100644 index 00000000000..8be23c10eaf --- /dev/null +++ b/tests/basic/ec/ec-badfd.c @@ -0,0 +1,124 @@ +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <limits.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +int +fill_iov(struct iovec *iov, char fillchar, int count) +{ + int ret = -1; + + iov->iov_base = malloc(count + 1); + if (iov->iov_base == NULL) { + return ret; + } else { + iov->iov_len = count; + ret = 0; + } + memset(iov->iov_base, fillchar, count); + memset(iov->iov_base + count, '\0', 1); + + return ret; +} + +int +write_sync(glfs_t *fs, glfs_fd_t *glfd, int char_count) +{ + ssize_t ret = -1; + int flags = O_RDWR; + struct iovec iov = {0}; + + ret = fill_iov(&iov, 'a', char_count); + if (ret) { + fprintf(stderr, "failed to create iov"); + goto out; + } + + ret = glfs_pwritev(glfd, &iov, 1, 0, flags); +out: + if (ret < 0) { + fprintf(stderr, "glfs_pwritev failed, %d", errno); + } + return ret; +} + +int +main(int argc, char *argv[]) +{ + glfs_t *fs = NULL; + glfs_fd_t *fd = NULL; + int ret = 1; + char volume_cmd[4096] = {0}; + + if (argc != 4) { + fprintf(stderr, "Syntax: %s <host> <volname> <file>\n", argv[0]); + return 1; + } + + fs = glfs_new(argv[2]); + if (!fs) { + fprintf(stderr, "glfs_new: returned NULL\n"); + return 1; + } + + ret = glfs_set_volfile_server(fs, "tcp", argv[1], 24007); + if (ret != 0) { + fprintf(stderr, "glfs_set_volfile_server: returned %d\n", ret); + goto out; + } + ret = glfs_set_logging(fs, "/tmp/ec-badfd.log", 7); + if (ret != 0) { + fprintf(stderr, "glfs_set_logging: returned %d\n", ret); + goto out; + } + ret = glfs_init(fs); + if (ret != 0) { + fprintf(stderr, "glfs_init: returned %d\n", ret); + goto out; + } + + fd = glfs_open(fs, argv[3], O_RDWR); + if (fd == NULL) { + fprintf(stderr, "glfs_open: returned NULL\n"); + goto out; + } + + ret = write_sync(fs, fd, 16); + if (ret < 0) { + fprintf(stderr, "write_sync failed\n"); + } + + snprintf(volume_cmd, sizeof(volume_cmd), + "gluster --mode=script volume stop %s", argv[2]); + /*Stop the volume so that update-size-version fails*/ + system(volume_cmd); + sleep(8); /* 3 seconds more than eager-lock-timeout*/ + snprintf(volume_cmd, sizeof(volume_cmd), + "gluster --mode=script volume start %s", argv[2]); + system(volume_cmd); + sleep(8); /*wait for bricks to come up*/ + ret = glfs_fsync(fd, NULL, NULL); + if (ret == 0) { + fprintf(stderr, "fsync succeeded on a BADFD\n"); + exit(1); + } + + ret = glfs_close(fd); + if (ret == 0) { + fprintf(stderr, "flush succeeded on a BADFD\n"); + exit(1); + } + ret = 0; + +out: + unlink("/tmp/ec-badfd.log"); + glfs_fini(fs); + + return ret; +} diff --git a/tests/basic/ec/ec-badfd.t b/tests/basic/ec/ec-badfd.t new file mode 100755 index 00000000000..56feb47f115 --- /dev/null +++ b/tests/basic/ec/ec-badfd.t @@ -0,0 +1,26 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{1..6} +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 disperse.eager-lock-timeout 5 + +TEST $CLI volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +TEST $GFS -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +TEST touch $M0/file + +TEST build_tester $(dirname $0)/ec-badfd.c -lgfapi -Wall -O2 +TEST $(dirname $0)/ec-badfd $H0 $V0 /file +cleanup_tester $(dirname ${0})/ec-badfd + +cleanup; diff --git a/tests/basic/ec/ec-common b/tests/basic/ec/ec-common new file mode 100644 index 00000000000..152e3b51236 --- /dev/null +++ b/tests/basic/ec/ec-common @@ -0,0 +1,145 @@ +SIZE_LIST="1048576 1000 12345 0" + +LAST_BRICK=$(($DISPERSE - 1)) + +CHUNK_SIZE=512 + +function fragment_size +{ + local fragments=$(($DISPERSE - $REDUNDANCY)) + local block_size=$(($CHUNK_SIZE * $fragments)) + local size=$(($1 + $block_size - 1)) + + echo $((( $size - ( $size ) % $block_size ) / $fragments)) +} + +cleanup + +tmp=`mktemp -d -t ${0##*/}.XXXXXX` +if [ ! -d $tmp ]; then + exit 1 +fi + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy $REDUNDANCY $H0:$B0/${V0}{0..$LAST_BRICK} +EXPECT 'Created' volinfo_field $V0 'Status' +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status' +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0 + +TEST dd if=/dev/urandom of=$tmp/small bs=1024 count=1 +TEST dd if=/dev/urandom of=$tmp/big bs=1024 count=4096 + +cs_small=$(sha1sum $tmp/small | awk '{ print $1 }') +cs_big=$(sha1sum $tmp/big | awk '{ print $1 }') +cp $tmp/small $tmp/small1 +for size in $SIZE_LIST; do + truncate -s $size $tmp/small1 + eval cs_small_truncate[$size]=$(sha1sum $tmp/small1 | awk '{ print $1 }') +done +cp $tmp/big $tmp/big1 +for size in $SIZE_LIST; do + truncate -s $size $tmp/big1 + eval cs_big_truncate[$size]=$(sha1sum $tmp/big1 | awk '{ print $1 }') +done + +TEST df -h $M0 +TEST stat $M0 + +for idx in `seq 0 $LAST_BRICK`; do + brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx) +done + +TEST stat $M0/ +TEST mkdir $M0/dir1 +TEST [ -d $M0/dir1 ] +TEST touch $M0/file1 +TEST [ -f $M0/file1 ] + +for dir in . dir1; do + TEST cp $tmp/small $M0/$dir/small + TEST [ -f $M0/$dir/small ] + fsize=$(fragment_size 1024) + EXPECT "1024" stat -c "%s" $M0/$dir/small + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/small + done + + EXPECT "$cs_small" echo $(sha1sum $M0/$dir/small | awk '{ print $1 }') + + TEST cp $tmp/big $M0/$dir/big + TEST [ -f $M0/$dir/big ] + fsize=$(fragment_size 4194304) + EXPECT "4194304" stat -c "%s" $M0/$dir/big + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/big + done + + EXPECT "$cs_big" echo $(sha1sum $M0/$dir/big | awk '{ print $1 }') + +# Give enough time for current operations to complete. Otherwise the +# following kill_brick can cause data corruption and self-heal will be +# needed, but this script is not prepared to handle self-healing. + sleep 2 + + for idx in `seq 0 $LAST_BRICK`; do + TEST kill_brick $V0 $H0 $B0/$V0$idx + + EXPECT "1024" stat -c "%s" $M0/$dir/small + EXPECT "4194304" stat -c "%s" $M0/$dir/big + + TEST $CLI volume start $V0 force + EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0 + done + + for size in $SIZE_LIST; do + TEST truncate -s $size $M0/$dir/small + TEST [ -f $M0/$dir/small ] + fsize=$(fragment_size $size) + EXPECT "$size" stat -c "%s" $M0/$dir/small + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/small + done + + EXPECT "${cs_small_truncate[$size]}" echo $(sha1sum $M0/$dir/small | awk '{ print $1 }') + + TEST truncate -s $size $M0/$dir/big + TEST [ -f $M0/$dir/big ] + EXPECT "$size" stat -c "%s" $M0/$dir/big + for idx in `seq 0 $LAST_BRICK`; do + EXPECT "$fsize" stat -c "%s" ${brick[$idx]}/$dir/big + done + + EXPECT "${cs_big_truncate[$size]}" echo $(sha1sum $M0/$dir/big | awk '{ print $1 }') + done + + TEST rm -f $M0/$dir/small + TEST [ ! -e $M0/$dir/small ] + for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/$dir/small ] + done + + TEST rm -f $M0/$dir/big + TEST [ ! -e $M0/$dir/big ] + for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/$dir/big ] + done +done + +TEST rmdir $M0/dir1 +TEST [ ! -e $M0/dir1 ] +for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/dir1 ] +done + +TEST rm -f $M0/file1 +TEST [ ! -e $M0/file1 ] +for idx in `seq 0 $LAST_BRICK`; do + TEST [ ! -e ${brick[$idx]}/file1 ] +done + +rm -rf $tmp + +cleanup diff --git a/tests/basic/ec/ec-cpu-extensions.t b/tests/basic/ec/ec-cpu-extensions.t new file mode 100644 index 00000000000..c9af27ea234 --- /dev/null +++ b/tests/basic/ec/ec-cpu-extensions.t @@ -0,0 +1,62 @@ +#!/bin/bash + +DISPERSE=18 +REDUNDANCY=2 + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +TESTS_EXPECTED_IN_LOOP=96 + +function check_contents +{ + local src=$1 + local cs=$2 + + TEST cp $src $M0/file + TEST [ -f $M0/file ] + + for ext in none x64 sse avx; do + EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + TEST $CLI volume set $V0 disperse.cpu-extensions $ext + TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 + EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0 + + EXPECT "$cs" echo $(sha1sum $M0/file | awk '{ print $1 }') + done + + TEST rm -f $M0/file +} + +cleanup + +tmp=`mktemp -p ${LOGDIR} -d -t ${0##*/}.XXXXXX` +if [ ! -d $tmp ]; then + exit 1 +fi + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy $REDUNDANCY $H0:$B0/${V0}{1..$DISPERSE} +TEST $CLI volume set $V0 performance.flush-behind off +TEST $CLI volume set $V0 disperse.read-policy round-robin +EXPECT 'Created' volinfo_field $V0 'Status' +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status' + +TEST dd if=/dev/urandom of=$tmp/file bs=1048576 count=1 +cs_file=$(sha1sum $tmp/file | awk '{ print $1 }') + +for ext in none x64 sse avx; do + TEST $CLI volume set $V0 disperse.cpu-extensions $ext + TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 + EXPECT_WITHIN $CHILD_UP_TIMEOUT "$DISPERSE" ec_child_up_count $V0 0 + + check_contents $tmp/file $cs_file + + EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +done + +TEST rm -rf $tmp + +cleanup diff --git a/tests/basic/ec/ec-data-heal.t b/tests/basic/ec/ec-data-heal.t new file mode 100755 index 00000000000..2672661c6b1 --- /dev/null +++ b/tests/basic/ec/ec-data-heal.t @@ -0,0 +1,75 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +#This test checks data corruption after heal while IO is going on + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +############ Start IO ########### +TEST touch $M0/file +#start background IO on file +dd if=/dev/urandom of=$M0/file conv=fdatasync & +iopid=$(echo $!) + + +############ Kill and start brick0 for heal ########### +TEST kill_brick $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +#sleep so that data can be written which will be healed later +sleep 10 +TEST $CLI volume start $V0 force +##wait for heal info to become 0 and kill IO +EXPECT_WITHIN $IO_HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +kill $iopid +EXPECT_WITHIN $IO_HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############### Check md5sum ######################### + +## unmount and mount get md5sum after killing brick0 + +TEST kill_brick $V0 $H0 $B0/${V0}0 + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +mdsum0=`md5sum $M0/file | awk '{print $1}'` +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +## unmount and mount get md5sum after killing brick1 + +TEST kill_brick $V0 $H0 $B0/${V0}1 + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +mdsum1=`md5sum $M0/file | awk '{print $1}'` +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +## unmount and mount get md5sum after killing brick2 + +TEST kill_brick $V0 $H0 $B0/${V0}2 + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +mdsum2=`md5sum $M0/file | awk '{print $1}'` +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +# compare all the three md5sums +EXPECT "$mdsum0" echo $mdsum1 +EXPECT "$mdsum0" echo $mdsum2 +EXPECT "$mdsum1" echo $mdsum2 + +cleanup diff --git a/tests/basic/ec/ec-dirty-flags.t b/tests/basic/ec/ec-dirty-flags.t new file mode 100644 index 00000000000..68e66103f08 --- /dev/null +++ b/tests/basic/ec/ec-dirty-flags.t @@ -0,0 +1,23 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This checks if the fop keeps the dirty flags settings correctly after +# finishing the fop. + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume heal $V0 disable +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +cd $M0 +for i in {1..1000}; do dd if=/dev/zero of=file-${i} bs=512k count=2; done +cd - +EXPECT "^0$" get_pending_heal_count $V0 + +cleanup diff --git a/tests/basic/ec/ec-discard.t b/tests/basic/ec/ec-discard.t new file mode 100644 index 00000000000..001f4498c86 --- /dev/null +++ b/tests/basic/ec/ec-discard.t @@ -0,0 +1,205 @@ +#!/bin/bash +# +# Test discard functionality +# +# Test that basic discard (hole punch) functionality works via the fallocate +# command line tool. Hole punch deallocates a region of a file, creating a hole +# and a zero-filled data region. We verify that hole punch works, frees blocks +# and that subsequent reads do not read stale data (caches are invalidated). +# +# NOTE: fuse fallocate is known to be broken with regard to cache invalidation +# up to 3.9.0 kernels. Therefore, FOPEN_KEEP_CACHE is not used in this +# test (opens will invalidate the fuse cache). +### + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../fallocate.rc +. $(dirname $0)/../../volume.rc + +cleanup + +#cleate and start volume +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume set $V0 disperse.optimistic-change-log on +TEST $CLI volume start $V0 + +#Mount the volume +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#Check for fallocate and hole punch support +require_fallocate -l 1m $M0/file +require_fallocate -p -l 512k $M0/file && rm -f $M0/file + +#Write some data, punch a hole and verify the file content changes +TEST dd if=/dev/urandom of=$M0/file bs=1024k count=1 +TEST cp $M0/file $M0/file.copy.pre +TEST fallocate -p -o 512k -l 128k $M0/file +TEST ! cmp $M0/file.copy.pre $M0/file +TEST rm -f $M0/file $M0/file.copy.pre + +#Allocate some blocks, punch a hole and verify block allocation +TEST fallocate -l 1m $M0/file +blksz=`stat -c %B $M0/file` +nblks=`stat -c %b $M0/file` +TEST [ $(($blksz * $nblks)) -ge 1048576 ] +TEST fallocate -p -o 512k -l 128k $M0/file +nblks=`stat -c %b $M0/file` +TEST [ $(($blksz * $nblks)) -lt $((933889)) ] +TEST unlink $M0/file + +###Punch hole test cases without fallocate +##With write +#Touching starting boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 0 -l 500 $B0/test_file +TEST fallocate -p -o 0 -l 500 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Touching boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 500 -l 1548 $B0/test_file +TEST fallocate -p -o 500 -l 1548 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Not touching boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 500 -l 1000 $B0/test_file +TEST fallocate -p -o 500 -l 1000 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Over boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 1500 -l 1000 $B0/test_file +TEST fallocate -p -o 1500 -l 1000 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +###Punch hole test cases with fallocate +##Without write + +#Zero size +TEST dd if=/dev/urandom of=$M0/test_file bs=1024 count=8 +TEST ! fallocate -p -o 1500 -l 0 $M0/test_file + +#Negative size +TEST ! fallocate -p -o 1500 -l -100 $M0/test_file +TEST rm -f $M0/test_file + +#Touching boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 2048 -l 2048 $B0/test_file +TEST fallocate -p -o 2048 -l 2048 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Touching boundary,multiple stripe +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 2048 -l 4096 $B0/test_file +TEST fallocate -p -o 2048 -l 4096 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +##With write + +#Size ends in boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 600 -l 3496 $B0/test_file +TEST fallocate -p -o 600 -l 3496 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Offset at boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 2048 -l 3072 $B0/test_file +TEST fallocate -p -o 2048 -l 3072 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Offset and Size not at boundary covering a stripe +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 1500 -l 3000 $B0/test_file +TEST fallocate -p -o 1500 -l 3000 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file +TEST rm -f $B0/test_file $M0/test_file + +#Offset and Size not at boundary +TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=8 +TEST cp $B0/test_file $M0/test_file +TEST fallocate -p -o 1000 -l 3072 $B0/test_file +TEST fallocate -p -o 1000 -l 3072 $M0/test_file +TEST md5_sum=`get_md5_sum $B0/test_file` +EXPECT $md5_sum get_md5_sum $M0/test_file + +#Data Corruption Tests +#Kill brick1 and brick2 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#Unmount and mount +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0; +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#verify md5 sum +EXPECT $md5_sum get_md5_sum $M0/test_file + +#Bring up the bricks +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#Kill brick3 and brick4 +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill_brick $V0 $H0 $B0/${V0}3 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#Unmount and mount +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0; +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#verify md5 sum +EXPECT $md5_sum get_md5_sum $M0/test_file + +#Bring up the bricks +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#Kill brick5 and brick6 +TEST kill_brick $V0 $H0 $B0/${V0}4 +TEST kill_brick $V0 $H0 $B0/${V0}5 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#Unmount and mount +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0; +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "4" ec_child_up_count $V0 0 + +#verify md5 sum +EXPECT $md5_sum get_md5_sum $M0/test_file + +cleanup diff --git a/tests/basic/ec/ec-fallocate.t b/tests/basic/ec/ec-fallocate.t new file mode 100644 index 00000000000..1b827eed7df --- /dev/null +++ b/tests/basic/ec/ec-fallocate.t @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Run several commands to verify basic fallocate functionality. We verify that +# fallocate creates and allocates blocks to a file. We also verify that the keep +# size option does not modify the file size. +### + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../fallocate.rc + +cleanup + +#cleate and start volume +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume start $V0 + +#Mount the volume +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +# check for fallocate support before continuing the test +require_fallocate -l 1m -n $M0/file && rm -f $M0/file + +# fallocate a file and verify blocks are allocated +TEST fallocate -l 1m $M0/file +blksz=`stat -c %b $M0/file` +nblks=`stat -c %B $M0/file` +TEST [ $(($blksz * $nblks)) -eq 1048576 ] + +TEST unlink $M0/file + +# truncate a file to a fixed size, fallocate and verify that the size does not +# change +TEST truncate -s 1M $M0/file +TEST fallocate -l 2m -n $M0/file +blksz=`stat -c %b $M0/file` +nblks=`stat -c %B $M0/file` +sz=`stat -c %s $M0/file` +TEST [ $sz -eq 1048576 ] +# Note that gluster currently incorporates a hack to limit the number of blocks +# reported as allocated to the file by the file size. We have allocated beyond the +# file size here. Just check for non-zero allocation to avoid setting a land mine +# for if/when that behavior might change. +TEST [ ! $(($blksz * $nblks)) -eq 0 ] +TEST unlink $M0/file + +# write some data, fallocate within and outside the range +# and check for data corruption. +TEST dd if=/dev/urandom of=$M0/file bs=1024k count=1 +TEST cp $M0/file $M0/file.copy.pre +TEST fallocate -o 512k -l 128k $M0/file +TEST cp $M0/file $M0/file.copy.post +TEST cmp $M0/file.copy.pre $M0/file.copy.post +TEST fallocate -o 1000k -l 128k $M0/file +TEST cp $M0/file $M0/file.copy.post2 +TEST ! cmp $M0/file.copy.pre $M0/file.copy.post2 +TEST truncate -s 1M $M0/file.copy.post2 +TEST cmp $M0/file.copy.pre $M0/file.copy.post2 +TEST unlink $M0/file + +#Make sure offset/size are modified so that 3 blocks are allocated +TEST touch $M0/f1 +TEST fallocate -o 1280 -l 1024 $M0/f1 +EXPECT "^2304$" stat -c "%s" $M0/f1 +EXPECT "^1536$" stat -c "%s" $B0/${V0}0/f1 + +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +cleanup; diff --git a/tests/basic/ec/ec-fast-fgetxattr.c b/tests/basic/ec/ec-fast-fgetxattr.c new file mode 100644 index 00000000000..bf982151861 --- /dev/null +++ b/tests/basic/ec/ec-fast-fgetxattr.c @@ -0,0 +1,129 @@ +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <limits.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +int cbk_complete = 0; +ssize_t cbk_ret_val = 0; +int +fill_iov(struct iovec *iov, char fillchar, int count) +{ + int ret = -1; + + iov->iov_base = malloc(count + 1); + if (iov->iov_base == NULL) { + return ret; + } else { + iov->iov_len = count; + ret = 0; + } + memset(iov->iov_base, fillchar, count); + memset(iov->iov_base + count, '\0', 1); + + return ret; +} + +void +write_async_cbk(glfs_fd_t *fd, ssize_t ret, struct stat *prestat, + struct stat *poststat, void *cookie) +{ + if (ret < 0) { + fprintf(stderr, "glfs_write failed"); + } + cbk_ret_val = ret; + cbk_complete = 1; +} + +int +write_async(glfs_t *fs, glfs_fd_t *glfd, int char_count) +{ + ssize_t ret = -1; + int flags = O_RDWR; + struct iovec iov = {0}; + + ret = fill_iov(&iov, 'a', char_count); + if (ret) { + fprintf(stderr, "failed to create iov"); + goto out; + } + + ret = glfs_pwritev_async(glfd, &iov, 1, 0, flags, write_async_cbk, NULL); +out: + if (ret < 0) { + fprintf(stderr, "glfs_pwritev async failed"); + } + return ret; +} + +int +main(int argc, char *argv[]) +{ + glfs_t *fs = NULL; + glfs_fd_t *fd = NULL; + int ret = 1; + char buf[1024] = {0}; + + if (argc != 4) { + fprintf(stderr, "Syntax: %s <host> <volname> <file>\n", argv[0]); + return 1; + } + + fs = glfs_new(argv[2]); + if (!fs) { + fprintf(stderr, "glfs_new: returned NULL\n"); + return 1; + } + + ret = glfs_set_volfile_server(fs, "tcp", argv[1], 24007); + if (ret != 0) { + fprintf(stderr, "glfs_set_volfile_server: returned %d\n", ret); + goto out; + } + ret = glfs_set_logging(fs, "/tmp/ec-fgetxattr.log", 7); + if (ret != 0) { + fprintf(stderr, "glfs_set_logging: returned %d\n", ret); + goto out; + } + ret = glfs_init(fs); + if (ret != 0) { + fprintf(stderr, "glfs_init: returned %d\n", ret); + goto out; + } + + fd = glfs_open(fs, argv[3], O_RDWR | O_TRUNC); + if (fd == NULL) { + fprintf(stderr, "glfs_open: returned NULL\n"); + goto out; + } + + ret = write_async(fs, fd, 16); + if (ret) { + fprintf(stderr, "write_async failed\n"); + } + + sleep(1); + ret = glfs_fgetxattr(fd, "trusted.glusterfs.abc", buf, sizeof buf); + while (cbk_complete != 1) { + /* ret will be -ve as xattr doesn't exist, and fgetxattr should + * return waaaayyy before writev */ + ret = 0; + sleep(1); + } + if (cbk_ret_val < 0) { + fprintf(stderr, "cbk_ret_val is -ve\n"); + ret = -1; + } + glfs_close(fd); + +out: + unlink("/tmp/ec-fgetxattr.log"); + glfs_fini(fs); + + return ret; +} diff --git a/tests/basic/ec/ec-fast-fgetxattr.t b/tests/basic/ec/ec-fast-fgetxattr.t new file mode 100755 index 00000000000..eb12fa4a0ba --- /dev/null +++ b/tests/basic/ec/ec-fast-fgetxattr.t @@ -0,0 +1,40 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{1..6} +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.client-io-threads off +TEST $CLI volume set $V0 brick-log-level DEBUG +TEST $CLI volume set $V0 delay-gen posix +TEST $CLI volume set $V0 delay-gen.delay-duration 10000000 +TEST $CLI volume set $V0 delay-gen.delay-percentage 100 +TEST $CLI volume set $V0 delay-gen.enable read,write + +TEST $CLI volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +TEST $GFS -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +TEST touch $M0/file + +# Perform two writes to make sure io-threads have enough threads to perform +# things in parallel when the test execution happens. +echo abc > $M0/file1 & +echo abc > $M0/file2 & +wait + +TEST build_tester $(dirname $0)/ec-fast-fgetxattr.c -lgfapi -Wall -O2 +TEST $(dirname $0)/ec-fast-fgetxattr $H0 $V0 /file +cleanup_tester $(dirname ${0})/ec-fast-fgetxattr + +cleanup; diff --git a/tests/basic/ec/ec-fix-openfd.t b/tests/basic/ec/ec-fix-openfd.t new file mode 100644 index 00000000000..04fdd802c62 --- /dev/null +++ b/tests/basic/ec/ec-fix-openfd.t @@ -0,0 +1,111 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../fileio.rc + +# This test checks for open fd heal on EC + +#Create Volume +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume set $V0 performance.read-after-open yes +TEST $CLI volume set $V0 performance.lazy-open no +TEST $CLI volume set $V0 performance.open-behind off +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume heal $V0 disable +TEST $CLI volume start $V0 + +#Mount the volume +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +#Touch a file +TEST touch "$M0/test_file" + +#Kill a brick +TEST kill_brick $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Open the file in write mode +TEST fd=`fd_available` +TEST fd_open $fd 'rw' "$M0/test_file" + +#Bring up the killed brick +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +sleep 1 + +#Test the fd count +EXPECT "0" get_fd_count $V0 $H0 $B0/${V0}0 test_file +EXPECT "1" get_fd_count $V0 $H0 $B0/${V0}1 test_file +EXPECT "1" get_fd_count $V0 $H0 $B0/${V0}2 test_file + +#Write to file +dd iflag=fullblock if=/dev/urandom bs=1024 count=2 >&$fd 2>/dev/null + +#Test the fd count +EXPECT "1" get_fd_count $V0 $H0 $B0/${V0}0 test_file + +#Close fd +TEST fd_close $fd + +#Stop the volume +TEST $CLI volume stop $V0 + +#Start the volume +TEST $CLI volume start $V0 + +#Kill brick1 +TEST kill_brick $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Unmount and mount +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0; +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Calculate md5 sum +md5sum0=`get_md5_sum "$M0/test_file"` + +#Bring up the brick +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +#Kill brick2 +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Unmount and mount +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Calculate md5 sum +md5sum1=`get_md5_sum "$M0/test_file"` + +#Bring up the brick +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +#Kill brick3 +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Unmount and mount +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 + +#Calculate md5 sum +md5sum2=`get_md5_sum "$M0/test_file"` + +#compare the md5sum +EXPECT "$md5sum0" echo $md5sum1 +EXPECT "$md5sum0" echo $md5sum2 +EXPECT "$md5sum1" echo $md5sum2 + +cleanup diff --git a/tests/basic/ec/ec-internal-xattrs.t b/tests/basic/ec/ec-internal-xattrs.t new file mode 100644 index 00000000000..5ef0502e016 --- /dev/null +++ b/tests/basic/ec/ec-internal-xattrs.t @@ -0,0 +1,45 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks internal xattr handling in ec +TESTS_EXPECTED_IN_LOOP=11 + +cleanup +function get_ec_xattrs +{ + getfattr -d -m. -e hex $1 | grep trusted.ec +} + +function get_xattr_count +{ + getfattr -d -m. -e hex $1 | grep "trusted" | wc -l +} + +declare -a xattrs=("trusted.ec.config" "trusted.ec.size" "trusted.ec.version" "trusted.ec.heal") + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +# Wait until all 6 children have been recognized by the ec xlator +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +TEST touch $M0/a + +#Check that internal xattrs are not modifiable or readable +for x in "${xattrs[@]}"; do + TEST_IN_LOOP ! setfattr -n $x "abc" $M0/a + TEST_IN_LOOP ! setfattr -x $x "abc" $M0/a + if [ $x != "trusted.ec.heal" ]; + then + TEST_IN_LOOP ! getfattr -n $x $M0/a + fi +done + +TEST ! get_ec_xattrs $M0/a +TEST setfattr -n trusted.abc -v 0x616263 $M0/a +EXPECT "616263" get_hex_xattr trusted.abc $M0/a +EXPECT "1" get_xattr_count $M0/a +cleanup diff --git a/tests/basic/ec/ec-new-entry.t b/tests/basic/ec/ec-new-entry.t new file mode 100644 index 00000000000..be97aecd8e2 --- /dev/null +++ b/tests/basic/ec/ec-new-entry.t @@ -0,0 +1,70 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup +function num_entries { + ls -l $1 | wc -l +} + +function get_md5sum { + md5sum $1 | awk '{print $1}' +} + +#after replace-brick immediately trusted.ec.version will be absent, so if it +#is present we can assume that heal attempted on root +function root_heal_attempted { + if [ -z $(get_hex_xattr trusted.ec.version $1) ]; + then + echo "N"; + else + echo "Y"; + fi +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +touch $M0/{1..10} +touch $M0/11 +TEST mknod $M0/char c 1 5 +TEST mknod $M0/block b 4 5 +for i in {1..10}; do dd if=/dev/zero of=$M0/$i bs=1M count=1; done +TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}5 $H0:$B0/${V0}6 commit force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count_shd $V0 0 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" root_heal_attempted $B0/${V0}6 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 +#ls -l gives "Total" line so number of lines will be 1 more +EXPECT "^14$" num_entries $B0/${V0}6 +EXPECT "^1 5$" stat -c "%t %T" $B0/${V0}6/char +EXPECT "^4 5$" stat -c "%t %T" $B0/${V0}6/block +ec_version=$(get_hex_xattr trusted.ec.version $B0/${V0}0) +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}1 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}2 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}3 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}4 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}6 +file_md5sum=$(get_md5sum $M0/1) +empty_md5sum=$(get_md5sum $M0/11) +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +echo $file_md5sum +EXPECT "$file_md5sum" get_md5sum $M0/1 +EXPECT "$file_md5sum" get_md5sum $M0/2 +EXPECT "$file_md5sum" get_md5sum $M0/3 +EXPECT "$file_md5sum" get_md5sum $M0/4 +EXPECT "$file_md5sum" get_md5sum $M0/5 +EXPECT "$file_md5sum" get_md5sum $M0/6 +EXPECT "$file_md5sum" get_md5sum $M0/7 +EXPECT "$file_md5sum" get_md5sum $M0/8 +EXPECT "$file_md5sum" get_md5sum $M0/9 +EXPECT "$file_md5sum" get_md5sum $M0/10 +EXPECT "$empty_md5sum" get_md5sum $M0/11 + +cleanup; diff --git a/tests/basic/ec/ec-notify.t b/tests/basic/ec/ec-notify.t new file mode 100644 index 00000000000..53290b7c798 --- /dev/null +++ b/tests/basic/ec/ec-notify.t @@ -0,0 +1,101 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks notify part of ec + +# We *know* some of these mounts will succeed but not be actually usable +# (terrible idea IMO), so speed things up and eliminate some noise by +# overriding this function. +_GFS () { + glusterfs "$@" +} + +ec_up_brick_count () { + local bricknum + for bricknum in $(seq 0 2); do + brick_up_status $V0 $H0 $B0/$V0$bricknum + done | grep -E '^1$' | wc -l +} + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "3" ec_up_brick_count + +#First time mount tests. +# When all the bricks are up, mount should succeed and up-children +# count should be 3 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST stat $M0 +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +# When the volume is stopped mount succeeds and up-children will be 0 +TEST $CLI volume stop $V0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +# Wait for 5 seconds even after that up_count should show 0 +sleep 5; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "0" ec_child_up_count $V0 0 +TEST ! stat $M0 +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +# When 2 bricks are up, mount should succeed and up-children +# count should be 2 + +TEST $CLI volume start $V0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" ec_up_brick_count +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +TEST stat $M0 +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +# When only 1 brick is up mount should fail. +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" ec_up_brick_count +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +# Wait for 5 seconds even after that up_count should show 1 +sleep 5 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_count $V0 0 +TEST ! stat $M0 +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +# Mount already succeeded. Test that the brick up down are leading to correct +# state changes in ec. +TEST $CLI volume stop $V0 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "3" ec_up_brick_count +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST touch $M0/a + +# kill 1 brick and the up_count should become 2, fops should still succeed +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" ec_up_brick_count +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +TEST touch $M0/b + +# kill one more brick and the up_count should become 1, fops should fail +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" ec_up_brick_count +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_count $V0 0 +TEST ! touch $M0/c + +# kill one more brick and the up_count should become 0, fops should still fail +TEST kill_brick $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" ec_up_brick_count +EXPECT_WITHIN $CHILD_UP_TIMEOUT "0" ec_child_up_count $V0 0 +TEST ! touch $M0/c + +# Bring up all the bricks up and see that up_count is 3 and fops are succeeding +# again. +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "3" ec_up_brick_count +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST touch $M0/c + +cleanup diff --git a/tests/basic/ec/ec-optimistic-changelog.t b/tests/basic/ec/ec-optimistic-changelog.t new file mode 100644 index 00000000000..a372cd39a64 --- /dev/null +++ b/tests/basic/ec/ec-optimistic-changelog.t @@ -0,0 +1,153 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks optimistic-change-log option + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume heal $V0 disable + +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume set $V0 disperse.optimistic-change-log off +TEST $CLI volume set $V0 disperse.eager-lock off +TEST $CLI volume set $V0 disperse.other-eager-lock off +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 background-heals +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "0" mount_get_option_value $M0 $V0-disperse-0 heal-wait-qlength + +TEST $CLI volume set $V0 disperse.background-heals 1 +TEST touch $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" count_sh_entries $B0/${V0}0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" count_sh_entries $B0/${V0}1 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" count_sh_entries $B0/${V0}2 + + + +### optimistic-change-log = off ; All bricks good. Test file operation +echo abc > $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = off ; Kill one brick . Test file operation +TEST kill_brick $V0 $H0 $B0/${V0}2 +echo abc > $M0/a +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +#Accessing file should heal the file now +EXPECT "abc" cat $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = off ; All bricks good. Test entry operation +TEST touch $M0/b +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = off ; All bricks good. Test metadata operation +TEST chmod 0777 $M0/b +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = off ; Kill one brick. Test entry operation + +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST touch $M0/c +EXPECT 4 get_pending_heal_count $V0 #two for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +getfattr -d -m. -e hex $M0 2>&1 > /dev/null +getfattr -d -m. -e hex $M0/c 2>&1 > /dev/null +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = off ; Kill one brick. Test metadata operation +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST chmod 0777 $M0/c +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +getfattr -d -m. -e hex $M0/c 2>&1 > /dev/null +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +TEST $CLI volume set $V0 disperse.optimistic-change-log on + +### optimistic-change-log = on ; All bricks good. Test file operation + +echo abc > $M0/aa +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = on ; Kill one brick. Test file operation + +TEST kill_brick $V0 $H0 $B0/${V0}2 +echo abc > $M0/aa +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +#Accessing file should heal the file now +getfattr -d -m. -e hex $M0/aa 2>&1 > /dev/null +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = on ; All bricks good. Test entry operation + +TEST touch $M0/bb +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = on ; All bricks good. Test metadata operation + +TEST chmod 0777 $M0/bb +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = on ; Kill one brick. Test entry operation + +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST touch $M0/cc +EXPECT 4 get_pending_heal_count $V0 #two for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +getfattr -d -m. -e hex $M0 2>&1 > /dev/null +getfattr -d -m. -e hex $M0/cc 2>&1 > /dev/null +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +## optimistic-change-log = on ; Kill one brick. Test metadata operation + +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST chmod 0777 $M0/cc +EXPECT 2 get_pending_heal_count $V0 #One for each active brick +$CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +getfattr -d -m. -e hex $M0/cc 2>&1 > /dev/null +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0 + +############################################################ + +cleanup diff --git a/tests/basic/ec/ec-quorum-count.t b/tests/basic/ec/ec-quorum-count.t new file mode 100644 index 00000000000..9310ebbb8f2 --- /dev/null +++ b/tests/basic/ec/ec-quorum-count.t @@ -0,0 +1,167 @@ + #!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../ec.rc + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume create $V1 $H0:$B0/${V1}{0..5} +TEST $CLI volume set $V0 disperse.eager-lock-timeout 5 +TEST $CLI volume set $V0 performance.flush-behind off +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume set $V0 disperse.heal-wait-qlength 0 + +#Should fail on non-disperse volume +TEST ! $CLI volume set $V1 disperse.quorum-count 5 + +#Should succeed on a valid range +TEST ! $CLI volume set $V0 disperse.quorum-count 0 +TEST ! $CLI volume set $V0 disperse.quorum-count -0 +TEST ! $CLI volume set $V0 disperse.quorum-count abc +TEST ! $CLI volume set $V0 disperse.quorum-count 10abc +TEST ! $CLI volume set $V0 disperse.quorum-count 1 +TEST ! $CLI volume set $V0 disperse.quorum-count 2 +TEST ! $CLI volume set $V0 disperse.quorum-count 3 +TEST $CLI volume set $V0 disperse.quorum-count 4 +TEST $CLI volume start $V0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#Test that the option is reflected in the mount +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^4$" ec_option_value $V0 $M0 0 quorum-count +TEST $CLI volume reset $V0 disperse.quorum-count +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^0$" ec_option_value $V0 $M0 0 quorum-count +TEST $CLI volume set $V0 disperse.quorum-count 6 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^6$" ec_option_value $V0 $M0 0 quorum-count + +TEST touch $M0/a +TEST touch $M0/data +TEST setfattr -n trusted.def -v def $M0/a +TEST touch $M0/src +TEST touch $M0/del-me +TEST mkdir $M0/dir1 +TEST dd if=/dev/zero of=$M0/read-file bs=1M count=1 oflag=direct +TEST dd if=/dev/zero of=$M0/del-file bs=1M count=1 oflag=direct +TEST gf_rm_file_and_gfid_link $B0/${V0}0 del-file +#modify operations should fail as the file is not in quorum +TEST ! dd if=/dev/zero of=$M0/del-file bs=1M count=1 oflag=direct +TEST kill_brick $V0 $H0 $B0/${V0}0 +#Read should succeed even when quorum-count is not met +TEST dd if=$M0/read-file of=/dev/null iflag=direct +TEST ! touch $M0/a2 +TEST ! mkdir $M0/dir2 +TEST ! mknod $M0/b2 b 4 5 +TEST ! ln -s $M0/a $M0/symlink +TEST ! ln $M0/a $M0/link +TEST ! mv $M0/src $M0/dst +TEST ! rm -f $M0/del-me +TEST ! rmdir $M0/dir1 +TEST ! dd if=/dev/zero of=$M0/a bs=1M count=1 conv=notrunc +TEST ! dd if=/dev/zero of=$M0/data bs=1M count=1 conv=notrunc +TEST ! truncate -s 0 $M0/a +TEST ! setfattr -n trusted.abc -v abc $M0/a +TEST ! setfattr -x trusted.def $M0/a +TEST ! chmod +x $M0/a +TEST ! fallocate -l 2m -n $M0/a +TEST ! fallocate -p -l 512k $M0/a +TEST $CLI volume start $V0 force +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0} + +# reset the option and check whether the default redundancy count is +# accepted or not. +TEST $CLI volume reset $V0 disperse.quorum-count +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^0$" ec_option_value $V0 $M0 0 quorum-count +TEST touch $M0/a1 +TEST touch $M0/data1 +TEST setfattr -n trusted.def -v def $M0/a1 +TEST touch $M0/src1 +TEST touch $M0/del-me1 +TEST mkdir $M0/dir11 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST touch $M0/a21 +TEST mkdir $M0/dir21 +TEST mknod $M0/b21 b 4 5 +TEST ln -s $M0/a1 $M0/symlink1 +TEST ln $M0/a1 $M0/link1 +TEST mv $M0/src1 $M0/dst1 +TEST rm -f $M0/del-me1 +TEST rmdir $M0/dir11 +TEST dd if=/dev/zero of=$M0/a1 bs=1M count=1 conv=notrunc +TEST dd if=/dev/zero of=$M0/data1 bs=1M count=1 conv=notrunc +TEST truncate -s 0 $M0/a1 +TEST setfattr -n trusted.abc -v abc $M0/a1 +TEST setfattr -x trusted.def $M0/a1 +TEST chmod +x $M0/a1 +TEST fallocate -l 2m -n $M0/a1 +TEST fallocate -p -l 512k $M0/a1 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +TEST touch $M0/a2 +TEST touch $M0/data2 +TEST setfattr -n trusted.def -v def $M0/a1 +TEST touch $M0/src2 +TEST touch $M0/del-me2 +TEST mkdir $M0/dir12 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST ! touch $M0/a22 +TEST ! mkdir $M0/dir22 +TEST ! mknod $M0/b22 b 4 5 +TEST ! ln -s $M0/a2 $M0/symlink2 +TEST ! ln $M0/a2 $M0/link2 +TEST ! mv $M0/src2 $M0/dst2 +TEST ! rm -f $M0/del-me2 +TEST ! rmdir $M0/dir12 +TEST ! dd if=/dev/zero of=$M0/a2 bs=1M count=1 conv=notrunc +TEST ! dd if=/dev/zero of=$M0/data2 bs=1M count=1 conv=notrunc +TEST ! truncate -s 0 $M0/a2 +TEST ! setfattr -n trusted.abc -v abc $M0/a2 +TEST ! setfattr -x trusted.def $M0/a2 +TEST ! chmod +x $M0/a2 +TEST ! fallocate -l 2m -n $M0/a2 +TEST ! fallocate -p -l 512k $M0/a2 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0} + +# Set quorum-count to 5 and kill 1 brick and the fops should pass +TEST $CLI volume set $V0 disperse.quorum-count 5 +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "^5$" ec_option_value $V0 $M0 0 quorum-count +TEST touch $M0/a3 +TEST touch $M0/data3 +TEST setfattr -n trusted.def -v def $M0/a3 +TEST touch $M0/src3 +TEST touch $M0/del-me3 +TEST mkdir $M0/dir13 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST touch $M0/a31 +TEST mkdir $M0/dir31 +TEST mknod $M0/b31 b 4 5 +TEST ln -s $M0/a3 $M0/symlink3 +TEST ln $M0/a3 $M0/link3 +TEST mv $M0/src3 $M0/dst3 +TEST rm -f $M0/del-me3 +TEST rmdir $M0/dir13 +TEST dd if=/dev/zero of=$M0/a3 bs=1M count=1 conv=notrunc +TEST dd if=/dev/zero of=$M0/data3 bs=1M count=1 conv=notrunc +TEST truncate -s 0 $M0/a3 +TEST setfattr -n trusted.abc -v abc $M0/a3 +TEST setfattr -x trusted.def $M0/a3 +TEST chmod +x $M0/a3 +TEST fallocate -l 2m -n $M0/a3 +TEST fallocate -p -l 512k $M0/a3 +TEST dd if=/dev/urandom of=$M0/heal-file bs=1M count=1 oflag=direct +cksum_before_heal="$(md5sum $M0/heal-file | awk '{print $1}')" +TEST $CLI volume start $V0 force +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0} +TEST kill_brick $V0 $H0 $B0/${V0}4 +TEST kill_brick $V0 $H0 $B0/${V0}5 +cksum_after_heal=$(dd if=$M0/heal-file iflag=direct | md5sum | awk '{print $1}') +TEST [[ $cksum_before_heal == $cksum_after_heal ]] +cleanup; diff --git a/tests/basic/ec/ec-read-mask.t b/tests/basic/ec/ec-read-mask.t new file mode 100644 index 00000000000..ddb556f2973 --- /dev/null +++ b/tests/basic/ec/ec-read-mask.t @@ -0,0 +1,114 @@ + #!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../ec.rc + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 + +#Empty read-mask should fail +TEST ! $GFS --xlator-option=*.ec-read-mask="" -s $H0 --volfile-id $V0 $M0 + +#Less than 4 number of bricks should fail +TEST ! $GFS --xlator-option="*.ec-read-mask=0" -s $H0 --volfile-id $V0 $M0 +TEST ! $GFS --xlator-option="*.ec-read-mask=0:1" -s $H0 --volfile-id $V0 $M0 +TEST ! $GFS --xlator-option=*.ec-read-mask="0:1:2" -s $H0 --volfile-id $V0 $M0 + +#ids greater than 5 should fail +TEST ! $GFS --xlator-option="*.ec-read-mask=0:1:2:6" -s $H0 --volfile-id $V0 $M0 + +#ids less than 0 should fail +TEST ! $GFS --xlator-option="*.ec-read-mask=0:-1:2:5" -s $H0 --volfile-id $V0 $M0 + +#read-mask with non-alphabet or comma should fail +TEST ! $GFS --xlator-option="*.ec-read-mask=0:1:2:5:abc" -s $H0 --volfile-id $V0 $M0 +TEST ! $GFS --xlator-option="*.ec-read-mask=0:1:2:5a" -s $H0 --volfile-id $V0 $M0 + +#mount with at least 4 read-mask-ids and all of them valid should pass +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5:4:3" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^111111$" ec_option_value $V0 $M0 0 read-mask +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask + +TEST dd if=/dev/urandom of=$M0/a bs=1M count=1 +md5=$(md5sum $M0/a | awk '{print $1}') +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +#Read on the file should fail if any of the read-mask is down when number of +#ids is data-count +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST ! dd if=$M0/a of=/dev/null +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST ! dd if=$M0/a of=/dev/null +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST ! dd if=$M0/a of=/dev/null +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}5 +TEST ! dd if=$M0/a of=/dev/null +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +#Read on file should succeed when non-read-mask bricks are down +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}3 +EXPECT "^$md5$" echo $(dd if=$M0/a | md5sum | awk '{print $1}') +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}4 +EXPECT "^$md5$" echo $(dd if=$M0/a | md5sum | awk '{print $1}') +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +TEST kill_brick $V0 $H0 $B0/${V0}3 +TEST kill_brick $V0 $H0 $B0/${V0}4 +EXPECT "^$md5$" echo $(dd if=$M0/a | md5sum | awk '{print $1}') +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $CLI volume start $V0 force + +#Deliberately corrupt chunks 3: 4 and check that reads still give correct data +TEST dd if=/dev/zero of=$B0/${V0}3/a bs=256k count=1 +TEST dd if=/dev/zero of=$B0/${V0}4/a bs=256k count=1 +TEST $GFS --xlator-option="*.ec-read-mask=0:1:2:5" -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +EXPECT "^100111$" ec_option_value $V0 $M0 0 read-mask +EXPECT "^$md5$" echo $(dd if=$M0/a | md5sum | awk '{print $1}') +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +cleanup; diff --git a/tests/basic/ec/ec-read-policy.t b/tests/basic/ec/ec-read-policy.t new file mode 100644 index 00000000000..fe6fe6576e7 --- /dev/null +++ b/tests/basic/ec/ec-read-policy.t @@ -0,0 +1,52 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume heal $V0 disable +TEST $CLI volume start $V0 + +#Disable all caching +TEST glusterfs --direct-io-mode=yes --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +#TEST volume operations work fine + +EXPECT "gfid-hash" mount_get_option_value $M0 $V0-disperse-0 read-policy +TEST $CLI volume set $V0 disperse.read-policy round-robin +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "round-robin" mount_get_option_value $M0 $V0-disperse-0 read-policy + +#TEST if the option gives the intended behavior. The way we perform this test +#is by performing reads from the mount and write to /dev/null. If the +#read-policy is round-robin, then all bricks should have read-fop where as +#with gfid-hash number of bricks with reads should be equal to (num-bricks - redundancy) +#count + +TEST $CLI volume profile $V0 start +TEST dd if=/dev/zero of=$M0/1 bs=1M count=4 +#Perform reads now from file on the mount, this only tests dispatch_min +TEST dd if=$M0/1 of=/dev/null bs=1M count=4 +#TEST that reads are executed on all bricks +rr_reads=$($CLI volume profile $V0 info cumulative| grep -w READ | wc -l) +EXPECT "^6$" echo $rr_reads +TEST $CLI volume profile $V0 info clear + +TEST $CLI volume set $V0 disperse.read-policy gfid-hash +EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "gfid-hash" mount_get_option_value $M0 $V0-disperse-0 read-policy + +#Perform reads now from file on the mount, this only tests dispatch_min +TEST dd if=$M0/1 of=/dev/null bs=1M count=4 +#TEST that reads are executed on all bricks +gh_reads=$($CLI volume profile $V0 info cumulative| grep -w READ | wc -l) +EXPECT "^4$" echo $gh_reads + +cleanup; diff --git a/tests/basic/ec/ec-readdir.t b/tests/basic/ec/ec-readdir.t new file mode 100644 index 00000000000..fad101bcabe --- /dev/null +++ b/tests/basic/ec/ec-readdir.t @@ -0,0 +1,46 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks that readdir works fine on distributed disperse volume + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..5} +TEST $CLI volume heal $V0 disable +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 1 + +TEST mkdir $M0/d +TEST touch $M0/d/{1..100} +EXPECT "100" echo $(ls $M0/d/* | wc -l) +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}3 +TEST rm -rf $M0/d/{1..100} +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 1 +#Do it 3 times so that even with all load balancing, readdir never falls +#on stale bricks +EXPECT "0" echo $(ls $M0/d/ | wc -l) +EXPECT "0" echo $(ls $M0/d/ | wc -l) +EXPECT "0" echo $(ls $M0/d/ | wc -l) +#Do the same test above for creation of entries +TEST mkdir $M0/d1 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}3 +TEST touch $M0/d1/{1..100} +TEST $CLI volume start $V0 force +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 1 +#Do it 3 times so that even with all load balancing, readdir never falls +#on stale bricks +EXPECT "100" echo $(ls $M0/d1/ | wc -l) +EXPECT "100" echo $(ls $M0/d1/ | wc -l) +EXPECT "100" echo $(ls $M0/d1/ | wc -l) +cleanup diff --git a/tests/basic/ec/ec-rebalance.t b/tests/basic/ec/ec-rebalance.t new file mode 100644 index 00000000000..6cda3a3e4be --- /dev/null +++ b/tests/basic/ec/ec-rebalance.t @@ -0,0 +1,61 @@ +#!/bin/bash +# +# This will test the rebalance failure reported in 1447559 +# +### + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../fallocate.rc + +cleanup + +#cleate and start volume +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume set $V0 lookup-optimize on +TEST $CLI volume start $V0 + +#Mount the volume +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +# Create files +for i in {1..10} +do + dd if=/dev/urandom of=$M0/file$i bs=1024k count=1 +done + +md5_1=$(md5sum $M0/file1 | awk '{print $1}') +md5_2=$(md5sum $M0/file2 | awk '{print $1}') +md5_3=$(md5sum $M0/file3 | awk '{print $1}') +md5_4=$(md5sum $M0/file4 | awk '{print $1}') +md5_5=$(md5sum $M0/file5 | awk '{print $1}') +md5_6=$(md5sum $M0/file6 | awk '{print $1}') +md5_7=$(md5sum $M0/file7 | awk '{print $1}') +md5_8=$(md5sum $M0/file8 | awk '{print $1}') +md5_9=$(md5sum $M0/file9 | awk '{print $1}') +md5_10=$(md5sum $M0/file10 | awk '{print $1}') +# Add brick +TEST $CLI volume add-brick $V0 $H0:$B0/${V0}{3..5} + +#Trigger rebalance +TEST $CLI volume rebalance $V0 start force +EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" rebalance_status_field $V0 + +#Remount to avoid any caches +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; +EXPECT "$md5_1" echo $(md5sum $M0/file1 | awk '{print $1}') +EXPECT "$md5_2" echo $(md5sum $M0/file2 | awk '{print $1}') +EXPECT "$md5_3" echo $(md5sum $M0/file3 | awk '{print $1}') +EXPECT "$md5_4" echo $(md5sum $M0/file4 | awk '{print $1}') +EXPECT "$md5_5" echo $(md5sum $M0/file5 | awk '{print $1}') +EXPECT "$md5_6" echo $(md5sum $M0/file6 | awk '{print $1}') +EXPECT "$md5_7" echo $(md5sum $M0/file7 | awk '{print $1}') +EXPECT "$md5_8" echo $(md5sum $M0/file8 | awk '{print $1}') +EXPECT "$md5_9" echo $(md5sum $M0/file9 | awk '{print $1}') +EXPECT "$md5_10" echo $(md5sum $M0/file10 | awk '{print $1}') + +cleanup; diff --git a/tests/basic/ec/ec-reset-brick.t b/tests/basic/ec/ec-reset-brick.t new file mode 100644 index 00000000000..f1a625df4ff --- /dev/null +++ b/tests/basic/ec/ec-reset-brick.t @@ -0,0 +1,50 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup +function num_entries { + ls -l $1 | wc -l +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +mkdir $M0/dir +touch $M0/dir/{1..10} + +mkdir $M0/dir/dir1 +touch $M0/dir/dir1/{1..10} + +#kill brick process +TEST $CLI volume reset-brick $V0 $H0:$B0/${V0}5 start +EXPECT_WITHIN $CHILD_UP_TIMEOUT "5" ec_child_up_count $V0 0 + +#reset-brick by removing all the data and create dir again +rm -rf $B0/${V0}5 +mkdir $B0/${V0}5 + +#start brick process and heal by commiting reset-brick +TEST $CLI volume reset-brick $V0 $H0:$B0/${V0}5 $H0:$B0/${V0}5 commit force + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count_shd $V0 0 + +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0} + +EXPECT "^12$" num_entries $B0/${V0}5/dir +EXPECT "^11$" num_entries $B0/${V0}5/dir/dir1 + +ec_version=$(get_hex_xattr trusted.ec.version $B0/${V0}0) +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}1 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}2 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}3 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}4 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}5 + +cleanup; diff --git a/tests/basic/ec/ec-root-heal.t b/tests/basic/ec/ec-root-heal.t new file mode 100644 index 00000000000..11ea7cdf9d4 --- /dev/null +++ b/tests/basic/ec/ec-root-heal.t @@ -0,0 +1,34 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup +function num_entries { + ls -l $1 | wc -l +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume start $V0 +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 +touch $M0/{1..10} +TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}5 $H0:$B0/${V0}6 commit force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count_shd $V0 0 + +# active heal +TEST $CLI volume heal $V0 full +#ls -l gives "Total" line so number of lines will be 1 more +EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count ${V0} +EXPECT "^11$" num_entries $B0/${V0}6 +ec_version=$(get_hex_xattr trusted.ec.version $B0/${V0}0) +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}1 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}2 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}3 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}4 +EXPECT "$ec_version" get_hex_xattr trusted.ec.version $B0/${V0}6 + +cleanup; diff --git a/tests/basic/ec/ec-seek.t b/tests/basic/ec/ec-seek.t new file mode 100644 index 00000000000..5a7d31b9f8f --- /dev/null +++ b/tests/basic/ec/ec-seek.t @@ -0,0 +1,58 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup + +SEEK=$(dirname $0)/seek +build_tester $(dirname $0)/../seek.c -o ${SEEK} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info + +TEST mkdir -p $B0/${V0}{0..2} +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} + +EXPECT "$V0" volinfo_field $V0 'Volume Name' +EXPECT 'Created' volinfo_field $V0 'Status' +EXPECT '3' brick_count $V0 + +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status' + +TEST $GFS -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +TEST ${SEEK} create ${M0}/test 0 1 1048576 1 +# Determine underlying filesystem allocation block size +BSIZE="$(($(${SEEK} scan ${M0}/test hole 0) * 2))" + +TEST ${SEEK} create ${M0}/test 0 ${BSIZE} $((${BSIZE} * 4 + 512)) ${BSIZE} + +EXPECT "^0$" ${SEEK} scan ${M0}/test data 0 +EXPECT "^$((${BSIZE} / 2))$" ${SEEK} scan ${M0}/test data $((${BSIZE} / 2)) +EXPECT "^$((${BSIZE} - 1))$" ${SEEK} scan ${M0}/test data $((${BSIZE} - 1)) +EXPECT "^$((${BSIZE} * 4))$" ${SEEK} scan ${M0}/test data ${BSIZE} +EXPECT "^$((${BSIZE} * 4))$" ${SEEK} scan ${M0}/test data $((${BSIZE} * 4)) +EXPECT "^$((${BSIZE} * 5))$" ${SEEK} scan ${M0}/test data $((${BSIZE} * 5)) +EXPECT "^$((${BSIZE} * 5 + 511))$" ${SEEK} scan ${M0}/test data $((${BSIZE} * 5 + 511)) +EXPECT "^ENXIO$" ${SEEK} scan ${M0}/test data $((${BSIZE} * 5 + 512)) +EXPECT "^ENXIO$" ${SEEK} scan ${M0}/test data $((${BSIZE} * 6)) + +EXPECT "^${BSIZE}$" ${SEEK} scan ${M0}/test hole 0 +EXPECT "^${BSIZE}$" ${SEEK} scan ${M0}/test hole $((${BSIZE} / 2)) +EXPECT "^${BSIZE}$" ${SEEK} scan ${M0}/test hole $((${BSIZE} - 1)) +EXPECT "^${BSIZE}$" ${SEEK} scan ${M0}/test hole ${BSIZE} +EXPECT "^$((${BSIZE} * 5 + 512))$" ${SEEK} scan ${M0}/test hole $((${BSIZE} * 4)) +EXPECT "^$((${BSIZE} * 5 + 512))$" ${SEEK} scan ${M0}/test hole $((${BSIZE} * 5)) +EXPECT "^$((${BSIZE} * 5 + 512))$" ${SEEK} scan ${M0}/test hole $((${BSIZE} * 5 + 511)) +EXPECT "^ENXIO$" ${SEEK} scan ${M0}/test hole $((${BSIZE} * 5 + 512)) +EXPECT "^ENXIO$" ${SEEK} scan ${M0}/test hole $((${BSIZE} * 6)) + +rm -f ${SEEK} +cleanup + +# Centos6 regression slaves seem to not support SEEK_DATA/SEEK_HOLE +#G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=000000 diff --git a/tests/basic/ec/ec-stripe.t b/tests/basic/ec/ec-stripe.t new file mode 100644 index 00000000000..98b92294feb --- /dev/null +++ b/tests/basic/ec/ec-stripe.t @@ -0,0 +1,227 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# These tests will check the stripe cache functionality of +# disperse volume + +test_index=0 +stripe_count=4 +loop_test=0 + +TESTS_EXPECTED_IN_LOOP=182 + +function get_mount_stripe_cache { + local sd=$1 + local field=$2 + local val=$(grep "$field" $sd | cut -f2 -d'=' | tail -1) + echo $val +} + +function get_stripes_in_cache { + local target=$1 + local count=$2 + local c=0 + for (( c=0; c<$count; c++ )) + do + let x=102+$c*1024 + echo yy | dd of=$target oflag=seek_bytes,sync seek=$x conv=notrunc + if [ $? != 0 ] + then + break + fi + done + echo "$c" +} +# tests in this loop = 7 +function mount_get_test_files { + let test_index+=1 + let loop_test+=7 + echo "Test Case $test_index" + local stripe_count=$1 + TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0; + EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + TEST dd if=/dev/urandom of=$B0/test_file bs=1024 count=20 + TEST cp $B0/test_file $M0/test_file + TEST dd if=/dev/urandom of=$B0/misc_file bs=1024 count=20 + EXPECT_WITHIN $UMOUNT_TIMEOUT "$stripe_count" get_stripes_in_cache $B0/test_file $stripe_count + EXPECT_WITHIN $UMOUNT_TIMEOUT "$stripe_count" get_stripes_in_cache $M0/test_file $stripe_count +} + +#check_statedump_md5sum (hitcount misscount) +#tests in this loop = 4 +function check_statedump_md5sum { + statedump=$(generate_mount_statedump $V0) + let loop_test+=4 + sleep 1 + nhits=$(get_mount_stripe_cache $statedump "hits") + nmisses=$(get_mount_stripe_cache $statedump "misses") + EXPECT "$1" echo $nhits + EXPECT "$2" echo $nmisses + TEST md5_sum=`get_md5_sum $B0/test_file` + EXPECT $md5_sum get_md5_sum $M0/test_file +} + +#tests in this loop = 2 +function clean_file_unmount { + let loop_test+=2 + TEST rm -f $B0/test_file $M0/test_file $B0/misc_file + cleanup_mount_statedump $V0 + EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +} + +cleanup +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume heal $V0 disable +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume set $V0 disperse.eager-lock on +TEST $CLI volume set $V0 disperse.other-eager-lock on +TEST $CLI volume set $V0 disperse.stripe-cache 8 +TEST $CLI volume start $V0 + +### 1 - offset and size in one stripes #### + +mount_get_test_files $stripe_count +# This should have 4 hits on cached stripes +get_stripes_in_cache $M0/test_file $stripe_count +check_statedump_md5sum 4 4 +clean_file_unmount + +### 2 - Length less than a stripe size, covering two stripes #### + +mount_get_test_files $stripe_count +TEST dd if=$B0/misc_file of=$B0/test_file bs=1022 count=1 oflag=seek_bytes,sync seek=102 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1022 count=1 oflag=seek_bytes,sync seek=102 conv=notrunc +check_statedump_md5sum 2 4 +clean_file_unmount + +### 3 -Length exactly equal to the stripe size, covering a single stripe #### + +mount_get_test_files $stripe_count +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=0 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=0 conv=notrunc +check_statedump_md5sum 0 4 +clean_file_unmount + +### 4 - Length exactly equal to the stripe size, covering two stripes #### + +mount_get_test_files $stripe_count +TEST dd if=$B0/misc_file of=$B0/test_file bs=2048 count=1 oflag=seek_bytes,sync seek=1024 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=2048 count=1 oflag=seek_bytes,sync seek=1024 conv=notrunc +check_statedump_md5sum 0 4 +clean_file_unmount + +### 5 - Length greater than a stripe size, covering two stripes #### + +mount_get_test_files $stripe_count +TEST dd if=$B0/misc_file of=$B0/test_file bs=1030 count=1 oflag=seek_bytes,sync seek=500 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1030 count=1 oflag=seek_bytes,sync seek=500 conv=notrunc +check_statedump_md5sum 2 4 +clean_file_unmount + +### 6 - Length greater than a stripe size, covering three stripes #### + +mount_get_test_files $stripe_count +TEST dd if=$B0/misc_file of=$B0/test_file bs=2078 count=1 oflag=seek_bytes,sync seek=1000 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=2078 count=1 oflag=seek_bytes,sync seek=1000 conv=notrunc +check_statedump_md5sum 2 4 +clean_file_unmount + +### 7 - Discard range - all stripe from cache should be invalidated complete stripes #### + +mount_get_test_files $stripe_count +TEST fallocate -p -o 0 -l 5120 $B0/test_file +TEST fallocate -p -o 0 -l 5120 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=6 oflag=seek_bytes,sync seek=1030 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=6 oflag=seek_bytes,sync seek=1030 conv=notrunc +check_statedump_md5sum 5 11 +clean_file_unmount + +### 8 - Discard range - starts in the middle of stripe, ends on the middle of next stripe#### + +mount_get_test_files $stripe_count +TEST fallocate -p -o 500 -l 1024 $B0/test_file +TEST fallocate -p -o 500 -l 1024 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=5 oflag=seek_bytes,sync seek=500 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=5 oflag=seek_bytes,sync seek=500 conv=notrunc +check_statedump_md5sum 10 6 +clean_file_unmount + +### 9 - Discard range - starts in the middle of stripe, ends on the middle of 3rd stripe##### + +mount_get_test_files $stripe_count +TEST fallocate -p -o 500 -l 2048 $B0/test_file +TEST fallocate -p -o 500 -l 2048 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=5 oflag=seek_bytes,sync seek=500 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=5 oflag=seek_bytes,sync seek=500 conv=notrunc +check_statedump_md5sum 9 7 +clean_file_unmount + +### 10 - Discard range - starts and end within one stripe #### + +mount_get_test_files $stripe_count +TEST fallocate -p -o 500 -l 100 $B0/test_file +TEST fallocate -p -o 500 -l 100 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=0 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=0 conv=notrunc +check_statedump_md5sum 1 4 +clean_file_unmount + +### 11 - Discard range - starts and end in one complete stripe #### + +mount_get_test_files $stripe_count +TEST fallocate -p -o 0 -l 1024 $B0/test_file +TEST fallocate -p -o 0 -l 1024 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=512 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=512 conv=notrunc +check_statedump_md5sum 1 5 +clean_file_unmount + +### 12 - Discard range - starts and end two complete stripe #### + +mount_get_test_files $stripe_count +TEST fallocate -p -o 0 -l 2048 $B0/test_file +TEST fallocate -p -o 0 -l 2048 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=4 oflag=seek_bytes,sync seek=300 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=4 oflag=seek_bytes,sync seek=300 conv=notrunc +check_statedump_md5sum 5 7 +clean_file_unmount + +### 13 - Truncate to invalidate all the stripe in cache #### + +mount_get_test_files $stripe_count +TEST truncate -s 0 $B0/test_file +TEST truncate -s 0 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1022 count=5 oflag=seek_bytes,sync seek=400 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1022 count=5 oflag=seek_bytes,sync seek=400 conv=notrunc +check_statedump_md5sum 4 4 +clean_file_unmount + +### 14 - Truncate to invalidate all but one the stripe in cache #### + +mount_get_test_files $stripe_count +TEST truncate -s 500 $B0/test_file +TEST truncate -s 500 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=525 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1024 count=1 oflag=seek_bytes,sync seek=525 conv=notrunc +check_statedump_md5sum 2 4 +clean_file_unmount + +### 15 - Truncate to invalidate all but one the stripe in cache #### +mount_get_test_files $stripe_count +TEST truncate -s 2148 $B0/test_file +TEST truncate -s 2148 $M0/test_file +TEST dd if=$B0/misc_file of=$B0/test_file bs=1000 count=1 oflag=seek_bytes,sync seek=2050 conv=notrunc +TEST dd if=$B0/misc_file of=$M0/test_file bs=1000 count=1 oflag=seek_bytes,sync seek=2050 conv=notrunc +check_statedump_md5sum 2 4 +clean_file_unmount +echo "Total loop tests $loop_test" +cleanup diff --git a/tests/basic/ec/ec-up.t b/tests/basic/ec/ec-up.t new file mode 100644 index 00000000000..d54e7e1d022 --- /dev/null +++ b/tests/basic/ec/ec-up.t @@ -0,0 +1,28 @@ +#!/bin/bash +#Tests that ec subvolume is up/down correctly + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +. $(dirname $0)/../../ec.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse-data 2 redundancy 1 $H0:$B0/${V0}{0,1,3,4,5,6} +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0; +EXPECT "1" ec_up_status $V0 $M0 0 +EXPECT "1" ec_up_status $V0 $M0 1 + +#kill two bricks in first disperse subvolume and check that ec_up_status is 0 for it +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 + +EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT "0" ec_up_status $V0 $M0 0 +EXPECT "1" ec_up_status $V0 $M0 1 + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" ec_up_status $V0 $M0 0 +EXPECT "1" ec_up_status $V0 $M0 1 +cleanup; diff --git a/tests/basic/ec/ec.t b/tests/basic/ec/ec.t new file mode 100644 index 00000000000..cc882771501 --- /dev/null +++ b/tests/basic/ec/ec.t @@ -0,0 +1,259 @@ +#!/bin/bash + +. $(dirname $0)/../../traps.rc +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +TEST_USER=test-ec-user +TEST_UID=27341 + +function my_getfattr { + getfattr --only-values -e text $* 2> /dev/null +} + +function get_rep_count { + v=$(my_getfattr -n trusted.jbr.rep-count $1) + #echo $v > /dev/tty + echo $v +} + +function create_file { + dd if=/dev/urandom of=$1 bs=4k count=$2 conv=sync 2> /dev/null +} + +function setup_perm_file { + mkdir $1/perm_dir || return 1 + chown ${TEST_USER} $1/perm_dir || return 1 + su ${TEST_USER} -c "touch $1/perm_dir/perm_file" || return 1 + return 0 +} + +# Functions to check repair for specific operation types. + +function check_create_write { + for b in $*; do + cmp $tmpdir/create-write $b/create-write || return 1 + done + return 0 +} + +function check_truncate { + truncate -s 8192 $tmpdir/truncate + for b in $*; do + cmp $tmpdir/truncate $b/truncate || return 1 + done + return 0 +} + +function check_hard_link { + stat $M0/hard-link-1 + stat $M0/hard-link-2 + for b in $*; do + inum1=$(ls -i $b/hard-link-1 | cut -d' ' -f1) + inum2=$(ls -i $b/hard-link-2 | cut -d' ' -f1) + if [ "$inum1" != "$inum2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_soft_link { + stat $M0/soft-link + for b in $*; do + if [ "$(readlink $b/soft-link)" != "soft-link-tgt" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_unlink { + stat $M0/unlink + for b in $*; do + if [ -e $b/unlink ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_mkdir { + getfattr -m. -d $M0/mkdir + for b in $*; do + if [ ! -d $b/mkdir ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_rmdir { + getfattr -m. -d $M0/rmdir + for b in $*; do + if [ -e $b/rmdir ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_setxattr { + getfattr -d -m. -e hex $M0/setxattr + for b in $*; do + v=$(my_getfattr -n user.foo $b/setxattr) + if [ "$v" != "ash_nazg_durbatuluk" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_removexattr { + getfattr -d -m. -e hex $M0/removexattr + for b in $*; do + my_getfattr -n user.bar $b/removexattr 2> /dev/null + if [ $? -eq 0 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_perm_file { + stat $M0/perm_dir/perm_file + getfattr -m. -d $M0/perm_dir + b1=$1 + shift 1 + ftext=$(stat -c "%u %g %a" $b1/perm_dir/perm_file) + #echo "first u/g/a = $ftext" > /dev/tty + for b in $*; do + btext=$(stat -c "%u %g %a" $b/perm_dir/perm_file) + #echo " next u/a/a = $btext" > /dev/tty + if [ x"$btext" != x"$ftext" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +cleanup + +TEST useradd -o -M -u ${TEST_UID} ${TEST_USER} +push_trapfunc "userdel --force ${TEST_USER}" + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info + +TEST mkdir -p $B0/${V0}{0,1,2,3,4,5,6,7,8,9} +TEST $CLI volume create $V0 disperse 10 redundancy 2 $H0:$B0/${V0}{0,1,2,3,4,5,6,7,8,9} + +EXPECT "$V0" volinfo_field $V0 'Volume Name' +EXPECT 'Created' volinfo_field $V0 'Status' +EXPECT '10' brick_count $V0 + +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status' + +# Mount FUSE with caching disabled +TEST $GFS -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0 + +# Create local files for comparisons etc. +tmpdir=$(mktemp -d -t ${0##*/}.XXXXXX) +push_trapfunc "rm -rf $tmpdir" +TEST create_file $tmpdir/create-write 10 +TEST create_file $tmpdir/truncate 10 + +# Prepare files and directories we'll need later. +TEST cp $tmpdir/truncate $M0/ +TEST touch $M0/hard-link-1 +TEST touch $M0/unlink +TEST mkdir $M0/rmdir +TEST touch $M0/setxattr +TEST touch $M0/removexattr +TEST setfattr -n user.bar -v "ash_nazg_gimbatul" $M0/removexattr + +sleep 2 + +# Kill a couple of bricks and allow some time for things to settle. +TEST kill_brick $V0 $H0 $B0/${V0}3 +TEST kill_brick $V0 $H0 $B0/${V0}8 +sleep 10 + +# Test create+write +TEST cp $tmpdir/create-write $M0/ +# Test truncate +TEST truncate -s 8192 $M0/truncate +# Test hard link +TEST ln $M0/hard-link-1 $M0/hard-link-2 +# Test soft link +TEST ln -s soft-link-tgt $M0/soft-link +# Test unlink +TEST rm $M0/unlink +# Test rmdir +TEST rmdir $M0/rmdir +# Test mkdir +TEST mkdir $M0/mkdir +# Test setxattr +TEST setfattr -n user.foo -v "ash_nazg_durbatuluk" $M0/setxattr +# Test removexattr +TEST setfattr -x user.bar $M0/removexattr +# Test uid/gid behavior +TEST setup_perm_file $M0 + +sleep 2 + +# Unmount/remount so that create/write and truncate don't see cached data. +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS -s $H0 --volfile-id $V0 $M1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "8" ec_child_up_count $V0 0 + +# Test create/write and truncate *before* the bricks are brought back. +TEST check_create_write $M1 +TEST check_truncate $M1 + +# Restart the bricks and allow repair to occur. +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status' +EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0 + +# Unmount/remount again, same reason as before. +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1 +TEST $GFS -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "10" ec_child_up_count $V0 0 + +# Make sure everything is as it should be. Most tests check for consistency +# between the bricks and the front end. This is not valid for disperse, so we +# check the mountpoint state instead. + +TEST check_create_write $M0 +TEST check_truncate $M0 + +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_hard_link $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_soft_link $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_unlink $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_rmdir $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mkdir $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_setxattr $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_removexattr $B0/${V0}{0..9} +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_perm_file $B0/${V0}{0..9} + +cleanup diff --git a/tests/basic/ec/gfapi-ec-open-truncate.c b/tests/basic/ec/gfapi-ec-open-truncate.c new file mode 100644 index 00000000000..fb16807003a --- /dev/null +++ b/tests/basic/ec/gfapi-ec-open-truncate.c @@ -0,0 +1,171 @@ +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <limits.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +#define LOG_ERR(msg) \ + do { \ + fprintf(stderr, "%s : Error (%s)\n", msg, strerror(errno)); \ + } while (0) + +int +fill_iov(struct iovec *iov, char fillchar, int count) +{ + int ret = -1; + + iov->iov_base = calloc(count + 1, sizeof(fillchar)); + if (iov->iov_base == NULL) { + return ret; + } else { + iov->iov_len = count; + ret = 0; + } + memset(iov->iov_base, fillchar, count); + memset(iov->iov_base + count, '\0', 1); + + return ret; +} + +glfs_t * +init_glfs(const char *hostname, const char *volname, const char *logfile) +{ + int ret = -1; + glfs_t *fs = NULL; + + fs = glfs_new(volname); + if (!fs) { + LOG_ERR("glfs_new failed"); + return NULL; + } + + ret = glfs_set_volfile_server(fs, "tcp", hostname, 24007); + if (ret < 0) { + LOG_ERR("glfs_set_volfile_server failed"); + goto out; + } + + ret = glfs_set_logging(fs, logfile, 7); + if (ret < 0) { + LOG_ERR("glfs_set_logging failed"); + goto out; + } + + ret = glfs_init(fs); + if (ret < 0) { + LOG_ERR("glfs_init failed"); + goto out; + } + + ret = 0; +out: + if (ret) { + glfs_fini(fs); + fs = NULL; + } + + return fs; +} + +int +main(int argc, char *argv[]) +{ + char *hostname = NULL; + char *volname = NULL; + char *logfile = NULL; + glfs_t *fs = NULL; + glfs_fd_t *glfd = NULL; + int ret = 0; + int i = 0; + int count = 200; + struct iovec iov = {0}; + int flags = O_RDWR; + int bricksup = 0; + int fdopen = 0; + + if (argc != 4) { + fprintf(stderr, "Invalid argument\n"); + exit(1); + } + + hostname = argv[1]; + volname = argv[2]; + logfile = argv[3]; + + fs = init_glfs(hostname, volname, logfile); + if (fs == NULL) { + LOG_ERR("init_glfs failed"); + return -1; + } + + /* Brick is down and we are opening a file to trigger fd heal. */ + /* Bypass Write-behind */ + glfd = glfs_open(fs, "a", O_WRONLY | O_TRUNC | O_SYNC); + if (glfd == NULL) { + LOG_ERR("glfs_open_truncate failed"); + exit(1); + } + system("gluster --mode=script volume start patchy force"); + /*CHILD_UP_TIMEOUT is 20 seconds*/ + for (i = 0; i < 20; i++) { + ret = system( + "[ $(gluster --mode=script volume status patchy | " + "grep \" Y \" | awk '{print $(NF-1)}' | wc -l) == 3 ]"); + if (WIFEXITED(ret) && WEXITSTATUS(ret)) { + printf("Ret value of system: %d\n, ifexited: %d, exitstatus: %d", + ret, WIFEXITED(ret), WEXITSTATUS(ret)); + sleep(1); + continue; + } + printf("Number of loops: %d\n", i); + bricksup = 1; + break; + } + if (!bricksup) { + system("gluster --mode=script volume status patchy"); + LOG_ERR("Bricks didn't come up\n"); + exit(1); + } + + /*Not sure how to check that the child-up reached EC, so sleep 3 for now*/ + sleep(3); + ret = fill_iov(&iov, 'a', 200); + if (ret) { + LOG_ERR("failed to create iov"); + exit(1); + } + + /*write will trigger re-open*/ + ret = glfs_pwritev(glfd, &iov, 1, 0, flags); + if (ret < 0) { + LOG_ERR("glfs_test_function failed"); + exit(1); + } + /*Check reopen happened by checking for open-fds on the brick*/ + for (i = 0; i < 20; i++) { + ret = system( + "[ $(for i in $(pgrep glusterfsd); do ls -l /proc/$i/fd | grep " + "\"[.]glusterfs\" | grep -v \".glusterfs/[0-9a-f][0-9a-f]\" | grep " + "-v health_check; done | wc -l) == 3 ]"); + if (WIFEXITED(ret) && WEXITSTATUS(ret)) { + printf("Ret value of system: %d\n, ifexited: %d, exitstatus: %d", + ret, WIFEXITED(ret), WEXITSTATUS(ret)); + sleep(1); + continue; + } + fdopen = 1; + break; + } + + if (!fdopen) { + LOG_ERR("fd reopen didn't succeed"); + exit(1); + } + + return 0; +} diff --git a/tests/basic/ec/gfapi-ec-open-truncate.t b/tests/basic/ec/gfapi-ec-open-truncate.t new file mode 100644 index 00000000000..e22562c6ea3 --- /dev/null +++ b/tests/basic/ec/gfapi-ec-open-truncate.t @@ -0,0 +1,48 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +#This .t tests the functionality of open-fd-heal when opened with O_TRUNC. +#If re-open is not done with O_TRUNC then the test will pass. + +cleanup + +TEST glusterd + +TEST $CLI volume create $V0 disperse 3 ${H0}:$B0/brick{1,2,3} +EXPECT 'Created' volinfo_field $V0 'Status' +#Disable heals to prevent any chance of heals masking the problem +TEST $CLI volume set $V0 disperse.background-heals 0 +TEST $CLI volume set $V0 disperse.heal-wait-qlength 0 +TEST $CLI volume set $V0 performance.write-behind off + +#We need truncate fop to go through before pre-op completes for the write-fop +#which triggers open-fd heal. Otherwise truncate won't be allowed on 'bad' brick +TEST $CLI volume set $V0 delay-gen posix +TEST $CLI volume set $V0 delay-gen.enable fxattrop +TEST $CLI volume set $V0 delay-gen.delay-percentage 100 +TEST $CLI volume set $V0 delay-gen.delay-duration 1000000 + +TEST $CLI volume heal $V0 disable + +TEST $CLI volume start $V0 +TEST $CLI volume profile $V0 start +EXPECT 'Started' volinfo_field $V0 'Status' +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST touch $M0/a +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +TEST kill_brick $V0 $H0 $B0/brick1 +logdir=`gluster --print-logdir` + +TEST build_tester $(dirname $0)/gfapi-ec-open-truncate.c -lgfapi + +TEST $CLI volume profile $V0 info clear +TEST ./$(dirname $0)/gfapi-ec-open-truncate ${H0} $V0 $logdir/gfapi-ec-open-truncate.log + +EXPECT "^2$" echo $($CLI volume profile $V0 info incremental | grep -i truncate | wc -l) +cleanup_tester $(dirname $0)/gfapi-ec-open-truncate + +cleanup diff --git a/tests/basic/ec/heal-info.t b/tests/basic/ec/heal-info.t new file mode 100644 index 00000000000..1549d5fcdb0 --- /dev/null +++ b/tests/basic/ec/heal-info.t @@ -0,0 +1,74 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks if heal info works as expected or not + +function create_files { + for i in {21..1000}; + do + dd if=/dev/zero of=$M0/$i bs=1M count=1 2>&1 > /dev/null; + done + rm -f $M0/lock +} + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume set $V0 client-log-level DEBUG +TEST $CLI volume heal $V0 disable +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 --direct-io-mode=yes $M0; +# Wait until all 6 childs have been recognized by the ec xlator +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +#heal info should give zero entries to be healed when I/O is going on +dd if=/dev/zero of=$M0/a bs=1M count=2048 & +dd_pid=$! +sleep 3 #Wait for I/O to proceed for some time +EXPECT "^0$" get_pending_heal_count $V0 +kill -9 $dd_pid +touch $M0/lock +create_files & + +total_heal_count=0 +while [ -f $M0/lock ]; +do + heal_count=$(get_pending_heal_count $V0) + total_heal_count=$((heal_count+total_heal_count)) +done +EXPECT "^0$" echo $total_heal_count + +#When only data heal is required it should print it +#There is no easy way to create this using commands so assigning xattrs directly +TEST setfattr -n trusted.ec.version -v 0x00000000000000020000000000000000 $B0/${V0}0/1000 +TEST setfattr -n trusted.ec.version -v 0x00000000000000020000000000000000 $B0/${V0}1/1000 +TEST setfattr -n trusted.ec.version -v 0x00000000000000020000000000000000 $B0/${V0}2/1000 +TEST setfattr -n trusted.ec.version -v 0x00000000000000020000000000000000 $B0/${V0}3/1000 +TEST setfattr -n trusted.ec.version -v 0x00000000000000020000000000000000 $B0/${V0}4/1000 +TEST setfattr -n trusted.ec.version -v 0x00000000000000010000000000000000 $B0/${V0}5/1000 +index_path=$B0/${V0}5/.glusterfs/indices/xattrop/$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}5/1000)) +while [ -f $index_path ]; do :; done +TEST touch $index_path +EXPECT "^1$" get_pending_heal_count $V0 +TEST rm -f $M0/1000 + +#When files/directories need heal test that it prints them +TEST touch $M0/{1..10} +TEST kill_brick $V0 $H0 $B0/${V0}0 +for i in {11..20}; +do + echo abc > $M0/$i #Data + entry + metadata heal +done +for i in {1..10}; +do + chmod +x $M0/$i; +done + +EXPECT "^105$" get_pending_heal_count $V0 + +cleanup +#G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=1533815 diff --git a/tests/basic/ec/lock-contention.t b/tests/basic/ec/lock-contention.t new file mode 100644 index 00000000000..8f86cee16ad --- /dev/null +++ b/tests/basic/ec/lock-contention.t @@ -0,0 +1,62 @@ +#!/bin/bash + +# This test verifies that when 'lock-notify-contention' option is enabled, +# locks xlator actually sends an upcall notification that causes the acquired +# lock from one client to be released before it's supposed to when another +# client accesses the file. + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +function elapsed_time() { + local start="`date +%s`" + + if [[ "test" == `cat "$1"` ]]; then + echo "$((`date +%s` - ${start}))" + fi +} + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0..2} +TEST $CLI volume set $V0 performance.stat-prefetch off +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.quick-read off +TEST $CLI volume set $V0 performance.read-ahead off +TEST $CLI volume set $V0 performance.io-cache off +TEST $CLI volume set $V0 features.locks-notify-contention off +TEST $CLI volume set $V0 disperse.eager-lock on +TEST $CLI volume set $V0 disperse.eager-lock-timeout 6 +TEST $CLI volume set $V0 disperse.other-eager-lock on +TEST $CLI volume set $V0 disperse.other-eager-lock-timeout 6 +TEST $CLI volume start $V0 + +TEST $GFS --direct-io-mode=yes --volfile-id=/$V0 --volfile-server=$H0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 $M0 + +TEST $GFS --direct-io-mode=yes --volfile-id=/$V0 --volfile-server=$H0 $M1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 $M1 + +TEST $(echo "test" >$M0/file) + +# With locks-notify-contention set to off, accessing the file from another +# client should take 6 seconds. Checking against 3 seconds to be safe. +elapsed="$(elapsed_time $M1/file)" +TEST [[ ${elapsed} -ge 3 ]] + +elapsed="$(elapsed_time $M0/file)" +TEST [[ ${elapsed} -ge 3 ]] + +TEST $CLI volume set $V0 features.locks-notify-contention on + +# With locks-notify-contention set to on, accessing the file from another +# client should be fast. Checking against 3 seconds to be safe. +elapsed="$(elapsed_time $M1/file)" +TEST [[ ${elapsed} -le 3 ]] + +elapsed="$(elapsed_time $M0/file)" +TEST [[ ${elapsed} -le 3 ]] + +cleanup diff --git a/tests/basic/ec/nfs.t b/tests/basic/ec/nfs.t new file mode 100755 index 00000000000..3f51a640ef7 --- /dev/null +++ b/tests/basic/ec/nfs.t @@ -0,0 +1,29 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../nfs.rc + +#G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +EXPECT "Created" volinfo_field $V0 'Status' +TEST $CLI volume set $V0 nfs.disable false +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status' + +EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; +TEST mount_nfs $H0:/$V0 $N0 nolock + +# The test below fails with "bs=1024k count=1k", but passes when "oflag=direct" +# is used. There also does not seem to be an issue on systems with sufficient +# memory. Reducing the "count" prevents hangs too. +TEST dd if=/dev/zero of=$N0/test bs=1024k count=32 + +## Before killing daemon to avoid deadlocks +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" umount_nfs $N0 + +cleanup diff --git a/tests/basic/ec/quota.t b/tests/basic/ec/quota.t new file mode 100755 index 00000000000..c9612c8b76a --- /dev/null +++ b/tests/basic/ec/quota.t @@ -0,0 +1,45 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup +QDD=$(dirname $0)/quota +# compile the test write program and run it +build_tester $(dirname $0)/../quota.c -o $QDD + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 disperse $H0:$B0/${V0}{0..2} +EXPECT 'Created' volinfo_field $V0 'Status' +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT 'Started' volinfo_field $V0 'Status' +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 + +TEST mkdir -p $M0/test + +TEST $CLI volume quota $V0 enable + +TEST $CLI volume quota $V0 limit-usage /test 10MB + +EXPECT "10.0MB" quota_hard_limit "/test"; +EXPECT "80%" quota_soft_limit "/test"; + +TEST $CLI volume quota $V0 soft-timeout 0 +TEST $CLI volume quota $V0 hard-timeout 0 + +TEST ! $QDD $M0/test/file1.txt 256 48 +TEST rm $M0/test/file1.txt + +EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "0Bytes" quotausage "/test" + +TEST $QDD $M0/test/file2.txt 256 32 +EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "8.0MB" quotausage "/test" + +TEST rm $M0/test/file2.txt +EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "0Bytes" quotausage "/test" +TEST $CLI volume stop $V0 + +rm -f $QDD +cleanup; diff --git a/tests/basic/ec/self-heal-read-write-fail.t b/tests/basic/ec/self-heal-read-write-fail.t new file mode 100644 index 00000000000..0ba591b5bb2 --- /dev/null +++ b/tests/basic/ec/self-heal-read-write-fail.t @@ -0,0 +1,69 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +#This test verifies that self-heal fails when read/write fails as part of heal +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info + +TEST $CLI volume create $V0 disperse 3 redundancy 1 $H0:$B0/${V0}{0,1,2} +TEST $CLI volume heal $V0 disable +TEST $CLI volume start $V0 + +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +TEST touch $M0/a +TEST kill_brick $V0 $H0 $B0/${V0}0 +echo abc >> $M0/a + +# Umount the volume to force all pending writes to reach the bricks +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 + +#Load error-gen and fail read fop and test that heal fails +TEST $CLI volume stop $V0 #Stop volume so that error-gen can be loaded +TEST $CLI volume set $V0 debug.error-gen posix +TEST $CLI volume set $V0 debug.error-fops read +TEST $CLI volume set $V0 debug.error-number EBADF +TEST $CLI volume set $V0 debug.error-failure 100 + +TEST $CLI volume start $V0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 +TEST ! getfattr -n trusted.ec.heal $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 + +#fail write fop and test that heal fails +TEST $CLI volume stop $V0 +TEST $CLI volume set $V0 debug.error-fops write + +TEST $CLI volume start $V0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 +TEST ! getfattr -n trusted.ec.heal $M0/a +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 + +TEST $CLI volume stop $V0 #Stop volume so that error-gen can be disabled +TEST $CLI volume reset $V0 debug.error-gen +TEST $CLI volume reset $V0 debug.error-fops +TEST $CLI volume reset $V0 debug.error-number +TEST $CLI volume reset $V0 debug.error-failure + +TEST $CLI volume start $V0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $HEAL_TIMEOUT "^2$" get_pending_heal_count $V0 +TEST getfattr -n trusted.ec.heal $M0/a +EXPECT "^0$" get_pending_heal_count $V0 + +#Test that heal worked as expected by forcing read from brick0 +#remount to make sure data is not served from any cache +EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 +TEST kill_brick $V0 $H0 $B0/${V0}2 +EXPECT "abc" cat $M0/a + +cleanup diff --git a/tests/basic/ec/self-heal.t b/tests/basic/ec/self-heal.t new file mode 100644 index 00000000000..6329bb60248 --- /dev/null +++ b/tests/basic/ec/self-heal.t @@ -0,0 +1,235 @@ +#!/bin/bash + +SCRIPT_TIMEOUT=300 + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +# This test checks self-healing feature of dispersed volumes + +cleanup + +function check_mount_dir +{ + getfattr -d -m. -e hex $M0 2>&1 > /dev/null + for i in {1..20}; do + ls -l $M0/ | grep "dir1" + if [ $? -ne 0 ]; then + return 1 + fi + done + + return 0 +} + +function check_size +{ + cat $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%s" $i/$1` + if [ "$res" != "$2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_mode +{ + getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%A" $i/$1` + if [ "$res" != "$2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_date +{ + getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%Y" $i/$1` + if [ "$res" != "$2" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_xattr +{ + getfattr -d -m. -e hex $M0/$1 2>&1 > /dev/null + for i in "${brick[@]}"; do + getfattr -n $2 $i/$1 2>/dev/null + if [ $? -eq 0 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_dir +{ + getfattr -m. -d $M0/dir1 2>&1 > /dev/null + for i in "${brick[@]}"; do + if [ ! -d $i/dir1 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_soft_link +{ + getfattr -d -m. -e hex $M0/test3 2>&1 > /dev/null + for i in "${brick[@]}"; do + if [ ! -h $i/test3 ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +function check_hard_link +{ + getfattr -d -m. -e hex $M0/test4 2>&1 > /dev/null + for i in "${brick[@]}"; do + res=`stat -c "%h" $i/test4` + if [ "$res" != "3" ]; then + echo "N" + return 0 + fi + done + echo "Y" + return 0 +} + +tmp=`mktemp -d -t ${0##*/}.XXXXXX` +if [ ! -d $tmp ]; then + exit 1 +fi + +TESTS_EXPECTED_IN_LOOP=194 + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy 2 $H0:$B0/${V0}{0..5} +TEST $CLI volume set $V0 client-log-level DEBUG +#Write-behind has a bug where lookup can race over write which leads to size mismatch on the mount after a 'cp' +TEST $CLI volume set $V0 performance.write-behind off +#md-cache can return stale stat due to default timeout being 1 sec +TEST $CLI volume set $V0 performance.stat-prefetch off +EXPECT "Created" volinfo_field $V0 'Status' +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status' +#direct-io-mode is to make sure 'cat' leads to READ fop which triggers heal +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 --direct-io-mode=yes $M0; +# Wait until all 6 childs have been recognized by the ec xlator +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +TEST dd if=/dev/urandom of=$tmp/test bs=1024 count=1024 + +cs=$(sha1sum $tmp/test | awk '{ print $1 }') + +TEST df -h $M0 +TEST stat $M0 + +for idx in {0..5}; do + brick[$idx]=$(gf_get_gfid_backend_file_path $B0/$V0$idx) +done + +TEST cp $tmp/test $M0/test +TEST chmod 644 $M0/test +TEST touch -d "@946681200" $M0/test +EXPECT "-rw-r--r--" stat -c "%A" $M0/test +EXPECT "946681200" stat -c "%Y" $M0/test + +for idx1 in {0..5}; do + TEST chmod 666 ${brick[$idx1]}/test + TEST truncate -s 0 ${brick[$idx1]}/test + TEST setfattr -n user.test -v "test1" ${brick[$idx1]}/test + EXPECT "-rw-r--r--" stat -c "%A" $M0/test + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test "262144" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test "-rw-r--r--" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_date test "946681200" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_xattr test "user.test" +done + +for idx1 in {0..4}; do + for idx2 in `seq $(($idx1 + 1)) 5`; do + if [ $idx1 -ne $idx2 ]; then + TEST chmod 666 ${brick[$idx1]}/test + TEST chmod 600 ${brick[$idx2]}/test + TEST truncate -s 0 ${brick[$idx1]}/test + TEST truncate -s 2097152 ${brick[$idx2]}/test + TEST setfattr -n user.test -v "test1" ${brick[$idx1]}/test + TEST setfattr -n user.test -v "test2" ${brick[$idx2]}/test + EXPECT "-rw-r--r--" stat -c "%A" $M0/test + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test "262144" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test "-rw-r--r--" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_date test "946681200" + EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_xattr test "user.test" + fi + done +done + +sleep 2 + +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST cp $tmp/test $M0/test2 +EXPECT "1048576" stat -c "%s" $M0/test2 +TEST chmod 777 $M0/test2 +EXPECT "-rwxrwxrwx" stat -c "%A" $M0/test2 + +TEST mkdir $M0/dir1 +TEST ls -al $M0/dir1 + +TEST ln -s test2 $M0/test3 +TEST [ -h $M0/test3 ] + +TEST ln $M0/test2 $M0/test4 +TEST [ -f $M0/test4 ] +EXPECT "2" stat -c "%h" $M0/test2 +EXPECT "2" stat -c "%h" $M0/test4 + +sleep 2 + +TEST $CLI volume start $V0 force +# Wait until the killed bricks have been started and recognized by the ec +# xlator +EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0 + +TEST check_mount_dir + +EXPECT "1048576" stat -c "%s" $M0/test2 +EXPECT "-rwxrwxrwx" stat -c "%A" $M0/test2 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_size test2 "262144" +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_mode test2 "-rwxrwxrwx" + +TEST ls -al $M0/dir1 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_dir + +TEST [ -h $M0/test3 ] +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_soft_link + +EXPECT "2" stat -c "%h" $M0/test4 +EXPECT_WITHIN $HEAL_TIMEOUT "Y" check_hard_link + +TEST rm -rf $tmp + +cleanup diff --git a/tests/basic/ec/statedump.t b/tests/basic/ec/statedump.t new file mode 100644 index 00000000000..8d311ec3d6d --- /dev/null +++ b/tests/basic/ec/statedump.t @@ -0,0 +1,28 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 redundancy 1 $H0:$B0/${V0}{0..2} +EXPECT "Created" volinfo_field $V0 'Status' +TEST $CLI volume start $V0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Started" volinfo_field $V0 'Status' +TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 + +EXPECT_WITHIN $CHILD_UP_TIMEOUT "3" ec_child_up_count $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_status $V0 0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_status $V0 0 1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_status $V0 0 2 + +TEST kill_brick $V0 $H0 $B0/${V0}0 + +EXPECT_WITHIN $CHILD_UP_TIMEOUT "2" ec_child_up_count $V0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "0" ec_child_up_status $V0 0 0 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_status $V0 0 1 +EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" ec_child_up_status $V0 0 2 + +cleanup |