From 097db13c11390174c5b9f11aa0fd87eca1735871 Mon Sep 17 00:00:00 2001 From: Amar Tumballi Date: Tue, 18 Aug 2020 14:08:20 +0530 Subject: tests: provide an option to mark tests as 'flaky' * also add some time gap in other tests to see if we get things properly * create a directory 'tests/000/', which can host any tests, which are flaky. * move all the tests mentioned in the issue to above directory. * as the above dir gets tested first, all flaky tests would be reported quickly. * change `run-tests.sh` to continue tests even if flaky tests fail. Reference: gluster/project-infrastructure#72 Updates: #1000 Change-Id: Ifdafa38d083ebd80f7ae3cbbc9aa3b68b6d21d0e Signed-off-by: Amar Tumballi --- tests/bugs/core/multiplex-limit-issue-151.t | 56 --------------- tests/bugs/distribute/bug-1117851.t | 101 ---------------------------- tests/bugs/distribute/bug-1122443.t | 60 ----------------- tests/bugs/glusterd/bug-857330/common.rc | 57 ---------------- tests/bugs/glusterd/bug-857330/normal.t | 69 ------------------- tests/bugs/glusterd/bug-857330/xml.t | 83 ----------------------- tests/bugs/glusterd/quorum-value-check.t | 37 ---------- tests/bugs/nfs/bug-1116503.t | 47 ------------- 8 files changed, 510 deletions(-) delete mode 100644 tests/bugs/core/multiplex-limit-issue-151.t delete mode 100755 tests/bugs/distribute/bug-1117851.t delete mode 100644 tests/bugs/distribute/bug-1122443.t delete mode 100644 tests/bugs/glusterd/bug-857330/common.rc delete mode 100755 tests/bugs/glusterd/bug-857330/normal.t delete mode 100755 tests/bugs/glusterd/bug-857330/xml.t delete mode 100755 tests/bugs/glusterd/quorum-value-check.t delete mode 100644 tests/bugs/nfs/bug-1116503.t (limited to 'tests/bugs') diff --git a/tests/bugs/core/multiplex-limit-issue-151.t b/tests/bugs/core/multiplex-limit-issue-151.t deleted file mode 100644 index dc9013061b0..00000000000 --- a/tests/bugs/core/multiplex-limit-issue-151.t +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../traps.rc -. $(dirname $0)/../../volume.rc - -function count_up_bricks { - $CLI --xml volume status all | grep '1' | wc -l -} - -function count_brick_processes { - pgrep glusterfsd | wc -l -} - -function count_brick_pids { - $CLI --xml volume status all | sed -n '/.*\([^<]*\).*/s//\1/p' \ - | grep -v "N/A" | sort | uniq | wc -l -} - -cleanup; - -TEST glusterd - -TEST $CLI volume set all cluster.brick-multiplex on -TEST ! $CLI volume set all cluster.max-bricks-per-process -1 -TEST ! $CLI volume set all cluster.max-bricks-per-process foobar -TEST $CLI volume set all cluster.max-bricks-per-process 3 - -TEST $CLI volume create $V0 $H0:$B0/brick{0..5} -TEST $CLI volume start $V0 - -EXPECT 2 count_brick_processes -EXPECT 2 count_brick_pids -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 count_up_bricks - -pkill gluster -TEST glusterd - -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_brick_processes -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_brick_pids -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 count_up_bricks - -TEST $CLI volume add-brick $V0 $H0:$B0/brick6 - -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_processes -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_pids -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 7 count_up_bricks - -TEST $CLI volume remove-brick $V0 $H0:$B0/brick3 start -TEST $CLI volume remove-brick $V0 $H0:$B0/brick3 force - -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_processes -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_pids -EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 count_up_bricks - -cleanup; diff --git a/tests/bugs/distribute/bug-1117851.t b/tests/bugs/distribute/bug-1117851.t deleted file mode 100755 index 62cb6b66ab4..00000000000 --- a/tests/bugs/distribute/bug-1117851.t +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -SCRIPT_TIMEOUT=250 - -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc - -create_files () { - for i in {1..1000}; do - orig=$(printf %s/abc%04d $1 $i) - real=$(printf %s/src%04d $1 $i) - # Make sure lots of these have linkfiles. - echo "This is file $i" > $orig - mv $orig $real - done - sync -} - -move_files_inner () { - sfile=$M0/status_$(basename $1) - for i in {1..1000}; do - src=$(printf %s/src%04d $1 $i) - dst=$(printf %s/dst%04d $1 $i) - mv $src $dst 2> /dev/null - done - echo "done" > $sfile -} - -move_files () { - #Create the status file here to prevent spurious failures - #caused by the file not being created in time by the - #background process - sfile=$M0/status_$(basename $1) - echo "running" > $sfile - move_files_inner $* & -} - -check_files () { - errors=0 - for i in {1..1000}; do - if [ ! -f $(printf %s/dst%04d $1 $i) ]; then - if [ -f $(printf %s/src%04d $1 $i) ]; then - echo "file $i didnt get moved" > /dev/stderr - else - echo "file $i is MISSING" > /dev/stderr - errors=$((errors+1)) - fi - fi - done - if [ $((errors)) != 0 ]; then - : ls -l $1 > /dev/stderr - fi - return $errors -} - -cleanup; - -TEST glusterd -TEST pidof glusterd -TEST $CLI volume info; - -TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4,5,6}; - -EXPECT "$V0" volinfo_field $V0 'Volume Name'; -EXPECT 'Created' volinfo_field $V0 'Status'; -EXPECT '6' brick_count $V0 - -TEST $CLI volume start $V0; -EXPECT 'Started' volinfo_field $V0 'Status'; - -## Mount FUSE with caching disabled (read-write) -TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0; - -TEST create_files $M0 - -## Mount FUSE with caching disabled (read-write) again -TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M1; - -TEST move_files $M0 -TEST move_files $M1 - -# It's regrettable that renaming 1000 files might take more than 30 seconds, -# but on our test systems sometimes it does, so double the time from what we'd -# use otherwise. There still seem to be some spurious failures, 1 in 20 when -# this does not complete, added an additional 60 seconds to take false reports -# out of the system, during test runs, especially on slower test systems. -EXPECT_WITHIN 120 "done" cat $M0/status_0 -EXPECT_WITHIN 120 "done" cat $M1/status_1 - -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1 -TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0; -TEST check_files $M0 - -TEST $CLI volume stop $V0; -EXPECT 'Stopped' volinfo_field $V0 'Status'; - -TEST $CLI volume delete $V0; -TEST ! $CLI volume info $V0; - -cleanup; diff --git a/tests/bugs/distribute/bug-1122443.t b/tests/bugs/distribute/bug-1122443.t deleted file mode 100644 index 906be7072bd..00000000000 --- a/tests/bugs/distribute/bug-1122443.t +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc -. $(dirname $0)/../../dht.rc - -make_files() { - mkdir $1 && \ - ln -s ../ $1/symlink && \ - mknod $1/special_b b 1 2 && \ - mknod $1/special_c c 3 4 && \ - mknod $1/special_u u 5 6 && \ - mknod $1/special_p p && \ - touch -h --date=@1 $1/symlink && \ - touch -h --date=@2 $1/special_b && - touch -h --date=@3 $1/special_c && - touch -h --date=@4 $1/special_u && - touch -h --date=@5 $1/special_p -} - -bug_1113050_workaround() { - # Test if graph change has settled (bug-1113050?) - test=$(stat -c "%n:%Y" $1 2>&1 | tr '\n' ',') - if [ $? -eq 0 ] ; then - echo RECONNECTED - else - echo WAITING - fi - return 0 -} - -cleanup - -TEST glusterd -TEST pidof glusterd - -TEST $CLI volume create $V0 $H0:$B0/${V0}0 -TEST $CLI volume start $V0 - -# Mount FUSE and create symlink -TEST glusterfs -s $H0 --volfile-id $V0 $M0 -TEST make_files $M0/subdir - -# Get mtime before migration -BEFORE="$(stat -c %n:%Y $M0/subdir/* | tr '\n' ',')" - -# Migrate brick -TEST $CLI volume add-brick $V0 $H0:$B0/${V0}1 -TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}0 start -EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field "$V0 $H0:$B0/${V0}0" -TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}0 commit - -# Get mtime after migration -EXPECT_WITHIN 5 RECONNECTED bug_1113050_workaround $M0/subdir/* -AFTER="$(stat -c %n:%Y $M0/subdir/* | tr '\n' ',')" - -# Check if mtime is unchanged -TEST [ "$AFTER" == "$BEFORE" ] - -cleanup diff --git a/tests/bugs/glusterd/bug-857330/common.rc b/tests/bugs/glusterd/bug-857330/common.rc deleted file mode 100644 index d0aa4b1a640..00000000000 --- a/tests/bugs/glusterd/bug-857330/common.rc +++ /dev/null @@ -1,57 +0,0 @@ -. $(dirname $0)/../../../include.rc - -UUID_REGEX='[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' - -TASK_ID="" -COMMAND="" -PATTERN="" - -function check-and-store-task-id() -{ - TASK_ID="" - - local task_id=$($CLI $COMMAND | grep $PATTERN | grep -o -E "$UUID_REGEX") - - if [ -z "$task_id" ] && [ "${task_id+asdf}" = "asdf" ]; then - return 1 - fi - - TASK_ID=$task_id - return 0; -} - -function get-task-id() -{ - $CLI $COMMAND | grep $PATTERN | grep -o -E "$UUID_REGEX" | tail -n1 - -} - -function check-and-store-task-id-xml() -{ - TASK_ID="" - - local task_id=$($CLI $COMMAND --xml | xmllint --format - | grep $PATTERN | grep -o -E "$UUID_REGEX") - - if [ -z "$task_id" ] && [ "${task_id+asdf}" = "asdf" ]; then - return 1 - fi - - TASK_ID=$task_id - return 0; -} - -function get-task-id-xml() -{ - $CLI $COMMAND --xml | xmllint --format - | grep $PATTERN | grep -o -E "$UUID_REGEX" -} - -function get-task-status() -{ - pattern=$1 - val=1 - test=$(gluster $COMMAND | grep -o $pattern 2>&1) - if [ $? -eq 0 ]; then - val=0 - fi - echo $val -} diff --git a/tests/bugs/glusterd/bug-857330/normal.t b/tests/bugs/glusterd/bug-857330/normal.t deleted file mode 100755 index ad0c8844fae..00000000000 --- a/tests/bugs/glusterd/bug-857330/normal.t +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -. $(dirname $0)/common.rc -. $(dirname $0)/../../../volume.rc -cleanup; - -TEST glusterd -TEST pidof glusterd -TEST $CLI volume info; - -TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}1 $H0:$B0/${V0}2; -TEST $CLI volume info $V0; -TEST $CLI volume start $V0; - -TEST glusterfs -s $H0 --volfile-id=$V0 $M0; - -TEST $PYTHON $(dirname $0)/../../../utils/create-files.py \ - --multi -b 10 -d 10 -n 10 $M0; - -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 - -############### -## Rebalance ## -############### -TEST $CLI volume add-brick $V0 replica 2 $H0:$B0/${V0}3 $H0:$B0/${V0}4; - -COMMAND="volume rebalance $V0 start" -PATTERN="ID:" -TEST check-and-store-task-id - -COMMAND="volume status $V0" -PATTERN="ID" -EXPECT $TASK_ID get-task-id - -COMMAND="volume rebalance $V0 status" -PATTERN="completed" -EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN - -################### -## Replace-brick ## -################### -REP_BRICK_PAIR="$H0:$B0/${V0}2 $H0:$B0/${V0}5" - -TEST $CLI volume replace-brick $V0 $REP_BRICK_PAIR commit force; - -################## -## Remove-brick ## -################## -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5 - -COMMAND="volume remove-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}5 start" -PATTERN="ID:" -TEST check-and-store-task-id - -COMMAND="volume status $V0" -PATTERN="ID" -EXPECT $TASK_ID get-task-id - -COMMAND="volume remove-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}5 status" -PATTERN="completed" -EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN - -TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}5 commit - -TEST $CLI volume stop $V0; -TEST $CLI volume delete $V0; -TEST ! $CLI volume info $V0; - -cleanup; diff --git a/tests/bugs/glusterd/bug-857330/xml.t b/tests/bugs/glusterd/bug-857330/xml.t deleted file mode 100755 index 8383d2a0711..00000000000 --- a/tests/bugs/glusterd/bug-857330/xml.t +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -. $(dirname $0)/common.rc -. $(dirname $0)/../../../volume.rc - -cleanup; - -TEST glusterd -TEST pidof glusterd -TEST $CLI volume info; - -TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}1 $H0:$B0/${V0}2; -TEST $CLI volume info $V0; -TEST $CLI volume start $V0; - -TEST glusterfs -s $H0 --volfile-id=$V0 $M0; - -TEST $PYTHON $(dirname $0)/../../../utils/create-files.py \ - --multi -b 10 -d 10 -n 10 $M0; - -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0 - - -############### -## Rebalance ## -############### -TEST $CLI volume add-brick $V0 replica 2 $H0:$B0/${V0}3 $H0:$B0/${V0}4; - -COMMAND="volume rebalance $V0 start" -PATTERN="task-id" -TEST check-and-store-task-id-xml - -COMMAND="volume status $V0" -PATTERN="id" -EXPECT $TASK_ID get-task-id-xml - -COMMAND="volume rebalance $V0 status" -PATTERN="task-id" -EXPECT $TASK_ID get-task-id-xml - -## TODO: Add tests for rebalance stop - -COMMAND="volume rebalance $V0 status" -PATTERN="completed" -EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN - -################### -## Replace-brick ## -################### -TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}4 $H0:$B0/${V0}5 commit force - -################## -## Remove-brick ## -################## -EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5 - -COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 start" -PATTERN="task-id" -TEST check-and-store-task-id-xml - -COMMAND="volume status $V0" -PATTERN="id" -EXPECT $TASK_ID get-task-id-xml - -COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 status" -PATTERN="task-id" -EXPECT $TASK_ID get-task-id-xml - -COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 status" -PATTERN="completed" -EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN - -## TODO: Add tests for remove-brick stop - -COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 commit" -PATTERN="task-id" -EXPECT $TASK_ID get-task-id-xml - -TEST $CLI volume stop $V0; -TEST $CLI volume delete $V0; -TEST ! $CLI volume info $V0; - -cleanup; diff --git a/tests/bugs/glusterd/quorum-value-check.t b/tests/bugs/glusterd/quorum-value-check.t deleted file mode 100755 index c701f08bbd5..00000000000 --- a/tests/bugs/glusterd/quorum-value-check.t +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc - -#G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST - -function check_quorum_nfs() { - local qnfs="$(less /var/lib/glusterd/nfs/nfs-server.vol | grep "quorum-count"| awk '{print $3}')" - local qinfo="$($CLI volume info $V0| grep "cluster.quorum-count"| awk '{print $2}')" - - if [ $qnfs = $qinfo ]; then - echo "Y" - else - echo "N" - fi -} - -cleanup; - -TEST glusterd -TEST pidof glusterd -TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2} -TEST $CLI volume set $V0 nfs.disable off -TEST $CLI volume set $V0 performance.write-behind off -TEST $CLI volume set $V0 cluster.self-heal-daemon off -TEST $CLI volume set $V0 cluster.quorum-type fixed -TEST $CLI volume start $V0 - -TEST $CLI volume set $V0 cluster.quorum-count 1 -EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "Y" check_quorum_nfs -TEST $CLI volume set $V0 cluster.quorum-count 2 -EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "Y" check_quorum_nfs -TEST $CLI volume set $V0 cluster.quorum-count 3 -EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "Y" check_quorum_nfs - -cleanup; diff --git a/tests/bugs/nfs/bug-1116503.t b/tests/bugs/nfs/bug-1116503.t deleted file mode 100644 index dd3998df150..00000000000 --- a/tests/bugs/nfs/bug-1116503.t +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# -# Verify that mounting NFS over UDP (MOUNT service only) works. -# - -. $(dirname $0)/../../include.rc -. $(dirname $0)/../../volume.rc -. $(dirname $0)/../../nfs.rc - -#G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST - -cleanup; -TEST glusterd -TEST pidof glusterd - -TEST $CLI volume create $V0 $H0:$B0/$V0 -TEST $CLI volume set $V0 nfs.disable false -TEST $CLI volume set $V0 nfs.mount-udp on - -TEST $CLI volume start $V0 -EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; - -TEST mount_nfs $H0:/$V0 $N0 nolock,mountproto=udp,proto=tcp; -TEST mkdir -p $N0/foo/bar -TEST ls $N0/foo -TEST ls $N0/foo/bar -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0 - -EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; -TEST mount_nfs $H0:/$V0/foo $N0 nolock,mountproto=udp,proto=tcp; -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0 - -EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; -TEST mount_nfs $H0:/$V0/foo/bar $N0 nolock,mountproto=udp,proto=tcp; -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0 - -TEST $CLI volume set $V0 nfs.addr-namelookup on -TEST $CLI volume set $V0 nfs.rpc-auth-allow $H0 -EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; -TEST mount_nfs $H0:/$V0/foo/bar $N0 nolock,mountproto=udp,proto=tcp; -EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0 - -TEST $CLI volume set $V0 nfs.rpc-auth-reject $H0 -EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available; -TEST ! mount_nfs $H0:/$V0/foo/bar $N0 nolock,mountproto=udp,proto=tcp; - -cleanup; -- cgit