tests: provide an option to mark tests as 'flaky'

* also add some time gap in other tests to see if we get things properly
* create a directory 'tests/000/', which can host any tests, which are flaky.
* move all the tests mentioned in the issue to above directory.
* as the above dir gets tested first, all flaky tests would be reported quickly.
* change `run-tests.sh` to continue tests even if flaky tests fail.

Reference: gluster/project-infrastructure#72
Updates: #1000
Change-Id: Ifdafa38d083ebd80f7ae3cbbc9aa3b68b6d21d0e
Signed-off-by: Amar Tumballi <amar@kadalu.io>

8 files changed, 0 insertions, 510 deletions

diff --git a/tests/bugs/core/multiplex-limit-issue-151.t b/tests/bugs/core/multiplex-limit-issue-151.t
deleted file mode 100644
index dc9013061b0..00000000000
--- a/tests/bugs/core/multiplex-limit-issue-151.t
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/../../include.rc
-. $(dirname $0)/../../traps.rc
-. $(dirname $0)/../../volume.rc
-
-function count_up_bricks {
-        $CLI --xml volume status all | grep '<status>1' | wc -l
-}
-
-function count_brick_processes {
-	pgrep glusterfsd | wc -l
-}
-
-function count_brick_pids {
-        $CLI --xml volume status all | sed -n '/.*<pid>\([^<]*\).*/s//\1/p' \
-                                     | grep -v "N/A" | sort | uniq | wc -l
-}
-
-cleanup;
-
-TEST glusterd
-
-TEST $CLI volume set all cluster.brick-multiplex on
-TEST ! $CLI volume set all cluster.max-bricks-per-process -1
-TEST ! $CLI volume set all cluster.max-bricks-per-process foobar
-TEST $CLI volume set all cluster.max-bricks-per-process 3
-
-TEST $CLI volume create $V0 $H0:$B0/brick{0..5}
-TEST $CLI volume start $V0
-
-EXPECT 2 count_brick_processes
-EXPECT 2 count_brick_pids
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 count_up_bricks
-
-pkill gluster
-TEST glusterd
-
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_brick_processes
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_brick_pids
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 count_up_bricks
-
-TEST $CLI volume add-brick $V0 $H0:$B0/brick6
-
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_processes
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_pids
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 7 count_up_bricks
-
-TEST $CLI volume remove-brick $V0 $H0:$B0/brick3 start
-TEST $CLI volume remove-brick $V0 $H0:$B0/brick3 force
-
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_processes
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 3 count_brick_pids
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT 6 count_up_bricks
-
-cleanup;
diff --git a/tests/bugs/distribute/bug-1117851.t b/tests/bugs/distribute/bug-1117851.t
deleted file mode 100755
index 62cb6b66ab4..00000000000
--- a/tests/bugs/distribute/bug-1117851.t
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-
-SCRIPT_TIMEOUT=250
-
-. $(dirname $0)/../../include.rc
-. $(dirname $0)/../../volume.rc
-
-create_files () {
-	for i in {1..1000}; do
-		orig=$(printf %s/abc%04d $1 $i)
-		real=$(printf %s/src%04d $1 $i)
-		# Make sure lots of these have linkfiles.
-		echo "This is file $i" > $orig
-		mv $orig $real
-	done
-	sync
-}
-
-move_files_inner () {
-	sfile=$M0/status_$(basename $1)
-	for i in {1..1000}; do
-		src=$(printf %s/src%04d $1 $i)
-		dst=$(printf %s/dst%04d $1 $i)
-		mv $src $dst 2> /dev/null
-	done
-	echo "done" > $sfile
-}
-
-move_files () {
-        #Create the status file here to prevent spurious failures
-        #caused by the file not being created in time by the
-        #background process
-	sfile=$M0/status_$(basename $1)
-	echo "running" > $sfile
-	move_files_inner $* &
-}
-
-check_files () {
-	errors=0
-	for i in {1..1000}; do
-		if [ ! -f $(printf %s/dst%04d $1 $i) ]; then
-			if [ -f $(printf %s/src%04d $1 $i) ]; then
-				echo "file $i didnt get moved" > /dev/stderr
-			else
-				echo "file $i is MISSING" > /dev/stderr
-				errors=$((errors+1))
-			fi
-		fi
-	done
-	if [ $((errors)) != 0 ]; then
-		: ls -l $1 > /dev/stderr
-	fi
-	return $errors
-}
-
-cleanup;
-
-TEST glusterd
-TEST pidof glusterd
-TEST $CLI volume info;
-
-TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4,5,6};
-
-EXPECT "$V0" volinfo_field $V0 'Volume Name';
-EXPECT 'Created' volinfo_field $V0 'Status';
-EXPECT '6' brick_count $V0
-
-TEST $CLI volume start $V0;
-EXPECT 'Started' volinfo_field $V0 'Status';
-
-## Mount FUSE with caching disabled (read-write)
-TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
-
-TEST create_files $M0
-
-## Mount FUSE with caching disabled (read-write) again
-TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M1;
-
-TEST move_files $M0
-TEST move_files $M1
-
-# It's regrettable that renaming 1000 files might take more than 30 seconds,
-# but on our test systems sometimes it does, so double the time from what we'd
-# use otherwise.  There still seem to be some spurious failures, 1 in 20 when
-# this does not complete, added an additional 60 seconds to take false reports
-# out of the system, during test runs, especially on slower test systems.
-EXPECT_WITHIN 120 "done" cat $M0/status_0
-EXPECT_WITHIN 120 "done" cat $M1/status_1
-
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M1
-TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0;
-TEST check_files $M0
-
-TEST $CLI volume stop $V0;
-EXPECT 'Stopped' volinfo_field $V0 'Status';
-
-TEST $CLI volume delete $V0;
-TEST ! $CLI volume info $V0;
-
-cleanup;
diff --git a/tests/bugs/distribute/bug-1122443.t b/tests/bugs/distribute/bug-1122443.t
deleted file mode 100644
index 906be7072bd..00000000000
--- a/tests/bugs/distribute/bug-1122443.t
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/../../include.rc
-. $(dirname $0)/../../volume.rc
-. $(dirname $0)/../../dht.rc
-
-make_files() {
-    mkdir $1 && \
-    ln -s ../ $1/symlink && \
-    mknod $1/special_b b 1 2 && \
-    mknod $1/special_c c 3 4 && \
-    mknod $1/special_u u 5 6 && \
-    mknod $1/special_p p && \
-    touch -h --date=@1 $1/symlink && \
-    touch -h --date=@2 $1/special_b &&
-    touch -h --date=@3 $1/special_c &&
-    touch -h --date=@4 $1/special_u &&
-    touch -h --date=@5 $1/special_p
-}
-
-bug_1113050_workaround() {
-    # Test if graph change has settled (bug-1113050?)
-    test=$(stat -c "%n:%Y" $1 2>&1 | tr '\n' ',')
-    if [ $? -eq 0 ] ; then
-	echo RECONNECTED
-    else
-	echo WAITING
-    fi
-    return 0
-}
-
-cleanup
-
-TEST glusterd
-TEST pidof glusterd
-
-TEST $CLI volume create $V0 $H0:$B0/${V0}0
-TEST $CLI volume start $V0
-
-# Mount FUSE and create symlink
-TEST glusterfs -s $H0 --volfile-id $V0 $M0
-TEST make_files $M0/subdir
-
-# Get mtime before migration
-BEFORE="$(stat -c %n:%Y $M0/subdir/* | tr '\n' ',')"
-
-# Migrate brick
-TEST $CLI volume add-brick $V0 $H0:$B0/${V0}1
-TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}0 start
-EXPECT_WITHIN $REBALANCE_TIMEOUT "completed" remove_brick_status_completed_field "$V0 $H0:$B0/${V0}0"
-TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}0 commit
-
-# Get mtime after migration
-EXPECT_WITHIN 5 RECONNECTED bug_1113050_workaround $M0/subdir/*
-AFTER="$(stat -c %n:%Y $M0/subdir/* | tr '\n' ',')"
-
-# Check if mtime is unchanged
-TEST [ "$AFTER" == "$BEFORE" ]
-
-cleanup
diff --git a/tests/bugs/glusterd/bug-857330/common.rc b/tests/bugs/glusterd/bug-857330/common.rc
deleted file mode 100644
index d0aa4b1a640..00000000000
--- a/tests/bugs/glusterd/bug-857330/common.rc
+++ /dev/null
@@ -1,57 +0,0 @@
-. $(dirname $0)/../../../include.rc
-
-UUID_REGEX='[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}'
-
-TASK_ID=""
-COMMAND=""
-PATTERN=""
-
-function check-and-store-task-id()
-{
-        TASK_ID=""
-
-        local task_id=$($CLI $COMMAND | grep $PATTERN | grep -o -E "$UUID_REGEX")
-
-        if [ -z "$task_id" ] && [ "${task_id+asdf}" = "asdf" ]; then
-                return 1
-        fi
-
-        TASK_ID=$task_id
-        return 0;
-}
-
-function get-task-id()
-{
-        $CLI $COMMAND | grep $PATTERN | grep -o -E "$UUID_REGEX" | tail -n1
-
-}
-
-function check-and-store-task-id-xml()
-{
-        TASK_ID=""
-
-        local task_id=$($CLI $COMMAND --xml | xmllint --format - | grep $PATTERN | grep -o -E "$UUID_REGEX")
-
-        if [ -z "$task_id" ] && [ "${task_id+asdf}" = "asdf" ]; then
-                return 1
-        fi
-
-        TASK_ID=$task_id
-        return 0;
-}
-
-function get-task-id-xml()
-{
-        $CLI $COMMAND --xml | xmllint --format - | grep $PATTERN | grep -o -E "$UUID_REGEX"
-}
-
-function get-task-status()
-{
-        pattern=$1
-        val=1
-        test=$(gluster $COMMAND | grep -o $pattern 2>&1)
-        if [ $? -eq 0 ]; then
-                 val=0
-        fi
-        echo $val
-}
diff --git a/tests/bugs/glusterd/bug-857330/normal.t b/tests/bugs/glusterd/bug-857330/normal.t
deleted file mode 100755
index ad0c8844fae..00000000000
--- a/tests/bugs/glusterd/bug-857330/normal.t
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/common.rc
-. $(dirname $0)/../../../volume.rc
-cleanup;
-
-TEST glusterd
-TEST pidof glusterd
-TEST $CLI volume info;
-
-TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}1 $H0:$B0/${V0}2;
-TEST $CLI volume info $V0;
-TEST $CLI volume start $V0;
-
-TEST glusterfs -s $H0 --volfile-id=$V0 $M0;
-
-TEST $PYTHON $(dirname $0)/../../../utils/create-files.py \
-             --multi -b 10 -d 10  -n 10 $M0;
-
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
-
-###############
-## Rebalance ##
-###############
-TEST $CLI volume add-brick $V0 replica 2 $H0:$B0/${V0}3 $H0:$B0/${V0}4;
-
-COMMAND="volume rebalance $V0 start"
-PATTERN="ID:"
-TEST check-and-store-task-id
-
-COMMAND="volume status $V0"
-PATTERN="ID"
-EXPECT $TASK_ID get-task-id
-
-COMMAND="volume rebalance $V0 status"
-PATTERN="completed"
-EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN
-
-###################
-## Replace-brick ##
-###################
-REP_BRICK_PAIR="$H0:$B0/${V0}2 $H0:$B0/${V0}5"
-
-TEST $CLI volume replace-brick $V0 $REP_BRICK_PAIR commit force;
-
-##################
-## Remove-brick ##
-##################
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5
-
-COMMAND="volume remove-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}5 start"
-PATTERN="ID:"
-TEST check-and-store-task-id
-
-COMMAND="volume status $V0"
-PATTERN="ID"
-EXPECT $TASK_ID get-task-id
-
-COMMAND="volume remove-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}5 status"
-PATTERN="completed"
-EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN
-
-TEST $CLI volume remove-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}5 commit
-
-TEST $CLI volume stop $V0;
-TEST $CLI volume delete $V0;
-TEST ! $CLI volume info $V0;
-
-cleanup;
diff --git a/tests/bugs/glusterd/bug-857330/xml.t b/tests/bugs/glusterd/bug-857330/xml.t
deleted file mode 100755
index 8383d2a0711..00000000000
--- a/tests/bugs/glusterd/bug-857330/xml.t
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/common.rc
-. $(dirname $0)/../../../volume.rc
-
-cleanup;
-
-TEST glusterd
-TEST pidof glusterd
-TEST $CLI volume info;
-
-TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}1 $H0:$B0/${V0}2;
-TEST $CLI volume info $V0;
-TEST $CLI volume start $V0;
-
-TEST glusterfs -s $H0 --volfile-id=$V0 $M0;
-
-TEST $PYTHON $(dirname $0)/../../../utils/create-files.py \
-             --multi -b 10 -d 10  -n 10 $M0;
-
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
-
-
-###############
-## Rebalance ##
-###############
-TEST $CLI volume add-brick $V0 replica 2 $H0:$B0/${V0}3 $H0:$B0/${V0}4;
-
-COMMAND="volume rebalance $V0 start"
-PATTERN="task-id"
-TEST check-and-store-task-id-xml
-
-COMMAND="volume status $V0"
-PATTERN="id"
-EXPECT $TASK_ID get-task-id-xml
-
-COMMAND="volume rebalance $V0 status"
-PATTERN="task-id"
-EXPECT $TASK_ID get-task-id-xml
-
-## TODO: Add tests for rebalance stop
-
-COMMAND="volume rebalance $V0 status"
-PATTERN="completed"
-EXPECT_WITHIN $REBALANCE_TIMEOUT "0" get-task-status $PATTERN
-
-###################
-## Replace-brick ##
-###################
-TEST $CLI volume replace-brick $V0 $H0:$B0/${V0}4 $H0:$B0/${V0}5 commit force
-
-##################
-## Remove-brick ##
-##################
-EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}5
-
-COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 start"
-PATTERN="task-id"
-TEST check-and-store-task-id-xml
-
-COMMAND="volume status $V0"
-PATTERN="id"
-EXPECT $TASK_ID get-task-id-xml
-
-COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 status"
-PATTERN="task-id"
-EXPECT $TASK_ID get-task-id-xml
-
-COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 status"
-PATTERN="completed"
-EXPECT_WITHIN  $REBALANCE_TIMEOUT "0" get-task-status $PATTERN
-
-## TODO: Add tests for remove-brick stop
-
-COMMAND="volume remove-brick $V0 $H0:$B0/${V0}3 $H0:$B0/${V0}5 commit"
-PATTERN="task-id"
-EXPECT $TASK_ID get-task-id-xml
-
-TEST $CLI volume stop $V0;
-TEST $CLI volume delete $V0;
-TEST ! $CLI volume info $V0;
-
-cleanup;
diff --git a/tests/bugs/glusterd/quorum-value-check.t b/tests/bugs/glusterd/quorum-value-check.t
deleted file mode 100755
index c701f08bbd5..00000000000
--- a/tests/bugs/glusterd/quorum-value-check.t
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/../../include.rc
-. $(dirname $0)/../../volume.rc
-
-#G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST
-
-function check_quorum_nfs() {
-    local qnfs="$(less /var/lib/glusterd/nfs/nfs-server.vol | grep "quorum-count"| awk '{print $3}')"
-    local qinfo="$($CLI volume info $V0| grep "cluster.quorum-count"| awk '{print $2}')"
-
-    if [ $qnfs = $qinfo ]; then
-        echo "Y"
-    else
-        echo "N"
-    fi
-}
-
-cleanup;
-
-TEST glusterd
-TEST pidof glusterd
-TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2}
-TEST $CLI volume set $V0 nfs.disable off
-TEST $CLI volume set $V0 performance.write-behind off
-TEST $CLI volume set $V0 cluster.self-heal-daemon off
-TEST $CLI volume set $V0 cluster.quorum-type fixed
-TEST $CLI volume start $V0
-
-TEST $CLI volume set $V0 cluster.quorum-count 1
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "Y" check_quorum_nfs
-TEST $CLI volume set $V0 cluster.quorum-count 2
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "Y" check_quorum_nfs
-TEST $CLI volume set $V0 cluster.quorum-count 3
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "Y" check_quorum_nfs
-
-cleanup;
diff --git a/tests/bugs/nfs/bug-1116503.t b/tests/bugs/nfs/bug-1116503.t
deleted file mode 100644
index dd3998df150..00000000000
--- a/tests/bugs/nfs/bug-1116503.t
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-#
-# Verify that mounting NFS over UDP (MOUNT service only) works.
-#
-
-. $(dirname $0)/../../include.rc
-. $(dirname $0)/../../volume.rc
-. $(dirname $0)/../../nfs.rc
-
-#G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TEST
-
-cleanup;
-TEST glusterd
-TEST pidof glusterd
-
-TEST $CLI volume create $V0 $H0:$B0/$V0
-TEST $CLI volume set $V0 nfs.disable false
-TEST $CLI volume set $V0 nfs.mount-udp on
-
-TEST $CLI volume start $V0
-EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
-
-TEST mount_nfs $H0:/$V0 $N0 nolock,mountproto=udp,proto=tcp;
-TEST mkdir -p $N0/foo/bar
-TEST ls $N0/foo
-TEST ls $N0/foo/bar
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
-
-EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
-TEST mount_nfs $H0:/$V0/foo $N0 nolock,mountproto=udp,proto=tcp;
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
-
-EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
-TEST mount_nfs $H0:/$V0/foo/bar $N0 nolock,mountproto=udp,proto=tcp;
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
-
-TEST $CLI volume set $V0 nfs.addr-namelookup on
-TEST $CLI volume set $V0 nfs.rpc-auth-allow $H0
-EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
-TEST mount_nfs $H0:/$V0/foo/bar $N0 nolock,mountproto=udp,proto=tcp;
-EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $N0
-
-TEST $CLI volume set $V0 nfs.rpc-auth-reject $H0
-EXPECT_WITHIN $NFS_EXPORT_TIMEOUT "1" is_nfs_export_available;
-TEST ! mount_nfs $H0:/$V0/foo/bar $N0 nolock,mountproto=udp,proto=tcp;
-
-cleanup;