From f6e5bc68d294cc67f1e7f4db043d2531d4cfec80 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Mon, 8 Feb 2016 12:13:54 +0530 Subject: tests: Add mechanism for disabled tests Requirements: Should be able to skip tests from run-tests.sh run. Should be granular enough to disable on subset of OSes. Solution: Tests can have special comment lines with some comma separated values within them. Key names used to determine test status are G_TESTDEF_TEST_STATUS_CENTOS6 G_TESTDEF_TEST_STATUS_NETBSD7 Some examples: G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456 G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444 G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555 You can change status of test to enabled or delete the line only if all the bugs are closed or modified or if the patch fixes it. Change-Id: Idee21fecaa5837fd4bd06e613f5c07a024f7b0c2 BUG: 1295704 Signed-off-by: Raghavendra Talur Reviewed-on: http://review.gluster.org/13393 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Rajesh Joseph Reviewed-by: Kaushal M --- run-tests.sh | 155 +++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 51 deletions(-) (limited to 'run-tests.sh') diff --git a/run-tests.sh b/run-tests.sh index 18dc3bc35e5..c681f7d727e 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -7,6 +7,9 @@ force="no" retry="no" tests="" exit_on_failure="yes" +skip_bad_tests="yes" +skip_known_bugs="yes" +section_separator="========================================" OSTYPE=$(uname -s) @@ -155,46 +158,68 @@ function match() return $match } -# If you're submitting a fix related to one of these tests and want its result -# to be considered, you'll need to remove it from the list as part of your -# patch. -function is_bad_test () +# Tests can have comment lines with some comma separated values within them. +# Key names used to determine test status are +# G_TESTDEF_TEST_STATUS_CENTOS6 +# G_TESTDEF_TEST_STATUS_NETBSD7 +# Some examples: +# G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456 +# G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444 +# G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555 +# You can change status of test to enabled or delete the line only if all the +# bugs are closed or modified or if the patch fixes it. +function get_test_status () { - local name=$1 - for bt in ./tests/basic/quota-anon-fd-nfs.t \ - ./tests/bugs/quota/bug-1235182.t \ - ./tests/basic/quota-nfs.t \ - ./tests/basic/tier/tier_lookup_heal.t \ - ./tests/basic/tier/bug-1214222-directories_missing_after_attach_tier.t \ - ./tests/basic/tier/fops-during-migration.t \ - ./tests/basic/tier/record-metadata-heat.t \ - ./tests/basic/tier/tier-snapshot.t \ - ./tests/bugs/snapshot/bug-1109889.t \ - ./tests/bugs/distribute/bug-1066798.t \ - ./tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t \ - ./tests/geo-rep/georep-basic-dr-rsync.t \ - ./tests/geo-rep/georep-basic-dr-tarssh.t \ - ./tests/bugs/fuse/bug-924726.t \ - ./tests/basic/afr/split-brain-healing.t \ - ./tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t \ - ./tests/bugs/tier/bug-1286974.t \ - ./tests/features/weighted-rebalance.t \ - ./tests/performance/open-behind.t \ - ./tests/basic/afr/self-heald.t \ - ./tests/basic/tier/tier-file-create.t \ - ; do - [ x"$name" = x"$bt" ] && return 0 # bash: zero means true/success - done - return 1 # bash: non-zero means false/failure + local test_name=$1 + local host_os="" + local result="" + + host_os=$(uname -s) + + case "$host_os" in + # Leaving out the logic to determine the particular distro and version + # for later. Why does the key have the distro and version then? + # Because changing the key in all test files would be very big process + # updating just this function with a better logic much simpler. + Linux) + result=$(grep -e "^#G_TESTDEF_TEST_STATUS_CENTOS6" $test_name | \ + awk -F"," {'print $1'} | awk -F"=" {'print $2'}) ;; + NetBSD) + result=$(grep -e "^#G_TESTDEF_TEST_STATUS_NETBSD7" $test_name | \ + awk -F"," {'print $1'} | awk -F"=" {'print $2'}) ;; + *) + result="ENABLED" ;; + esac + + echo "$result" + } -function is_unsupported_test() +function get_bug_list_for_disabled_test () { - if [ x"$OSTYPE" != x"NetBSD" ]; then - return 1 - fi + local test_name=$1 + local host_os="" + local result="" + + host_os=$(uname -s) + + case "$host_os" in + # Leaving out the logic to determine the particular distro and version + # for later. Why does the key have the distro and version then? + # Because changing the key in all test files would be very big process + # updating just this function with a better logic much simpler. + Linux) + result=$(grep -e "^#G_TESTDEF_TEST_STATUS_CENTOS6" $test_name | \ + awk -F"," {'print $2'} | awk -F"=" {'print $2'}) ;; + NetBSD) + result=$(grep -e "^#G_TESTDEF_TEST_STATUS_NETBSD7" $test_name | \ + awk -F"," {'print $2'} | awk -F"=" {'print $2'}) ;; + *) + result="0000000" ;; + esac + + echo "$result" - grep -iqs tier $1 } function run_tests() @@ -202,6 +227,11 @@ function run_tests() RES=0 FAILED='' GENERATED_CORE='' + total_tests=0 + selected_tests=0 + skipped_bad_tests=0 + skipped_known_issue_tests=0 + total_run_tests=0 # key = path of .t file; value = time taken to run the .t file declare -A ELAPSEDTIMEMAP @@ -209,24 +239,35 @@ function run_tests() for t in $(find ${regression_testsdir}/tests -name '*.t' \ | LC_COLLATE=C sort) ; do old_cores=$(ls /core.* 2> /dev/null | wc -l) + total_tests=$((total_tests+1)) if match $t "$@" ; then + selected_tests=$((selected_tests+1)) echo - echo "==================================================" - if is_bad_test $t; then + echo $section_separator$section_separator + if [[ $(get_test_status $t) == "BAD_TEST" ]] && \ + [[ $skip_bad_tests == "yes" ]] + then + skipped_bad_tests=$((skipped_bad_tests+1)) echo "Skipping bad test file $t" - echo "==================================================" + echo "Reason: bug(s):" $(get_bug_list_for_disabled_test $t) + echo $section_separator$section_separator echo continue fi - if is_unsupported_test $t; then - echo "Skipping test file $t (feature unsupported on platform)" - echo "==================================================" + if [[ $(get_test_status $t) == "KNOWN_ISSUE" ]] && \ + [[ $skip_known_bugs == "yes" ]] + then + skipped_known_issue_tests=$((skipped_known_issue_tests+1)) + echo "Skipping test file $t due to known issue" + echo "Reason: bug(s):" $(get_bug_list_for_disabled_test $t) + echo $section_separator$section_separator echo continue fi - echo "Running tests in file $t" + total_run_tests=$((total_run_tests+1)) + echo "[$(date +%H:%M:%S)] Running tests in file $t" starttime="$(date +%s)" - prove -mf --timer $t + prove -vf $t TMP_RES=$? ELAPSEDTIMEMAP[$t]=`expr $(date +%s) - $starttime` if [ ${TMP_RES} -ne 0 ] && [ "x${retry}" = "xyes" ] ; then @@ -238,7 +279,7 @@ function run_tests() echo " * we got some spurous failures *" echo " *********************************" echo "" - prove -mf --timer $t + prove -vf $t TMP_RES=$? fi if [ ${TMP_RES} -ne 0 ] ; then @@ -253,7 +294,7 @@ function run_tests() GENERATED_CORE="${GENERATED_CORE}${t} " fi echo "End of test $t" - echo "==================================================" + echo $section_separator$section_separator echo if [ $RES -ne 0 ] && [ x"$exit_on_failure" = "xyes" ] ; then break; @@ -262,33 +303,45 @@ function run_tests() done echo echo "Run complete" + echo $section_separator$section_separator + echo "Number of tests found: $total_tests" + echo "Number of tests selected for run based on pattern: $selected_tests" + echo "Number of tests skipped as they were marked bad: $skipped_bad_tests" + echo "Number of tests skipped because of known_issues: $skipped_known_issue_tests" + echo "Number of tests that were run: $total_run_tests" if [ ${RES} -ne 0 ] ; then FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u ) FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' ) - echo -e "$FAILED_COUNT test(s) failed \n${FAILED}" + echo -e "\n$FAILED_COUNT test(s) failed \n${FAILED}" GENERATED_CORE=$( echo ${GENERATED_CORE} | tr ' ' '\n' | sort -u ) GENERATED_CORE_COUNT=$( echo -n "${GENERATED_CORE}" | grep -c '^' ) - echo -e "$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}" + echo -e "\n$GENERATED_CORE_COUNT test(s) generated core \n${GENERATED_CORE}" fi - echo "Slowest 10 tests: " + echo + echo "Tests ordered by time taken, slowest to fastest: " + echo $section_separator$section_separator for key in "${!ELAPSEDTIMEMAP[@]}" do - echo $key ' - ' ${ELAPSEDTIMEMAP["$key"]} - done | sort -rn -k3 | head + echo "$key - ${ELAPSEDTIMEMAP["$key"]} second" + done | sort -rn -k3 + echo echo "Result is $RES" + echo return ${RES} } function parse_args () { - args=`getopt frc "$@"` + args=`getopt frcbk "$@"` set -- $args while [ $# -gt 0 ]; do case "$1" in -f) force="yes" ;; -r) retry="yes" ;; -c) exit_on_failure="no" ;; + -b) skip_bad_tests="no" ;; + -k) skip_known_bugs="no" ;; --) shift; break;; esac shift -- cgit