diff options
Diffstat (limited to 'run-tests.sh')
| -rwxr-xr-x | run-tests.sh | 171 |
1 files changed, 144 insertions, 27 deletions
diff --git a/run-tests.sh b/run-tests.sh index 5efdc201517..e2a1655d8e0 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -2,6 +2,14 @@ # Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com> # +# As many tests are designed to take values of variables from 'env.rc', +# it is good to source the file. While it is also required to source the +# file individually in each tests (as it should be possible to run the +# tests separately), exporting variables from env.rc is not harmful if +# done here + +source ./tests/env.rc + export TZ=UTC force="no" head="yes" @@ -14,6 +22,15 @@ result_output="/tmp/gluster_regression.txt" section_separator="========================================" run_timeout=200 kill_after_time=5 +nfs_tests=$RUN_NFS_TESTS + +# Option below preserves log tarballs for each run of a test separately +# named: <test>-iteration-<n>.tar +# If set to any other value, then log tarball is just named after the test and +# overwritten in each iteration (saves space) +# named: <test>.tar +# Use option -p to override default behavior +skip_preserve_logs="yes" OSTYPE=$(uname -s) @@ -24,31 +41,67 @@ OSTYPE=$(uname -s) # $3 extension: Extension string that would be appended to the generated # filename # Out: -# string of next available filename with appended "-<n>" if applicable +# string of next available filename with appended "-<n>" # Example: -# Interested routines that want to create a file name, say foo.txt at +# Interested routines that want to create a file name, say foo-<n>.txt at # location /var/log/gluster would pass in "/var/log/gluster" "foo" "txt" -# and be returned next available foo.txt filename to create. If foo.txt -# is available then foo is returned, else foo-<n> (where n is the next -# integer) is returned for use" +# and be returned next available foo-<n> filename to create. # Notes: # Function will not accept empty extension, and will return the same name -# over and over (which can be fixed when there is a use-case for it) +# over and over (which can be fixed when there is a need for it) function get_next_filename() { local basepath=$1 local filename=$2 local extension=$3 - local next=2 - local tfilename=${filename} + local next=1 + local tfilename="${filename}-${next}" while [ -e "${basepath}/${tfilename}.${extension}" ]; do - tfilename="${filename}-${next}" next=$((next+1)) + tfilename="${filename}-${next}" done echo "$tfilename" } +# Tar the gluster logs and generate a tarball named after the first parameter +# passed in to the function. Ideally the test name is passed to this function +# to generate the required tarball. +# Tarball name is further controlled by the variable skip_preserve_logs +function tar_logs() +{ + t=$1 + + logdir=$(gluster --print-logdir) + basetarname=$(basename "$t" .t) + + if [ -n "$logdir" ] + then + if [[ $skip_preserve_logs == "yes" ]]; then + savetarname=$(get_next_filename "${logdir}" \ + "${basetarname}-iteration" "tar" \ + | tail -1) + else + savetarname="$basetarname" + fi + + # Can't use --exclude here because NetBSD doesn't have it. + # However, both it and Linux have -X to take patterns from + # a file, so use that. + (echo '*.tar'; echo .notar) > "${logdir}"/.notar \ + && \ + tar -cf "${logdir}"/"${savetarname}".tar -X "${logdir}"/.notar \ + "${logdir}"/* 2> /dev/null \ + && \ + find "$logdir"/* -maxdepth 0 -name '*.tar' -prune \ + -o -exec rm -rf '{}' ';' + + echo "Logs preserved in tarball $savetarname.tar" + else + echo "Logs not preserved, as logdir is not set" + fi +} + function check_dependencies() { ## Check all dependencies are present @@ -212,8 +265,10 @@ function match() # G_TESTDEF_TEST_STATUS_NETBSD7 # Some examples: # G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456 +# G_TESTDEF_TEST_STATUS_CENTOS6=BRICK_MUX_BAD_TEST,BUG=123456 # G_TESTDEF_TEST_STATUS_NETBSD7=KNOWN_ISSUE,BUG=4444444 # G_TESTDEF_TEST_STATUS_CENTOS6=BAD_TEST,BUG=123456;555555 +# G_TESTDEF_TEST_STATUS_CENTOS6=NFS_TESTS,BUG=1385758 # You can change status of test to enabled or delete the line only if all the # bugs are closed or modified or if the patch fixes it. function get_test_status () @@ -273,6 +328,7 @@ function get_bug_list_for_disabled_test () function run_tests() { RES=0 + FLAKY='' FAILED='' TESTS_NEEDED_RETRY='' GENERATED_CORE='' @@ -294,15 +350,16 @@ function run_tests() timeout_cmd_exists="no" fi - for t in $(find ${regression_testsdir}/tests -name '*.t' \ - | LC_COLLATE=C sort) ; do + all_tests=($(find ${regression_testsdir}/tests -name '*.t' | sort)) + all_tests_cnt=${#all_tests[@]} + for t in "${all_tests[@]}" ; do old_cores=$(ls /*-*.core 2> /dev/null | wc -l) total_tests=$((total_tests+1)) if match $t "$@" ; then selected_tests=$((selected_tests+1)) echo - echo $section_separator$section_separator - if [[ $(get_test_status $t) == "BAD_TEST" ]] && \ + echo $section_separator "(${total_tests} / ${all_tests_cnt})" $section_separator + if [[ $(get_test_status $t) =~ "BAD_TEST" ]] && \ [[ $skip_bad_tests == "yes" ]] then skipped_bad_tests=$((skipped_bad_tests+1)) @@ -322,6 +379,14 @@ function run_tests() echo continue fi + if [[ $(get_test_status $t) == "NFS_TEST" ]] && \ + [[ $nfs_tests == "no" ]] + then + echo "Skipping nfs test file $t" + echo $section_separator$section_separator + echo + continue + fi total_run_tests=$((total_run_tests+1)) echo "[$(date +%H:%M:%S)] Running tests in file $t" starttime="$(date +%s)" @@ -332,12 +397,14 @@ function run_tests() cmd_timeout=$(grep "SCRIPT_TIMEOUT=" ${t} | cut -f2 -d'='); echo "Timeout set is ${cmd_timeout}, default ${run_timeout}" fi - timeout -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t} + timeout --foreground -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t} else prove -vmfe '/bin/bash' ${t} fi TMP_RES=$? ELAPSEDTIMEMAP[$t]=`expr $(date +%s) - $starttime` + tar_logs "$t" + # timeout always return 124 if it is actually a timeout. if ((${TMP_RES} == 124)); then echo "${t} timed out after ${cmd_timeout} seconds" @@ -352,28 +419,32 @@ function run_tests() echo " * we got some spurious failures *" echo " *********************************" echo "" - # backup old tar ball with time stamp - # TODO: Using gluster CLI here is possibly not the best thing! - logdir=$(gluster --print-logdir) - basetarname=$(basename "$t" .t) - savetarname=$(get_next_filename "${logdir}" "${basetarname}-$(date +%H:%M:%S)" "tar" | tail -1) - mv "$logdir"/"$basetarname".tar "$logdir"/"$savetarname".tar if [ ${timeout_cmd_exists} == "yes" ]; then - timeout -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t} + timeout --foreground -k ${kill_after_time} ${cmd_timeout} prove -vmfe '/bin/bash' ${t} else prove -vmfe '/bin/bash' ${t} fi TMP_RES=$? + tar_logs "$t" + if ((${TMP_RES} == 124)); then echo "${t} timed out after ${cmd_timeout} seconds" fi TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} " fi + + if [ ${TMP_RES} -ne 0 ] ; then - RES=${TMP_RES} - FAILED="${FAILED}${t} " + if [[ "$t" == *"tests/000-flaky/"* ]]; then + FLAKY="${FLAKY}${t} " + echo "FAILURE -> SUCCESS: Flaky test" + TMP_RES=0 + else + RES=${TMP_RES} + FAILED="${FAILED}${t} " + fi fi new_cores=$(ls /*-*.core 2> /dev/null | wc -l) @@ -408,8 +479,10 @@ function run_tests() echo "$key - ${ELAPSEDTIMEMAP["$key"]} second" done | sort -rn -k3 - # Output the errors into a file + # initialize the output file echo > "${result_output}" + + # Output the errors into a file if [ ${RES} -ne 0 ] ; then FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u ) FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' ) @@ -422,7 +495,13 @@ function run_tests() TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u ) RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' ) if [ ${RETRY_COUNT} -ne 0 ] ; then - echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" + echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}" + fi + + FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u ) + RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' ) + if [ ${RETRY_COUNT} -ne 0 ] ; then + echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}" fi echo @@ -451,8 +530,38 @@ function run_head_tests() run_tests "$htests" } -function parse_args () { - args=`getopt frcbkhHo:t: "$@"` +function show_usage () +{ + cat <<EOF +Usage: $0 <opts> [<glob>|<bzid>]... + +Options: + +-f force +-h skip tests altering from HEAD +-H run only tests altering from HEAD +-r retry failed tests +-R do not retry failed tests +-c dont't exit on failure +-b don't skip bad tests +-k don't skip known bugs +-p don't keep logs from preceding runs +-o OUTPUT +-t TIMEOUT +-n skip NFS tests +--help +EOF +} + +usage="no" + +function parse_args () +{ + args=`getopt -u -l help frRcbkphHno:t: "$@"` + if ! [ $? -eq 0 ]; then + show_usage + exit 1 + fi set -- $args while [ $# -gt 0 ]; do case "$1" in @@ -460,11 +569,15 @@ function parse_args () { -h) head="no" ;; -H) head="only" ;; -r) retry="yes" ;; + -R) retry="no" ;; -c) exit_on_failure="no" ;; -b) skip_bad_tests="no" ;; -k) skip_known_bugs="no" ;; + -p) skip_preserve_logs="no" ;; -o) result_output="$2"; shift;; -t) run_timeout="$2"; shift;; + -n) nfs_tests="no";; + --help) usage="yes" ;; --) shift; break;; esac shift @@ -479,6 +592,10 @@ echo # Get user options parse_args "$@" +if [ x"$usage" == x"yes" ]; then + show_usage + exit 0 +fi # Make sure we're running as the root user check_user |