From 5bf1e64744b6363e89222a4e10948b2253da202a Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Wed, 15 Apr 2015 22:58:59 -0400 Subject: tests: ignore results from some spuriously-failing tests These tests were selected based on a survey of the last 50 apparently spurious regression failures involving only one test, as of April 21, 2015. They were responsible for the following number of failures out of those 50. volume-snapshot.t: 24 uss.t: 8 glupy.t: 5 In other words, they were responsible for 74% of those failures. Until they're fixed (or we find some generic problem to which they're particularly vulnerable for some reason), ignoring their results should significantly improve our regression-test success rate. (As part of the rebase from master, it made more sense to bring along the early-termination code in run_all than to attempt surgery on it.) BUG: 1163543 Change-Id: I2727735cd57174618c56a9cdcb4bd4bf58301f05 Signed-off-by: Jeff Darcy Reviewed-on: http://review.gluster.org/10351 Reviewed-by: Vijay Bellur Tested-by: NetBSD Build System Tested-by: Vijay Bellur --- run-tests.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/run-tests.sh b/run-tests.sh index d6de6c6c31a..32b6e8e1d0b 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -186,11 +186,55 @@ function run_tests() return ${RES} } +# If you're submitting a fix related to one of these tests and want its result +# to be considered, you'll need to remove it from the list as part of your +# patch. +function is_bad_test () +{ + local name=$1 + for bt in ./tests/basic/volume-snapshot-clone.t \ + ./tests/basic/uss.t \ + ./tests/features/glupy.t; do + [ x"$name" = x"$bt" ] && return 0 # bash: zero means true/success + done + return 1 # bash: non-zero means false/failure +} + +function run_all () +{ + find ${regression_testsdir}/tests -name '*.t' \ + | LC_COLLATE=C sort \ + | while read t; do + old_cores=$(ls /core.* 2> /dev/null | wc -l) + retval=0 + prove -f --timer $t + TMP_RES=$? + if [ ${TMP_RES} -ne 0 ] ; then + echo "$t: bad status $TMP_RES" + retval=$((retval+1)) + fi + new_cores=$(ls /core.* 2> /dev/null | wc -l) + if [ x"$new_cores" != x"$old_cores" ]; then + core_diff=$((new_cores-old_cores)) + echo "$t: $core_diff new core files" + retval=$((retval+2)) + fi + if [ $retval -ne 0 ]; then + if is_bad_test $t; then + echo "Ignoring failure from known-bad test $t" + else + return $retval + fi + fi + done +} + function main() { if [ $# -lt 1 ]; then - echo "Running all the regression test cases" - prove -rf --timer ${regression_testsdir}/tests; + echo "Running all the regression test cases (new way)" + #prove -rf --timer ${regression_testsdir}/tests; + run_all else run_tests "$@" fi -- cgit