summaryrefslogtreecommitdiffstats
path: root/run-tests.sh
diff options
context:
space:
mode:
authorJeff Darcy <jdarcy@redhat.com>2015-04-15 22:58:59 -0400
committerVijay Bellur <vbellur@redhat.com>2015-04-23 11:14:21 -0700
commit5bf1e64744b6363e89222a4e10948b2253da202a (patch)
tree2478a402d829a09ba68a824b30de396f89634361 /run-tests.sh
parent14d75b9ded15c36ddd9b470f8ed2739bc94454fd (diff)
tests: ignore results from some spuriously-failing tests
These tests were selected based on a survey of the last 50 apparently spurious regression failures involving only one test, as of April 21, 2015. They were responsible for the following number of failures out of those 50. volume-snapshot.t: 24 uss.t: 8 glupy.t: 5 In other words, they were responsible for 74% of those failures. Until they're fixed (or we find some generic problem to which they're particularly vulnerable for some reason), ignoring their results should significantly improve our regression-test success rate. (As part of the rebase from master, it made more sense to bring along the early-termination code in run_all than to attempt surgery on it.) BUG: 1163543 Change-Id: I2727735cd57174618c56a9cdcb4bd4bf58301f05 Signed-off-by: Jeff Darcy <jdarcy@redhat.com> Reviewed-on: http://review.gluster.org/10351 Reviewed-by: Vijay Bellur <vbellur@redhat.com> Tested-by: NetBSD Build System Tested-by: Vijay Bellur <vbellur@redhat.com>
Diffstat (limited to 'run-tests.sh')
-rwxr-xr-xrun-tests.sh48
1 files changed, 46 insertions, 2 deletions
diff --git a/run-tests.sh b/run-tests.sh
index d6de6c6c31a..32b6e8e1d0b 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -186,11 +186,55 @@ function run_tests()
return ${RES}
}
+# If you're submitting a fix related to one of these tests and want its result
+# to be considered, you'll need to remove it from the list as part of your
+# patch.
+function is_bad_test ()
+{
+ local name=$1
+ for bt in ./tests/basic/volume-snapshot-clone.t \
+ ./tests/basic/uss.t \
+ ./tests/features/glupy.t; do
+ [ x"$name" = x"$bt" ] && return 0 # bash: zero means true/success
+ done
+ return 1 # bash: non-zero means false/failure
+}
+
+function run_all ()
+{
+ find ${regression_testsdir}/tests -name '*.t' \
+ | LC_COLLATE=C sort \
+ | while read t; do
+ old_cores=$(ls /core.* 2> /dev/null | wc -l)
+ retval=0
+ prove -f --timer $t
+ TMP_RES=$?
+ if [ ${TMP_RES} -ne 0 ] ; then
+ echo "$t: bad status $TMP_RES"
+ retval=$((retval+1))
+ fi
+ new_cores=$(ls /core.* 2> /dev/null | wc -l)
+ if [ x"$new_cores" != x"$old_cores" ]; then
+ core_diff=$((new_cores-old_cores))
+ echo "$t: $core_diff new core files"
+ retval=$((retval+2))
+ fi
+ if [ $retval -ne 0 ]; then
+ if is_bad_test $t; then
+ echo "Ignoring failure from known-bad test $t"
+ else
+ return $retval
+ fi
+ fi
+ done
+}
+
function main()
{
if [ $# -lt 1 ]; then
- echo "Running all the regression test cases"
- prove -rf --timer ${regression_testsdir}/tests;
+ echo "Running all the regression test cases (new way)"
+ #prove -rf --timer ${regression_testsdir}/tests;
+ run_all
else
run_tests "$@"
fi