summaryrefslogtreecommitdiffstats
path: root/run-tests.sh
diff options
context:
space:
mode:
authorAmar Tumballi <amarts@redhat.com>2017-12-01 08:33:50 -0500
committerAmar Tumballi <amarts@redhat.com>2017-12-05 14:22:52 +0000
commit4e798113229eab68e9e02364bb094a4ab8274c65 (patch)
treefec01b9cd20de498448c5c825de6030b319720a1 /run-tests.sh
parentdc1258bfe46d30059119a3294285a114ec2bcd36 (diff)
run-tests.sh: provide retry count option
and default it to 5. With this change, if any test fails due to timing issues (because of slower/faster VM etc), there is a retry logic which gets triggered. Many times, if a test successfully fails more than 5 times, there may be a genuine issue with it, one should take a serious look at the failing test. Change-Id: Ia88e3293fd2724b0f9f09d8bc026560792fc1578 BUG: 1517961 Signed-off-by: Amar Tumballi <amarts@redhat.com>
Diffstat (limited to 'run-tests.sh')
-rwxr-xr-xrun-tests.sh39
1 files changed, 27 insertions, 12 deletions
diff --git a/run-tests.sh b/run-tests.sh
index dbf4a5e..afeb350 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -6,6 +6,7 @@ export TZ=UTC
force="no"
head="yes"
retry="yes"
+retry_count=5
tests=""
exit_on_failure="yes"
skip_bad_tests="yes"
@@ -236,6 +237,7 @@ function run_tests()
# key = path of .t file; value = time taken to run the .t file
declare -A ELAPSEDTIMEMAP
+ declare -A TESTS_NEEDED_RETRY
for t in $(find ${regression_testsdir}/tests -name '*.t' \
| LC_COLLATE=C sort) ; do
@@ -271,18 +273,24 @@ function run_tests()
prove -vfe '/bin/bash' $t
TMP_RES=$?
ELAPSEDTIMEMAP[$t]=`expr $(date +%s) - $starttime`
- if [ ${TMP_RES} -ne 0 ] && [ "x${retry}" = "xyes" ] ; then
- echo "$t: bad status $TMP_RES"
- echo ""
- echo " *********************************"
- echo " * REGRESSION FAILED *"
- echo " * Retrying failed tests in case *"
- echo " * we got some spurious failures *"
- echo " *********************************"
- echo ""
- prove -vfe '/bin/bash' $t
- TMP_RES=$?
- fi
+ cnt=0
+ while [ $cnt -lt $retry_count ];
+ do
+ cnt=$((cnt+1))
+ if [ ${TMP_RES} -ne 0 ] && [ "x${retry}" = "xyes" ] ; then
+ echo "$t: bad status $TMP_RES (count: $cnt)"
+ echo ""
+ echo " *********************************"
+ echo " * REGRESSION FAILED *"
+ echo " * Retrying failed tests in case *"
+ echo " * we got some spurious failures *"
+ echo " *********************************"
+ echo ""
+ prove -vfe '/bin/bash' $t
+ TESTS_NEEDED_RETRY[$t]=$cnt
+ TMP_RES=$?
+ fi
+ done
if [ ${TMP_RES} -ne 0 ] ; then
RES=${TMP_RES}
FAILED="${FAILED}${t} "
@@ -329,6 +337,13 @@ function run_tests()
fi
echo
+ echo "tests which needed retry, ordered on the number of times they needed to be retried"
+ for key in "${!TESTS_NEEDED_RETRY[@]}"
+ do
+ echo "$key - ${TESTS_NEEDED_RETRY["$key"]} times"
+ done | sort -rn -k3
+
+ echo
echo "Result is $RES"
echo
return ${RES}