summaryrefslogtreecommitdiffstats
path: root/run-tests.sh
diff options
context:
space:
mode:
authorAmar Tumballi <amar@kadalu.io>2020-08-18 14:08:20 +0530
committerXavi Hernandez <xhernandez@redhat.com>2020-08-20 08:01:07 +0000
commit097db13c11390174c5b9f11aa0fd87eca1735871 (patch)
tree06469032e199cef35dacfdc53972fd934e7e9437 /run-tests.sh
parentf9b5074394e3d2f3b6728aab97230ba620879426 (diff)
tests: provide an option to mark tests as 'flaky'
* also add some time gap in other tests to see if we get things properly * create a directory 'tests/000/', which can host any tests, which are flaky. * move all the tests mentioned in the issue to above directory. * as the above dir gets tested first, all flaky tests would be reported quickly. * change `run-tests.sh` to continue tests even if flaky tests fail. Reference: gluster/project-infrastructure#72 Updates: #1000 Change-Id: Ifdafa38d083ebd80f7ae3cbbc9aa3b68b6d21d0e Signed-off-by: Amar Tumballi <amar@kadalu.io>
Diffstat (limited to 'run-tests.sh')
-rwxr-xr-xrun-tests.sh27
1 files changed, 22 insertions, 5 deletions
diff --git a/run-tests.sh b/run-tests.sh
index 6d48ecf..e2a1655 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -328,6 +328,7 @@ function get_bug_list_for_disabled_test ()
function run_tests()
{
RES=0
+ FLAKY=''
FAILED=''
TESTS_NEEDED_RETRY=''
GENERATED_CORE=''
@@ -349,7 +350,7 @@ function run_tests()
timeout_cmd_exists="no"
fi
- all_tests=($(find ${regression_testsdir}/tests -name '*.t' | LC_COLLATE=C sort))
+ all_tests=($(find ${regression_testsdir}/tests -name '*.t' | sort))
all_tests_cnt=${#all_tests[@]}
for t in "${all_tests[@]}" ; do
old_cores=$(ls /*-*.core 2> /dev/null | wc -l)
@@ -433,9 +434,17 @@ function run_tests()
TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} "
fi
+
+
if [ ${TMP_RES} -ne 0 ] ; then
- RES=${TMP_RES}
- FAILED="${FAILED}${t} "
+ if [[ "$t" == *"tests/000-flaky/"* ]]; then
+ FLAKY="${FLAKY}${t} "
+ echo "FAILURE -> SUCCESS: Flaky test"
+ TMP_RES=0
+ else
+ RES=${TMP_RES}
+ FAILED="${FAILED}${t} "
+ fi
fi
new_cores=$(ls /*-*.core 2> /dev/null | wc -l)
@@ -470,8 +479,10 @@ function run_tests()
echo "$key - ${ELAPSEDTIMEMAP["$key"]} second"
done | sort -rn -k3
- # Output the errors into a file
+ # initialize the output file
echo > "${result_output}"
+
+ # Output the errors into a file
if [ ${RES} -ne 0 ] ; then
FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u )
FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' )
@@ -484,7 +495,13 @@ function run_tests()
TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u )
RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' )
if [ ${RETRY_COUNT} -ne 0 ] ; then
- echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}"
+ echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}"
+ fi
+
+ FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u )
+ RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' )
+ if [ ${RETRY_COUNT} -ne 0 ] ; then
+ echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}"
fi
echo