diff options
Diffstat (limited to 'run-tests.sh')
| -rwxr-xr-x | run-tests.sh | 27 |
1 files changed, 22 insertions, 5 deletions
diff --git a/run-tests.sh b/run-tests.sh index 6d48ecfcb03..e2a1655d8e0 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -328,6 +328,7 @@ function get_bug_list_for_disabled_test () function run_tests() { RES=0 + FLAKY='' FAILED='' TESTS_NEEDED_RETRY='' GENERATED_CORE='' @@ -349,7 +350,7 @@ function run_tests() timeout_cmd_exists="no" fi - all_tests=($(find ${regression_testsdir}/tests -name '*.t' | LC_COLLATE=C sort)) + all_tests=($(find ${regression_testsdir}/tests -name '*.t' | sort)) all_tests_cnt=${#all_tests[@]} for t in "${all_tests[@]}" ; do old_cores=$(ls /*-*.core 2> /dev/null | wc -l) @@ -433,9 +434,17 @@ function run_tests() TESTS_NEEDED_RETRY="${TESTS_NEEDED_RETRY}${t} " fi + + if [ ${TMP_RES} -ne 0 ] ; then - RES=${TMP_RES} - FAILED="${FAILED}${t} " + if [[ "$t" == *"tests/000-flaky/"* ]]; then + FLAKY="${FLAKY}${t} " + echo "FAILURE -> SUCCESS: Flaky test" + TMP_RES=0 + else + RES=${TMP_RES} + FAILED="${FAILED}${t} " + fi fi new_cores=$(ls /*-*.core 2> /dev/null | wc -l) @@ -470,8 +479,10 @@ function run_tests() echo "$key - ${ELAPSEDTIMEMAP["$key"]} second" done | sort -rn -k3 - # Output the errors into a file + # initialize the output file echo > "${result_output}" + + # Output the errors into a file if [ ${RES} -ne 0 ] ; then FAILED=$( echo ${FAILED} | tr ' ' '\n' | sort -u ) FAILED_COUNT=$( echo -n "${FAILED}" | grep -c '^' ) @@ -484,7 +495,13 @@ function run_tests() TESTS_NEEDED_RETRY=$( echo ${TESTS_NEEDED_RETRY} | tr ' ' '\n' | sort -u ) RETRY_COUNT=$( echo -n "${TESTS_NEEDED_RETRY}" | grep -c '^' ) if [ ${RETRY_COUNT} -ne 0 ] ; then - echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" + echo -e "\n${RETRY_COUNT} test(s) needed retry \n${TESTS_NEEDED_RETRY}" >> "${result_output}" + fi + + FLAKY_TESTS_FAILED=$( echo ${FLAKY} | tr ' ' '\n' | sort -u ) + RETRY_COUNT=$( echo -n "${FLAKY_TESTS_FAILED}" | grep -c '^' ) + if [ ${RETRY_COUNT} -ne 0 ] ; then + echo -e "\n${RETRY_COUNT} flaky test(s) marked as success even though they failed \n${FLAKY_TESTS_FAILED}" >> "${result_output}" fi echo |