tests: don't kill the process directly with KILL signal

Instead send the SIGTERM (default, 15) first, and at the end
send SIGKILL. If SIGKILL is sent directly, we miss many tests
like valgrind, lcov etc., not able to process the information
properly.

BUG: 1549000
Change-Id: I664de12ee7dbf47eb98b8141004cd51f6006b314
Signed-off-by: Amar Tumballi <amarts@redhat.com>

2 files changed, 86 insertions, 4 deletions

diff --git a/tests/cluster.rc b/tests/cluster.rc
index e258b586847..1c5247ca43f 100644
--- a/tests/cluster.rc
+++ b/tests/cluster.rc
@@ -96,7 +96,7 @@ function kill_node() {
 
     h="H$index";
 
-    kill -9 $(ps -ef | grep gluster | grep ${!h} | awk '{print $2}');
+    terminate_pids $(ps -ef | grep gluster | grep ${!h} | awk '{print $2}')
     find $B0/$index/glusterd/vols -name '*.pid' | xargs rm -f
 }
 
diff --git a/tests/include.rc b/tests/include.rc
index f3b94336179..f02bba1442b 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -100,6 +100,24 @@ CLI_NO_FORCE="gluster --mode=script";
 # root partition is ignored while running the command in a "no force" mode
 CLI_IGNORE_PARTITION="gluster --mode=script --wignore-partition"
 
+function wait_delay() {
+        local delay="$1"
+        local interval="$2"
+        shift 2
+        local deadline="$(($(date +%s%N) + ${delay}000000000))"
+
+        $*
+        while [[ $? -ne 0 ]]; do
+                if [[ $(date +%s%N) -ge ${deadline} ]]; then
+                        return 1
+                fi
+                sleep ${interval}
+                $*
+        done
+
+        return 0
+}
+
 _GFS () {
 	glusterfs "$@"
 	local mount_ret=$?
@@ -463,6 +481,68 @@ stat -c %s /dev/null > /dev/null 2>&1 || {
   }
 }
 
+function signal_pids() {
+        local sig="$1"
+        shift
+        local pids=($*)
+
+        if [[ ${#pids[@]} -gt 0 ]]; then
+                kill -${sig} ${pids[@]} 2>/dev/null || true
+        fi
+}
+
+function check_pids() {
+        local pids=($*)
+        local tmp=()
+        local pid
+
+        for pid in "${pids[@]}"; do
+                kill -0 "${pid}" 2>/dev/null && tmp+=(${pid})
+        done
+
+        echo "${tmp[@]}"
+}
+
+function pids_alive() {
+        local pids=($*)
+
+        if [[ "$(check_pids ${pids[@]})" != "" ]]; then
+                return 1;
+        fi
+
+        return 0
+}
+
+function terminate_pids() {
+        local pids=($*)
+
+        signal_pids TERM ${pids[@]}
+        wait_delay ${PROCESS_DOWN_TIMEOUT} 0.1 pids_alive ${pids[@]}
+        if [[ $? -ne 0 ]]; then
+                pids=($(check_pids ${pids[@]}))
+                signal_pids KILL ${pids[@]}
+                wait_delay 1 0.1 pids_alive ${pids[@]}
+                if [[ $? -ne 0 ]]; then
+                        return 2
+                fi
+
+                return 1
+        fi
+
+        return 0
+}
+
+function process_pids() {
+        local proc
+        local pids=()
+
+        for proc in $*; do
+                pids+=($(pgrep ${proc}))
+        done
+
+        echo "${pids[@]}"
+}
+
 function cleanup()
 {
 
@@ -500,8 +580,9 @@ function cleanup()
         umount $flag /tmp/mnt* 2>/dev/null
 
 
-        # Send SIGKILL to all gluster processes and rpc.statd that are still running
-        killall -9 glusterfs glusterfsd glusterd rpc.statd 2>/dev/null || true;
+        # Send SIGTERM to all gluster processes and rpc.statd that are still running
+        terminate_pids $(process_pids glusterfs glusterfsd glusterd rpc.statd)
+
         test x"$OSTYPE" = x"NetBSD" && pkill -9 perfused || true
 
         # unregister nfs and related services from portmapper/rpcbind
@@ -622,7 +703,8 @@ function cleanup()
 	# above to fail, promoting that into a failure of the whole test (and
 	# thus of an entire regression-test run) seems a bit excessive.  Make
 	# sure we return good status anyway.
-	return 0
+
+        return 0
 }
 
 function force_terminate () {