tests: fix online_brick_count for multiplexing

The number of brick processes no longer matches the number of bricks,
therefore counting processes doesn't work.  Counting *pidfiles* does.
Ironically, the fix broke multiplex.t which used this function, so it
now uses a different function with the old process-counting behavior.
Also had to fix online_brick_count and kill_node in cluster.rc to be
consistent with the new reality.

Change-Id: I4e81a6633b93227e10604f53e18a0b802c75cbcc
BUG: 1385758
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: https://review.gluster.org/16527
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 24 insertions, 8 deletions

diff --git a/tests/basic/multiplex.t b/tests/basic/multiplex.t
index bff3efb0a2c..d503bf5a232 100644
--- a/tests/basic/multiplex.t
+++ b/tests/basic/multiplex.t
@@ -8,6 +8,10 @@ function count_up_bricks {
         $CLI --xml volume status $V0 | grep '<status>1' | wc -l
 }
 
+function count_brick_processes {
+	pgrep glusterfsd | wc -l
+}
+
 function count_brick_pids {
         $CLI --xml volume status $V0 | sed -n '/.*<pid>\([^<]*\).*/s//\1/p' \
                                      | grep -v "N/A" | sort | uniq | wc -l
@@ -22,31 +26,31 @@ TEST $CLI volume create $V0 $H0:$B0/brick{0,1}
 TEST $CLI volume start $V0
 # Without multiplexing, there would be two.
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
-EXPECT 1 online_brick_count
+EXPECT 1 count_brick_processes
 
 TEST $CLI volume stop $V0
-EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 0 online_brick_count
+EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 0 count_brick_processes
 TEST $CLI volume start $V0
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
-EXPECT 1 online_brick_count
+EXPECT 1 count_brick_processes
 
 TEST kill_brick $V0 $H0 $B0/brick1
 EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 1 count_up_bricks
 # Make sure the whole process didn't go away.
-EXPECT 1 online_brick_count
+EXPECT 1 count_brick_processes
 
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
-EXPECT 1 online_brick_count
+EXPECT 1 count_brick_processes
 
 # Killing the first brick is a bit more of a challenge due to socket-path
 # issues.
 TEST kill_brick $V0 $H0 $B0/brick0
 EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 1 count_up_bricks
-EXPECT 1 online_brick_count
+EXPECT 1 count_brick_processes
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT 2 count_up_bricks
-EXPECT 1 online_brick_count
+EXPECT 1 count_brick_processes
 
 # Make sure that the two bricks show the same PID.
 EXPECT 1 count_brick_pids
diff --git a/tests/cluster.rc b/tests/cluster.rc
index 467bbcb06e1..6dece8182e4 100644
--- a/tests/cluster.rc
+++ b/tests/cluster.rc
@@ -95,6 +95,7 @@ function kill_node() {
     h="H$index";
 
     kill -9 $(ps -ef | grep gluster | grep ${!h} | awk '{print $2}');
+    find $B0/$index/glusterd/vols -name '*.pid' | xargs rm -f
 }
 
 
@@ -177,3 +178,14 @@ function brick_up_status_1 {
         local brick=$3
         $CLI_1 volume status $vol $host:$brick --xml | sed -ne 's/.*<status>\([01]\)<\/status>/\1/p'
 }
+
+function online_brick_count {
+	local bricks
+	local total=0
+	local i
+	for i in $(seq 1 $CLUSTER_COUNT); do
+		bricks=$(find $B0/$i/glusterd/vols -name '*.pid' | wc -l)
+		total=$((total+bricks))
+	done
+	echo $total
+}
diff --git a/tests/volume.rc b/tests/volume.rc
index 2062f42940e..3a649e850a6 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -23,7 +23,7 @@ function brick_count()
 
 function online_brick_count ()
 {
-    pgrep glusterfsd | wc -l
+    find $GLUSTERD_WORKDIR/vols/ -name '*.pid' | wc -l
 }
 
 function brick_up_status {