summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSanju Rakonde <srakonde@redhat.com>2018-06-14 11:48:36 +0530
committerJonathan Holloway <jholloway@redhat.com>2018-06-19 23:12:55 +0000
commit37fab28f7bd20875bb7a1dcaa70a5e9f6907e355 (patch)
tree499312460ab2ea134da1932b1d2ed400204e658d
parent316f6a58fc1a22278a2dbc15b2cd7829ec9d7b99 (diff)
glusto-tests/glusterd: remove brick should fail, when server quorum is not met
Steps followed are: 1. Create and a start a volume 2. Set cluster.server-quorum-type as server 3. Set cluster.server-quorum-ratio as 95% 3. Bring down glusterd in half of the nodes 4. Confirm that quorum is not met, by check whether the bricks are down. 5. Perform a remove brick operation, which should fail. Change-Id: I69525651727ec92dce2f346ad706ab0943490a2d Signed-off-by: Sanju Rakonde <srakonde@redhat.com>
-rw-r--r--tests/functional/glusterd/test_remove_brick_when_quorum_not_met.py140
1 files changed, 140 insertions, 0 deletions
diff --git a/tests/functional/glusterd/test_remove_brick_when_quorum_not_met.py b/tests/functional/glusterd/test_remove_brick_when_quorum_not_met.py
new file mode 100644
index 000000000..50f7c5e7b
--- /dev/null
+++ b/tests/functional/glusterd/test_remove_brick_when_quorum_not_met.py
@@ -0,0 +1,140 @@
+# Copyright (C) 2017-2018 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from time import sleep
+from glusto.core import Glusto as g
+from glustolibs.gluster.gluster_base_class import runs_on, GlusterBaseClass
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.volume_libs import setup_volume
+from glustolibs.gluster.volume_ops import set_volume_options, get_volume_status
+from glustolibs.gluster.gluster_init import (stop_glusterd, start_glusterd,
+ is_glusterd_running)
+from glustolibs.gluster.brick_libs import get_all_bricks, are_bricks_offline
+from glustolibs.gluster.brick_ops import remove_brick
+
+
+@runs_on([['distributed'], ['glusterfs']])
+class TestRemoveBrickWhenQuorumNotMet(GlusterBaseClass):
+
+ def setUp(self):
+
+ GlusterBaseClass.setUp.im_func(self)
+
+ # check whether peers are in connected state
+ ret = self.validate_peers_are_connected()
+ if not ret:
+ raise ExecutionError("Peers are not in connected state")
+
+ def tearDown(self):
+
+ ret = is_glusterd_running(self.servers)
+ if ret:
+ ret = start_glusterd(self.servers)
+ if not ret:
+ raise ExecutionError("Failed to start glusterd on servers")
+
+ # checking for peer status from every node
+ count = 0
+ while count < 80:
+ ret = self.validate_peers_are_connected()
+ if ret:
+ break
+ sleep(2)
+ count += 1
+
+ if not ret:
+ raise ExecutionError("Servers are not in connected state")
+
+ # stopping the volume and Cleaning up the volume
+ ret = self.cleanup_volume()
+ if not ret:
+ raise ExecutionError("Failed to Cleanup the Volume %s"
+ % self.volname)
+ g.log.info("Volume deleted successfully : %s", self.volname)
+
+ GlusterBaseClass.tearDown.im_func(self)
+
+ def test_remove_brick_when_quorum_not_met(self):
+
+ # create and start a volume
+ ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
+ self.assertTrue(ret, ("Failed to create "
+ "and start volume %s" % self.volname))
+
+ # set cluster.server-quorum-type as server
+ ret = set_volume_options(self.mnode, self.volname,
+ {'cluster.server-quorum-type': 'server'})
+ self.assertTrue(ret, ("Failed to set the quorum type as a server"
+ " on volume %s", self.volname))
+
+ # Setting quorum ratio to 95%
+ ret = set_volume_options(self.mnode, 'all',
+ {'cluster.server-quorum-ratio': '95%'})
+ self.assertTrue(ret, "Failed to set server quorum ratio on %s"
+ % self.volname)
+
+ # bring down glusterd of half nodes
+ num_of_servers = len(self.servers)
+ num_of_nodes_to_bring_down = num_of_servers/2
+
+ for node in range(num_of_nodes_to_bring_down, num_of_servers):
+ ret = stop_glusterd(self.servers[node])
+ self.assertTrue(ret, ("Failed to stop glusterd on %s"
+ % self.servers[node]))
+
+ for node in range(num_of_nodes_to_bring_down, num_of_servers):
+ count = 0
+ while count < 80:
+ ret = is_glusterd_running(self.servers[node])
+ if ret:
+ break
+ sleep(2)
+ count += 1
+ self.assertNotEqual(ret, 0, "glusterd is still running on %s"
+ % self.servers[node])
+
+ # Verifying node count in volume status after glusterd stopped
+ # on half of the servers, Its not possible to check the brick status
+ # immediately in volume status after glusterd stop
+ count = 0
+ while count < 100:
+ vol_status = get_volume_status(self.mnode, self.volname)
+ servers_count = len(vol_status[self.volname].keys())
+ if servers_count == num_of_servers - num_of_nodes_to_bring_down:
+ break
+ sleep(2)
+ count += 1
+
+ # confirm that quorum is not met, brick process should be down
+ bricks_list = get_all_bricks(self.mnode, self.volname)
+ self.assertIsNotNone(bricks_list, "Failed to get the brick list")
+ bricks_to_check = bricks_list[0:num_of_nodes_to_bring_down]
+ ret = are_bricks_offline(self.mnode, self.volname, bricks_to_check)
+ self.assertTrue(ret, "Server quorum is not met, Bricks are up")
+
+ # try remove brick operation, which should fail
+ brick1 = bricks_list[0]
+ ret, _, _ = remove_brick(self.mnode, self.volname, [brick1], "start")
+ self.assertNotEqual(ret, 0, ("remove brick is success, when quorum is"
+ " not met"))
+
+ # confirm that, removed brick is still part of volume
+ bricks_list = get_all_bricks(self.mnode, self.volname)
+ self.assertIsNotNone(bricks_list, "Failed to get the brick list")
+ if brick1 not in bricks_list:
+ ret = False
+ self.assertTrue(ret, ("remove brick is success, when quorum is"
+ " not met"))