summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkshithijiyer <kshithij.ki@gmail.com>2019-05-13 14:06:17 +0530
committerBala Konda Reddy M <bmekala@redhat.com>2019-11-21 09:05:47 +0000
commit6ac0650e14a1dcf2ea42504b78e4d55c1dc6205c (patch)
tree1aae5cdffa20c58ba9326cf74acefd509df1f7ba
parent736ada3540d174c64d6c10424af65cf69a8da4e9 (diff)
[TC] Resubmitting testcase test_glusterd_quorum_validation after bug fix
As the below mentioned bug is fixed resubmitting testcase: https://bugzilla.redhat.com/show_bug.cgi?id=1690753 Test case: -> Creating two volumes and starting them, stop the second volume -> Set the server quorum and set the ratio to 90 -> Stop the glusterd in one of the node, so the quorum won't meet -> Peer probing a new node should fail -> Volume stop will fail -> Volume delete will fail -> Volume reset will fail -> Start the glusterd on the node where it is stopped -> Volume stop, start, delete will succeed once quorum is met Change-Id: Ic9dea44364d4cb84b6170eb1f1cfeff1398b7a9b Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>
-rw-r--r--tests/functional/glusterd/test_glusterd_quorum.py301
1 files changed, 301 insertions, 0 deletions
diff --git a/tests/functional/glusterd/test_glusterd_quorum.py b/tests/functional/glusterd/test_glusterd_quorum.py
new file mode 100644
index 0000000..e60397c
--- /dev/null
+++ b/tests/functional/glusterd/test_glusterd_quorum.py
@@ -0,0 +1,301 @@
+# Copyright (C) 2017-2018 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from time import sleep
+import pytest
+from glusto.core import Glusto as g
+from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.brick_libs import wait_for_bricks_to_be_online
+from glustolibs.gluster.volume_libs import (setup_volume, volume_exists,
+ cleanup_volume)
+from glustolibs.gluster.gluster_init import (stop_glusterd, start_glusterd,
+ is_glusterd_running)
+from glustolibs.gluster.volume_ops import (set_volume_options, volume_start,
+ volume_stop, volume_delete,
+ get_volume_list, volume_reset)
+from glustolibs.gluster.peer_ops import (is_peer_connected, peer_probe_servers,
+ peer_detach_servers, peer_probe)
+
+
+@runs_on([['distributed-replicated', 'replicated'], ['glusterfs']])
+class TestServerQuorum(GlusterBaseClass):
+
+ @classmethod
+ def setUpClass(cls):
+ # Calling GlusterBaseClass setUpClass
+ GlusterBaseClass.setUpClass.im_func(cls)
+ ret = volume_exists(cls.mnode, cls.volname)
+ if ret:
+ ret = cleanup_volume(cls.mnode, cls.volname)
+ if not ret:
+ raise ExecutionError("Unable to delete volume")
+ g.log.info("Successfully deleted volume % s", cls.volname)
+
+ # Check if peer is connected state or not and detach all the nodes
+ for server in cls.servers:
+ ret = is_peer_connected(server, cls.servers)
+ if ret:
+ ret = peer_detach_servers(server, cls.servers)
+ if not ret:
+ raise ExecutionError(
+ "Detach failed from all the servers from the node.")
+ g.log.info("Peer detach SUCCESSFUL.")
+
+ # Before starting the testcase, proceed only it has minimum of 4 nodes
+ if len(cls.servers) < 4:
+ raise ExecutionError("Minimun four nodes required for this "
+ " testcase to execute")
+
+ @classmethod
+ def tearDownClass(cls):
+
+ # Setting quorum ratio to 51%
+ ret = set_volume_options(cls.mnode, 'all',
+ {'cluster.server-quorum-ratio': '51%'})
+ if not ret:
+ raise ExecutionError("Failed to set server quorum ratio on %s"
+ % cls.volname)
+
+ vol_list = get_volume_list(cls.mnode)
+ if vol_list is None:
+ raise ExecutionError("Failed to get volume list")
+
+ for volume in vol_list:
+ ret = cleanup_volume(cls.mnode, volume)
+ if not ret:
+ raise ExecutionError("Failed Cleanup the volume")
+ g.log.info("Volume deleted successfully %s", volume)
+
+ # Peer probe servers since we are doing peer detach in setUpClass
+ for server in cls.servers:
+ ret = is_peer_connected(server, cls.servers)
+ if not ret:
+ ret = peer_probe_servers(server, cls.servers)
+ if not ret:
+ raise ExecutionError(
+ "Peer probe failed to one of the node")
+ g.log.info("Peer probe successful")
+
+ GlusterBaseClass.tearDownClass.im_func(cls)
+
+ @pytest.mark.test_glusterd_quorum_validation
+ def test_glusterd_quorum_validation(self):
+ """
+ -> Creating two volumes and starting them, stop the second volume
+ -> set the server quorum and set the ratio to 90
+ -> Stop the glusterd in one of the node, so the quorum won't meet
+ -> Peer probing a new node should fail
+ -> Volume stop will fail
+ -> volume delete will fail
+ -> volume reset will fail
+ -> Start the glusterd on the node where it is stopped
+ -> Volume stop, start, delete will succeed once quorum is met
+ """
+ # pylint: disable=too-many-statements, too-many-branches
+
+ # Peer probe first 3 servers
+ servers_info_from_three_nodes = {}
+ for server in self.servers[0:3]:
+ servers_info_from_three_nodes[
+ server] = self.all_servers_info[server]
+
+ # Peer probe the first 3 servers
+ ret, _, _ = peer_probe(self.mnode, server)
+ self.assertEqual(ret, 0,
+ ("Peer probe failed to one of the server"))
+ g.log.info("Peer probe to first 3 nodes succeeded")
+
+ self.volume['servers'] = self.servers[0:3]
+ # Create a volume using the first 3 nodes
+ ret = setup_volume(self.mnode, servers_info_from_three_nodes,
+ self.volume, force=True)
+ self.assertTrue(ret, ("Failed to create and start volume"))
+ g.log.info("Volume created and started successfully")
+
+ # Creating another volume and stopping it
+ second_volume = "second_volume"
+ self.volume['name'] = second_volume
+ ret = setup_volume(self.mnode, servers_info_from_three_nodes,
+ self.volume, force=True)
+ self.assertTrue(ret, ("Failed to create and start volume"))
+ g.log.info("Volume created and started succssfully")
+
+ # stopping the second volume
+ g.log.info("Stopping the second volume %s", second_volume)
+ ret, _, _ = volume_stop(self.mnode, second_volume)
+ self.assertEqual(ret, 0, ("Failed to stop the volume"))
+ g.log.info("Successfully stopped second volume %s", second_volume)
+
+ # Setting the server-quorum-type as server
+ self.options = {"cluster.server-quorum-type": "server"}
+ vol_list = get_volume_list(self.mnode)
+ self.assertIsNotNone(vol_list, "Failed to get the volume list")
+ g.log.info("Fetched the volume list")
+ for volume in vol_list:
+ g.log.info("Setting the server-quorum-type as server"
+ " on volume %s", volume)
+ ret = set_volume_options(self.mnode, volume, self.options)
+ self.assertTrue(ret, ("Failed to set the quorum type as a server"
+ " on volume %s", volume))
+ g.log.info("Server Quorum type is set as a server")
+
+ # Setting the server quorum ratio to 90
+ self.quorum_perecent = {'cluster.server-quorum-ratio': '90%'}
+ ret = set_volume_options(self.mnode, 'all', self.quorum_perecent)
+ self.assertTrue(ret, ("Failed to set the server quorum ratio "
+ "to 90 on servers"))
+ g.log.info("Successfully set server quorum ratio to 90% on servers")
+
+ # Stop glusterd on one of the node
+ ret = stop_glusterd(self.servers[2])
+ self.assertTrue(ret, ("Failed to stop glusterd on "
+ "node %s", self.servers[2]))
+ g.log.info("Glusterd stop on the nodes : %s"
+ " succeeded", self.servers[2])
+
+ # Check glusterd is stopped
+ ret = is_glusterd_running(self.servers[2])
+ self.assertEqual(ret, 1, "Unexpected: Glusterd is running on node")
+ g.log.info("Expected: Glusterd stopped on node %s", self.servers[2])
+
+ # Adding a new peer will fail as quorum not met
+ ret, _, _ = peer_probe(self.mnode, self.servers[3])
+ self.assertNotEqual(ret, 0, (
+ "Unexpected:"
+ "Succeeded to peer probe new node %s when quorum "
+ "is not met", self.servers[3]))
+ g.log.info("Failed to peer probe new node as expected"
+ " when quorum not met")
+
+ # Stopping an already started volume should fail as quorum is not met
+ ret, _, _ = volume_start(self.mnode, second_volume)
+ self.assertNotEqual(ret, 0, "Unexpected: Successfuly started "
+ "volume even when quorum not met.")
+ g.log.info("Volume start %s failed as expected when quorum "
+ "is not met", second_volume)
+
+ # Stopping a volume should fail stop the first volume
+ ret, _, _ = volume_stop(self.mnode, self.volname)
+ self.assertEqual(ret, 1, "Unexpected: Successfully stopped"
+ " volume even when quourm is not met")
+ g.log.info("volume stop %s failed as expected when quorum "
+ "is not met", self.volname)
+
+ # Stopping a volume with force option should fail
+ ret, _, _ = volume_stop(self.mnode, self.volname, force=True)
+ self.assertNotEqual(ret, 0, "Unexpected: Successfully "
+ "stopped volume with force. Expected: "
+ "Volume stop should fail when quourm is not met")
+ g.log.info("volume stop failed as expected when quorum is not met")
+
+ # Deleting a volume should fail. Deleting the second volume.
+ ret = volume_delete(self.mnode, second_volume)
+ self.assertFalse(ret, "Unexpected: Volume delete was "
+ "successful even when quourm is not met")
+ g.log.info("volume delete failed as expected when quorum is not met")
+
+ # Volume reset should fail when quorum is not met
+ ret, _, _ = volume_reset(self.mnode, self.volname)
+ self.assertNotEqual(ret, 0, "Unexpected: Volume reset was "
+ "successful even when quorum is not met")
+ g.log.info("volume reset failed as expected when quorum is not met")
+
+ # Volume reset should fail even with force when quourum is not met
+ ret, _, _ = volume_reset(self.mnode, self.volname, force=True)
+ self.assertNotEqual(ret, 0, "Unexpected: Volume reset was "
+ "successful with force even "
+ "when quourm is not met")
+ g.log.info("volume reset failed as expected when quorum is not met")
+
+ # Start glusterd on the node where glusterd is stopped
+ ret = start_glusterd(self.servers[2])
+ self.assertTrue(ret, "Failed to start glusterd on one node")
+ g.log.info("Started glusterd on server"
+ " %s successfully", self.servers[2])
+
+ ret = is_glusterd_running(self.servers[2])
+ self.assertEqual(ret, 0, ("glusterd is not running on "
+ "node %s", self.servers[2]))
+ g.log.info("glusterd is running on node"
+ " %s ", self.servers[2])
+
+ # Check peer status whether all peer are in connected state none of the
+ # nodes should be in peer rejected state
+ halt, counter, _rc = 30, 0, False
+ g.log.info("Wait for some seconds, right after glusterd start it "
+ "will create two daemon process it need few seconds "
+ "(like 3-5) to initialize the glusterd")
+ while counter < halt:
+ ret = is_peer_connected(self.mnode, self.servers[0:3])
+ if not ret:
+ g.log.info("Peers are not connected state,"
+ " Retry after 2 seconds .......")
+ sleep(2)
+ counter = counter + 2
+ else:
+ _rc = True
+ g.log.info("Peers are in connected state in the cluster")
+ break
+
+ self.assertTrue(_rc, ("Peers are not connected state after "
+ "bringing back glusterd online on the "
+ "nodes in which previously glusterd "
+ "had been stopped"))
+
+ # Check all bricks are online or wait for the bricks to be online
+ ret = wait_for_bricks_to_be_online(self.mnode, self.volname)
+ self.assertTrue(ret, "All bricks are not online")
+ g.log.info("All bricks of the volume %s are online", self.volname)
+
+ # Once quorum is met should be able to cleanup the volume
+ ret = volume_delete(self.mnode, second_volume)
+ self.assertTrue(ret, "Volume delete failed even when quorum is met")
+ g.log.info("volume delete succeed without any issues")
+
+ # Volume stop should succeed
+ ret, _, _ = volume_stop(self.mnode, self.volname)
+ self.assertEqual(ret, 0, "Volume stop failed")
+ g.log.info("succeeded stopping the volume as expected")
+
+ # volume reset should succeed
+ ret, _, _ = volume_reset(self.mnode, self.volname)
+ self.assertEqual(ret, 0, "Volume reset failed ")
+ g.log.info("volume reset succeeded as expected when quorum is not met")
+
+ # Peer probe new node should succeed
+ ret, _, _ = peer_probe(self.mnode, self.servers[3])
+ self.assertEqual(ret, 0, (
+ "Failed to peer probe new node even when quorum is met"))
+ g.log.info("Succeeded to peer probe new node when quorum met")
+
+ # Check peer status whether all peer are in connected state none of the
+ # nodes should be in peer rejected state
+ halt, counter, _rc = 30, 0, False
+ g.log.info("Wait for some seconds, right after peer probe")
+ while counter < halt:
+ ret = is_peer_connected(self.mnode, self.servers[0:3])
+ if not ret:
+ g.log.info("Peers are not connected state,"
+ " Retry after 2 seconds .......")
+ sleep(2)
+ counter = counter + 2
+ else:
+ _rc = True
+ g.log.info("Peers are in connected state in the cluster")
+ break
+
+ self.assertTrue(_rc, ("Peers are not connected state"))