diff options
author | Rajesh Madaka <rmadaka@redhat.com> | 2018-02-01 19:00:40 +0530 |
---|---|---|
committer | Rajesh Madaka <rmadaka@redhat.com> | 2018-02-08 16:07:20 +0530 |
commit | 6db9d9e4d4e8442e99339e9bc6fbc7bfb4784819 (patch) | |
tree | a0f1afc07e9536663c68567ddda261d9f0192216 | |
parent | 634a6c40ad3323b0050548025a0b4018a6477c34 (diff) |
quorum related messages in logs
desc:
create two volumes
Set server quorum to both the volumes
set server quorum ratio 90%
stop glusterd service any one of the node
quorum regain message should be recorded with message id - 106002
for both the volumes in /var/log/messages and
/var/log/glusterfs/glusterd.log
start the glusterd service of same node
quorum regain message should be recorded with message id - 106003
for both the volumes in /var/log/messages and
/var/log/glusterfs/glusterd.log
Change-Id: I9ecab59b6131fc9c4c58bb972b3a41f15af1b87c
Signed-off-by: Rajesh Madaka <rmadaka@redhat.com>
-rw-r--r-- | tests/functional/glusterd/test_quorum_related_messages_in_syslog.py | 292 |
1 files changed, 292 insertions, 0 deletions
diff --git a/tests/functional/glusterd/test_quorum_related_messages_in_syslog.py b/tests/functional/glusterd/test_quorum_related_messages_in_syslog.py new file mode 100644 index 000000000..2b21a2a29 --- /dev/null +++ b/tests/functional/glusterd/test_quorum_related_messages_in_syslog.py @@ -0,0 +1,292 @@ +# Copyright (C) 2017-2018 Red Hat, Inc. <http://www.redhat.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" Description: + Test Cases in this module related to quorum + related messages in syslog, when there are more volumes. +""" + +from glusto.core import Glusto as g +from glustolibs.gluster.exceptions import ExecutionError +from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on +from glustolibs.gluster.volume_libs import (setup_volume, cleanup_volume) +from glustolibs.gluster.volume_ops import set_volume_options +from glustolibs.gluster.gluster_init import (stop_glusterd, start_glusterd, + is_glusterd_running) +from time import sleep +import re + + +@runs_on([['distributed', 'replicated', 'distributed-replicated', + 'dispersed', 'distributed-dispersed'], ['glusterfs']]) +class TestQuorumRelatedMessagesInSyslog(GlusterBaseClass): + @classmethod + def setUpClass(cls): + GlusterBaseClass.setUpClass.im_func(cls) + + # checking for peer status from every node + ret = cls.validate_peers_are_connected() + if not ret: + raise ExecutionError("Servers are not in peer probed state") + + def setUp(self): + """ + setUp method for every test + """ + # calling GlusterBaseClass setUp + GlusterBaseClass.setUp.im_func(self) + self.volume_list = [] + # create a volume + ret = setup_volume(self.mnode, self.all_servers_info, + self.volume) + self.volume_list.append(self.volname) + if not ret: + raise ExecutionError("Volume creation failed: %s" % self.volname) + + # Creating another volume + second_volume = "second_volume" + self.volume['name'] = second_volume + ret = setup_volume(self.mnode, self.all_servers_info, + self.volume) + self.volume_list.append(second_volume) + if not ret: + raise ExecutionError("Volume creation failed: %s" % second_volume) + + def tearDown(self): + """ + tearDown for every test + """ + if not self.glusterd_service: + ret = start_glusterd(self.servers[1]) + if not ret: + raise ExecutionError("Failed to start glusterd services " + "for : %s" % self.servers[1]) + + # Checking glusterd service running or not + ret = is_glusterd_running(self.servers[1]) + if (ret == 0): + g.log.info("glusterd running on :%s" % self.servers[1]) + else: + raise ExecutionError("glusterd not running on :%s" + % self.servers[1]) + + # In this test case performing quorum operations, + # deleting volumes immediately after glusterd services start, volume + # deletions are failing with quorum not met, + # that's the reason verifying peers are connected or not before + # deleting volumes + peers_not_connected = True + count = 0 + while(count < 10): + ret = self.validate_peers_are_connected() + if ret: + peers_not_connected = False + break + count += 1 + sleep(5) + if (peers_not_connected): + raise ExecutionError("Servers are not in peer probed state") + + # stopping the volume and Cleaning up the volume + for volume in self.volume_list: + ret = cleanup_volume(self.mnode, volume) + if not ret: + raise ExecutionError("Failed to Cleanup the " + "Volume %s" % volume) + g.log.info("Volume deleted successfully : %s" % volume) + + # Calling GlusterBaseClass tearDown + GlusterBaseClass.tearDown.im_func(self) + + def test_quorum_messages_in_syslog_with_more_volumes(self): + """ + create two volumes + Set server quorum to both the volumes + set server quorum ratio 90% + stop glusterd service any one of the node + quorum regain message should be recorded with message id - 106002 + for both the volumes in /var/log/messages and + /var/log/glusterfs/glusterd.log + start the glusterd service of same node + quorum regain message should be recorded with message id - 106003 + for both the volumes in /var/log/messages and + /var/log/glusterfs/glusterd.log + """ + + self.log_messages = "/var/log/messages" + self.log_glusterd = "/var/log/glusterfs/glusterd.log" + + # Enabling server quorum all volumes + self.quorum_options = {'cluster.server-quorum-type': 'server'} + for volume in self.volume_list: + ret = set_volume_options(self.mnode, volume, self.quorum_options) + self.assertTrue(ret, "gluster volume set %s cluster.server" + "-quorum-type server Failed" % self.volname) + g.log.info("gluster volume set %s cluster.server-quorum" + "-type server enabled successfully" % self.volname) + + # Setting Quorum ratio in percentage + self.quorum_perecent = {'cluster.server-quorum-ratio': '91%'} + ret = set_volume_options(self.mnode, 'all', self.quorum_perecent) + self.assertTrue(ret, "gluster volume set all cluster.server-quorum-" + "ratio percentage Failed :%s" % self.servers) + g.log.info("gluster volume set all cluster.server-quorum-ratio 91 " + "percentage enabled successfully :%s" % self.servers) + + # counting quorum regain messages-id '106002' in /var/log/messages + # file, before glusterd services stop + cmd_messages = ' '.join(['grep -o', '106002', self.log_messages, + '| wc -l']) + ret, before_glusterd_stop_msgid_count, _ = g.run(self.mnode, + cmd_messages) + self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " + "106002 count in : %s" % self.log_messages) + + # counting quorum regain messages-id '106002' in + # /var/log/glusterfs/glusterd.log file, before glusterd services stop + cmd_glusterd = ' '.join(['grep -o', '106002', self.log_glusterd, + '| wc -l']) + ret, before_glusterd_stop_glusterd_id_count, _ = g.run(self.mnode, + cmd_glusterd) + self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " + "106002 count in :%s" % self.log_glusterd) + + # Stopping glusterd services + ret = stop_glusterd(self.servers[1]) + self.glusterd_service = False + self.assertTrue(ret, "Failed stop glusterd services : %s" + % self.servers[1]) + g.log.info("Stopped glusterd services successfully on: %s" + % self.servers[1]) + + # checking glusterd service stopped or not + ret = is_glusterd_running(self.servers[1]) + self.assertEqual(ret, 1, "glusterd service should be stopped") + + # counting quorum regain messages-id '106002' in /var/log/messages file + # after glusterd services stop. + count = 0 + msg_count = False + expected_msg_id_count = int(before_glusterd_stop_msgid_count) + 2 + while (count <= 10): + ret, after_glusterd_stop_msgid_count, _ = g.run(self.mnode, + cmd_messages) + if(re.search(r'\b' + str(expected_msg_id_count) + r'\b', + after_glusterd_stop_msgid_count)): + msg_count = True + break + sleep(5) + count += 1 + self.assertTrue(msg_count, "Failed to grep quorum regain message-id " + "106002 count in :%s" % self.log_messages) + + # counting quorum regain messages-id '106002' in + # /var/log/glusterfs/glusterd.log file after glusterd services stop + ret, after_glusterd_stop_glusterd_id_count, _ = g.run(self.mnode, + cmd_glusterd) + self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " + "106002 count in :%s" % self.log_glusterd) + + # Finding quorum regain message-id count difference between before + # and after glusterd services stop in /var/log/messages + count_diff = (int(after_glusterd_stop_msgid_count) - + int(before_glusterd_stop_msgid_count)) + + self.assertEqual(count_diff, 2, "Failed to record regain messages " + "in : %s" % self.log_messages) + g.log.info("regain messages recorded for two volumes " + "successfully after glusterd services stop " + ":%s" % self.log_messages) + + # Finding quorum regain message-id count difference between before + # and after glusterd services stop in /var/log/glusterfs/glusterd.log + count_diff = (int(after_glusterd_stop_glusterd_id_count) - + int(before_glusterd_stop_glusterd_id_count)) + self.assertEqual(count_diff, 2, "Failed to record regain messages in " + ": %s" % self.log_glusterd) + g.log.info("regain messages recorded for two volumes successfully " + "after glusterd services stop :%s" % self.log_glusterd) + + # counting quorum messages-id '106003' in a /var/log/messages file + # before glusterd services start + cmd_messages = ' '.join(['grep -o', '106003', self.log_messages, + '| wc -l']) + ret, before_glusterd_start_msgid_count, _ = g.run(self.mnode, + cmd_messages) + self.assertEqual(ret, 0, "Failed to grep quorum message-id 106003 " + "count in :%s" % self.log_messages) + + # counting quorum regain messages-id '106003' in + # /var/log/glusterfs/glusterd.log file before glusterd services start + cmd_glusterd = ' '.join(['grep -o', '106003', self.log_glusterd, + '| wc -l']) + ret, before_glusterd_start_glusterd_id_count, _ = g.run(self.mnode, + cmd_glusterd) + self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " + "106003 count in :%s" % self.log_glusterd) + + # Startin glusterd services + ret = start_glusterd(self.servers[1]) + self.glusterd_service = True + self.assertTrue(ret, "Failed to start glusterd " + "services: %s" % self.servers[1]) + + # Checking glusterd service running or not + ret = is_glusterd_running(self.servers[1]) + self.assertEqual(ret, 0, "glusterd service should be running") + + # counting quorum messages-id '106003' in a file in a + # /var/log/messages file after glusterd service start + count = 0 + expected_msg_id_count = int(before_glusterd_start_msgid_count) + 2 + msg_count = False + while(count <= 10): + ret, after_glusterd_start_msgid_count, _ = g.run(self.mnode, + cmd_messages) + if (re.search(r'\b' + str(expected_msg_id_count) + r'\b', + after_glusterd_start_msgid_count)): + msg_count = True + break + sleep(5) + count += 1 + + self.assertTrue(msg_count, "Failed to grep quorum message-id 106003 " + "count in :%s" % self.log_messages) + + # counting quorum regain messages-id '106003' in + # /var/log/glusterfs/glusterd.log file after glusterd services start + ret, after_glusterd_start_glusterd_id_count, _ = g.run(self.mnode, + cmd_glusterd) + self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " + "106003 count in :%s" % self.log_glusterd) + + # Finding quorum regain message-id count difference between before + # and after glusterd services start in /var/log/messages + count_diff = (int(after_glusterd_start_msgid_count) - + int(before_glusterd_start_msgid_count)) + self.assertEqual(count_diff, 2, "Failed to record regain " + "messages in :%s" % self.log_messages) + g.log.info("regain messages recorded for two volumes successfully " + "after glusterd services start in :%s" % self.log_messages) + + # Finding quorum regain message-id count difference between before + # and after glusterd services start in /var/log/glusterfs/glusterd.log + count_diff = (int(after_glusterd_start_glusterd_id_count) - + int(before_glusterd_start_glusterd_id_count)) + self.assertEqual(count_diff, 2, "Failed to record regain messages " + "in : %s" % self.log_glusterd) + g.log.info("regain messages recorded for two volumes successfully " + "after glusterd services start :%s" % self.log_glusterd) |