# Copyright (C) 2017-2018 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. from time import sleep import re from glusto.core import Glusto as g from glustolibs.gluster.exceptions import ExecutionError from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on from glustolibs.gluster.volume_libs import (setup_volume, cleanup_volume) from glustolibs.gluster.volume_ops import set_volume_options from glustolibs.gluster.gluster_init import (stop_glusterd, start_glusterd, is_glusterd_running) @runs_on([['distributed', 'replicated', 'distributed-replicated', 'dispersed', 'distributed-dispersed'], ['glusterfs']]) class TestQuorumRelatedMessagesInSyslog(GlusterBaseClass): """ Test Cases in this module related to quorum related messages in syslog, when there are more volumes. """ @classmethod def setUpClass(cls): cls.get_super_method(cls, 'setUpClass')() # checking for peer status from every node ret = cls.validate_peers_are_connected() if not ret: raise ExecutionError("Servers are not in peer probed state") def setUp(self): """ setUp method for every test """ # calling GlusterBaseClass setUp self.get_super_method(self, 'setUp')() self.volume_list = [] # create a volume ret = setup_volume(self.mnode, self.all_servers_info, self.volume) self.volume_list.append(self.volname) if not ret: raise ExecutionError("Volume creation failed: %s" % self.volname) # Creating another volume second_volume = "second_volume" self.volume['name'] = second_volume ret = setup_volume(self.mnode, self.all_servers_info, self.volume) self.volume_list.append(second_volume) if not ret: raise ExecutionError("Volume creation failed: %s" % second_volume) def tearDown(self): """ tearDown for every test """ if not self.glusterd_service: ret = start_glusterd(self.servers[1]) if not ret: raise ExecutionError("Failed to start glusterd services " "for : %s" % self.servers[1]) # Checking glusterd service running or not ret = is_glusterd_running(self.servers[1]) if ret == 0: g.log.info("glusterd running on :%s", self.servers[1]) else: raise ExecutionError("glusterd not running on :%s" % self.servers[1]) # In this test case performing quorum operations, # deleting volumes immediately after glusterd services start, volume # deletions are failing with quorum not met, # that's the reason verifying peers are connected or not before # deleting volumes peers_not_connected = True count = 0 while count < 10: ret = self.validate_peers_are_connected() if ret: peers_not_connected = False break count += 1 sleep(5) if peers_not_connected: raise ExecutionError("Servers are not in peer probed state") # Reverting back the quorum ratio to 51% self.quorum_perecent = {'cluster.server-quorum-ratio': '51%'} ret = set_volume_options(self.mnode, 'all', self.quorum_perecent) if not ret: raise ExecutionError(ret, "gluster volume set all cluster" ".server-quorum- ratio percentage Failed" " :%s" % self.servers) g.log.info("gluster volume set all cluster.server-quorum-ratio 51" "percentage enabled successfully :%s", self.servers) # stopping the volume and Cleaning up the volume for volume in self.volume_list: ret = cleanup_volume(self.mnode, volume) if not ret: raise ExecutionError("Failed to Cleanup the " "Volume %s" % volume) g.log.info("Volume deleted successfully : %s", volume) # Calling GlusterBaseClass tearDown self.get_super_method(self, 'tearDown')() def test_quorum_messages_in_syslog_with_more_volumes(self): """ create two volumes Set server quorum to both the volumes set server quorum ratio 90% stop glusterd service any one of the node quorum regain message should be recorded with message id - 106002 for both the volumes in /var/log/messages and /var/log/glusterfs/glusterd.log start the glusterd service of same node quorum regain message should be recorded with message id - 106003 for both the volumes in /var/log/messages and /var/log/glusterfs/glusterd.log """ # pylint: disable=too-many-locals # pylint: disable=too-many-statements self.log_messages = "/var/log/messages" self.log_glusterd = "/var/log/glusterfs/glusterd.log" # Enabling server quorum all volumes self.quorum_options = {'cluster.server-quorum-type': 'server'} for volume in self.volume_list: ret = set_volume_options(self.mnode, volume, self.quorum_options) self.assertTrue(ret, "gluster volume set %s cluster.server" "-quorum-type server Failed" % self.volname) g.log.info("gluster volume set %s cluster.server-quorum" "-type server enabled successfully", self.volname) # Setting Quorum ratio in percentage self.quorum_perecent = {'cluster.server-quorum-ratio': '91%'} ret = set_volume_options(self.mnode, 'all', self.quorum_perecent) self.assertTrue(ret, "gluster volume set all cluster.server-quorum-" "ratio percentage Failed :%s" % self.servers) g.log.info("gluster volume set all cluster.server-quorum-ratio 91 " "percentage enabled successfully :%s", self.servers) # counting quorum regain messages-id '106002' in /var/log/messages # file, before glusterd services stop cmd_messages = ' '.join(['grep -o', '106002', self.log_messages, '| wc -l']) ret, before_glusterd_stop_msgid_count, _ = g.run(self.mnode, cmd_messages) self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " "106002 count in : %s" % self.log_messages) # counting quorum regain messages-id '106002' in # /var/log/glusterfs/glusterd.log file, before glusterd services stop cmd_glusterd = ' '.join(['grep -o', '106002', self.log_glusterd, '| wc -l']) ret, before_glusterd_stop_glusterd_id_count, _ = g.run(self.mnode, cmd_glusterd) self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " "106002 count in :%s" % self.log_glusterd) # Stopping glusterd services ret = stop_glusterd(self.servers[1]) self.glusterd_service = False self.assertTrue(ret, "Failed stop glusterd services : %s" % self.servers[1]) g.log.info("Stopped glusterd services successfully on: %s", self.servers[1]) # checking glusterd service stopped or not ret = is_glusterd_running(self.servers[1]) self.assertEqual(ret, 1, "glusterd service should be stopped") # counting quorum regain messages-id '106002' in /var/log/messages file # after glusterd services stop. count = 0 msg_count = False expected_msg_id_count = int(before_glusterd_stop_msgid_count) + 2 while count <= 10: ret, after_glusterd_stop_msgid_count, _ = g.run(self.mnode, cmd_messages) if(re.search(r'\b' + str(expected_msg_id_count) + r'\b', after_glusterd_stop_msgid_count)): msg_count = True break sleep(5) count += 1 self.assertTrue(msg_count, "Failed to grep quorum regain message-id " "106002 count in :%s" % self.log_messages) # counting quorum regain messages-id '106002' in # /var/log/glusterfs/glusterd.log file after glusterd services stop ret, after_glusterd_stop_glusterd_id_count, _ = g.run(self.mnode, cmd_glusterd) self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " "106002 count in :%s" % self.log_glusterd) # Finding quorum regain message-id count difference between before # and after glusterd services stop in /var/log/messages count_diff = (int(after_glusterd_stop_msgid_count) - int(before_glusterd_stop_msgid_count)) self.assertEqual(count_diff, 2, "Failed to record regain messages " "in : %s" % self.log_messages) g.log.info("regain messages recorded for two volumes " "successfully after glusterd services stop " ":%s", self.log_messages) # Finding quorum regain message-id count difference between before # and after glusterd services stop in /var/log/glusterfs/glusterd.log count_diff = (int(after_glusterd_stop_glusterd_id_count) - int(before_glusterd_stop_glusterd_id_count)) self.assertEqual(count_diff, 2, "Failed to record regain messages in " ": %s" % self.log_glusterd) g.log.info("regain messages recorded for two volumes successfully " "after glusterd services stop :%s", self.log_glusterd) # counting quorum messages-id '106003' in a /var/log/messages file # before glusterd services start cmd_messages = ' '.join(['grep -o', '106003', self.log_messages, '| wc -l']) ret, before_glusterd_start_msgid_count, _ = g.run(self.mnode, cmd_messages) self.assertEqual(ret, 0, "Failed to grep quorum message-id 106003 " "count in :%s" % self.log_messages) # counting quorum regain messages-id '106003' in # /var/log/glusterfs/glusterd.log file before glusterd services start cmd_glusterd = ' '.join(['grep -o', '106003', self.log_glusterd, '| wc -l']) ret, before_glusterd_start_glusterd_id_count, _ = g.run(self.mnode, cmd_glusterd) self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " "106003 count in :%s" % self.log_glusterd) # Startin glusterd services ret = start_glusterd(self.servers[1]) self.glusterd_service = True self.assertTrue(ret, "Failed to start glusterd " "services: %s" % self.servers[1]) # Checking glusterd service running or not ret = is_glusterd_running(self.servers[1]) self.assertEqual(ret, 0, "glusterd service should be running") # counting quorum messages-id '106003' in a file in a # /var/log/messages file after glusterd service start count = 0 expected_msg_id_count = int(before_glusterd_start_msgid_count) + 2 msg_count = False while count <= 10: ret, after_glusterd_start_msgid_count, _ = g.run(self.mnode, cmd_messages) if (re.search(r'\b' + str(expected_msg_id_count) + r'\b', after_glusterd_start_msgid_count)): msg_count = True break sleep(5) count += 1 self.assertTrue(msg_count, "Failed to grep quorum message-id 106003 " "count in :%s" % self.log_messages) # counting quorum regain messages-id '106003' in # /var/log/glusterfs/glusterd.log file after glusterd services start ret, after_glusterd_start_glusterd_id_count, _ = g.run(self.mnode, cmd_glusterd) self.assertEqual(ret, 0, "Failed to grep quorum regain message-id " "106003 count in :%s" % self.log_glusterd) # Finding quorum regain message-id count difference between before # and after glusterd services start in /var/log/messages count_diff = (int(after_glusterd_start_msgid_count) - int(before_glusterd_start_msgid_count)) self.assertEqual(count_diff, 2, "Failed to record regain " "messages in :%s" % self.log_messages) g.log.info("regain messages recorded for two volumes successfully " "after glusterd services start in :%s", self.log_messages) # Finding quorum regain message-id count difference between before # and after glusterd services start in /var/log/glusterfs/glusterd.log count_diff = (int(after_glusterd_start_glusterd_id_count) - int(before_glusterd_start_glusterd_id_count)) self.assertEqual(count_diff, 2, "Failed to record regain messages " "in : %s" % self.log_glusterd) g.log.info("regain messages recorded for two volumes successfully " "after glusterd services start :%s", self.log_glusterd)