diff options
Diffstat (limited to 'glustolibs-gluster/glustolibs/gluster/gluster_base_class.py')
| -rwxr-xr-x[-rw-r--r--] | glustolibs-gluster/glustolibs/gluster/gluster_base_class.py | 1491 |
1 files changed, 712 insertions, 779 deletions
diff --git a/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py b/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py index 97a9fe365..65061cb13 100644..100755 --- a/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py +++ b/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Red Hat, Inc. <http://www.redhat.com> +# Copyright (C) 2018-2021 Red Hat, Inc. <http://www.redhat.com> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,35 +19,54 @@ variables necessary for tests. """ -import unittest -import os -import random -import copy -import datetime -import time -import socket +from copy import deepcopy +from datetime import datetime +from inspect import isclass +from os.path import join as path_join +from random import choice as random_choice +from socket import ( + gethostbyname, + gaierror, +) +from unittest import TestCase +from time import sleep + from glusto.core import Glusto as g -from glustolibs.gluster.exceptions import ConfigError, ExecutionError -from glustolibs.gluster.peer_ops import is_peer_connected, peer_status -from glustolibs.gluster.volume_ops import set_volume_options -from glustolibs.gluster.block_ops import block_delete -from glustolibs.gluster.block_libs import (setup_block, if_block_exists, - get_block_list, - get_block_info) -from glustolibs.gluster.volume_libs import (setup_volume, - cleanup_volume, - log_volume_info_and_status) -from glustolibs.gluster.volume_libs import ( - wait_for_volume_process_to_be_online) -from glustolibs.gluster.samba_libs import share_volume_over_smb -from glustolibs.gluster.nfs_libs import export_volume_through_nfs + +from glustolibs.gluster.exceptions import ( + ConfigError, + ExecutionError, +) +from glustolibs.gluster.lib_utils import inject_msg_in_logs from glustolibs.gluster.mount_ops import create_mount_objs +from glustolibs.gluster.nfs_libs import export_volume_through_nfs +from glustolibs.gluster.peer_ops import ( + is_peer_connected, + peer_probe_servers, peer_status +) +from glustolibs.gluster.gluster_init import ( + restart_glusterd, stop_glusterd, wait_for_glusterd_to_start) +from glustolibs.gluster.samba_libs import share_volume_over_smb +from glustolibs.gluster.shared_storage_ops import is_shared_volume_mounted +from glustolibs.gluster.volume_libs import ( + cleanup_volume, + log_volume_info_and_status, + setup_volume, + wait_for_volume_process_to_be_online, +) +from glustolibs.gluster.brick_libs import ( + wait_for_bricks_to_be_online, get_offline_bricks_list) +from glustolibs.gluster.volume_ops import ( + set_volume_options, volume_reset, volume_start) from glustolibs.io.utils import log_mounts_info -from glustolibs.gluster.lib_utils import inject_msg_in_logs +from glustolibs.gluster.geo_rep_libs import setup_master_and_slave_volumes +from glustolibs.gluster.nfs_ganesha_ops import ( + teardown_nfs_ganesha_cluster) +from glustolibs.misc.misc_libs import kill_process class runs_on(g.CarteTestClass): - """Decorator providing runs_on capability for standard unittest script""" + """Decorator providing runs_on capability for standard unittest script.""" def __init__(self, value): # the names of the class attributes set by the runs_on decorator @@ -56,21 +75,19 @@ class runs_on(g.CarteTestClass): # the options to replace 'ALL' in selections self.available_options = [['distributed', 'replicated', 'distributed-replicated', - 'dispersed', 'distributed-dispersed'], + 'dispersed', 'distributed-dispersed', + 'arbiter', 'distributed-arbiter'], ['glusterfs', 'nfs', 'cifs', 'smb']] # these are the volume and mount options to run and set in config # what do runs_on_volumes and runs_on_mounts need to be named???? - run_on_volumes = self.available_options[0] - run_on_mounts = self.available_options[1] - if 'gluster' in g.config and g.config['gluster']: - if ('running_on_volumes' in g.config['gluster'] and - g.config['gluster']['running_on_volumes']): - run_on_volumes = g.config['gluster']['running_on_volumes'] - - if ('running_on_mounts' in g.config['gluster'] and - g.config['gluster']['running_on_mounts']): - run_on_mounts = g.config['gluster']['running_on_mounts'] + run_on_volumes, run_on_mounts = self.available_options[0:2] + if g.config.get('gluster', {}).get('running_on_volumes'): + run_on_volumes = g.config['gluster']['running_on_volumes'] + + if g.config.get('gluster', {}).get('running_on_mounts'): + run_on_mounts = g.config['gluster']['running_on_mounts'] + # selections is the above info from the run that is intersected with # the limits from the test script self.selections = [run_on_volumes, run_on_mounts] @@ -79,7 +96,7 @@ class runs_on(g.CarteTestClass): self.limits = value -class GlusterBaseClass(unittest.TestCase): +class GlusterBaseClass(TestCase): """GlusterBaseClass to be subclassed by Gluster Tests. This class reads the config for variable values that will be used in gluster tests. If variable values are not specified in the config file, @@ -89,6 +106,36 @@ class GlusterBaseClass(unittest.TestCase): # defaults in setUpClass() volume_type = None mount_type = None + error_or_failure_exists = False + + @staticmethod + def get_super_method(obj, method_name): + """PY2/3 compatible method for getting proper parent's (super) methods. + + Useful for test classes wrapped by 'runs_on' decorator which has + duplicated original test class [py3] as parent instead of the + base class as it is expected. + + Example for calling 'setUp()' method of the base class from the + 'setUp' method of a test class which was decorated with 'runs_on': + + @runs_on([['distributed'], ['glusterfs']]) + class TestDecoratedClass(GlusterBaseClass): + ... + @classmethod + def setUpClass(cls): + cls.get_super_method(cls, 'setUpClass')() + ... + def setUp(self): + self.get_super_method(self, 'setUp')() + ... + + """ + current_type = obj if isclass(obj) else obj.__class__ + while getattr(super(current_type, obj), method_name) == getattr( + obj, method_name): + current_type = current_type.__base__ + return getattr(super(current_type, obj), method_name) @classmethod def inject_msg_in_gluster_logs(cls, msg): @@ -129,12 +176,12 @@ class GlusterBaseClass(unittest.TestCase): list: List of IP's corresponding to the hostnames of nodes. """ nodes_ips = [] - if isinstance(nodes, str): + if not isinstance(nodes, list): nodes = [nodes] for node in nodes: try: - ip = socket.gethostbyname(node) - except socket.gaierror as e: + ip = gethostbyname(node) + except gaierror as e: g.log.error("Failed to get the IP of Host: %s : %s", node, e.strerror) ip = None @@ -149,6 +196,11 @@ class GlusterBaseClass(unittest.TestCase): Returns (bool): True if all peers are in connected with other peers. False otherwise. """ + + # If the setup has single node server, by pass this validation. + if len(cls.servers) == 1: + return True + # Validate if peer is connected from all the servers g.log.info("Validating if servers %s are connected from other servers " "in the cluster", cls.servers) @@ -171,11 +223,121 @@ class GlusterBaseClass(unittest.TestCase): return True + def _is_error_or_failure_exists(self): + """Function to get execution error in case of + failures in testcases + """ + if hasattr(self, '_outcome'): + # Python 3.4+ + result = self.defaultTestResult() + self._feedErrorsToResult(result, self._outcome.errors) + else: + # Python 2.7-3.3 + result = getattr( + self, '_outcomeForDoCleanups', self._resultForDoCleanups) + ok_result = True + for attr in ('errors', 'failures'): + if not hasattr(result, attr): + continue + exc_list = getattr(result, attr) + if exc_list and exc_list[-1][0] is self: + ok_result = ok_result and not exc_list[-1][1] + if hasattr(result, '_excinfo'): + ok_result = ok_result and not result._excinfo + if ok_result: + return False + self.error_or_failure_exists = True + GlusterBaseClass.error_or_failure_exists = True + return True + + @classmethod + def scratch_cleanup(cls, error_or_failure_exists): + """ + This scratch_cleanup script will run only when the code + currently running goes into execution or assertion error. + + Args: + error_or_failure_exists (bool): If set True will cleanup setup + atlast of testcase only if exectution or assertion error in + teststeps. False will skip this scratch cleanup step. + + Returns (bool): True if setup cleanup is successful. + False otherwise. + """ + if error_or_failure_exists: + shared_storage_mounted = False + if is_shared_volume_mounted(cls.mnode): + shared_storage_mounted = True + ret = stop_glusterd(cls.servers) + if not ret: + g.log.error("Failed to stop glusterd") + cmd_list = ("pkill `pidof glusterd`", + "rm /var/run/glusterd.socket") + for server in cls.servers: + for cmd in cmd_list: + ret, _, _ = g.run(server, cmd, "root") + if ret: + g.log.error("Failed to stop glusterd") + return False + for server in cls.servers: + ret, out, _ = g.run(server, "pgrep glusterfsd", "root") + if not ret: + ret = kill_process(server, + process_ids=out.strip().split('\n')) + if not ret: + g.log.error("Unable to kill process {}".format( + out.strip().split('\n'))) + return False + if not shared_storage_mounted: + cmd_list = ( + "rm -rf /var/lib/glusterd/vols/*", + "rm -rf /var/lib/glusterd/snaps/*", + "rm -rf /var/lib/glusterd/peers/*", + "rm -rf {}/*/*".format( + cls.all_servers_info[server]['brick_root'])) + else: + cmd_list = ( + "for vol in `ls /var/lib/glusterd/vols/ | " + "grep -v gluster_shared_storage`;do " + "rm -rf /var/lib/glusterd/vols/$vol;done", + "rm -rf /var/lib/glusterd/snaps/*" + "rm -rf {}/*/*".format( + cls.all_servers_info[server]['brick_root'])) + for cmd in cmd_list: + ret, _, _ = g.run(server, cmd, "root") + if ret: + g.log.error( + "failed to cleanup server {}".format(server)) + return False + ret = restart_glusterd(cls.servers) + if not ret: + g.log.error("Failed to start glusterd") + return False + sleep(2) + ret = wait_for_glusterd_to_start(cls.servers) + if not ret: + g.log.error("Failed to bring glusterd up") + return False + if not shared_storage_mounted: + ret = peer_probe_servers(cls.mnode, cls.servers) + if not ret: + g.log.error("Failed to peer probe servers") + return False + for client in cls.clients: + cmd_list = ("umount /mnt/*", "rm -rf /mnt/*") + for cmd in cmd_list: + ret = g.run(client, cmd, "root") + if ret: + g.log.error( + "failed to unmount/already unmounted {}" + .format(client)) + return True + @classmethod - def setup_volume(cls, volume_create_force=False): + def setup_volume(cls, volume_create_force=False, only_volume_create=False): """Setup the volume: - Create the volume, Start volume, Set volume - options, enable snapshot/quota/tier if specified in the config + options, enable snapshot/quota if specified in the config file. - Wait for volume processes to be online - Export volume as NFS/SMB share if mount_type is NFS or SMB @@ -184,6 +346,9 @@ class GlusterBaseClass(unittest.TestCase): Args: volume_create_force(bool): True if create_volume should be executed with 'force' option. + only_volume_create(bool): True, only volume creation is needed + False, by default volume creation and + start. Returns (bool): True if all the steps mentioned in the descriptions passes. False otherwise. @@ -206,12 +371,19 @@ class GlusterBaseClass(unittest.TestCase): g.log.info("Setting up volume %s", cls.volname) ret = setup_volume(mnode=cls.mnode, all_servers_info=cls.all_servers_info, - volume_config=cls.volume, force=force_volume_create) + volume_config=cls.volume, force=force_volume_create, + create_only=only_volume_create) if not ret: g.log.error("Failed to Setup volume %s", cls.volname) return False g.log.info("Successful in setting up volume %s", cls.volname) + # Returning the value without proceeding for next steps + if only_volume_create and ret: + g.log.info("Setup volume with volume creation {} " + "successful".format(cls.volname)) + return True + # Wait for volume processes to be online g.log.info("Wait for volume %s processes to be online", cls.volname) ret = wait_for_volume_process_to_be_online(cls.mnode, cls.volname) @@ -302,6 +474,9 @@ class GlusterBaseClass(unittest.TestCase): """ g.log.info("Starting to mount volume %s", cls.volname) for mount_obj in mounts: + # For nfs-ganesha, mount is done via vip + if cls.enable_nfs_ganesha: + mount_obj.server_system = cls.vips[0] g.log.info("Mounting volume '%s:%s' on '%s:%s'", mount_obj.server_system, mount_obj.volname, mount_obj.client_system, mount_obj.mountpoint) @@ -350,6 +525,53 @@ class GlusterBaseClass(unittest.TestCase): return True @classmethod + def bricks_online_and_volume_reset(cls): + """ + reset the volume if any bricks are offline. + waits for all bricks to be online and resets + volume options set + """ + bricks_offline = get_offline_bricks_list(cls.mnode, cls.volname) + if bricks_offline is not None: + ret = volume_start(cls.mnode, cls.volname, force=True) + if not ret: + raise ExecutionError("Failed to force start volume" + "%s" % cls.volname) + ret = wait_for_bricks_to_be_online(cls.mnode, cls.volname) + if not ret: + raise ExecutionError("Failed to bring bricks online" + "for volume %s" % cls.volname) + + ret, _, _ = volume_reset(cls.mnode, cls.volname, force=True) + if ret: + raise ExecutionError("Failed to reset volume %s" % cls.volname) + g.log.info("Successful in volume reset %s", cls.volname) + + @classmethod + def setup_and_mount_geo_rep_master_and_slave_volumes(cls, force=False): + """Setup geo-rep master and slave volumes. + + Returns (bool): True if cleanup volume is successful. False otherwise. + """ + # Creating and starting master and slave volume. + ret = setup_master_and_slave_volumes( + cls.mode, cls.all_servers_info, cls.master_volume, + cls.snode, cls.all_slaves_info, cls.slave_volume, + force) + if not ret: + g.log.error('Failed to create master and slave volumes.') + return False + + # Mounting master and slave volumes + for mount in [cls.master_mounts, cls.slave_mounts]: + ret = cls.mount_volume(cls, mount) + if not ret: + g.log.error('Failed to mount volume %s.', + mount['volname']) + return False + return True + + @classmethod def unmount_volume(cls, mounts): """Unmount all mounts for the volume @@ -376,13 +598,24 @@ class GlusterBaseClass(unittest.TestCase): log_mounts_info(cls.mounts) return False - else: - g.log.info("Successful in unmounting volume '%s:%s' on " - "'%s:%s'", mount_obj.server_system, - mount_obj.volname, mount_obj.client_system, - mount_obj.mountpoint) - g.log.info("Successful in unmounting all mount objs for the volume %s", - cls.volname) + + g.log.info("Starting to delete the directory path used for " + "mounting") + cmd = ('rm -rf %s' % mount_obj.mountpoint) + ret, _, err = g.run( + mount_obj.client_system, cmd, user=mount_obj.user) + if ret: + g.log.error( + "failed to delete the directory path used for " + "mounting %s: %s" % (mount_obj.mountpoint, err)) + return False + + g.log.info( + "Successful in deleting the directory path used for " + "mounting '%s:%s' on '%s:%s'" % ( + mount_obj.server_system, + mount_obj.volname, mount_obj.client_system, + mount_obj.mountpoint)) # Get mounts info g.log.info("Get mounts Info:") @@ -391,11 +624,30 @@ class GlusterBaseClass(unittest.TestCase): return True @classmethod + def get_unique_lv_list_from_all_servers(cls): + """Get all unique lv path from all servers + + Returns: List of all unique lv path in all servers. None otherwise. + """ + cmd = "lvs --noheadings -o lv_path | awk '{if ($1) print $1}'" + lv_list = [] + for server in cls.servers: + ret, out, _ = g.run(server, cmd, "root") + current_lv_list = out.splitlines() + if current_lv_list: + lv_list.extend(current_lv_list) + if ret: + g.log.error("failed to execute command %s" % cmd) + raise ExecutionError("Failed to execute %s cmd" % cmd) + return list(set(lv_list)) + + @classmethod def cleanup_volume(cls): """Cleanup the volume Returns (bool): True if cleanup volume is successful. False otherwise. """ + cls.bricks_online_and_volume_reset() g.log.info("Cleanup Volume %s", cls.volname) ret = cleanup_volume(mnode=cls.mnode, volname=cls.volname) if not ret: @@ -407,7 +659,28 @@ class GlusterBaseClass(unittest.TestCase): g.log.info("Log Volume %s Info and Status", cls.volname) log_volume_info_and_status(cls.mnode, cls.volname) - return ret + # compare and remove additional lv created, skip otherwise + new_lv_list = cls.get_unique_lv_list_from_all_servers() + if cls.lv_list != new_lv_list: + cmd = ("for mnt in `mount | grep 'run/gluster/snaps' |" + "awk '{print $3}'`; do umount $mnt; done") + for server in cls.servers: + ret, _, err = g.run(server, cmd, "root") + if ret: + g.log.error("Failed to remove snap " + "bricks from mountpoint %s" % err) + return False + new_lv_list = cls.get_unique_lv_list_from_all_servers() + lv_remove_list = list(set(new_lv_list) - set(cls.lv_list)) + for server in cls.servers: + for lv in lv_remove_list: + cmd = ("lvremove %s --force" % lv) + ret, _, err = g.run(server, cmd, "root") + if ret: + g.log.error("failed to remove lv: %s" % err) + g.log.info("Expected error msg '%s'" % err) + g.log.info("Successfully cleaned-up volumes") + return True @classmethod def unmount_volume_and_cleanup_volume(cls, mounts): @@ -433,8 +706,7 @@ class GlusterBaseClass(unittest.TestCase): @classmethod def setUpClass(cls): - """Initialize all the variables necessary for testing Gluster - """ + """Initialize all the variables necessary for testing Gluster.""" # Get all servers cls.all_servers = None if 'servers' in g.config and g.config['servers']: @@ -445,17 +717,16 @@ class GlusterBaseClass(unittest.TestCase): # Get all slaves cls.slaves = None - if 'slaves' in g.config and g.config['slaves']: + if g.config.get('slaves'): cls.slaves = g.config['slaves'] # Set mnode_slave : Node on which slave commands are executed cls.mnode_slave = cls.slaves[0] # Slave IP's - cls.slaves_ip = [] cls.slaves_ip = cls.get_ip_from_hostname(cls.slaves) # Get all clients cls.all_clients = None - if 'clients' in g.config and g.config['clients']: + if g.config.get('clients'): cls.all_clients = g.config['clients'] cls.clients = cls.all_clients else: @@ -463,29 +734,31 @@ class GlusterBaseClass(unittest.TestCase): # Get all servers info cls.all_servers_info = None - if 'servers_info' in g.config and g.config['servers_info']: + if g.config.get('servers_info'): cls.all_servers_info = g.config['servers_info'] else: raise ConfigError("'servers_info' not defined in the global " "config") # Get all slaves info cls.all_slaves_info = None - if 'slaves_info' in g.config and g.config['slaves_info']: + if g.config.get('slaves_info'): cls.all_slaves_info = g.config['slaves_info'] # All clients_info cls.all_clients_info = None - if 'clients_info' in g.config and g.config['clients_info']: + if g.config.get('clients_info'): cls.all_clients_info = g.config['clients_info'] else: raise ConfigError("'clients_info' not defined in the global " "config") + # get lv list + cls.lv_list = cls.get_unique_lv_list_from_all_servers() + # Set mnode : Node on which gluster commands are executed cls.mnode = cls.all_servers[0] # Server IP's - cls.servers_ips = [] cls.servers_ips = cls.get_ip_from_hostname(cls.servers) # SMB Cluster info @@ -500,58 +773,82 @@ class GlusterBaseClass(unittest.TestCase): # NFS-Ganesha Cluster info try: - cls.enable_nfs_ganesha = bool(g.config['gluster']['cluster_config'] - ['nfs_ganesha']['enable']) - cls.num_of_nfs_ganesha_nodes = (g.config['gluster'] - ['cluster_config']['nfs_ganesha'] - ['num_of_nfs_ganesha_nodes']) - cls.vips = (g.config['gluster']['cluster_config']['nfs_ganesha'] - ['vips']) + cls.enable_nfs_ganesha = ( + g.config['gluster']['cluster_config']['nfs_ganesha']['enable'] + in ('TRUE', 'True', 'true', 'YES', 'Yes', 'yes', '1', 1) + ) + cls.num_of_nfs_ganesha_nodes = g.config['gluster'][ + 'cluster_config']['nfs_ganesha']['num_of_nfs_ganesha_nodes'] + cls.vips = ( + g.config['gluster']['cluster_config']['nfs_ganesha']['vips']) except KeyError: cls.enable_nfs_ganesha = False cls.num_of_nfs_ganesha_nodes = None cls.vips = [] + # Geo-rep Cluster information + try: + cls.geo_rep_info = (g.config['gluster']['geo_rep'] + ['cluster_config']) + except KeyError: + cls.geo_rep_info = {} + cls.geo_rep_info['root'] = {} + cls.geo_rep_info['user'] = {} + cls.geo_rep_info['root']['password'] = '' + cls.geo_rep_info['user']['name'] = '' + cls.geo_rep_info['user']['password'] = '' + cls.geo_rep_info['user']['group'] = '' + # Defining default volume_types configuration. cls.default_volume_type_config = { 'replicated': { 'type': 'replicated', 'replica_count': 3, - 'transport': 'tcp' - }, + 'transport': 'tcp', + }, 'dispersed': { 'type': 'dispersed', 'disperse_count': 6, 'redundancy_count': 2, - 'transport': 'tcp' - }, + 'transport': 'tcp', + }, 'distributed': { 'type': 'distributed', 'dist_count': 4, - 'transport': 'tcp' - }, + 'transport': 'tcp', + }, 'distributed-replicated': { 'type': 'distributed-replicated', 'dist_count': 2, 'replica_count': 3, - 'transport': 'tcp' - }, + 'transport': 'tcp', + }, 'distributed-dispersed': { 'type': 'distributed-dispersed', 'dist_count': 2, 'disperse_count': 6, 'redundancy_count': 2, - 'transport': 'tcp' - } + 'transport': 'tcp', + }, + 'arbiter': { + 'type': 'arbiter', + 'replica_count': 3, + 'arbiter_count': 1, + 'transport': 'tcp', + }, + 'distributed-arbiter': { + 'type': 'distributed-arbiter', + 'dist_count': 2, + 'replica_count': 3, + 'arbiter_count': 1, + 'tranport': 'tcp', } + } - # Check if default volume_type configuration is provided in - # config yml - if (g.config.get('gluster') and - g.config['gluster'].get('volume_types')): + # Check if default volume_type configuration is provided in config yml + if g.config.get('gluster', {}).get('volume_types'): default_volume_type_from_config = ( g.config['gluster']['volume_types']) - for volume_type in default_volume_type_from_config.keys(): if default_volume_type_from_config[volume_type]: if volume_type in cls.default_volume_type_config: @@ -560,53 +857,40 @@ class GlusterBaseClass(unittest.TestCase): # Create Volume with force option cls.volume_create_force = False - if (g.config.get('gluster') and - g.config['gluster'].get('volume_create_force')): + if g.config.get('gluster', {}).get('volume_create_force'): cls.volume_create_force = ( g.config['gluster']['volume_create_force']) # Default volume options which is applicable for all the volumes cls.volume_options = {} - if (g.config.get('gluster') and - g.config['gluster'].get('volume_options')): + if g.config.get('gluster', {}).get('volume_options'): cls.volume_options = g.config['gluster']['volume_options'] # If the volume is exported as SMB Share, then set the following # volume options on the share. cls.smb_share_options = {} - if (g.config.get('gluster') and - g.config['gluster'].get('smb_share_options')): - cls.smb_share_options = ( - g.config['gluster']['smb_share_options']) + if g.config.get('gluster', {}).get('smb_share_options'): + cls.smb_share_options = g.config['gluster']['smb_share_options'] # If the volume is exported as NFS-Ganesha export, # then set the following volume options on the export. cls.nfs_ganesha_export_options = {} - if (g.config.get('gluster') and - g.config['gluster'].get('nfs_ganesha_export_options')): + if g.config.get('gluster', {}).get('nfs_ganesha_export_options'): cls.nfs_ganesha_export_options = ( g.config['gluster']['nfs_ganesha_export_options']) # Get the volume configuration. cls.volume = {} if cls.volume_type: - found_volume = False - if 'gluster' in g.config: - if 'volumes' in g.config['gluster']: - for volume in g.config['gluster']['volumes']: - if volume['voltype']['type'] == cls.volume_type: - cls.volume = copy.deepcopy(volume) - found_volume = True - break - - if found_volume: - if 'name' not in cls.volume: - cls.volume['name'] = 'testvol_%s' % cls.volume_type - - if 'servers' not in cls.volume: - cls.volume['servers'] = cls.all_servers - - if not found_volume: + for volume in g.config.get('gluster', {}).get('volumes', []): + if volume['voltype']['type'] == cls.volume_type: + cls.volume = deepcopy(volume) + if 'name' not in cls.volume: + cls.volume['name'] = 'testvol_%s' % cls.volume_type + if 'servers' not in cls.volume: + cls.volume['servers'] = cls.all_servers + break + else: try: if g.config['gluster']['volume_types'][cls.volume_type]: cls.volume['voltype'] = (g.config['gluster'] @@ -614,8 +898,8 @@ class GlusterBaseClass(unittest.TestCase): [cls.volume_type]) except KeyError: try: - cls.volume['voltype'] = (cls.default_volume_type_config - [cls.volume_type]) + cls.volume['voltype'] = ( + cls.default_volume_type_config[cls.volume_type]) except KeyError: raise ConfigError("Unable to get configs of volume " "type: %s", cls.volume_type) @@ -633,75 +917,89 @@ class GlusterBaseClass(unittest.TestCase): cls.mnode = cls.servers[0] cls.vol_options = cls.volume['options'] + # Define useful variable for geo-rep volumes. + if cls.slaves: + # For master volume + cls.master_volume = cls.volume + cls.master_volume['name'] = ('master_testvol_%s' + % cls.volume_type) + cls.master_volname = cls.master_volume['name'] + cls.master_voltype = (cls.master_volume['voltype'] + ['type']) + + # For slave volume + cls.slave_volume = deepcopy(cls.volume) + cls.slave_volume['name'] = ('slave_testvol_%s' + % cls.volume_type) + cls.slave_volume['servers'] = cls.slaves + cls.slave_volname = cls.slave_volume['name'] + cls.slave_voltype = (cls.slave_volume['voltype'] + ['type']) + # Get the mount configuration. cls.mounts = [] if cls.mount_type: cls.mounts_dict_list = [] - found_mount = False - if 'gluster' in g.config: - if 'mounts' in g.config['gluster']: - for mount in g.config['gluster']['mounts']: - if mount['protocol'] == cls.mount_type: - temp_mount = {} - temp_mount['protocol'] = cls.mount_type - if 'volname' in mount and mount['volname']: - if mount['volname'] == cls.volname: - temp_mount = copy.deepcopy(mount) - else: - continue - else: - temp_mount['volname'] = cls.volname - if ('server' not in mount or - (not mount['server'])): - temp_mount['server'] = cls.mnode - else: - temp_mount['server'] = mount['server'] - if ('mountpoint' not in mount or - (not mount['mountpoint'])): - temp_mount['mountpoint'] = (os.path.join( - "/mnt", '_'.join([cls.volname, - cls.mount_type]))) - else: - temp_mount['mountpoint'] = mount['mountpoint'] - if ('client' not in mount or - (not mount['client'])): - temp_mount['client'] = ( - cls.all_clients_info[ - random.choice( - cls.all_clients_info.keys())] - ) - else: - temp_mount['client'] = mount['client'] - if 'options' in mount and mount['options']: - temp_mount['options'] = mount['options'] - else: - temp_mount['options'] = '' - cls.mounts_dict_list.append(temp_mount) - found_mount = True - - if not found_mount: + for mount in g.config.get('gluster', {}).get('mounts', []): + if mount['protocol'] != cls.mount_type: + continue + temp_mount = { + 'protocol': cls.mount_type, + 'volname': cls.volname, + } + if mount.get('volname'): + if mount['volname'] == cls.volname: + temp_mount = deepcopy(mount) + else: + continue + temp_mount.update({ + 'server': mount.get('server', cls.mnode), + 'mountpoint': mount.get('mountpoint', path_join( + "/mnt", '_'.join([cls.volname, cls.mount_type]))), + 'client': mount.get('client', cls.all_clients_info[ + random_choice(list(cls.all_clients_info.keys()))]), + 'options': mount.get('options', ''), + }) + cls.mounts_dict_list.append(temp_mount) + + if not cls.mounts_dict_list: for client in cls.all_clients_info.keys(): - mount = { + cls.mounts_dict_list.append({ 'protocol': cls.mount_type, 'server': cls.mnode, 'volname': cls.volname, 'client': cls.all_clients_info[client], - 'mountpoint': (os.path.join( - "/mnt", '_'.join([cls.volname, cls.mount_type]))), - 'options': '' - } - cls.mounts_dict_list.append(mount) + 'mountpoint': path_join( + "/mnt", '_'.join([cls.volname, cls.mount_type])), + 'options': '', + }) if cls.mount_type == 'cifs' or cls.mount_type == 'smb': for mount in cls.mounts_dict_list: if 'smbuser' not in mount: - mount['smbuser'] = random.choice( - cls.smb_users_info.keys()) + mount['smbuser'] = random_choice( + list(cls.smb_users_info.keys())) mount['smbpasswd'] = ( cls.smb_users_info[mount['smbuser']]['password']) cls.mounts = create_mount_objs(cls.mounts_dict_list) + # Setting mounts for geo-rep volumes. + if cls.slaves: + + # For master volume mount + cls.master_mounts = cls.mounts + + # For slave volume mount + slave_mount_dict_list = deepcopy(cls.mounts_dict_list) + for mount_dict in slave_mount_dict_list: + mount_dict['volname'] = cls.slave_volume + mount_dict['server'] = cls.mnode_slave + mount_dict['mountpoint'] = path_join( + "/mnt", '_'.join([cls.slave_volname, + cls.mount_type])) + cls.slave_mounts = create_mount_objs(slave_mount_dict_list) + # Defining clients from mounts. cls.clients = [] for mount in cls.mounts_dict_list: @@ -709,37 +1007,26 @@ class GlusterBaseClass(unittest.TestCase): cls.clients = list(set(cls.clients)) # Gluster Logs info - cls.server_gluster_logs_dirs = ["/var/log/glusterfs", - "/var/log/samba"] + cls.server_gluster_logs_dirs = ["/var/log/glusterfs", "/var/log/samba"] cls.server_gluster_logs_files = ["/var/log/ganesha.log", "/var/log/ganesha-gfapi.log"] - if ('gluster' in g.config and - 'server_gluster_logs_info' in g.config['gluster']): + if g.config.get('gluster', {}).get('server_gluster_logs_info'): server_gluster_logs_info = ( g.config['gluster']['server_gluster_logs_info']) - if ('dirs' in server_gluster_logs_info and - server_gluster_logs_info['dirs']): - cls.server_gluster_logs_dirs = ( - server_gluster_logs_info['dirs']) - - if ('files' in server_gluster_logs_info and - server_gluster_logs_info['files']): + if server_gluster_logs_info.get('dirs'): + cls.server_gluster_logs_dirs = server_gluster_logs_info['dirs'] + if server_gluster_logs_info.get('files'): cls.server_gluster_logs_files = ( server_gluster_logs_info['files']) cls.client_gluster_logs_dirs = ["/var/log/glusterfs"] cls.client_gluster_logs_files = [] - if ('gluster' in g.config and - 'client_gluster_logs_info' in g.config['gluster']): + if g.config.get('gluster', {}).get('client_gluster_logs_info'): client_gluster_logs_info = ( g.config['gluster']['client_gluster_logs_info']) - if ('dirs' in client_gluster_logs_info and - client_gluster_logs_info['dirs']): - cls.client_gluster_logs_dirs = ( - client_gluster_logs_info['dirs']) - - if ('files' in client_gluster_logs_info and - client_gluster_logs_info['files']): + if client_gluster_logs_info.get('dirs'): + cls.client_gluster_logs_dirs = client_gluster_logs_info['dirs'] + if client_gluster_logs_info.get('files'): cls.client_gluster_logs_files = ( client_gluster_logs_info['files']) @@ -747,9 +1034,34 @@ class GlusterBaseClass(unittest.TestCase): # gluster logs if 'glustotest_run_id' not in g.config: g.config['glustotest_run_id'] = ( - datetime.datetime.now().strftime('%H_%M_%d_%m_%Y')) + datetime.now().strftime('%H_%M_%d_%m_%Y')) cls.glustotest_run_id = g.config['glustotest_run_id'] + if cls.enable_nfs_ganesha: + g.log.info("Setup NFS_Ganesha") + cls.num_of_nfs_ganesha_nodes = int(cls.num_of_nfs_ganesha_nodes) + cls.servers_in_nfs_ganesha_cluster = ( + cls.servers[:cls.num_of_nfs_ganesha_nodes]) + cls.vips_in_nfs_ganesha_cluster = ( + cls.vips[:cls.num_of_nfs_ganesha_nodes]) + + # Obtain hostname of servers in ganesha cluster + cls.ganesha_servers_hostname = [] + for ganesha_server in cls.servers_in_nfs_ganesha_cluster: + ret, hostname, _ = g.run(ganesha_server, "hostname") + if ret: + raise ExecutionError("Failed to obtain hostname of %s" + % ganesha_server) + hostname = hostname.strip() + g.log.info("Obtained hostname: IP- %s, hostname- %s", + ganesha_server, hostname) + cls.ganesha_servers_hostname.append(hostname) + from glustolibs.gluster.nfs_ganesha_libs import setup_nfs_ganesha + ret = setup_nfs_ganesha(cls) + if not ret: + raise ExecutionError("Failed to setup nfs ganesha") + g.log.info("Successful in setting up NFS Ganesha Cluster") + msg = "Setupclass: %s : %s" % (cls.__name__, cls.glustotest_run_id) g.log.info(msg) cls.inject_msg_in_gluster_logs(msg) @@ -773,642 +1085,263 @@ class GlusterBaseClass(unittest.TestCase): g.log.info(msg) cls.inject_msg_in_gluster_logs(msg) + def doCleanups(self): + if (self.error_or_failure_exists or + self._is_error_or_failure_exists()): + ret = self.scratch_cleanup(self.error_or_failure_exists) + g.log.info(ret) + return self.get_super_method(self, 'doCleanups')() -class GlusterBlockBaseClass(GlusterBaseClass): - """GlusterBlockBaseClass sets up the volume and blocks. - """ @classmethod - def setup_blocks(cls, blocknames): - """Create blocks and calls the methods: - update_block_info_dict and create_client_block_map - - Args: - blocknames(list): Blocks to be create - Returns: - bool: False if block creation is unsuccessful and - true if all blocks created. - - """ - if not isinstance(blocknames, list): - blocknames = [blocknames] - - g.log.info("Creating block devices on volume %s", cls.volname) - for blockname in blocknames: - each_block = cls.gluster_block_args_info.get(blockname) - if each_block: - # Form a dict for keyargs - block_args_info = {} - block_args_info['ha'] = each_block['ha'] - block_args_info['auth'] = each_block['auth'] - block_args_info['prealloc'] = each_block['prealloc'] - block_args_info['storage'] = each_block['storage'] - block_args_info['ring-buffer'] = each_block['ring-buffer'] - - _rc = setup_block( - mnode=cls.mnode, - volname=each_block['volname'], - blockname=each_block['blockname'], - servers=each_block['servers'], - size=each_block['size'], - **block_args_info) - if not _rc: - g.log.error("Failed to create block on volume " - "%s: \n%s", cls.volname, each_block) - return False - g.log.info("Successfully created block on volume " - "%s: \n%s", cls.volname, each_block) - else: - g.log.error("Unable to get args info for block %s on " - "volume %s", blockname, cls.volname) - return False - - # Check if all the blocks are listed in block list command - for blockname in blocknames: - each_block = cls.gluster_block_args_info.get(blockname) - _rc = if_block_exists(cls.mnode, each_block['volname'], blockname) - if not _rc: - return False - - # Update the block info dict - cls.update_block_info_dict() - # Create client-block map - cls.create_client_block_map(cls.blocknames) - return True + def doClassCleanups(cls): + if (GlusterBaseClass.error_or_failure_exists or + cls._is_error_or_failure_exists()): + ret = cls.scratch_cleanup( + GlusterBaseClass.error_or_failure_exists) + g.log.info(ret) + return cls.get_super_method(cls, 'doClassCleanups')() @classmethod - def update_block_info_dict(cls): - """Updates the class's block_info_dict variable - Calls the gluster-block info command and updates the block info. - """ - # Get Block dict - cls.blocknames = get_block_list(cls.mnode, cls.volname) - - if cls.blocknames: - for blockname in cls.blocknames: - cls.block_info_dict[blockname] = (get_block_info(cls.mnode, - cls.volname, - blockname)) - if cls.block_info_dict[blockname] is None: - g.log.error("Could not get block info") - return False - # Update total_number_of_blocks - cls.total_num_of_blocks = len(cls.blocknames) - - # Log the block_info_dict - g.log.info("Logging Block Info:") - for key, value in cls.block_info_dict.iteritems(): - g.log.info("Glusto block info: %s\n %s" % (key, value)) - - return True + def delete_nfs_ganesha_cluster(cls): + ret = teardown_nfs_ganesha_cluster( + cls.servers_in_nfs_ganesha_cluster) + if not ret: + g.log.error("Teardown got failed. Hence, cleaning up " + "nfs-ganesha cluster forcefully") + ret = teardown_nfs_ganesha_cluster( + cls.servers_in_nfs_ganesha_cluster, force=True) + if not ret: + raise ExecutionError("Force cleanup of nfs-ganesha " + "cluster failed") + g.log.info("Teardown nfs ganesha cluster succeeded") @classmethod - def discover_blocks_on_clients(cls, blocknames): - """Discover blocks on all the clients - """ - # List all the block devices on clients (Logging) - - if not isinstance(blocknames, list): - blocknames = [blocknames] - - # results = g.run_parallel(cls.clients, "lsblk -S") - server_list = [] - for blockname in blocknames: - block_info = get_block_info(cls.mnode, cls.volname, blockname) - if block_info: - servers_to_add = block_info.get("EXPORTED ON") - for server_ip in servers_to_add: - if server_ip not in server_list: - server_list.append(server_ip) - else: - g.log.error("Failed to get block info for block %s" - " on volume %s", blockname, cls.volname) - return False + def start_memory_and_cpu_usage_logging(cls, test_id, interval=60, + count=100): + """Upload logger script and start logging usage on cluster - g.log.info("Server list %s", server_list) - # Discover the block devices from clients - for client in cls.clients: - for server in server_list: - cmd = ("iscsiadm -m discovery -t st -p %s" % - server) - ret, out, err = g.run(client, cmd) - if ret != 0: - g.log.error("Failed to discover blocks on " - "client %s: %s", client, err) - return False - g.log.info("Discovered blocks on client %s: %s", - client, out) - return True + Args: + test_id(str): ID of the test running fetched from self.id() - @classmethod - def get_iqn_of_blocks_on_clients(cls, blocknames): - """Get iqn number of each block on it's respective client. + Kawrgs: + interval(int): Time interval after which logs are to be collected + (Default: 60) + count(int): Number of samples to be collected(Default: 100) - Args: - blocknames: list Returns: - bool: True if iqn of all blocks is obtained. False otherwise + proc_dict(dict):Dictionary of logging processes """ - if not isinstance(blocknames, list): - blocknames = [blocknames] - for blockname in blocknames: + # imports are added inside function to make it them + # optional and not cause breakage on installation + # which don't use the resource leak library + from glustolibs.io.memory_and_cpu_utils import ( + check_upload_memory_and_cpu_logger_script, + log_memory_and_cpu_usage_on_cluster) - try: - block_gbid = cls.block_info_dict[blockname]['GBID'] - except KeyError: - g.log.error("Failed to get GBID of block %s on volume %s", - blockname, cls.volname) - return False + # Checking if script is present on servers or not if not then + # upload it to servers. + if not check_upload_memory_and_cpu_logger_script(cls.servers): + return None - try: - block_mapped_client = ( - cls.clients_blocks_map[blockname]['client']) - except KeyError: - g.log.error("Failed to get the client which mounts the block " - "%s on the volume %s", blockname, cls.volname) - return False + # Checking if script is present on clients or not if not then + # upload it to clients. + if not check_upload_memory_and_cpu_logger_script(cls.clients): + return None - # Get the servers where the blocks are exported - server_ip = cls.block_info_dict[blockname]['EXPORTED ON'][0] - - # Get iqn from gbid - cmd = ("iscsiadm -m discovery -t st -p %s | grep -F %s | " - "tail -1 | cut -d ' ' -f2" % - (server_ip, block_gbid)) - - # Not using async here as if two processes execute the above - # command at the same time in background, it will cause: - # 'iscsiadm: Connection to Discovery Address' error - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - g.log.error("Failed to get iqn of block %s on client %s: %s", - block_gbid, block_mapped_client, err) - return False - g.log.info("Iqn for gbid '%s' on client %s : '%s'", - block_gbid, block_mapped_client, out) - block_iqn = out.strip() - cls.clients_blocks_map[blockname]['iqn'] = block_iqn + # Start logging on servers and clients + proc_dict = log_memory_and_cpu_usage_on_cluster( + cls.servers, cls.clients, test_id, interval, count) - return True + return proc_dict @classmethod - def login_to_iqn_on_clients(cls, blocknames): - """Login the blocks on their clients/initiator. + def compute_and_print_usage_stats(cls, test_id, proc_dict, + kill_proc=False): + """Compute and print CPU and memory usage statistics - Return: - Either bool or Execution error. - """ - if not isinstance(blocknames, list): - blocknames = [blocknames] - - ret = cls.update_block_info_dict() - if not ret: - return False - - result = cls.get_iqn_of_blocks_on_clients(blocknames) - if not result: - return False - - # ret_value = True - # Login to the block from the client - for blockname in blocknames: - block_gbid = cls.block_info_dict[blockname]['GBID'] - block_mapped_client = ( - cls.clients_blocks_map[blockname]['client']) - - if not cls.clients_blocks_map[blockname].get('iqn'): - g.log.error("Iqn info for block %s not there. So can't login", - blockname) - return False - - block_iqn = cls.clients_blocks_map[blockname]['iqn'] - cls.clients_blocks_map[blockname]['logged_in'] = False - - if cls.block_info_dict[blockname]['PASSWORD']: - block_password = cls.block_info_dict[blockname]['PASSWORD'] - cmd = ("iscsiadm -m node -T %s -o update -n " - "node.session.auth.authmethod -v CHAP -n " - "node.session.auth.username -v %s -n " - "node.session.auth.password -v %s " % - (block_iqn, block_gbid, block_password)) - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - g.log.error("Unable to update login credentials for " - "iqn %s on %s: %s", - block_iqn, block_mapped_client, err) - return False - g.log.info("Credentials for iqn %s updated successfully " - "on %s", - block_iqn, block_mapped_client) - - # Login to iqn - if not cls.clients_blocks_map[blockname].get('logged_in'): - cmd = "iscsiadm -m node -T %s -l" % block_iqn - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - raise ExecutionError("Failed to login to iqn %s on " - "%s: %s Command o/p: %s ", - block_iqn, block_mapped_client, - err, out) - - g.log.info("Successfully logged in to iqn %s on %s: %s", - block_iqn, block_mapped_client, out) - cls.clients_blocks_map[blockname]['logged_in'] = True - - return True + Args: + proc_dict(dict):Dictionary of logging processes + test_id(str): ID of the test running fetched from self.id() - @classmethod - def logout_iqn_on_clients(cls, blocknames): - """Logout each block from the initiator + Kwargs: + kill_proc(bool): Kill logging process if true else wait + for process to complete execution """ - # Convert string or unicode type to list - if not isinstance(blocknames, list): - blocknames = [blocknames] - - for blockname in blocknames: - block_mapped_client = ( - cls.clients_blocks_map[blockname]['client']) - block_iqn = cls.clients_blocks_map[blockname]['iqn'] - cmd = "iscsiadm -m node -T %s -u" % block_iqn - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - g.log.error("Failed to logout of iqn %s on %s: %s" - " Command o/p: %s", - block_iqn, block_mapped_client, err, out) - return False - g.log.info("Successfully logged out of iqn %s on %s: %s", - block_iqn, block_mapped_client, out) + # imports are added inside function to make it them + # optional and not cause breakage on installation + # which don't use the resource leak library + from glustolibs.io.memory_and_cpu_utils import ( + wait_for_logging_processes_to_stop, kill_all_logging_processes, + compute_data_usage_stats_on_servers, + compute_data_usage_stats_on_clients) + + # Wait or kill running logging process + if kill_proc: + nodes = cls.servers + cls.clients + ret = kill_all_logging_processes(proc_dict, nodes, cluster=True) + if not ret: + g.log.error("Unable to stop logging processes.") + else: + ret = wait_for_logging_processes_to_stop(proc_dict, cluster=True) + if not ret: + g.log.error("Processes didn't complete still running.") - return True + # Compute and print stats for servers + ret = compute_data_usage_stats_on_servers(cls.servers, test_id) + g.log.info('*' * 50) + g.log.info(ret) # TODO: Make logged message more structured + g.log.info('*' * 50) + + # Compute and print stats for clients + ret = compute_data_usage_stats_on_clients(cls.clients, test_id) + g.log.info('*' * 50) + g.log.info(ret) # TODO: Make logged message more structured + g.log.info('*' * 50) @classmethod - def get_mpath_of_iqn_on_clients(cls, blocknames): - """Get mpath of the logged in blocks + def check_for_memory_leaks_and_oom_kills_on_servers(cls, test_id, + gain=30.0): + """Check for memory leaks and OOM kills on servers - Return: - True if successful and execution error if getting mpath fails. - """ - # Get the mpath for iqn - # Donot forget to install 'sg3_utils' - # Convert string or unicode type to list - if not isinstance(blocknames, list): - blocknames = [blocknames] - - for blockname in blocknames: - block_gbid = cls.block_info_dict[blockname]['GBID'] - block_mapped_client = cls.clients_blocks_map[blockname]['client'] - block_iqn = cls.clients_blocks_map[blockname]['iqn'] - if not cls.clients_blocks_map[blockname].get('mpath'): - cmd = ("for i in `/dev/mapper/mpath*` ; do " - "sg_inq -i $i | grep %s > /dev/null ; " - "if [[ $? -eq 0 ]] ; then echo $i ; fi ; done" % - (block_gbid)) - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - raise ExecutionError("Failed to get mpath for iqn %s on " - "client %s: %s", block_iqn, - block_mapped_client, err) - block_mpath = out.strip() - g.log.info("Successfully got mpath '%s' for iqn '%s' on " - "client %s", block_mpath, block_iqn, - block_mapped_client) - cls.clients_blocks_map[blockname]['mpath'] = block_mpath - time.sleep(1) + Args: + test_id(str): ID of the test running fetched from self.id() - return True + Kwargs: + gain(float): Accepted amount of leak for a given testcase in MB + (Default:30) - @classmethod - def create_client_block_map(cls, blocknames): - """ - Mapping a single block to a client. - Select a client randomly from the list + Returns: + bool: True if memory leaks or OOM kills are observed else false """ - if not isinstance(blocknames, list): - blocknames = [blocknames] - - tmp_client_list = cls.clients[:] - for blockname in blocknames: - if blockname not in cls.clients_blocks_map: - if len(tmp_client_list) == 0: - tmp_client_list = cls.clients[:] - client_to_map = random.choice(tmp_client_list) - tmp_client_list.remove(client_to_map) - cls.clients_blocks_map[blockname] = { - 'client': client_to_map, - 'iqn': '', - 'logged_in': False, - 'mpath': '', - 'is_formatted': False, - 'is_mounted': False, - } - g.log.info("Blocks mapped to clients. Each block is mapped to a " - "randomly selected client") - for blockname in blocknames: - g.log.info("Block %s mapped to %s", blockname, - cls.clients_blocks_map[blockname]['client']) + # imports are added inside function to make it them + # optional and not cause breakage on installation + # which don't use the resource leak library + from glustolibs.io.memory_and_cpu_utils import ( + check_for_memory_leaks_in_glusterd, + check_for_memory_leaks_in_glusterfs, + check_for_memory_leaks_in_glusterfsd, + check_for_oom_killers_on_servers) + + # Check for memory leaks on glusterd + if check_for_memory_leaks_in_glusterd(cls.servers, test_id, gain): + g.log.error("Memory leak on glusterd.") + return True + + if cls.volume_type != "distributed": + # Check for memory leaks on shd + if check_for_memory_leaks_in_glusterfs(cls.servers, test_id, + gain): + g.log.error("Memory leak on shd.") + return True + + # Check for memory leaks on brick processes + if check_for_memory_leaks_in_glusterfsd(cls.servers, test_id, gain): + g.log.error("Memory leak on brick process.") + return True + + # Check OOM kills on servers for all gluster server processes + if check_for_oom_killers_on_servers(cls.servers): + g.log.error('OOM kills present on servers.') + return True + return False @classmethod - def mount_blocks(cls, blocknames, filesystem='xfs'): - """Mount the blocks on their clients - """ - if not isinstance(blocknames, list): - blocknames = [blocknames] - # Discover the block on client - ret = cls.discover_blocks_on_clients(blocknames) - if not ret: - return False - - for blockname in blocknames: - if not cls.clients_blocks_map[blockname]['logged_in']: + def check_for_memory_leaks_and_oom_kills_on_clients(cls, test_id, gain=30): + """Check for memory leaks and OOM kills on clients - # Login inside the block on client - ret = cls.login_to_iqn_on_clients(blockname) - if not ret: - return False - - # time.sleep added because the path /dev/mapper/mapth* - # is getting read even before the logging is completed. - time.sleep(2) - # Get mpath of block on it's client - ret = cls.get_mpath_of_iqn_on_clients(blockname) - if not ret: - return False - - # make fs - block_mpath = cls.clients_blocks_map[blockname]['mpath'] - block_mapped_client = cls.clients_blocks_map[blockname]['client'] - if not cls.clients_blocks_map[blockname].get('is_formatted'): - cmd = "mkfs.%s -f %s" % (filesystem, block_mpath) - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - raise ExecutionError("Failed to make fs on %s on client " - "%s: %s", block_mpath, - block_mapped_client, err) - g.log.info("Successfully created fs on %s on client %s: %s", - block_mpath, block_mapped_client, out) - cls.clients_blocks_map[blockname]['is_formatted'] = True - - # mount the block - if not cls.clients_blocks_map[blockname].get('is_mounted'): - temp_mount = { - 'protocol': 'xfs', - 'client': { - 'host': cls.clients_blocks_map[blockname]['client'], - }, - 'volname': cls.clients_blocks_map[blockname]['mpath'], - 'mountpoint': "/mnt/%s" % blockname - } - mount_obj = create_mount_objs([temp_mount]).pop() - - g.log.info("Mount Obj %s", mount_obj) - g.log.info("Mounting the device %s on %s:%s" % - (mount_obj.volname, mount_obj.client_system, - mount_obj.mountpoint)) - - # The function is_mounted will give an error in log file: - # "Missing arguments for mount" - # Because this is also used for mounting glusterfs volumes and - # a server name is needed But here mounting does not - # require a server name and therefore the argument check - # for server fails and an error is reported in the log file. - # But that will not affect the block mounting. - # So, we can live with it for now. - ret = mount_obj.mount() - if not ret: - raise ExecutionError("Unable to mount the " - "device %s on %s:%s" % - (mount_obj.volname, - mount_obj.client_system, - mount_obj.mountpoint)) - g.log.info("Successfully mounted the device %s on %s:%s" % - (mount_obj.volname, mount_obj.client_system, - mount_obj.mountpoint)) - cls.mount_blocks_list.append(mount_obj) - cls.clients_blocks_map[blockname]['is_mounted'] = True + Args: + test_id(str): ID of the test running fetched from self.id() - return True + Kwargs: + gain(float): Accepted amount of leak for a given testcase in MB + (Default:30) - @classmethod - def setup_block_mount_block(cls, blocknames): - """Create and mount the blocks + Returns: + bool: True if memory leaks or OOM kills are observed else false """ - # Setup block - g.log.info("Setting up blocks") - ret = cls.setup_blocks(blocknames) - if not ret: - raise ExecutionError("Failed to setup blocks") - g.log.info("Successful in setting up blocks") + # imports are added inside function to make it them + # optional and not cause breakage on installation + # which don't use the resource leak library + from glustolibs.io.memory_and_cpu_utils import ( + check_for_memory_leaks_in_glusterfs_fuse, + check_for_oom_killers_on_clients) + + # Check for memory leak on glusterfs fuse process + if check_for_memory_leaks_in_glusterfs_fuse(cls.clients, test_id, + gain): + g.log.error("Memory leaks observed on FUSE clients.") + return True + + # Check for oom kills on clients + if check_for_oom_killers_on_clients(cls.clients): + g.log.error("OOM kills present on clients.") + return True + return False - # Mount Blocks - g.log.info("Mounting the blocks on initiator nodes") - ret = cls.mount_blocks(blocknames) - if not ret: - raise ExecutionError("Failed to mount the blocks of volume %s", - cls.volname) - g.log.info("Successful in mounting the blocks of the volume %s", - cls.volname) + @classmethod + def check_for_cpu_usage_spikes_on_servers(cls, test_id, threshold=3): + """Check for CPU usage spikes on servers - return True + Args: + test_id(str): ID of the test running fetched from self.id() - @classmethod - def get_block_args_info_from_config(cls): - """Created the dict gluster_block_args_info which helps in - providing block information during block creation + Kwargs: + threshold(int): Accepted amount of instances of 100% CPU usage + (Default:3) + Returns: + bool: True if CPU spikes are more than threshold else False """ - # Get gluster block info from config file - if g.config.get('gluster_block_args_info'): - cls.gluster_block_args_info = {} - blocks_count = 0 - each_block_info = g.config['gluster_block_args_info'] - # for i, each_block_info in enumerate( - # g.config['gluster_block_args_info']): - # volname - block_on_volume = cls.volname - if each_block_info.get('volname'): - block_on_volume = each_block_info['volname'] - - # Block name - block_base_name = "gluster_block" - if each_block_info.get('blockname'): - block_base_name = each_block_info['blockname'] - - # servers - block_servers = cls.servers - if each_block_info.get('servers'): - block_servers = each_block_info['servers'] - if not filter(None, block_servers): - block_servers = cls.servers - - # Block size - block_size = "1GiB" - if each_block_info.get('size'): - block_size = each_block_info['size'] - - # HA - block_ha = 3 - if each_block_info.get('ha'): - block_ha = each_block_info['ha'] - - # auth - block_auth = None - if each_block_info.get('auth'): - block_auth = each_block_info['auth'] - - # prealloc - block_prealloc = None - if each_block_info.get('prealloc'): - block_prealloc = each_block_info['prealloc'] - - # ring-buffer - block_ring_buffer = None - if each_block_info.get('ring-buffer'): - block_ring_buffer = each_block_info['ring-buffer'] - - # Number of blocks - num_of_blocks = 1 - if each_block_info.get('num_of_blocks'): - num_of_blocks = int(each_block_info['num_of_blocks']) - - # for count in range(blocks_count,num_of_blocks +blocks_count): - for count in range(blocks_count, num_of_blocks): - # blocks_count = int(count) + i - - if block_ha: - selected_block_servers = random.sample(block_servers, - block_ha) - else: - selected_block_servers = random.choice(block_servers) - - block_name = "_".join([block_base_name, - str(count + 1)]) - - cls.gluster_block_args_info[block_name] = ( - {'volname': block_on_volume, - 'blockname': block_name, - 'servers': cls.get_ip_from_hostname( - selected_block_servers), - 'size': block_size, - 'ha': block_ha, - 'auth': block_auth, - 'prealloc': block_prealloc, - 'storage': None, - 'ring-buffer': block_ring_buffer} - ) - - for key in cls.gluster_block_args_info.keys(): - value = cls.gluster_block_args_info[key] - g.log.info("Gluster-Block args info: %s\n %s" % (key, value)) + # imports are added inside function to make it them + # optional and not cause breakage on installation + # which don't use the resource leak library + from glustolibs.io.memory_and_cpu_utils import ( + check_for_cpu_usage_spikes_on_glusterd, + check_for_cpu_usage_spikes_on_glusterfs, + check_for_cpu_usage_spikes_on_glusterfsd) + + # Check for CPU usage spikes on glusterd + if check_for_cpu_usage_spikes_on_glusterd(cls.servers, test_id, + threshold): + g.log.error("CPU usage spikes observed more than threshold " + "on glusterd.") + return True + + if cls.volume_type != "distributed": + # Check for CPU usage spikes on shd + if check_for_cpu_usage_spikes_on_glusterfs(cls.servers, test_id, + threshold): + g.log.error("CPU usage spikes observed more than threshold " + "on shd.") + return True + + # Check for CPU usage spikes on brick processes + if check_for_cpu_usage_spikes_on_glusterfsd(cls.servers, test_id, + threshold): + g.log.error("CPU usage spikes observed more than threshold " + "on shd.") + return True + return False @classmethod - def setUpClass(cls, setup_vol=True, setup_blk=True, mount_blk=True): - """Setup volume, create blocks, mount the blocks if specified. - """ - GlusterBaseClass.setUpClass.im_func(cls) - - cls.mount_blocks_list = [] - cls.total_num_of_blocks = 0 - cls.block_info_dict = {} - cls.clients_blocks_map = {} - cls.blocknames = [] - - # Default gluster block info - cls.gluster_block_args_info = { - 'gluster_block_%d' % (cls.total_num_of_blocks + 1): { - 'volname': cls.volname, - 'blockname': 'gluster_block_%d' - % (cls.total_num_of_blocks + 1), - 'servers': random.sample(cls.servers_ips, 2), - 'size': '1GiB', - 'ha': 2, - 'auth': None, - 'prealloc': None, - 'storage': None, - 'ring-buffer': None - } - } + def check_for_cpu_spikes_on_clients(cls, test_id, threshold=3): + """Check for CPU usage spikes on clients - if g.config.get('gluster_block_args_info'): - cls.get_block_args_info_from_config() + Args: + test_id(str): ID of the test running fetched from self.id() - @classmethod - def tearDownClass(cls, umount_blocks=True, cleanup_blocks=True, - cleanup_vol=True, unlink_storage="yes"): - """Teardown the mounts, deletes blocks, gluster volume. + Kwargs: + threshold(int): Accepted amount of instances of 100% CPU usage + (Default:3) + Returns: + bool: True if CPU spikes are more than threshold else False """ - # Unmount volume - if umount_blocks: - _rc = True - g.log.info("Starting to UnMount Blocks") - for mount_obj in cls.mount_blocks_list: - ret = mount_obj.unmount() - if not ret: - g.log.error("Unable to unmount block '%s on cleint %s " - "at %s'", - mount_obj.volname, mount_obj.client_system, - mount_obj.mountpoint) - _rc = False - if not _rc: - raise ExecutionError("Unmount of all mounts are not " - "successful") - else: - g.log.info("Successful in unmounting volume on all clients") - else: - g.log.info("Not Unmounting the Volume as 'umount_vol' is set " - "to %s", umount_blocks) - - # Logout the blocks - for blockname in cls.clients_blocks_map: - block_iqn = cls.clients_blocks_map[blockname]['iqn'] - block_mapped_client = ( - cls.clients_blocks_map[blockname]['client']) - g.log.info("Logging out iqn %s on client %s", block_iqn, - block_mapped_client) - cmd = "iscsiadm -m node -T %s -u" % block_iqn - ret, out, err = g.run(block_mapped_client, cmd) - if ret != 0: - raise ExecutionError("Failed to logout iqn %s on client %s " - ":%s", block_iqn, block_mapped_client, - err) - g.log.info("Successfully logged out iqn %s on client %s: %s", - block_iqn, block_mapped_client, out) - - # Restarting multipathd on all clients - g.log.info("Restarting multipathd on all clients") - cmd = "service multipathd restart && service multipathd status" - results = g.run_parallel(cls.clients, cmd) - for client in results: - ret, out, err = results[client] - if ret != 0: - raise ExecutionError("Failed to restart multipathd on " - "client %s: %s", client, err) - g.log.info("Successfully restarted multipathd on client %s: %s", - client, out) - - # Cleanup blocks - if cleanup_blocks: - blocknames = get_block_list(cls.mnode, cls.volname) - if blocknames: - g.log.info("Listing blocks before deleting:\n%s", - '\n'.join(blocknames)) - for blockname in blocknames: - ret, out, err = block_delete(cls.mnode, cls.volname, - blockname, unlink_storage) - if ret != 0: - raise ExecutionError("Failed to delete the block " - "%s on volume %s", blockname, - cls.volname) - g.log.info("Successfully deleted the block %s on " - "volume %s", blockname, cls.volname) - - # Cleanup volume - if cleanup_vol: - g.log.info("Cleanup Volume %s", cls.volname) - ret = cleanup_volume(mnode=cls.mnode, volname=cls.volname) - if not ret: - raise ExecutionError("cleanup volume %s failed", cls.volname) - else: - g.log.info("Successfully cleaned-up volume") - else: - g.log.info("Not Cleaning-Up volume as 'cleanup_vol' is %s", - cleanup_vol) - - GlusterBaseClass.tearDownClass.im_func(cls) + # imports are added inside function to make it them + # optional and not cause breakage on installation + # which don't use the resource leak library + from glustolibs.io.memory_and_cpu_utils import ( + check_for_cpu_usage_spikes_on_glusterfs_fuse) + + ret = check_for_cpu_usage_spikes_on_glusterfs_fuse(cls.clients, + test_id, + threshold) + return ret |