From 8f09d6d30b5745bd2abf8b73b5d8df8c63b79f63 Mon Sep 17 00:00:00 2001 From: Shwetha Panduranga Date: Mon, 20 Mar 2017 14:39:05 +0530 Subject: Adding a sanity case to test shrinking volume. i.e remove-brick Remove brick Sanity case covers testing of remove-brick of a subvolume, waiting for rebalance to complete, commiting the operation and validate IO is successful on the mount. Change-Id: I5912f62b3df5dfb5bf5339de036967f83b6a5117 Signed-off-by: Shwetha Panduranga --- glustolibs-gluster/glustolibs/gluster/brick_ops.py | 30 +- .../glustolibs/gluster/volume_libs.py | 312 ++++++++++++++++++++- 2 files changed, 322 insertions(+), 20 deletions(-) (limited to 'glustolibs-gluster') diff --git a/glustolibs-gluster/glustolibs/gluster/brick_ops.py b/glustolibs-gluster/glustolibs/gluster/brick_ops.py index 016569f15..0e99b955e 100644 --- a/glustolibs-gluster/glustolibs/gluster/brick_ops.py +++ b/glustolibs-gluster/glustolibs/gluster/brick_ops.py @@ -74,7 +74,7 @@ def add_brick(mnode, volname, bricks_list, force=False, **kwargs): return g.run(mnode, cmd) -def remove_brick(mnode, volname, bricks_list, option, replica=None): +def remove_brick(mnode, volname, bricks_list, option, xml=False, **kwargs): """Remove bricks specified in the bricks_list from the volume. Args: @@ -84,8 +84,10 @@ def remove_brick(mnode, volname, bricks_list, option, replica=None): option (str): Remove brick options: Kwargs: - replica (int): Replica count to increase the replica count of - the volume. + xml (bool): if xml is True, get xml output of command execution. + **kwargs + The keys, values in kwargs are: + - replica_count : (int)|None Returns: tuple: Tuple containing three elements (ret, out, err). @@ -101,13 +103,21 @@ def remove_brick(mnode, volname, bricks_list, option, replica=None): if option == "commit" or option == "force": option = option + " --mode=script" - if replica is None: - cmd = ("gluster volume remove-brick %s %s %s" % - (volname, ' '.join(bricks_list), option)) - else: - cmd = ("gluster volume remove-brick %s replica %d %s force " - "--mode=script" % (volname, int(replica), - ' '.join(bricks_list))) + replica_count = None + replica = '' + + if 'replica_count' in kwargs: + replica_count = int(kwargs['replica_count']) + + if replica_count is not None: + replica = "replica %d" % replica_count + + xml_str = '' + if xml: + xml_str = "--xml" + + cmd = ("gluster volume remove-brick %s %s %s %s %s" % + (volname, replica, ' '.join(bricks_list), option, xml_str)) return g.run(mnode, cmd) diff --git a/glustolibs-gluster/glustolibs/gluster/volume_libs.py b/glustolibs-gluster/glustolibs/gluster/volume_libs.py index 22ac7e375..7b9dd5647 100644 --- a/glustolibs-gluster/glustolibs/gluster/volume_libs.py +++ b/glustolibs-gluster/glustolibs/gluster/volume_libs.py @@ -16,9 +16,13 @@ """ Description: Module for gluster volume related helper functions. """ - -from glusto.core import Glusto as g import time +import random +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree +from glusto.core import Glusto as g from glustolibs.gluster.lib_utils import form_bricks_list from glustolibs.gluster.volume_ops import (volume_create, volume_start, set_volume_options, get_volume_info, @@ -35,7 +39,7 @@ from glustolibs.gluster.uss_ops import enable_uss, is_uss_enabled from glustolibs.gluster.snap_ops import snap_delete_by_volumename from glustolibs.gluster.brick_libs import are_bricks_online, get_all_bricks from glustolibs.gluster.heal_libs import are_all_self_heal_daemons_are_online -from glustolibs.gluster.brick_ops import add_brick +from glustolibs.gluster.brick_ops import add_brick, remove_brick def volume_exists(mnode, volname): @@ -171,6 +175,10 @@ def setup_volume(mnode, all_servers_info, volume_config, force=False): g.log.error("Replica count not specified in the volume config") return False + if 'arbiter_count' in volume_config['voltype']: + kwargs['arbiter_count'] = (volume_config['voltype'] + ['arbiter_count']) + number_of_bricks = (kwargs['dist_count'] * kwargs['replica_count']) elif volume_type == 'dispersed': @@ -664,10 +672,11 @@ def get_volume_type_info(mnode, volname): 'hot_tier_type_info': {}, 'cold_tier_type_info': {}, 'volume_type_info': { - 'typeStr': 'Disperse' - 'replicaCount': 1 - 'stripeCount': 1 - 'disperseCount': '3' + 'typeStr': 'Disperse', + 'replicaCount': '1', + 'arbiterCount': '0', + 'stripeCount': '1', + 'disperseCount': '3', 'redundancyCount': '1' } } @@ -676,12 +685,13 @@ def get_volume_type_info(mnode, volname): 'is_tier': True, 'hot_tier_type_info': { 'hotBrickType': 'Distribute', - 'hotreplicaCount': 1 + 'hotreplicaCount': '1' }, 'cold_tier_type_info': { 'coldBrickType': 'Disperse', - 'coldreplicaCount': 1, - 'colddisperseCount':3, + 'coldreplicaCount': '1', + 'coldarbiterCount': '0', + 'colddisperseCount': '3', 'numberOfBricks':1 }, 'volume_type_info': {} @@ -715,6 +725,7 @@ def get_volume_type_info(mnode, volname): non_tiered_volume_type_info = { 'typeStr': '', 'replicaCount': '', + 'arbiterCount': '', 'stripeCount': '', 'disperseCount': '', 'redundancyCount': '' @@ -744,6 +755,7 @@ def get_cold_tier_type_info(mnode, volname): cold_tier_type_info = { 'coldBrickType': 'Disperse', 'coldreplicaCount': '1', + 'coldarbiterCount': '0', 'colddisperseCount': '3', 'numberOfBricks': '3' } @@ -762,6 +774,7 @@ def get_cold_tier_type_info(mnode, volname): cold_tier_type_info = { 'coldBrickType': '', 'coldreplicaCount': '', + 'coldarbiterCount': '', 'colddisperseCount': '', 'numberOfBricks': '' } @@ -1160,7 +1173,14 @@ def expand_volume(mnode, volname, servers, all_servers_info, force=False, Returns: bool: True of expanding volumes is successful. False otherwise. + + NOTE: adding bricks to hot tier is yet to be added in this function. """ + # Check if volume exists + if not volume_exists(mnode, volname): + g.log.error("Volume %s doesn't exists.", volname) + return False + # Check whether we need to increase the replica count of the volume if 'replica_count' in kwargs: new_replica_count = int(kwargs['replica_count']) @@ -1246,6 +1266,7 @@ def expand_volume(mnode, volname, servers, all_servers_info, force=False, return False # Add bricks to the volume + g.log.info("Adding bricks to the volume: %s", volname) ret, out, err = add_brick(mnode, volname, bricks_list, force=force, **kwargs) if ret != 0: @@ -1253,3 +1274,274 @@ def expand_volume(mnode, volname, servers, all_servers_info, force=False, return False g.log.info("Successfully added bricks to the volume: %s", out) return True + + +def shrink_volume(mnode, volname, subvol_num=None, replica_num=None, + force=False, rebalance_timeout=300, delete_bricks=True, + remove_from_hot_tier=False, **kwargs): + """Remove bricks from the volume. + + Args: + mnode (str): Node on which commands has to be executed + volname (str): volume name + + Kwargs: + subvol_num (list): List of sub volumes number to remove. + For example: If subvol_num = [2, 5], Then we will be removing + bricks from 2nd and 5th sub-volume of the given volume. + The sub-volume number starts from 0. + + replica_num (int): Specify which replica brick to remove. + If replica_num = 0, then 1st brick from each subvolume is removed. + the replica_num starts from 0. + + force (bool): If this option is set to True, then remove-brick command + will get executed with force option. If it is set to False, + then remove-brick is executed with 'start' and 'commit' when the + remove-brick 'status' becomes completed. + + rebalance_timeout (int): Wait time for remove-brick to complete in + seconds. Default is 5 minutes. + + delete_bricks (bool): After remove-brick delete the removed bricks. + + remove_from_hot_tier (bool): True If bricks are to be removed from + hot_tier. False otherwise. Defaults to False. + + **kwargs + The keys, values in kwargs are: + - replica_count : (int)|None. Specify the replica count to + reduce + - distribute_count: (int)|None. Specify the distribute count to + reduce. + Returns: + bool: True if removing bricks from the volume is successful. + False otherwise. + + NOTE: remove-bricks from hot-tier is yet to be added in this function. + """ + # Check if volume exists + if not volume_exists(mnode, volname): + g.log.error("Volume %s doesn't exists.", volname) + return False + + # Get Subvols + subvols_info = get_subvols(mnode, volname) + + # Initialize bricks to remove + bricks_list_to_remove = [] + + # remove bricks by reducing replica count of the volume + if 'replica_count' in kwargs: + new_replica_count = int(kwargs['replica_count']) + + # Get replica count info. + replica_count_info = get_replica_count(mnode, volname) + + # Get volume type info + volume_type_info = get_volume_type_info(mnode, volname) + + # Set is_arbiter to False + is_arbiter = False + + # Calculate bricks to remove + if subvols_info['is_tier']: + if remove_from_hot_tier: + current_replica_count = ( + int(replica_count_info['hot_tier_replica_count'])) + subvols_list = subvols_info['hot_tier_subvols'] + else: + current_replica_count = ( + int(replica_count_info['cold_tier_replica_count'])) + subvols_list = subvols_info['cold_tier_subvols'] + arbiter_count = int(volume_type_info['cold_tier_type_info'] + ['coldarbiterCount']) + if arbiter_count == 1: + is_arbiter = True + else: + current_replica_count = ( + int(replica_count_info['volume_replica_count'])) + subvols_list = subvols_info['volume_subvols'] + arbiter_count = int(volume_type_info['volume_type_info'] + ['arbiterCount']) + if arbiter_count == 1: + is_arbiter = True + + if new_replica_count >= current_replica_count: + g.log.error("Provided replica count '%d' is greater than or " + "equal to the Existing replica count '%d' of the " + "volume %s. Hence cannot proceed with remove-brick", + new_replica_count, current_replica_count, volname) + return False + + # If replica_num is specified select the bricks of that replica number + # from all the subvolumes. + if replica_num: + try: + bricks_list_to_remove = ([subvol[replica_num] + for subvol in subvols_list]) + except IndexError: + g.log.error("Invalid replica number: %d specified " + "for removing the replica brick from the volume: " + "%s", replica_num, volname) + return False + + # If arbiter_volume, always remove the 3rd brick (arbiter brick) + elif is_arbiter: + bricks_list_to_remove = [subvol[-1] for subvol in subvols_list] + + # If replica_num is not specified not it is arbiter volume, randomly + # select the bricks to remove. + else: + num_of_bricks_per_subvol_to_remove = (current_replica_count - + new_replica_count) + bricks_list_to_remove = ( + [random.sample(subvol, num_of_bricks_per_subvol_to_remove) + for subvol in subvols_list]) + + # remove bricks from sub-volumes + else: + if subvols_info['is_tier']: + if remove_from_hot_tier: + subvols_list = subvols_info['hot_tier_subvols'] + else: + subvols_list = subvols_info['cold_tier_subvols'] + else: + subvols_list = subvols_info['volume_subvols'] + + if not subvols_list: + g.log.error("No Sub-Volumes available for the volume %s", volname) + return False + + # select bricks of subvol_num specified as argument to this function. + if subvol_num: + try: + bricks_list_to_remove = subvols_list[subvol_num] + except IndexError: + g.log.error("Invalid sub volume number: %d specified " + "for removing the subvolume from the " + "volume: %s", subvol_num, volname) + return False + + # select bricks from multiple subvols with number of + # subvolumes specified as distribute_count argument. + elif 'distribute_count' in kwargs: + distribute_count = int(kwargs['distribute_count']) + bricks_list_to_remove = random.sample(subvols_list, + distribute_count) + + # randomly choose a subvolume to remove-bricks from. + else: + bricks_list_to_remove = random.choice(subvols_list) + + # If force, then remove-bricks with force option + if force: + g.log.info("Removing bricks %s from volume %s with force option", + bricks_list_to_remove, volname) + ret, _, _ = remove_brick(mnode, volname, bricks_list_to_remove, + option="force") + if ret != 0: + g.log.error("Failed to remove bricks %s from the volume %s with " + "force option", bricks_list_to_remove, volname) + return False + g.log.info("Successfully removed bricks %s from the volume %s with " + "force option", bricks_list_to_remove, volname) + return True + + # remove-brick start + g.log.info("Start Removing bricks %s from the volume %s", + bricks_list_to_remove, volname) + ret, _, _ = remove_brick(mnode, volname, bricks_list_to_remove, + option="start") + if ret != 0: + g.log.error("Failed to start remove-brick of bricks %s on the volume " + "%s", bricks_list_to_remove, volname) + return False + g.log.info("Successfully started removal of bricks %s from the volume %s", + bricks_list_to_remove, volname) + + # remove-brick status + g.log.info("Checking remove-brick status of bricks %s on the volume %s", + bricks_list_to_remove, volname) + ret, _, _ = remove_brick(mnode, volname, bricks_list_to_remove, + option="status") + if ret != 0: + g.log.error("Failed to get status of remove-brick of bricks %s on " + "volume %s", bricks_list_to_remove, volname) + g.log.info("Successfully got remove-brick status of bricks %s on " + "volume %s", bricks_list_to_remove, volname) + + # Wait for rebalance started by remove-brick to complete + _rc = False + while (rebalance_timeout > 0): + ret, out, _ = remove_brick(mnode, volname, bricks_list_to_remove, + option="status", xml=True) + if ret != 0: + g.log.error("Failed to get xml output remove-brick status of " + "bricks %s on volume %s", bricks_list_to_remove, + volname) + return False + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster remove-brick status " + "xml output.") + return False + remove_brick_aggregate_status = {} + for info in root.findall("volRemoveBrick"): + for element in info.getchildren(): + if element.tag == "aggregate": + for elmt in element.getchildren(): + remove_brick_aggregate_status[elmt.tag] = elmt.text + if "completed" in remove_brick_aggregate_status['statusStr']: + _rc = True + break + + elif "in progress" in remove_brick_aggregate_status['statusStr']: + rebalance_timeout = rebalance_timeout - 30 + time.sleep(30) + continue + else: + g.log.error("Invalid status string in remove brick status") + return False + + if not _rc: + g.log.error("Rebalance started by remove-brick is not yet complete " + "on the volume %s", volname) + return False + g.log.info("Rebalance started by remove-brick is successfully complete " + "on the volume %s", volname) + + # remove-brick status after rebalance is complete + g.log.info("Checking remove-brick status of bricks %s on the volume %s " + "after rebalance is complete", bricks_list_to_remove, volname) + ret, _, _ = remove_brick(mnode, volname, bricks_list_to_remove, + option="status") + if ret != 0: + g.log.error("Failed to get status of remove-brick of bricks %s on " + "volume %s after rebalance is complete", + bricks_list_to_remove, volname) + g.log.info("Successfully got remove-brick status of bricks %s on " + "volume %s after rebalance is complete", bricks_list_to_remove, + volname) + + # Commit remove-brick + g.log.info("Commit remove-brick of bricks %s on volume %s", + bricks_list_to_remove, volname) + ret, _, _ = remove_brick(mnode, volname, bricks_list_to_remove, + option="commit") + if ret != 0: + g.log.error("Failed to commit remove-brick of bricks %s on volume %s", + bricks_list_to_remove, volname) + return False + g.log.info("Successfully commited remove-bricks of bricks %s on volume %s", + bricks_list_to_remove, volname) + + # Delete the removed bricks + if delete_bricks: + for brick in bricks_list_to_remove: + brick_node, brick_path = brick.split(":") + _, _, _ = g.run(brick_node, "rm -rf %s" % brick_path) + + return True -- cgit