From aa56eab484f11801c12a31f03c3b17b1f093641c Mon Sep 17 00:00:00 2001 From: Arthy Loganathan Date: Tue, 4 Oct 2016 13:29:27 +0530 Subject: Adding library functions for bitrot, quota, rebalance, tiering, snap and volume ops Change-Id: Iadb0772fc338f5dffd485b2526babaffb3494be6 Signed-off-by: Arthy Loganathan --- .../glustolibs/gluster/bitrot_ops.py | 614 ++++++++++++ glustolibs-gluster/glustolibs/gluster/quota_ops.py | 520 ++++++++++ .../glustolibs/gluster/rebalance_ops.py | 304 ++++++ glustolibs-gluster/glustolibs/gluster/snap_ops.py | 898 +++++++++++++++++ .../glustolibs/gluster/tiering_ops.py | 1019 ++++++++++++++++++++ .../glustolibs/gluster/volume_ops.py | 705 ++++++++++++++ 6 files changed, 4060 insertions(+) create mode 100644 glustolibs-gluster/glustolibs/gluster/bitrot_ops.py create mode 100644 glustolibs-gluster/glustolibs/gluster/quota_ops.py create mode 100644 glustolibs-gluster/glustolibs/gluster/rebalance_ops.py create mode 100644 glustolibs-gluster/glustolibs/gluster/snap_ops.py create mode 100644 glustolibs-gluster/glustolibs/gluster/tiering_ops.py create mode 100644 glustolibs-gluster/glustolibs/gluster/volume_ops.py (limited to 'glustolibs-gluster/glustolibs') diff --git a/glustolibs-gluster/glustolibs/gluster/bitrot_ops.py b/glustolibs-gluster/glustolibs/gluster/bitrot_ops.py new file mode 100644 index 000000000..6eed10e19 --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/bitrot_ops.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Description: Library for gluster bitrot operations. +""" + +from glusto.core import Glusto as g +from glustolibs.gluster.volume_ops import get_volume_options, get_volume_status +from glustolibs.gluster.lib_utils import (get_pathinfo, + calculate_checksum, + get_extended_attributes_info) +import time +import re + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree + + +# Global variables +SCRUBBER_TIMEOUT = 100 + + +def enable_bitrot(mnode, volname): + """Enables bitrot for given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + enable_bitrot("abc.com", testvol) + """ + + cmd = "gluster volume bitrot %s enable" % volname + return g.run(mnode, cmd) + + +def disable_bitrot(mnode, volname): + """Disables bitrot for given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + disable_bitrot("abc.com", testvol) + """ + + cmd = "gluster volume bitrot %s disable" % volname + return g.run(mnode, cmd) + + +def is_bitrot_enabled(mnode, volname): + """Checks if bitrot is enabled on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + True on success, False otherwise + + Example: + is_bitrot_enabled("abc.com", testvol) + """ + + output = get_volume_options(mnode, volname, "features.bitrot") + if output is None: + return False + + g.log.info("Bitrot Status in volume %s: %s" + % (volname, output["features.bitrot"])) + if output["features.bitrot"] != 'on': + return False + + return True + + +def is_file_signed(mnode, filename, volname, expected_file_version=None): + """Verifies if the given file is signed + + Args: + mnode (str): Node on which cmd has to be executed. + filename (str): relative path of filename to be verified + volname (str): volume name + + Kwargs: + expected_file_version (str): file version to check with getfattr output + If this option is set, this function + will check file versioning as part of signing verification. + If this option is set to None, function will not check + for file versioning. Defaults to None. + + Returns: + True on success, False otherwise + + Example: + is_file_signed("abc.com", 'file1', "testvol", + expected_file_version='2') + """ + + # Getting file path in the rhs node + file_location = get_pathinfo(mnode, filename, volname) + if file_location is None: + g.log.error("Failed to get backend file path in is_file_signed()") + return False + + path_info = file_location[0].split(':') + + expected_file_signature = (calculate_checksum(path_info[0], + [path_info[1]]) + [path_info[1]]) + + attr_info = get_extended_attributes_info(path_info[0], + [path_info[1]]) + if attr_info is None: + g.log.error("Failed to get attribute info in is_file_signed()") + return False + + if 'trusted.bit-rot.signature' in attr_info[path_info[1]]: + file_signature = attr_info[path_info[1]]['trusted.bit-rot.signature'] + else: + g.log.error("trusted.bit-rot.signature attribute not present " + " for file %s" % filename) + return False + + if expected_file_version is not None: + expected_file_version = ('{0:02d}'.format(int( + expected_file_version))).ljust(16, '0') + actual_signature_file_version = re.findall('.{16}', + file_signature[4:]).pop(0) + + # Verifying file version after signing + if actual_signature_file_version != expected_file_version: + g.log.error("File version mismatch in signature.Filename: %s ." + "Expected file version: %s.Actual file version: %s" + % (filename, expected_file_version, + actual_signature_file_version)) + return False + + actual_file_signature = ''.join(re.findall('.{16}', + file_signature[4:])[1:]) + + # Verifying file signature + if actual_file_signature != expected_file_signature: + g.log.error("File signature mismatch. File name: %s . Expected " + "file signature: %s. Actual file signature: %s" + % (filename, expected_file_signature, + actual_file_signature)) + return False + return True + + +def is_file_bad(mnode, filename): + """Verifies if scrubber identifies bad file + Args: + filename (str): absolute path of the file in mnode + mnode (str): Node on which cmd has to be executed. + + Returns: + True on success, False otherwise + + Example: + is_file_bad("abc.xyz.com", "/bricks/file1") + """ + ret = True + count = 0 + flag = 0 + while (count < SCRUBBER_TIMEOUT): + attr_info = get_extended_attributes_info(mnode, [filename]) + if attr_info is None: + ret = False + + if 'trusted.bit-rot.bad-file' in attr_info[filename]: + flag = 1 + break + + time.sleep(10) + count = count + 10 + if not flag: + g.log.error("Scrubber failed to identify bad file") + ret = False + + return ret + + +def bring_down_bitd(mnode): + """Brings down bitd process + Args: + mnode (str): Node on which cmd has to be executed. + + Returns: + True on success, False otherwise + + Example: + bring_down_bitd("abc.xyz.com") + """ + + kill_cmd = ("pid=`cat /var/lib/glusterd/bitd/run/bitd.pid` && " + "kill -15 $pid || kill -9 $pid") + ret, _, _ = g.run(mnode, kill_cmd) + if ret != 0: + g.log.error("Unable to kill the bitd for %s" + % mnode) + return False + else: + return True + + +def bring_down_scrub_process(mnode): + """Brings down scrub process + Args: + mnode (str): Node on which cmd has to be executed. + + Returns: + True on success, False otherwise + + Example: + bring_down_scrub_process("abc.xyz.com") + """ + + kill_cmd = ("pid=`cat /var/lib/glusterd/scrub/run/scrub.pid` && " + "kill -15 $pid || kill -9 $pid") + + ret, _, _ = g.run(mnode, kill_cmd) + if ret != 0: + g.log.error("Unable to kill the scrub process for %s" + % mnode) + return False + else: + return True + + +def set_scrub_throttle(mnode, volname, throttle_type='lazy'): + """Sets scrub throttle + + Args: + volname (str): volume name + mnode (str): Node on which cmd has to be executed. + + Kwargs: + throttle_type (str): throttling type (lazy|normal|aggressive) + Defaults to 'lazy' + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + set_scrub_throttle(mnode, testvol) + """ + + cmd = ("gluster volume bitrot %s scrub-throttle %s" + % (volname, throttle_type)) + return g.run(mnode, cmd) + + +def set_scrub_frequency(mnode, volname, frequency_type='biweekly'): + """Sets scrub frequency + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + frequency_type (str): frequency type (hourly|daily|weekly|biweekly| + monthly). Defaults to 'biweekly' + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + set_scrub_frequency("abc.com", testvol) + """ + + cmd = ("gluster volume bitrot %s scrub-frequency %s" + % (volname, frequency_type)) + return g.run(mnode, cmd) + + +def pause_scrub(mnode, volname): + """Pauses scrub + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + pause_scrub("abc.com", testvol) + """ + + cmd = "gluster volume bitrot %s scrub pause" % volname + return g.run(mnode, cmd) + + +def resume_scrub(mnode, volname): + """Resumes scrub + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + resume_scrub("abc.com", testvol) + """ + + cmd = "gluster volume bitrot %s scrub resume" % volname + return g.run(mnode, cmd) + + +def get_bitd_pid(mnode): + """Gets bitd process id for the given node + Args: + mnode (str): Node on which cmd has to be executed. + + Returns: + str: pid of the bitd process on success + NoneType: None if command execution fails, errors. + + Example: + get_bitd_pid("abc.com") + """ + + cmd = ("cat /var/lib/glusterd/bitd/run/bitd.pid") + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Unable to get bitd pid for %s" + % mnode) + return None + + return out.strip("\n") + + +def get_scrub_process_pid(mnode): + """Gets scrub process id for the given node + Args: + mnode (str): Node on which cmd has to be executed. + + Returns: + str: pid of the scrub process on success + NoneType: None if command execution fails, errors. + + Example: + get_scrub_process_pid("abc.com") + """ + + cmd = ("cat /var/lib/glusterd/scrub/run/scrub.pid") + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Unable to get scrub pid for %s" + % mnode) + return None + + return out.strip("\n") + + +def is_bitd_running(mnode, volname): + """Checks if bitd is running on the given node + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + True on success, False otherwise + + Example: + is_bitd_running("abc.com", "testvol") + """ + + vol_status = get_volume_status(mnode, volname=volname) + if vol_status is None: + g.log.error("Failed to get volume status in is_bitd_running()") + return False + + if 'Bitrot Daemon' not in vol_status[volname][mnode]: + g.log.error("Bitrot is not enabled in volume %s" + % volname) + return False + + bitd_status = vol_status[volname][mnode]['Bitrot Daemon']['status'] + if bitd_status != '1': + g.log.error("Bitrot Daemon is not running in node %s" + % mnode) + return False + return True + + +def is_scrub_process_running(mnode, volname): + """Checks if scrub process is running on the given node + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + True on success, False otherwise + + Example: + is_scrub_process_running("abc.com", "testvol") + """ + + vol_status = get_volume_status(mnode, volname=volname) + if vol_status is None: + g.log.error("Failed to get volume status in " + "is_scrub_process_running()") + return False + + if 'Scrubber Daemon' not in vol_status[volname][mnode]: + g.log.error("Bitrot is not enabled in volume %s" + % volname) + return False + + bitd_status = vol_status[volname][mnode]['Scrubber Daemon']['status'] + if bitd_status != '1': + g.log.error("Scrubber Daemon is not running in node %s" + % mnode) + return False + return True + + +def scrub_status(mnode, volname): + """Executes gluster bitrot scrub status command + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + scrub_status("abc.com", testvol) + """ + + cmd = "gluster volume bitrot %s scrub status" % volname + return g.run(mnode, cmd) + + +def get_scrub_status(mnode, volname): + """Parse the output of gluster bitrot scrub status command + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + dict: scrub status in dict format + NoneType: None if command execution fails, errors. + + Example: + >>>get_scrub_status("abc.com", testvol) + {'State of scrub': 'Active', 'Bitrot error log location': + '/var/log/glusterfs/bitd.log', 'Scrub impact': 'aggressive', + 'Scrub frequency': 'hourly', 'status_info': {'localhost': + {'Duration of last scrub (D:M:H:M:S)': '0:0:0:0', 'corrupted_gfid': + ['475ca13f-577f-460c-a5d7-ea18bb0e7779'], 'Error count': '1', + 'Last completed scrub time': '2016-06-21 12:46:19', + 'Number of Skipped files': '0', 'Number of Scrubbed files': '0'}, + '10.70.47.118': {'Duration of last scrub (D:M:H:M:S)': '0:0:0:1', + 'corrupted_gfid': ['19e62b26-5942-4867-a2f6-e354cd166da9', + 'fab55c36-0580-4d11-9ac0-d8e4e51f39a0'], 'Error count': '2', + 'Last completed scrub time': '2016-06-21 12:46:03', + 'Number of Skipped files': '0', 'Number of Scrubbed files': '2'}}, + 'Volume name': 'testvol', 'Scrubber error log location': + '/var/log/glusterfs/scrub.log'} + """ + + cmd = "gluster volume bitrot %s scrub status" % volname + ret, out, err = g.run(mnode, cmd) + if ret != 0: + g.log.error("Unable to get scrub status for volume %s" + % volname) + return None + + match = re.search('(.*?)(==.*==.*)', out, re.S) + if match is None: + g.log.error("Mismatch in regex. Scrub status raw output is not" + " in expected format") + return None + info = match.group(2).replace('\n\n', '\n') + + if "Corrupted object's [GFID]" in info: + info = info.replace("Corrupted object's [GFID]:\n", + "Corrupted object's [GFID]:") + regex = 'Node(?:(?!Node).)*?Corrupted object.*?:.*?\n+=' + temp_list = re.findall(regex, info, re.S) + corrupt_list = [] + for node in temp_list: + tmp_reg = ('Node: (\S+)\n.*Error count.*' + + 'Corrupted object.*?:(.*)\n=.*') + m = re.search(tmp_reg, node, re.S) + if m is None: + g.log.error("Mismatch in cli output when bad file" + "is identified") + return None + corrupt_list.append(m.groups()) + else: + corrupt_list = [] + info_list = re.findall('Node:.*?\n.*:.*\n.*:.*\n.*:.*\n.*:.*\n.*:.*\n+', + info) + temp_list = [] + for item in info_list: + item = item.replace('\n\n', '') + temp_list.append(item) + + tmp_dict1 = {} + for item in temp_list: + tmp = item.split('\n') + tmp_0 = tmp[0].split(':') + tmp.pop(0) + tmp_dict = {} + for tmp_item in tmp[:-1]: + tmp_1 = tmp_item.split(': ') + tmp_dict[tmp_1[0].strip(' ')] = tmp_1[1].strip(' ') + tmp_dict1[tmp_0[1].strip(' ')] = tmp_dict + status_dict = {} + for item in match.group(1).split('\n\n')[:-1]: + elmt = item.split(':') + tmp_elmt = elmt[1].strip(' ').strip('\n') + status_dict[elmt[0].strip(' ').strip('\n')] = tmp_elmt + + status_dict['status_info'] = tmp_dict1 + for elmt in corrupt_list: + if elmt[0].strip(' ') in status_dict['status_info'].keys(): + val = elmt[1].split('\n') + val = filter(None, val) + gfid = "corrupted_gfid" + status_dict['status_info'][elmt[0].strip(' ')][gfid] = val + return status_dict diff --git a/glustolibs-gluster/glustolibs/gluster/quota_ops.py b/glustolibs-gluster/glustolibs/gluster/quota_ops.py new file mode 100644 index 000000000..f7121b05f --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/quota_ops.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Description: Library for gluster quota operations. +""" + +from glusto.core import Glusto as g +import re +import time + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree + + +def enable_quota(mnode, volname): + """Enables quota on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + enable_quota("abc.xyz.com", testvol) + """ + + cmd = "gluster volume quota %s enable" % volname + return g.run(mnode, cmd) + + +def disable_quota(mnode, volname): + """Disables quota on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + disable_quota("abc.xyz.com", testvol) + """ + + cmd = "gluster volume quota %s disable --mode=script" % volname + return g.run(mnode, cmd) + + +def is_quota_enabled(mnode, volname): + """Checks if quota is enabled on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + bool: True, if quota is enabled + False, if quota is disabled + + Example: + is_quota_enabled(mnode, testvol) + """ + + from glustolibs.gluster.volume_ops import get_volume_options + + output = get_volume_options(mnode, volname, "features.quota") + if output is None: + return False + + g.log.info("Quota Status in volume %s %s" + % (volname, output["features.quota"])) + if output["features.quota"] != 'on': + return False + + return True + + +def quota_list(mnode, volname, path=None): + """Executes quota list command for given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + path (str): Quota path + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + quota_list(mnode, testvol) + """ + + if not path: + path = '' + + cmd = "gluster volume quota %s list %s" % (volname, path) + ret = g.run(mnode, cmd) + return ret + + +def set_quota_limit_usage(mnode, volname, path='/', limit='100GB', + soft_limit=''): + """Sets limit-usage on the path of the specified volume to + specified limit + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + path (str): path to which quota limit usage is set. + Defaults to /. + limit (str): quota limit usage. defaults to 100GB + soft_limit (str): quota soft limit to be set + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> set_quota_limit_usage("abc.com", "testvol") + + """ + + cmd = ("gluster volume quota %s limit-usage %s %s %s --mode=script" + % (volname, path, limit, soft_limit)) + return g.run(mnode, cmd) + + +def get_quota_list(mnode, volname, path=None): + """Parse the output of 'gluster quota list' command. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Kwargs: + path (str): Quota path + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. + + Examples: + >>> get_quota_list('abc.lab.eng.xyz.com', "testvol") + {'/': {'used_space': '0', 'hl_exceeded': 'No', 'soft_limit_percent': + '60%', 'avail_space': '2147483648', 'soft_limit_value': '1288490188', + 'sl_exceeded': 'No', 'hard_limit': '2147483648'}} + """ + if not path: + path = '' + + cmd = "gluster volume quota %s list %s --xml" % (volname, path) + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'quota list' on node %s. " + "Hence failed to get the quota list.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster quota list xml output.") + return None + + quotalist = {} + for path in root.findall("volQuota/limit"): + for elem in path.getchildren(): + if elem.tag == "path": + path = elem.text + quotalist[path] = {} + else: + quotalist[path][elem.tag] = elem.text + return quotalist + + +def set_quota_limit_objects(mnode, volname, path='/', limit='10', + soft_limit=''): + """Sets limit-objects on the path of the specified volume to + specified limit + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Kwargs: + path (str): path to which quota limit usage is set. + Defaults to /. + limit (str): quota limit objects. defaults to 10. + soft_limit (str): quota soft limit to be set + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> set_quota_limit_objects("abc.com", "testvol") + + """ + + cmd = ("gluster volume quota %s limit-objects %s %s %s --mode=script" + % (volname, path, limit, soft_limit)) + return g.run(mnode, cmd) + + +def quota_list_objects(mnode, volname, path=None): + """Executes quota list command for given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + path (str): Quota path + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + quota_list_objects("abc.com", testvol) + + """ + + if not path: + path = '' + + cmd = "gluster volume quota %s list-objects %s" % (volname, path) + ret = g.run(mnode, cmd) + return ret + + +def get_quota_list_objects(mnode, volname, path=None): + """Parse the output of 'gluster quota list-objects' command. + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + path (str): Quota path + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict of dict on success. + + Examples: + >>> get_quota_list_objects('abc.lab.eng.xyz.com', "testvol") + {'/': {'available': '7', 'hl_exceeded': 'No', 'soft_limit_percent': + '80%', 'soft_limit_value': '8', 'dir_count': '3', 'sl_exceeded': + 'No', 'file_count': '0', 'hard_limit': '10'}} + """ + + if not path: + path = '' + + cmd = "gluster volume quota %s list-objects %s --xml" % (volname, path) + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'quota list' on node %s. " + "Hence failed to get the quota list.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster quota list xml output.") + return None + + quotalist = {} + for path in root.findall("volQuota/limit"): + for elem in path.getchildren(): + if elem.tag == "path": + path = elem.text + quotalist[path] = {} + else: + quotalist[path][elem.tag] = elem.text + return quotalist + + +def set_quota_alert_time(mnode, volname, time): + """Sets quota alert time + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + time (str): quota alert time in seconds + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> set_quota_alert_time("abc.com", "testvol", ) + + """ + + cmd = ("gluster volume quota %s alert-time %s --mode=script" + % (volname, time)) + return g.run(mnode, cmd) + + +def set_quota_soft_timeout(mnode, volname, timeout): + """Sets quota soft timeout + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + timeout (str): quota soft limit timeout value + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> set_quota_soft_timeout("abc.com", "testvol", ) + + """ + + cmd = ("gluster volume quota %s soft-timeout %s --mode=script" + % (volname, timeout)) + return g.run(mnode, cmd) + + +def set_quota_hard_timeout(mnode, volname, timeout): + """Sets quota hard timeout + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + timeout (str): quota hard limit timeout value + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> set_quota_hard_timeout("abc.com", "testvol", ) + + """ + + cmd = ("gluster volume quota %s hard-timeout %s --mode=script" + % (volname, timeout)) + return g.run(mnode, cmd) + + +def set_quota_default_soft_limit(mnode, volname, timeout): + """Sets quota default soft limit + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + timeout (str): quota soft limit timeout value + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> set_quota_default_soft_limit("abc.com", "testvol", + ) + + """ + + cmd = ("gluster volume quota %s default-soft-limit %s --mode=script" + % (volname, timeout)) + return g.run(mnode, cmd) + + +def remove_quota(mnode, volname, path): + """Removes quota for the given path + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + path (str): path to which quota limit usage is set. + Defaults to /. + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> remove_quota("abc.com", "testvol", ) + + """ + + cmd = "gluster volume quota %s remove %s --mode=script" % (volname, path) + return g.run(mnode, cmd) + + +def remove_quota_objects(mnode, volname, path): + """Removes quota objects for the given path + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + path (str): path to which quota limit usage is set. + Defaults to /. + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Examples: + >>> remove_quota_objects("abc.com", "testvol", ) + + """ + + cmd = ("gluster volume quota %s remove-objects %s --mode=script" + % (volname, path)) + return g.run(mnode, cmd) diff --git a/glustolibs-gluster/glustolibs/gluster/rebalance_ops.py b/glustolibs-gluster/glustolibs/gluster/rebalance_ops.py new file mode 100644 index 000000000..108a246af --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/rebalance_ops.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Description: Library for gluster rebalance operations. +""" + +from glusto.core import Glusto as g +import re +import time + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree + + +def rebalance_start(mnode, volname, fix_layout=False, force=False): + """Starts rebalance on the given volume. + + Example: + rebalance_start("abc.com", testvol) + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + fix_layout (bool) : If this option is set to True, then rebalance + start will get execute with fix-layout option. If set to False, + then rebalance start will get executed without fix-layout option + force (bool): If this option is set to True, then rebalance + start will get execute with force option. If it is set to False, + then rebalance start will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + """ + + flayout = '' + if fix_layout: + flayout = "fix-layout" + + frce = '' + if force: + frce = 'force' + + if fix_layout and force: + g.log.warning("Both fix-layout and force option is specified." + "Ignoring force option") + frce = '' + + cmd = "gluster volume rebalance %s %s start %s" % (volname, flayout, frce) + ret = g.run(mnode, cmd) + return ret + + +def rebalance_stop(mnode, volname): + """Stops rebalance on the given volume. + + Example: + rebalance_stop("abc.com", testvol) + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + """ + + cmd = "gluster volume rebalance %s stop" % volname + ret = g.run(mnode, cmd) + return ret + + +def rebalance_status(mnode, volname): + """Executes rebalance status on the given volume. + + Example: + rebalance_status("abc.com", testvol) + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + """ + + cmd = "gluster volume rebalance %s status" % volname + ret = g.run(mnode, cmd) + return ret + + +def get_rebalance_status(mnode, volname): + """Parse the output of 'gluster vol rebalance status' command + for the given volume + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. rebalance status will be + in dict format + + Examples: + >>> get_rebalance_status('abc.lab.eng.xyz.com', testvol) + {'node': [{'files': '0', 'status': '3', 'lookups': '0', 'skipped': '0', + 'nodeName': 'localhost', 'failures': '0', 'runtime': '0.00', 'id': + '11336017-9561-4e88-9ac3-a94d4b403340', 'statusStr': 'completed', + 'size': '0'}, {'files': '0', 'status': '1', 'lookups': '0', 'skipped': + '0', 'nodeName': '10.70.47.16', 'failures': '0', 'runtime': '0.00', + 'id': 'a2b88b10-eba2-4f97-add2-8dc37df08b27', 'statusStr': + 'in progress', 'size': '0'}, {'files': '0', 'status': '3', + 'lookups': '0', 'skipped': '0', 'nodeName': '10.70.47.152', + 'failures': '0', 'runtime': '0.00', 'id': + 'b15b8337-9f8e-4ec3-8bdb-200d6a67ae12', 'statusStr': 'completed', + 'size': '0'}, {'files': '0', 'status': '3', 'lookups': '0', 'skipped': + '0', 'nodeName': '10.70.46.52', 'failures': '0', 'runtime': '0.00', + 'id': '77dc299a-32f7-43d8-9977-7345a344c398', 'statusStr': 'completed', + 'size': '0'}], 'task-id': 'a16f99d1-e165-40e7-9960-30508506529b', + 'aggregate': {'files': '0', 'status': '1', 'lookups': '0', 'skipped': + '0', 'failures': '0', 'runtime': '0.00', 'statusStr': 'in progress', + 'size': '0'}, 'nodeCount': '4', 'op': '3'} + """ + + cmd = "gluster volume rebalance %s status --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'rebalance status' on node %s. " + "Hence failed to get the rebalance status.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster rebalance status " + "xml output.") + return None + + rebal_status = {} + rebal_status["node"] = [] + for info in root.findall("volRebalance"): + for element in info.getchildren(): + if element.tag == "node": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + rebal_status[element.tag].append(status_info) + elif element.tag == "aggregate": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + rebal_status[element.tag] = status_info + else: + rebal_status[element.tag] = element.text + return rebal_status + + +def rebalance_stop_and_get_status(mnode, volname): + """Parse the output of 'gluster vol rebalance stop' command + for the given volume + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. rebalance status will be + in dict format + + Examples: + >>> rebalance_stop_and_get_status('abc.xyz.com', testvol) + {'node': [{'files': '0', 'status': '3', 'lookups': '0', 'skipped': '0', + 'nodeName': 'localhost', 'failures': '0', 'runtime': '0.00', 'id': + '11336017-9561-4e88-9ac3-a94d4b403340', 'statusStr': 'completed', + 'size': '0'}, {'files': '0', 'status': '1', 'lookups': '0', 'skipped': + '0', 'nodeName': '10.70.47.16', 'failures': '0', 'runtime': '0.00', + 'id': 'a2b88b10-eba2-4f97-add2-8dc37df08b27', 'statusStr': + 'in progress', 'size': '0'}, {'files': '0', 'status': '3', + 'lookups': '0', 'skipped': '0', 'nodeName': '10.70.47.152', + 'failures': '0', 'runtime': '0.00', 'id': + 'b15b8337-9f8e-4ec3-8bdb-200d6a67ae12', 'statusStr': 'completed', + 'size': '0'}, {'files': '0', 'status': '3', 'lookups': '0', 'skipped': + '0', 'nodeName': '10.70.46.52', 'failures': '0', 'runtime': '0.00', + 'id': '77dc299a-32f7-43d8-9977-7345a344c398', 'statusStr': 'completed', + 'size': '0'}], 'task-id': 'a16f99d1-e165-40e7-9960-30508506529b', + 'aggregate': {'files': '0', 'status': '1', 'lookups': '0', 'skipped': + '0', 'failures': '0', 'runtime': '0.00', 'statusStr': 'in progress', + 'size': '0'}, 'nodeCount': '4', 'op': '3'} + """ + + cmd = "gluster volume rebalance %s stop --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'rebalance stop' on node %s. " + "Hence failed to parse the rebalance status.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse gluster rebalance stop xml output.") + return None + + rebal_status = {} + rebal_status["node"] = [] + for info in root.findall("volRebalance"): + for element in info.getchildren(): + if element.tag == "node": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + rebal_status[element.tag].append(status_info) + elif element.tag == "aggregate": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + rebal_status[element.tag] = status_info + else: + rebal_status[element.tag] = element.text + return rebal_status + + +def wait_for_rebalance_to_complete(mnode, volname, timeout=300): + """Waits for the rebalance to complete + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Kwargs: + timeout (int): timeout value in seconds to wait for rebalance + to complete + + Returns: + True on success, False otherwise + + Examples: + >>> wait_for_rebalance_to_complete("abc.com", "testvol") + """ + + count = 0 + flag = 0 + while (count < timeout): + status_info = get_rebalance_status(mnode, volname) + if status_info is None: + return False + + status = status_info['aggregate']['statusStr'] + if status == 'completed': + flag = 1 + break + + time.sleep(10) + count = count + 10 + if not flag: + g.log.error("rebalance is not completed") + return False + else: + g.log.info("rebalance is successfully completed") + return True diff --git a/glustolibs-gluster/glustolibs/gluster/snap_ops.py b/glustolibs-gluster/glustolibs/gluster/snap_ops.py new file mode 100644 index 000000000..5d58f23be --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/snap_ops.py @@ -0,0 +1,898 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Description: Library for gluster snapshot operations. +""" + +from glusto.core import Glusto as g +from glustolibs.gluster.volume_ops import volume_start, volume_stop +import re +import time + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree + + +def snap_create(mnode, volname, snapname, timestamp=False, + description='', force=False): + """Creates snapshot for the given volume. + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + snapname (str): snapshot name + + Kwargs: + timestamp (bool): If this option is set to True, then + timestamps will get appended to the snapname. If this option + is set to False, then timestamps will not be appended to snapname. + description (str): description for snapshot creation + force (bool): If this option is set to True, then snap + create will get execute with force option. If it is set to False, + then snap create will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_create("abc.com", testvol, testsnap) + + """ + + if description != '': + description = "description '%s'" % description + + tstamp = '' + if not timestamp: + tstamp = "no-timestamp" + + frce = '' + if force: + frce = 'force' + + cmd = ("gluster snapshot create %s %s %s %s %s" + % (snapname, volname, tstamp, description, frce)) + return g.run(mnode, cmd) + + +def snap_clone(mnode, snapname, clonename): + """Clones the given snapshot + + Args: + mnode (str): Node on which cmd has to be executed. + snapname (str): snapshot name to be cloned + clonename (str): clone name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_clone("abc.com", testsnap, clone1) + + """ + cmd = "gluster snapshot clone %s %s --mode=script" % (clonename, snapname) + return g.run(mnode, cmd) + + +def snap_restore(mnode, snapname): + """Executes snap restore cli for the given snapshot + + Args: + mnode (str): Node on which cmd has to be executed. + snapname (str): snapshot name to be cloned + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_restore(mnode, testsnap) + + """ + + cmd = "gluster snapshot restore %s --mode=script" % snapname + return g.run(mnode, cmd) + + +def snap_restore_complete(mnode, volname, snapname): + """stops the volume restore the snapshot and starts the volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + snapname (str): snapshot name + + Returns: + bool: True on success, False on failure + + Example: + snap_restore_complete(mnode, testvol, testsnap) + + """ + + # Stopping volume before snap restore + ret = volume_stop(mnode, volname) + if not ret: + g.log.error("Failed to stop volume %s before restoring snapshot " + "%s in node %s" % (volname, snapname, mnode)) + return False + ret, _, _ = snap_restore(mnode, snapname) + if ret != 0: + g.log.error("snapshot restore cli execution failed") + return False + + # Starting volume after snap restore + ret = volume_start(mnode, volname) + if not ret: + g.log.error("Failed to start volume %s after restoring snapshot " + "%s in node %s" % (volname, snapname, mnode)) + return False + return True + + +def snap_status(mnode, snapname="", volname=""): + """Runs 'gluster snapshot status' on specific node + + Args: + mnode (str): Node on which cmd has to be executed. + + Kwargs: + snapname (str): snapshot name + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_status("abc.com") + + """ + + if snapname != "" and volname != "": + g.log.error("Incorrect cmd. snap status cli accepts either " + "snapname or volname") + return (-1, None, None) + + if volname != '': + volname = "volume %s" % volname + + cmd = "gluster snapshot status %s %s" % (snapname, volname) + return g.run(mnode, cmd) + + +def get_snap_status(mnode): + """Parse the output of 'gluster snapshot status' command. + + Args: + mnode (str): Node on which command has to be executed. + + Returns: + NoneType: None if command execution fails, parse errors. + list: list of dict on success. Each snap status will be + in dict format + + Examples: + >>> get_snap_status('abc.lab.eng.xyz.com') + [{'volCount': '1', 'volume': {'brick': [{'path': '10.70.47.11: + testvol_brick0', 'pid': '26747', 'lvUsage': '3.52', 'volumeGroup': + 'RHS_vg0', 'lvSize': '9.95g'}, {'path': '10.70.47.16:/testvol_brick1', + 'pid': '25497', 'lvUsage': '3.52', 'volumeGroup': 'RHS_vg0', + 'lvSize': '9.95g'}], 'brickCount': '2'}, 'name': 'snap2', 'uuid': + '56a39a92-c339-47cc-a8b2-9e54bb2a6324'}, {'volCount': '1', 'volume': + {'brick': [{'path': '10.70.47.11:testvol_next_brick0', 'pid': '26719', + 'lvUsage': '4.93', 'volumeGroup': 'RHS_vg1', 'lvSize': '9.95g'}], + 'brickCount': '1'}, 'name': 'next_snap1', + 'uuid': 'dcf0cd31-c0db-47ad-92ec-f72af2d7b385'}] + """ + + ret, out, _ = g.run(mnode, "gluster snapshot status --xml") + if ret != 0: + g.log.error("Failed to execute 'snapshot status' on node %s. " + "Hence failed to get the snapshot status.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster snapshot " + "status xml output.") + return None + + snap_status_list = [] + for snap in root.findall("snapStatus/snapshots/snapshot"): + snap_status = {} + for element in snap.getchildren(): + if element.tag == "volume": + status = {} + status["brick"] = [] + for elmt in element.getchildren(): + if elmt.tag == "brick": + brick_info = {} + for el in elmt.getchildren(): + brick_info[el.tag] = el.text + status["brick"].append(brick_info) + else: + status[elmt.tag] = elmt.text + + snap_status[element.tag] = status + else: + snap_status[element.tag] = element.text + snap_status_list.append(snap_status) + return snap_status_list + + +def get_snap_status_by_snapname(mnode, snapname): + """Parse the output of 'gluster snapshot status' command + for the given snapshot. + + Args: + mnode (str): Node on which command has to be executed. + snapname (str): snapshot name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: on success. + + Examples: + >>> get_snap_status_by_snapname('abc.lab.eng.xyz.com', + 'snap1') + {'volCount': '1', 'volume': {'brick': [{'path': '10.70.47.11: + testvol_brick0', 'pid': '26747', 'lvUsage': '3.52', 'volumeGroup': + 'RHS_vg0', 'lvSize': '9.95g'}, {'path': '10.70.47.16:/testvol_brick1', + 'pid': '25497', 'lvUsage': '3.52', 'volumeGroup': 'RHS_vg0', + 'lvSize': '9.95g'}], 'brickCount': '2'}, 'name': 'snap2', 'uuid': + '56a39a92-c339-47cc-a8b2-9e54bb2a6324'} + """ + + snap_status_list = get_snap_status(mnode) + if not snap_status_list: + g.log.error("Failed to parse snap status in " + "get_snap_status_by_snapname()") + return None + + for snap_status in snap_status_list: + if "name" in snap_status: + if snap_status["name"] == snapname: + return snap_status + g.log.error("The snap %s not found" % (snapname)) + return None + + +def get_snap_status_by_volname(mnode, volname): + """Parse the output of 'gluster snapshot status' command + for the given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): snapshot name + + Returns: + NoneType: None if command execution fails, parse errors. + list: list of dicts on success. + + Examples: + >>> get_snap_status_by_volname('abc.lab.eng.xyz.com', + 'testvol') + [{'volCount': '1', 'volume': {'brick': [{'path': '10.70.47.11: + testvol_brick0', 'pid': '26747', 'lvUsage': '3.52', 'volumeGroup': + 'RHS_vg0', 'lvSize': '9.95g'}, {'path': '10.70.47.16:/testvol_brick1', + 'pid': '25497', 'lvUsage': '3.52', 'volumeGroup': 'RHS_vg0', + 'lvSize': '9.95g'}], 'brickCount': '2'}, 'name': 'snap2', 'uuid': + '56a39a92-c339-47cc-a8b2-9e54bb2a6324'}, {'volCount': '1', 'volume': + {'brick': [{'path': '10.70.47.11:testvol_next_brick0', 'pid': '26719', + 'lvUsage': '4.93', 'volumeGroup': 'RHS_vg1', 'lvSize': '9.95g'}], + 'brickCount': '1'}, 'name': 'next_snap1', + 'uuid': 'dcf0cd31-c0db-47ad-92ec-f72af2d7b385'}] + """ + + cmd = "gluster snapshot status volume %s --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'snapshot status' on node %s. " + "Hence failed to get the snapshot status.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster snapshot " + "status xml output.") + return None + + snap_status_list = [] + for snap in root.findall("snapStatus/snapshots/snapshot"): + snap_status = {} + for element in snap.getchildren(): + if element.tag == "volume": + status = {} + status["brick"] = [] + for elmt in element.getchildren(): + if elmt.tag == "brick": + brick_info = {} + for el in elmt.getchildren(): + brick_info[el.tag] = el.text + status["brick"].append(brick_info) + else: + status[elmt.tag] = elmt.text + + snap_status[element.tag] = status + else: + snap_status[element.tag] = element.text + snap_status_list.append(snap_status) + return snap_status_list + + +def snap_info(mnode, snapname="", volname=""): + """Runs 'gluster snapshot info' on specific node + + Args: + mnode (str): Node on which cmd has to be executed. + + Kwargs: + snapname (str): snapshot name + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_info("abc.com") + + """ + + if snapname != "" and volname != "": + g.log.error("Incorrect cmd. snap info cli accepts either " + "snapname or volname") + return (-1, None, None) + + if volname != '': + volname = "volume %s" % volname + + cmd = "gluster snapshot info %s %s" % (snapname, volname) + return g.run(mnode, cmd) + + +def get_snap_info(mnode): + """Parse the output of 'gluster snapshot info' command. + + Args: + mnode (str): Node on which command has to be executed. + + Returns: + NoneType: None if command execution fails, parse errors. + list: list of dicts on success. + + Examples: + >>> get_snap_info('abc.lab.eng.xyz.com') + [{'description': 'This is snap2', 'uuid': + '56a39a92-c339-47cc-a8b2-9e54bb2a6324', 'volCount': '1', + 'snapVolume': {'status': 'Stopped', 'name': + 'df1882d3f86d48738e69f298096f3810'}, 'createTime': + '2016-04-07 12:01:21', 'name': 'snap2'}, {'description': None, + 'uuid': 'a322d93a-2732-447d-ab88-b943fa402fd2', 'volCount': '1', + 'snapVolume': {'status': 'Stopped', 'name': + '2c790e6132e447e79168d9708d4abfe7'}, 'createTime': + '2016-04-07 13:59:43', 'name': 'snap1'}] + """ + + ret, out, _ = g.run(mnode, "gluster snapshot info --xml") + if ret != 0: + g.log.error("Failed to execute 'snapshot info' on node %s. " + "Hence failed to get the snapshot info.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster snapshot " + "info xml output.") + return None + + snap_info_list = [] + for snap in root.findall("snapInfo/snapshots/snapshot"): + snap_info = {} + for element in snap.getchildren(): + if element.tag == "snapVolume": + info = {} + for elmt in element.getchildren(): + if elmt.tag == "originVolume": + info["originVolume"] = {} + for el in elmt.getchildren(): + info[elmt.tag][el.tag] = el.text + else: + info[elmt.tag] = elmt.text + snap_info[element.tag] = info + else: + snap_info[element.tag] = element.text + snap_info_list.append(snap_info) + return snap_info_list + + +def get_snap_info_by_snapname(mnode, snapname): + """Parse the output of 'gluster snapshot info' command + for the given snapshot. + + Args: + mnode (str): Node on which command has to be executed. + snapname (str): snapshot name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: on success. + + Examples: + >>> get_snap_info_by_snapname('abc.lab.eng.xyz.com', 'snap1') + {'description': 'This is snap2', 'uuid': + '56a39a92-c339-47cc-a8b2-9e54bb2a6324', 'volCount': '1', + 'snapVolume': {'status': 'Stopped', 'name': + 'df1882d3f86d48738e69f298096f3810'} + """ + + snap_info_list = get_snap_info(mnode) + if not snap_info_list: + g.log.error("Failed to parse snap info in " + "get_snap_info_by_snapname()") + return None + + for snap_info in snap_info_list: + if "name" in snap_info: + if snap_info["name"] == snapname: + return snap_info + g.log.error("The snap %s not found" % (snapname)) + return None + + +def get_snap_info_by_volname(mnode, volname): + """Parse the output of 'gluster snapshot info' command + for the given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): snapshot name + + Returns: + NoneType: None if command execution fails, parse errors. + list: list of dicts on success. + + Examples: + >>> get_snap_info_by_volname('abc.lab.eng.xyz.com', + 'testvol') + {'originVolume': {'snapCount': '1', 'name': 'testvol', + 'snapRemaining': '255'}, 'count': '1', 'snapshots': + [{'description': 'This is next snap1', 'uuid': + 'dcf0cd31-c0db-47ad-92ec-f72af2d7b385', 'volCount': '1', + 'snapVolume': {'status': 'Stopped', 'name': + '49c290d6e8b74205adb3cce1206b5bc5'}, 'createTime': + '2016-04-07 12:03:11', 'name': 'next_snap1'}]} + """ + + cmd = "gluster snapshot info volume %s --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'snapshot info' on node %s. " + "Hence failed to get the snapshot info.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster snapshot " + "info xml output.") + return None + + snap_vol_info = {} + + for snap in root.findall("snapInfo"): + for element in snap.getchildren(): + if element.tag == "originVolume": + info = {} + for elmt in element.getchildren(): + info[elmt.tag] = elmt.text + snap_vol_info[element.tag] = info + else: + snap_vol_info[element.tag] = element.text + + snap_info_list = [] + for snap in root.findall("snapInfo/snapshots/snapshot"): + snap_info = {} + for element in snap.getchildren(): + if element.tag == "snapVolume": + info = {} + for elmt in element.getchildren(): + if elmt.tag == "originVolume": + info["originVolume"] = {} + for el in elmt.getchildren(): + info[elmt.tag][el.tag] = el.text + else: + info[elmt.tag] = elmt.text + snap_info[element.tag] = info + else: + snap_info[element.tag] = element.text + snap_info_list.append(snap_info) + snap_vol_info["snapshots"] = snap_info_list + return snap_vol_info + + +def snap_list(mnode): + """Lists the snapshots + + Args: + mnode (str): Node on which cmd has to be executed. + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_list("abc.com") + + """ + + cmd = "gluster snapshot list" + return g.run(mnode, cmd) + + +def get_snap_list(mnode): + """Parse the output of 'gluster snapshot list' command. + + Args: + mnode (str): Node on which command has to be executed. + + Returns: + NoneType: None if command execution fails, parse errors. + list: list of snapshots on success. + + Examples: + >>> get_snap_list('abc.lab.eng.xyz.com') + ['snap1', 'snap2'] + """ + + ret, out, _ = g.run(mnode, "gluster snapshot list --xml") + if ret != 0: + g.log.error("Failed to execute 'snapshot list' on node %s. " + "Hence failed to get the snapshot list.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster snapshot " + "list xml output.") + return None + + snap_list = [] + for snap in root.findall("snapList/snapshot"): + snap_list.append(snap.text) + + return snap_list + + +def snap_config(mnode, volname=None): + """Runs 'gluster snapshot config' on specific node + + Args: + mnode (str): Node on which cmd has to be executed. + + Kwargs: + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_config(mnode) + + """ + + if volname is None: + volname = "" + + cmd = "gluster snapshot config %s" % volname + return g.run(mnode, cmd) + + +def get_snap_config(mnode, volname=None): + """Parse the output of 'gluster snapshot config' command. + + Args: + mnode (str): Node on which command has to be executed. + + Kwargs: + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: on success. + + Examples: + >>> get_snap_config('abc.com') + {'volumeConfig': [{'softLimit': '230', 'effectiveHardLimit': '256', + 'name': 'testvol', 'hardLimit': '256'}, {'softLimit': '230', + 'effectiveHardLimit': '256', 'name': 'testvol_next', + 'hardLimit': '256'}], 'systemConfig': {'softLimit': '90%', + 'activateOnCreate': 'disable', 'hardLimit': '256', + 'autoDelete': 'disable'}} + """ + + ret, out, _ = g.run(mnode, "gluster snapshot config --xml") + if ret != 0: + g.log.error("Failed to execute 'snapshot config' on node %s. " + "Hence failed to get the snapshot config.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster snapshot " + "config xml output.") + return None + + snap_config = {} + for config in root.findall("snapConfig/systemConfig"): + sys_config = {} + for element in config.getchildren(): + sys_config[element.tag] = element.text + snap_config["systemConfig"] = sys_config + + volume_config = [] + for config in root.findall("snapConfig/volumeConfig/volume"): + vol_config = {} + for element in config.getchildren(): + vol_config[element.tag] = element.text + + if volname is not None: + if volname == vol_config["name"]: + volume_config.append(vol_config) + else: + volume_config.append(vol_config) + + snap_config["volumeConfig"] = volume_config + return snap_config + + +def set_snap_config(mnode, option, volname=None): + """Sets given snap config on the given node + + Args: + mnode (str): Node on which cmd has to be executed. + option (dict): dict of single snap config option + + Kwargs: + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + >>>option={'snap-max-hard-limit':'200'} + set_snap_config(mnode, option) + + """ + + if volname is None: + volname = "" + + cmd = ("gluster snapshot config %s %s %s --mode=script" + % (volname, option.keys()[0], option.values()[0])) + return g.run(mnode, cmd) + + +def snap_delete(mnode, snapname): + """Deletes the given snapshot + + Args: + mnode (str): Node on which cmd has to be executed. + snapname (str): snapshot name to be deleted + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_delete(mnode, testsnap) + + """ + + cmd = "gluster snapshot delete %s --mode=script" % snapname + return g.run(mnode, cmd) + + +def snap_delete_by_volumename(mnode, volname): + """Deletes the given snapshot + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_delete_by_volumename(mnode, testvol) + + """ + + cmd = "gluster snapshot delete volume %s --mode=script" % volname + return g.run(mnode, cmd) + + +def snap_delete_all(mnode): + """Deletes all the snapshot in the cluster + + Args: + mnode (str): Node on which cmd has to be executed. + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_delete_all("abc.com") + + """ + cmd = "gluster snapshot delete all --mode=script" + return g.run(mnode, cmd) + + +def snap_activate(mnode, snapname, force=False): + """Activates the given snapshot + + Args: + mnode (str): Node on which cmd has to be executed. + snapname (str): snapshot name to be cloned + + Kwargs: + force (bool): If this option is set to True, then snap + activate will get execute with force option. If it is set to False, + then snap activate will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_activate("abc.com", testsnap) + + """ + + frce = '' + if force: + frce = 'force' + + cmd = "gluster snapshot activate %s %s --mode=script" % (snapname, frce) + return g.run(mnode, cmd) + + +def snap_deactivate(mnode, snapname): + """Deactivates the given snapshot + + Args: + mnode (str): Node on which cmd has to be executed. + snapname (str): snapshot name to be cloned + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + snap_deactivate("abc.com", testsnap) + + """ + + cmd = "gluster snapshot deactivate %s --mode=script" % snapname + return g.run(mnode, cmd) diff --git a/glustolibs-gluster/glustolibs/gluster/tiering_ops.py b/glustolibs-gluster/glustolibs/gluster/tiering_ops.py new file mode 100644 index 000000000..38890f5ca --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/tiering_ops.py @@ -0,0 +1,1019 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Description: Library for gluster tiering operations. +""" + +import re +from glusto.core import Glusto as g +from glustolibs.gluster.peer_ops import peer_probe_servers +from glustolibs.gluster.gluster_init import start_glusterd +from glustolibs.gluster.lib_utils import list_files + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree + + +def add_extra_servers_to_cluster(mnode, extra_servers): + """Adds the given extra servers to cluster + + Args: + mnode (str): Node on which cmd has to be executed. + extra_servers (list) : list of extra servers to be attached to cluster + + Returns: + bool: True, if extra servers are attached to cluster + False, otherwise + + Example: + add_extra_servers_to_cluster("abc.com", ['peer_node1','peer_node2']) + """ + + if not isinstance(extra_servers, list): + extra_servers = [extra_servers] + + ret = start_glusterd(servers=extra_servers) + if not ret: + g.log.error("glusterd did not start in peer nodes") + return False + + ret = peer_probe_servers(mnode, servers=extra_servers) + if not ret: + g.log.error("Unable to do peer probe on extra server machines") + return False + + return True + + +def tier_attach(mnode, volname, num_bricks_to_add, extra_servers, + extra_servers_info, replica=1, force=False): + """Attaches tier to the volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + num_bricks_to_add (str): number of bricks to be added as hot tier + extra_servers (list): from these servers, hot tier will be added + to volume + extra_servers_info (dict): dict of server info of each extra servers + + Kwargs: + replica (str): replica count of the hot tier + force (bool): If this option is set to True, then attach tier + will get executed with force option. If it is set to False, + then attach tier will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_attach("abc.com", testvol, '2', ['extra_server1','extra_server2'], + extra_server_info) + """ + + replica = int(replica) + repc = '' + if replica != 1: + repc = "replica %d" % replica + + frce = '' + if force: + frce = 'force' + + num_bricks_to_add = int(num_bricks_to_add) + + from glustolibs.gluster.lib_utils import form_bricks_list + bricks_list = form_bricks_list(mnode, volname, num_bricks_to_add, + extra_servers[:], extra_servers_info) + if bricks_list is None: + g.log.error("number of bricks required are greater than " + "unused bricks") + return (-1, '', '') + + bricks_path = ' '.join(bricks_list) + bricks_path = [re.sub(r"(.*\/\S+\_)brick(\d+)", r"\1tier\2", item) + for item in bricks_path.split() if item] + tier_bricks_path = " ".join(bricks_path) + cmd = ("gluster volume tier %s attach %s %s %s --mode=script" + % (volname, repc, tier_bricks_path, frce)) + + return g.run(mnode, cmd) + + +def tier_start(mnode, volname, force=False): + """Starts the tier volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + force (bool): If this option is set to True, then attach tier + will get executed with force option. If it is set to False, + then attach tier will get executed without force option + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_start("abc.com", testvol) + """ + + frce = '' + if force: + frce = 'force' + + cmd = ("gluster volume tier %s start %s --mode=script" + % (volname, frce)) + return g.run(mnode, cmd) + + +def tier_status(mnode, volname): + """executes tier status command + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_status("abc.com", testvol) + """ + + cmd = "gluster volume tier %s status" % volname + ret = g.run(mnode, cmd) + + return ret + + +def get_tier_status(mnode, volname): + """Parse the output of 'gluster tier status' command. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. + + Examples: + >>> get_tier_status('abc.lab.eng.xyz.com', 'testvol') + {'node': [{'promotedFiles': '0', 'demotedFiles': '0', 'nodeName': + 'localhost', 'statusStr': 'in progress'}, {'promotedFiles': '0', + 'demotedFiles': '0', 'nodeName': '10.70.47.16', 'statusStr': + 'in progress'}], 'task-id': '2ed28cbd-4246-493a-87b8-1fdcce313b34', + 'nodeCount': '4', 'op': '7'} + """ + + cmd = "gluster volume tier %s status --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'tier status' on node %s. " + "Hence failed to get tier status.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster tier status xml output.") + return None + + tier_status = {} + tier_status["node"] = [] + for info in root.findall("volRebalance"): + for element in info.getchildren(): + if element.tag == "node": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + tier_status[element.tag].append(status_info) + else: + tier_status[element.tag] = element.text + return tier_status + + +def tier_detach_start(mnode, volname): + """starts detaching tier on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_detach_start("abc.com", testvol) + + """ + + cmd = "gluster volume tier %s detach start --mode=script" % volname + return g.run(mnode, cmd) + + +def tier_detach_status(mnode, volname): + """executes detach tier status on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_detach_status("abc.com", testvol) + + """ + + cmd = "gluster volume tier %s detach status --mode=script" % volname + return g.run(mnode, cmd) + + +def tier_detach_stop(mnode, volname): + """stops detaching tier on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_detach_stop("abc.com", testvol) + + """ + + cmd = "gluster volume tier %s detach stop --mode=script" % volname + return g.run(mnode, cmd) + + +def tier_detach_commit(mnode, volname): + """commits detach tier on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_detach_commit("abc.com", testvol) + + """ + + cmd = "gluster volume tier %s detach commit --mode=script" % volname + return g.run(mnode, cmd) + + +def tier_detach_force(mnode, volname): + """detaches tier forcefully on given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + tier_detach_force("abc.com", testvol) + + """ + + cmd = "gluster volume tier %s detach force --mode=script" % volname + return g.run(mnode, cmd) + + +def get_detach_tier_status(mnode, volname): + """Parse the output of 'gluster volume tier detach status' command. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. + + Examples: + >>> get_detach_tier_status('abc.lab.eng.xyz.com', "testvol") + {'node': [{'files': '0', 'status': '3', 'lookups': '1', 'skipped': '0', + 'nodeName': 'localhost', 'failures': '0', 'runtime': '0.00', 'id': + '11336017-9561-4e88-9ac3-a94d4b403340', 'statusStr': 'completed', + 'size': '0'}, {'files': '0', 'status': '3', 'lookups': '0', 'skipped': + '0', 'nodeName': '10.70.47.16', 'failures': '0', 'runtime': '0.00', + 'id': 'a2b88b10-eba2-4f97-add2-8dc37df08b27', 'statusStr': 'completed', + 'size': '0'}], 'nodeCount': '4', 'aggregate': {'files': '0', 'status': + '3', 'lookups': '1', 'skipped': '0', 'failures': '0', 'runtime': '0.0', + 'statusStr': 'completed', 'size': '0'}} + """ + + cmd = "gluster volume tier %s detach status --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'detach tier status' on node %s. " + "Hence failed to get detach tier status.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the detach tier status xml output.") + return None + + tier_status = {} + tier_status["node"] = [] + for info in root.findall("volDetachTier"): + for element in info.getchildren(): + if element.tag == "node": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + tier_status[element.tag].append(status_info) + elif element.tag == "aggregate": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + tier_status[element.tag] = status_info + else: + tier_status[element.tag] = element.text + return tier_status + + +def tier_detach_start_and_get_taskid(mnode, volname): + """Parse the output of 'gluster volume tier detach start' command. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. + + Examples: + >>> tier_detach_start_and_get_taskid('abc.lab.eng.xyz.com', + "testvol") + {'task-id': '8020835c-ff0d-4ea1-9f07-62dd067e92d4'} + """ + + cmd = "gluster volume tier %s detach start --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'detach tier start' on node %s. " + "Hence failed to parse the detach tier start.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster detach tier " + "start xml output.") + return None + + tier_status = {} + for info in root.findall("volDetachTier"): + for element in info.getchildren(): + tier_status[element.tag] = element.text + return tier_status + + +def tier_detach_stop_and_get_status(mnode, volname): + """Parse the output of 'gluster volume tier detach stop' command. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: dict on success. + + Examples: + >>> tier_detach_stop_and_get_status('abc.lab.eng.xyz.com', + "testvol") + {'node': [{'files': '0', 'status': '3', 'lookups': '1', 'skipped': '0', + 'nodeName': 'localhost', 'failures': '0', 'runtime': '0.00', 'id': + '11336017-9561-4e88-9ac3-a94d4b403340', 'statusStr': 'completed', + 'size': '0'}, {'files': '0', 'status': '3', 'lookups': '0', 'skipped': + '0', 'nodeName': '10.70.47.16', 'failures': '0', 'runtime': '0.00', + 'id': 'a2b88b12-eba2-4f97-add2-8dc37df08b27', 'statusStr': 'completed', + 'size': '0'}], 'nodeCount': '4', 'aggregate': {'files': '0', 'status': + '3', 'lookups': '1', 'skipped': '0', 'failures': '0', 'runtime': '0.0', + 'statusStr': 'completed', 'size': '0'}} + """ + + cmd = "gluster volume tier %s detach stop --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'tier start' on node %s. " + "Hence failed to parse the tier start.", mnode) + return None + + try: + root = etree.XML(out) + except etree.ParseError: + g.log.error("Failed to parse the gluster detach tier stop" + " xml output.") + return None + + tier_status = {} + tier_status["node"] = [] + for info in root.findall("volDetachTier"): + for element in info.getchildren(): + if element.tag == "node": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + tier_status[element.tag].append(status_info) + elif element.tag == "aggregate": + status_info = {} + for elmt in element.getchildren(): + status_info[elmt.tag] = elmt.text + tier_status[element.tag] = status_info + else: + tier_status[element.tag] = element.text + return tier_status + + +def wait_for_detach_tier_to_complete(mnode, volname, timeout=300): + """Waits for the detach tier to complete + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Kwargs: + timeout (int): timeout value to wait for detach tier to complete + + Returns: + True on success, False otherwise + + Examples: + >>> wait_for_detach_tier_to_complete(mnode, "testvol") + """ + + count = 0 + flag = 0 + while (count < timeout): + status_info = get_detach_tier_status(mnode, volname) + if status_info is None: + return False + + status = status_info['aggregate']['statusStr'] + if status == 'completed': + flag = 1 + break + + time.sleep(10) + count = count + 10 + if not flag: + g.log.error("detach tier is not completed") + return False + else: + g.log.info("detach tier is successfully completed") + return True + + +def get_files_from_hot_tier(mnode, volname): + """Lists files from hot tier for the given volume + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + Emptylist: if there are no files in hot tier. + list: list of files in hot tier on success. + + Examples: + >>>get_files_from_hot_tier(mnode, "testvol") + """ + + files = [] + from glustolibs.gluster.volume_libs import get_subvols + subvols = get_subvols(mnode, volname) + for subvol in subvols['hot_tier_subvols']: + info = subvol[0].split(':') + file_list = list_files(info[0], info[1]) + for file in file_list: + if ".glusterfs" not in file: + files.append(file) + + return files + + +def get_files_from_cold_tier(mnode, volname): + """Lists files from cold tier for the given volume + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + Emptylist: if there are no files in cold tier. + list: list of files in cold tier on success. + + Examples: + >>>get_files_from_hot_tier("testvol") + """ + + files = [] + from glustolibs.gluster.volume_libs import get_subvols + subvols = get_subvols(mnode, volname) + for subvol in subvols['cold_tier_subvols']: + info = subvol[0].split(':') + file_list = list_files(info[0], info[1]) + for file in file_list: + if ".glusterfs" not in file: + files.append(file) + + return files + + +def get_tier_promote_frequency(mnode, volname): + """Gets tier promote frequency value for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: promote frequency value on success. + + Examples: + >>>get_tier_promote_frequency("abc.com", "testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.tier-promote-frequency'] + + +def get_tier_demote_frequency(mnode, volname): + """Gets tier demote frequency value for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: demote frequency value on success. + + Examples: + >>>get_tier_demote_frequency("abc.com", "testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.tier-demote-frequency'] + + +def get_tier_mode(mnode, volname): + """Gets tier mode for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: tier mode on success. + + Examples: + >>>get_tier_mode("testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.tier-mode'] + + +def get_tier_max_mb(mnode, volname): + """Gets tier max mb for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: tier max mb on success. + + Examples: + >>>get_tier_max_mb("abc.com", "testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.tier-max-mb'] + + +def get_tier_max_files(mnode, volname): + """Gets tier max files for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: tier max files on success. + + Examples: + >>>get_tier_max_files("abc.com", "testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.tier-max-files'] + + +def get_tier_watermark_high_limit(mnode, volname): + """Gets tier watermark high limit for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: tier watermark high limit on success. + + Examples: + >>>get_tier_watermark_high_limit(mnode, "testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.watermark-hi'] + + +def get_tier_watermark_low_limit(mnode, volname): + """Gets tier watermark low limit for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: tier watermark low limit on success. + + Examples: + >>>get_tier_watermark_low_limit("abc.com", "testvol") + """ + + from glustolibs.gluster.volume_ops import get_volume_options + vol_options = get_volume_options(mnode, volname) + if vol_options is None: + g.log.error("Failed to get volume options") + return None + + return vol_options['cluster.watermark-low'] + + +def set_tier_promote_frequency(mnode, volname, value): + """Sets tier promote frequency value for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): promote frequency value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_promote_frequency("abc.com", "testvol", '1000') + """ + + option = {'cluster.tier-promote-frequency': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set promote frequency to %s" + % value) + return False + + return True + + +def set_tier_demote_frequency(mnode, volname, value): + """Sets tier demote frequency value for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): demote frequency value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_demote_frequency("abc.com", "testvol", "500") + """ + + option = {'cluster.tier-demote-frequency': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set demote frequency to %s" + % value) + return False + + return True + + +def set_tier_mode(mnode, volname, value): + """Sets tier mode for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): tier mode value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_mode("abc.com", "testvol", "cache") + """ + + option = {'cluster.tier-mode': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set tier mode to %s" + % value) + return False + + return True + + +def set_tier_max_mb(mnode, volname, value): + """Sets tier max mb for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): tier mode value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_max_mb("abc.com", "testvol", "50") + """ + + option = {'cluster.tier-max-mb': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set tier max mb to %s" + % value) + return False + + return True + + +def set_tier_max_files(mnode, volname, value): + """Sets tier max files for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): tier mode value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_max_files("abc.com", "testvol", "10") + """ + + option = {'cluster.tier-max-files': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set tier max files to %s" + % value) + return False + + return True + + +def set_tier_watermark_high_limit(mnode, volname, value): + """Sets tier watermark high limit for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): tier mode value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_watermark_high_limit("abc.com", "testvol", "95") + """ + + option = {'cluster.watermark-hi': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set tier watermark high limit to %s" + % value) + return False + + return True + + +def set_tier_watermark_low_limit(mnode, volname, value): + """Sets tier watermark low limit for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + value (str): tier mode value + + Returns: + bool: True on success, False Otherwise + + Examples: + >>>set_tier_watermark_low_limit("abc.com", "testvol", "40") + """ + + option = {'cluster.watermark-low': value} + + from glustolibs.gluster.volume_ops import set_volume_options + if not set_volume_options(mnode, volname, + options=option): + g.log.error("Failed to set tier watermark low limit to %s" + % value) + return False + + return True + + +def get_tier_pid(mnode, volname): + """Gets tier pid for given volume. + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + str: pid of tier process on success. + + Examples: + >>>get_tier_pid("abc.xyz.com", "testvol") + """ + + cmd = ("ps -ef | grep -v grep | grep '/var/log/glusterfs/%s-tier.log' |" + "awk '{print $2}'" % volname) + ret, out, err = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute 'ps' cmd") + return None + + return out.strip("\n") + + +def is_tier_process_running(mnode, volname): + """Checks whether tier process is running + + Args: + mnode (str): Node on which command has to be executed. + volname (str): volume name + + Returns: + True on success, False otherwise + + Examples: + >>>is_tier_process_running("abc.xyz.com", "testvol") + """ + + pid = get_tier_pid(mnode, volname) + if pid == '': + return False + return True diff --git a/glustolibs-gluster/glustolibs/gluster/volume_ops.py b/glustolibs-gluster/glustolibs/gluster/volume_ops.py new file mode 100644 index 000000000..603bcfc0d --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/volume_ops.py @@ -0,0 +1,705 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import re +import time +from glusto.core import Glusto as g +from pprint import pformat +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree +from glustolibs.gluster.mount_ops import mount_volume +from glustolibs.gluster.gluster_init import env_setup_servers, start_glusterd +from glustolibs.gluster.peer_ops import (peer_probe_servers, + nodes_from_pool_list) + +""" + This file contains the gluster volume operations like create volume, + start/stop volume etc +""" + + +def volume_create(mnode, volname, bricks_list, force=False, **kwargs): + """Create the gluster volume with specified configuration + + Args: + mnode(str): server on which command has to be executed + volname(str): volume name that has to be created + bricks_list (list): List of bricks to use for creating volume. + Example: + from glustolibs.gluster.lib_utils import form_bricks_list + bricks_list = form_bricks_list(mnode, volname, num_of_bricks, + servers, servers_info) + Kwargs: + force (bool): If this option is set to True, then create volume + will get executed with force option. If it is set to False, + then create volume will get executed without force option + + **kwargs + The keys, values in kwargs are: + - replica_count : (int)|None + - arbiter_count : (int)|None + - stripe_count : (int)|None + - disperse_count : (int)|None + - disperse_data_count : (int)|None + - redundancy_count : (int)|None + - transport_type : tcp|rdma|tcp,rdma|None + - ... + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + (-1, '', ''): If not enough bricks are available to create volume. + (ret, out, err): As returned by volume create command execution. + + Example: + volume_create(mnode, volname, bricks_list) + """ + replica_count = arbiter_count = stripe_count = None + disperse_count = disperse_data_count = redundancy_count = None + transport_type = None + + if 'replica_count' in kwargs: + replica_count = int(kwargs['replica_count']) + + if 'arbiter_count' in kwargs: + arbiter_count = int(kwargs['arbiter_count']) + + if 'stripe_count' in kwargs: + stripe_count = int(kwargs['stripe_count']) + + if 'disperse_count' in kwargs: + disperse_count = int(kwargs['disperse_count']) + + if 'disperse_data_count' in kwargs: + disperse_data_count = int(kwargs['disperse_data_count']) + + if 'redundancy_count' in kwargs: + redundancy_count = int(kwargs['redundancy_count']) + + if 'transport_type' in kwargs: + transport_type = kwargs['transport_type'] + + replica = arbiter = stripe = disperse = disperse_data = redundancy = '' + transport = '' + if replica_count is not None: + replica = "replica %d" % replica_count + + if arbiter_count is not None: + arbiter = "arbiter %d" % arbiter_count + + if stripe_count is not None: + stripe = "stripe %d" % stripe_count + + if disperse_count is not None: + disperse = "disperse %d" % disperse_count + + if disperse_data_count is not None: + disperse_data = "disperse-data %d" % disperse_data_count + + if redundancy_count is not None: + redundancy = "redundancy %d" % redundancy_count + + if transport_type is not None: + transport = "transport %s" % transport_type + + cmd = ("gluster volume create %s %s %s %s %s %s %s %s %s " + "--mode=script" % (volname, replica, arbiter, stripe, + disperse, disperse_data, redundancy, + transport, ' '.join(bricks_list))) + + if force: + cmd = cmd + " force" + + return g.run(mnode, cmd) + +def volume_start(mnode, volname, force=False): + """Starts the gluster volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + force (bool): If this option is set to True, then start volume + will get executed with force option. If it is set to False, + then start volume will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + volume_start("testvol") + """ + if force: + cmd = "gluster volume start %s force --mode=script" % volname + else: + cmd = "gluster volume start %s --mode=script" % volname + return g.run(mnode, cmd) + + +def volume_stop(mnode, volname, force=False): + """Stops the gluster volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + force (bool): If this option is set to True, then stop volume + will get executed with force option. If it is set to False, + then stop volume will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + volume_stop(mnode, "testvol") + """ + if force: + cmd = "gluster volume stop %s force --mode=script" % volname + else: + cmd = "gluster volume stop %s --mode=script" % volname + return g.run(mnode, cmd) + + +def volume_delete(mnode, volname): + """Deletes the gluster volume if given volume exists in + gluster and deletes the directories in the bricks + associated with the given volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Returns: + bool: True, if volume is deleted + False, otherwise + + Example: + volume_delete("abc.xyz.com", "testvol") + """ + + volinfo = get_volume_info(mnode, volname) + if volinfo is None or volname not in volinfo: + g.log.info("Volume %s does not exist in %s" % (volname, mnode)) + return True + + if volinfo[volname]['typeStr'] == 'Tier': + tmp_hot_brick = volinfo[volname]["bricks"]["hotBricks"]["brick"] + hot_bricks = [x["name"] for x in tmp_hot_brick if "name" in x] + tmp_cold_brick = volinfo[volname]["bricks"]["coldBricks"]["brick"] + cold_bricks = [x["name"] for x in tmp_cold_brick if "name" in x] + bricks = hot_bricks + cold_bricks + else: + bricks = [x["name"] for x in volinfo[volname]["bricks"]["brick"] + if "name" in x] + ret, _, _ = g.run(mnode, "gluster volume delete %s --mode=script" + % volname) + if ret != 0: + return False + + for brick in bricks: + node, vol_dir = brick.split(":") + ret = g.run(node, "rm -rf %s" % vol_dir) + + return True + + +def volume_reset(mnode, volname, force=False): + """Resets the gluster volume + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + force (bool): If this option is set to True, then reset volume + will get executed with force option. If it is set to False, + then reset volume will get executed without force option + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + volume_reset("abc.xyz.com", "testvol") + """ + if force: + cmd = "gluster volume reset %s force --mode=script" % volname + else: + cmd = "gluster volume reset %s --mode=script" % volname + return g.run(mnode, cmd) + + +def volume_status(mnode, volname='all', service='', options=''): + """Executes gluster volume status cli command + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + volname (str): volume name. Defaults to 'all' + service (str): name of the service to get status. + serivce can be, [nfs|shd||quotad]], If not given, + the function returns all the services + options (str): options can be, + [detail|clients|mem|inode|fd|callpool|tasks]. If not given, + the function returns the output of gluster volume status + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + volume_status("abc.xyz.com") + """ + cmd = "gluster vol status %s %s %s" % (volname, service, options) + return g.run(mnode, cmd) + + +def _parse_volume_status_xml(root_xml): + """ + Helper module for get_volume_status. It takes root xml object as input, + parses and returns the 'volume' tag xml object. + """ + + for element in root_xml: + if element.findall("volume"): + return element.findall("volume") + root_vol = _parse_volume_status_xml(element) + if root_vol is not None: + return root_vol + + +def parse_xml(tag_obj): + """ + This helper module takes any xml element object and parses all the child + nodes and returns the parsed data in dictionary format + """ + node_dict = {} + for tag in tag_obj: + if re.search(r'\n\s+', tag.text) is not None: + port_dict = {} + port_dict = parse_xml(tag) + node_dict[tag.tag] = port_dict + else: + node_dict[tag.tag] = tag.text + return node_dict + + +def get_volume_status(mnode, volname='all', service='', options=''): + """This module gets the status of all or specified volume(s)/brick + + Args: + mnode (str): Node on which cmd has to be executed. + + Kwargs: + volname (str): volume name. Defaults to 'all' + service (str): name of the service to get status. + serivce can be, [nfs|shd||quotad]], If not given, + the function returns all the services + options (str): options can be, + [detail|clients|mem|inode|fd|callpool|tasks]. If not given, + the function returns the output of gluster volume status + Returns: + dict: volume status in dict of dictionary format, on success + NoneType: on failure + + Example: + get_volume_status("10.70.47.89", volname="testvol") + >>>{'testvol': {'10.70.47.89': {'/bricks/brick1/a11': {'status': '1', + 'pid': '28963', 'bricktype': 'cold', 'port': '49163', 'peerid': + '7fc9015e-8134-4753-b837-54cbc6030c98', 'ports': {'rdma': 'N/A', + 'tcp': '49163'}}, '/bricks/brick2/a31': {'status': '1', 'pid': + '28982', 'bricktype': 'cold', 'port': '49164', 'peerid': + '7fc9015e-8134-4753-b837-54cbc6030c98', 'ports': {'rdma': 'N/A', + 'tcp': '49164'}}, 'NFS Server': {'status': '1', 'pid': '30525', + 'port': '2049', 'peerid': '7fc9015e-8134-4753-b837-54cbc6030c98', + 'ports': {'rdma': 'N/A', 'tcp': '2049'}}, '/bricks/brick1/a12': + {'status': '1', 'pid': '30505', 'bricktype': 'hot', 'port': '49165', + 'peerid': '7fc9015e-8134-4753-b837-54cbc6030c98', 'ports': {'rdma': + 'N/A', 'tcp': '49165'}}}, '10.70.47.118': {'/bricks/brick1/a21': + {'status': '1', 'pid': '5427', 'bricktype': 'cold', 'port': '49162', + 'peerid': '5397d8f5-2986-453a-b0b5-5c40a9bb87ff', 'ports': {'rdma': + 'N/A', 'tcp': '49162'}}, '/bricks/brick2/a41': {'status': '1', 'pid': + '5446', 'bricktype': 'cold', 'port': '49163', 'peerid': + '5397d8f5-2986-453a-b0b5-5c40a9bb87ff', 'ports': {'rdma': 'N/A', + 'tcp': '49163'}}, 'NFS Server': {'status': '1', 'pid': '6397', 'port': + '2049', 'peerid': '5397d8f5-2986-453a-b0b5-5c40a9bb87ff', 'ports': + {'rdma': 'N/A', 'tcp': '2049'}}}}} + """ + + cmd = "gluster vol status %s %s %s --xml" % (volname, service, options) + + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute gluster volume status command") + return None + + root = etree.XML(out) + volume_list = _parse_volume_status_xml(root) + if volume_list is None: + g.log.error("Failed to parse the XML output of volume status for " + "volume %s" % volname) + return None + + vol_status = {} + for volume in volume_list: + tmp_dict1 = {} + tmp_dict2 = {} + hot_bricks = [] + cold_bricks = [] + vol_name = [vol.text for vol in volume if vol.tag == "volName"] + + # parsing volume status xml output + if options == 'tasks': + tasks = volume.findall("tasks") + for each_task in tasks: + tmp_dict3 = parse_xml(each_task) + node_name = 'task_status' + if 'task' in tmp_dict3.keys(): + if node_name in tmp_dict2.keys(): + tmp_dict2[node_name].append(tmp_dict3['task']) + else: + tmp_dict2[node_name] = [tmp_dict3['task']] + else: + tmp_dict2[node_name] = [tmp_dict3] + else: + elem_tag = [] + for elem in volume.getchildren(): + elem_tag.append(elem.tag) + if ('hotBricks' in elem_tag) or ('coldBricks' in elem_tag): + for elem in volume.getchildren(): + if (elem.tag == 'hotBricks'): + nodes = elem.findall("node") + hot_bricks = [node.find('path').text + for node in nodes + if ( + node.find('path').text.startswith('/'))] + if (elem.tag == 'coldBricks'): + for n in elem.findall("node"): + nodes.append(n) + cold_bricks = [node.find('path').text + for node in nodes + if ( + (node.find('path'). + text.startswith('/')))] + else: + nodes = volume.findall("node") + + for each_node in nodes: + if each_node.find('path').text.startswith('/'): + node_name = each_node.find('hostname').text + elif each_node.find('path').text == 'localhost': + node_name = mnode + else: + node_name = each_node.find('path').text + node_dict = parse_xml(each_node) + tmp_dict3 = {} + if "hostname" in node_dict.keys(): + if node_dict['path'].startswith('/'): + if node_dict['path'] in hot_bricks: + node_dict["bricktype"] = 'hot' + elif node_dict['path'] in cold_bricks: + node_dict["bricktype"] = 'cold' + else: + node_dict["bricktype"] = 'None' + tmp = node_dict["path"] + tmp_dict3[node_dict["path"]] = node_dict + else: + tmp_dict3[node_dict["hostname"]] = node_dict + tmp = node_dict["hostname"] + del tmp_dict3[tmp]["path"] + del tmp_dict3[tmp]["hostname"] + if node_name in tmp_dict1.keys(): + tmp_dict1[node_name].append(tmp_dict3) + else: + tmp_dict1[node_name] = [tmp_dict3] + + tmp_dict4 = {} + for item in tmp_dict1[node_name]: + for key, val in item.items(): + tmp_dict4[key] = val + tmp_dict2[node_name] = tmp_dict4 + + vol_status[vol_name[0]] = tmp_dict2 + g.log.debug("Volume status output: %s" + % pformat(vol_status, indent=10)) + return vol_status + + +def get_volume_options(mnode, volname, option='all'): + """gets the option values for the given volume. + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + + Kwargs: + option (str): volume option to get status. + If not given, the function returns all the options for + the given volume + + Returns: + dict: value for the given volume option in dict format, on success + NoneType: on failure + + Example: + get_volume_options(mnode, "testvol") + """ + + cmd = "gluster volume get %s %s" % (volname, option) + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to execute gluster volume get command" + "for volume %s" % volname) + return None + + volume_option = {} + raw_output = out.split("\n") + for line in raw_output[2:-1]: + match = re.search(r'^(\S+)(.*)', line.strip()) + if match is None: + g.log.error("gluster get volume output is not in " + "expected format") + return None + + volume_option[match.group(1)] = match.group(2).strip() + + return volume_option + + +def set_volume_options(mnode, volname, options): + """Sets the option values for the given volume. + + Args: + mnode (str): Node on which cmd has to be executed. + volname (str): volume name + options (dict): volume options in key + value format + + Returns: + bool: True, if the volume option is set + False, on failure + + Example: + options = {"user.cifs":"enable","user.smb":"enable"} + set_volume_option("abc.com", "testvol", options) + """ + _rc = True + for option in options: + cmd = ("gluster volume set %s %s %s" + % (volname, option, options[option])) + ret, _, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("Unable to set value %s for option %s" + % (options[option], option)) + _rc = False + return _rc + + +def volume_info(mnode, volname='all'): + """Executes gluster volume info cli command + + Args: + mnode (str): Node on which cmd has to be executed. + + Kwargs: + volname (str): volume name. Defaults to 'all' + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + volume_status("abc.com") + """ + + cmd = "gluster volume info %s" % volname + return g.run(mnode, cmd) + + +def get_volume_info(mnode, volname='all'): + """Fetches the volume information as displayed in the volume info. + Uses xml output of volume info and parses the into to a dict + + Args: + mnode (str): Node on which cmd has to be executed. + + Kwargs: + volname (str): volume name. Defaults to 'all' + + Returns: + NoneType: If there are errors + dict: volume info in dict of dicts + + Example: + get_volume_info("abc.com", volname="testvol") + >>>{'testvol': {'status': '1', 'xlators': None, 'disperseCount': '0', + 'bricks': {'coldBricks': {'colddisperseCount': '0', + 'coldarbiterCount': '0', 'coldBrickType': 'Distribute', + 'coldbrickCount': '4', 'numberOfBricks': '4', 'brick': + [{'isArbiter': '0', 'name': '10.70.47.89:/bricks/brick1/a11', + 'hostUuid': '7fc9015e-8134-4753-b837-54cbc6030c98'}, {'isArbiter': + '0', 'name': '10.70.47.118:/bricks/brick1/a21', 'hostUuid': + '7fc9015e-8134-4753-b837-54cbc6030c98'}, {'isArbiter': '0', 'name': + '10.70.47.89:/bricks/brick2/a31', 'hostUuid': + '7fc9015e-8134-4753-b837-54cbc6030c98'}, {'isArbiter': '0', + 'name': '10.70.47.118:/bricks/brick2/a41', 'hostUuid': + '7fc9015e-8134-4753-b837-54cbc6030c98'}], 'coldreplicaCount': '1'}, + 'hotBricks': {'hotBrickType': 'Distribute', 'numberOfBricks': '1', + 'brick': [{'name': '10.70.47.89:/bricks/brick1/a12', 'hostUuid': + '7fc9015e-8134-4753-b837-54cbc6030c98'}], 'hotbrickCount': '1', + 'hotreplicaCount': '1'}}, 'type': '5', 'distCount': '1', + 'replicaCount': '1', 'brickCount': '5', 'options': + {'cluster.tier-mode': 'cache', 'performance.readdir-ahead': 'on', + 'features.ctr-enabled': 'on'}, 'redundancyCount': '0', 'transport': + '0', 'typeStr': 'Tier', 'stripeCount': '1', 'arbiterCount': '0', + 'id': 'ffa8a8d1-546f-4ebf-8e82-fcc96c7e4e05', 'statusStr': 'Started', + 'optCount': '3'}} + """ + + cmd = "gluster volume info %s --xml" % volname + ret, out, _ = g.run(mnode, cmd) + if ret != 0: + g.log.error("volume info returned error") + return None + root = etree.XML(out) + volinfo = {} + for volume in root.findall("volInfo/volumes/volume"): + for elem in volume.getchildren(): + if elem.tag == "name": + volname = elem.text + volinfo[volname] = {} + elif elem.tag == "bricks": + volinfo[volname]["bricks"] = {} + tag_list = [x.tag for x in elem.getchildren() if x] + if 'brick' in tag_list: + volinfo[volname]["bricks"]["brick"] = [] + for el in elem.getchildren(): + if el.tag == 'brick': + brick_info_dict = {} + for elmt in el.getchildren(): + brick_info_dict[elmt.tag] = elmt.text + (volinfo[volname]["bricks"]["brick"]. + append(brick_info_dict)) + + if el.tag == "hotBricks" or el.tag == "coldBricks": + volinfo[volname]["bricks"][el.tag] = {} + volinfo[volname]["bricks"][el.tag]["brick"] = [] + for elmt in el.getchildren(): + if elmt.tag == 'brick': + brick_info_dict = {} + for el_brk in elmt.getchildren(): + brick_info_dict[el_brk.tag] = el_brk.text + (volinfo[volname]["bricks"][el.tag]["brick"]. + append(brick_info_dict)) + else: + volinfo[volname]["bricks"][el.tag][elmt.tag] = elmt.text + elif elem.tag == "options": + volinfo[volname]["options"] = {} + for option in elem.findall("option"): + for el in option.getchildren(): + if el.tag == "name": + opt = el.text + if el.tag == "value": + volinfo[volname]["options"][opt] = el.text + else: + volinfo[volname][elem.tag] = elem.text + + g.log.debug("Volume info output: %s" + % pformat(volinfo, indent=10)) + + return volinfo + + +def volume_sync(mnode, hostname, volname="all"): + """syncs the volume to the specified host + + Args: + mnode (str): Node on which cmd has to be executed. + hostname (str): host name to which volume has to be sync'ed + + Kwargs: + volname (str): volume name. Defaults to 'all'. + + Returns: + tuple: Tuple containing three elements (ret, out, err). + The first element 'ret' is of type 'int' and is the return value + of command execution. + + The second element 'out' is of type 'str' and is the stdout value + of the command execution. + + The third element 'err' is of type 'str' and is the stderr value + of the command execution. + + Example: + volume_sync("abc.xyz.com",volname="testvol") + """ + + cmd = "gluster volume sync %s %s --mode=script" % (hostname, volname) + return g.run(mnode, cmd) -- cgit