From 9fd735480ad1d714fb73d64533159077ed9542f9 Mon Sep 17 00:00:00 2001 From: Arthy Loganathan Date: Fri, 21 Oct 2016 11:21:13 +0530 Subject: Added lib_utils.py Change-Id: I4cd5061d9df470aed1a620eb3f67cab236645215 Signed-off-by: Arthy Loganathan --- glustolibs-gluster/glustolibs/gluster/lib_utils.py | 662 +++++++++++++++++++++ 1 file changed, 662 insertions(+) create mode 100644 glustolibs-gluster/glustolibs/gluster/lib_utils.py (limited to 'glustolibs-gluster/glustolibs/gluster/lib_utils.py') diff --git a/glustolibs-gluster/glustolibs/gluster/lib_utils.py b/glustolibs-gluster/glustolibs/gluster/lib_utils.py new file mode 100644 index 000000000..19587894f --- /dev/null +++ b/glustolibs-gluster/glustolibs/gluster/lib_utils.py @@ -0,0 +1,662 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Description: Helper library for gluster modules. +""" + +from glusto.core import Glusto as g +from glustolibs.gluster.volume_ops import get_volume_info +from glustolibs.gluster.mount_ops import mount_volume, umount_volume +import re +import time +from collections import OrderedDict +import tempfile +import subprocess +import random + +ONE_GB_BYTES = 1073741824.0 + +try: + import xml.etree.cElementTree as etree +except ImportError: + import xml.etree.ElementTree as etree + + +def append_string_to_file(mnode, filename, str_to_add_in_file, + user="root"): + """Appends the given string in the file. + + Example: + append_string_to_file("abc.def.com", "/var/log/messages", + "test_1_string") + + Args: + mnode (str): Node on which cmd has to be executed. + filename (str): absolute file path to append the string + str_to_add_in_file (str): string to be added in the file, + which is used as a start and stop string for parsing + the file in search_pattern_in_file(). + + Kwargs: + user (str): username. Defaults to 'root' user. + + Returns: + True, on success, False otherwise + """ + try: + conn = g.rpyc_get_connection(mnode, user=user) + if conn is None: + g.log.error("Unable to get connection to 'root' of node %s" + " in append_string_to_file()" % mnode) + return False + + with conn.builtin.open(filename, 'a') as _filehandle: + _filehandle.write(str_to_add_in_file) + + return True + except: + g.log.error("Exception occured while adding string to " + "file %s in append_string_to_file()" % filename) + return False + finally: + g.rpyc_close_connection(host=mnode, user=user) + + +def search_pattern_in_file(mnode, search_pattern, filename, start_str_to_parse, + end_str_to_parse): + """checks if the given search pattern exists in the file + in between 'start_str_to_parse' and 'end_str_to_parse' string. + + Example: + search_pattern = r'.*scrub.*' + search_log("abc.def.com", search_pattern, "/var/log/messages", + "start_pattern", "end_pattern") + + Args: + mnode (str): Node on which cmd has to be executed. + search_pattern (str): regex string to be matched in the file + filename (str): absolute file path to search given string + start_str_to_parse (str): this will be as start string in the + file from which this method will check + if the given search string is present. + end_str_to_parse (str): this will be as end string in the + file whithin which this method will check + if the given search string is present. + + Returns: + True, if search_pattern is present in the file + False, otherwise + """ + + cmd = ("awk '{a[NR]=$0}/" + start_str_to_parse + "/{s=NR}/" + + end_str_to_parse + "/{e=NR}END{for(i=s;i<=e;++i)print " + "a[i]}' " + filename) + + ret, out, err = g.run(mnode, cmd) + if ret != 0: + g.log.error("Failed to match start and end pattern in file" + % filename) + return False + + if not re.search(search_pattern, str(out), re.S): + g.log.error("file %s did not have the expected message" + % filename) + return False + + return True + + +def calculate_checksum(mnode, file_list, chksum_type='sha256sum'): + """This module calculates given checksum for the given file list + + Example: + calculate_checksum("abc.com", [file1, file2]) + + Args: + mnode (str): Node on which cmd has to be executed. + file_list (list): absolute file names for which checksum + to be calculated + + Kwargs: + chksum_type (str): type of the checksum algorithm. + Defaults to sha256sum + + Returns: + NoneType: None if command execution fails, parse errors. + dict: checksum value for each file in the given file list + """ + + cmd = chksum_type + " %s" % ' '.join(file_list) + ret = g.run(mnode, cmd) + if ret[0] != 0: + g.log.error("Failed to execute checksum command in server %s" + % mnode) + return None + + checksum_dict = {} + for line in ret[1].split('\n')[:-1]: + match = re.search(r'^(\S+)\s+(\S+)', line.strip()) + if match is None: + g.log.error("checksum output is not in expected format") + return None + + checksum_dict[match.group(2)] = match.group(1) + + return checksum_dict + + +def get_extended_attributes_info(mnode, file_list, encoding='hex', + attr_name=''): + """This module gets extended attribute info for the given file list + + Example: + get_extended_attributes_info("abc.com", [file1, file2]) + + Args: + mnode (str): Node on which cmd has to be executed. + file_list (list): absolute file names for which extended + attributes to be fetched + + Kwargs: + encoding (str): encoding format + attr_name (str): extended attribute name + + Returns: + NoneType: None if command execution fails, parse errors. + dict: extended attribute for each file in the given file list + """ + + if attr_name == '': + cmd = "getfattr -d -m . -e %s %s" % (encoding, ' '.join(file_list)) + else: + cmd = ("getfattr -d -m . -e %s -n %s %s" + % (encoding, attr_name, ' '.join(file_list))) + + ret = g.run(mnode, cmd) + if ret[0] != 0: + g.log.error("Failed to execute getfattr command in server %s" + % mnode) + return None + + attr_dict = {} + for each_attr in ret[1].split('\n\n')[:-1]: + for line in each_attr.split('\n'): + if line.startswith('#'): + match = re.search(r'.*file:\s(\S+).*', line) + if match is None: + g.log.error("getfattr output is not in expected format") + return None + key = "/" + match.group(1) + attr_dict[key] = {} + else: + output = line.split('=') + attr_dict[key][output[0]] = output[1] + return attr_dict + + +def get_pathinfo(mnode, filename, volname): + """This module gets filepath of the given file in gluster server. + + Example: + get_pathinfo(mnode, "file1", "testvol") + + Args: + mnode (str): Node on which cmd has to be executed. + filename (str): relative path of file + volname (str): volume name + + Returns: + NoneType: None if command execution fails, parse errors. + list: file path for the given file in gluster server + """ + + mount_point = tempfile.mkdtemp() + + # Performing glusterfs mount because only with glusterfs mount + # the file location in gluster server can be identified + ret, _, _ = mount_volume(volname, mtype='glusterfs', + mpoint=mount_point, + mserver=mnode, + mclient=mnode) + if ret != 0: + g.log.error("Failed to do gluster mount on volume %s to fetch" + "pathinfo from server %s" + % (volname, mnode)) + return None + + filename = mount_point + '/' + filename + attr_name = 'trusted.glusterfs.pathinfo' + output = get_extended_attributes_info(mnode, [filename], + attr_name=attr_name) + if output is None: + g.log.error("Failed to get path info for %s" % filename) + return None + + pathinfo = output[filename][attr_name] + + umount_volume(mnode, mount_point) + g.run(mnode, "rm -rf " + mount_point) + + return re.findall(".*?POSIX.*?:(\S+)\>", pathinfo) + + +def list_files(mnode, dir_path, parse_str="", user="root"): + """This module list files from the given file path + + Example: + list_files("/root/dir1/") + + Args: + mnode (str): Node on which cmd has to be executed. + dir_path (str): directory path name + + Kwargs: + parse_str (str): sub string of the filename to be fetched + user (str): username. Defaults to 'root' user. + + Returns: + NoneType: None if command execution fails, parse errors. + list: files with absolute name + """ + + try: + conn = g.rpyc_get_connection(mnode, user=user) + if conn is None: + g.log.error("Unable to get connection to 'root' of node %s" + % mnode) + return None + + filepaths = [] + for root, directories, files in conn.modules.os.walk(dir_path): + for filename in files: + if parse_str != "": + if parse_str in filename: + filepath = conn.modules.os.path.join(root, filename) + filepaths.append(filepath) + else: + filepath = conn.modules.os.path.join(root, filename) + filepaths.append(filepath) + return filepaths + except: + g.log.error("Exception occured in list_files()") + return None + + finally: + g.rpyc_close_connection(host=mnode, user=user) + + +def get_servers_bricks_dict(servers, servers_info): + """This module returns servers_bricks dictionary. + Args: + servers (list): List of servers for which we need the + list of bricks available on it. + servers_info (dict): dict of server info of each servers + Returns: + OrderedDict: key - server + value - list of bricks + Example: + get_servers_bricks_dict(g.config['servers'], g.config['servers_info']) + """ + servers_bricks_dict = OrderedDict() + if not isinstance(servers, list): + servers = [servers] + for server in servers: + server_info = servers_info[server] + brick_root = server_info["brick_root"] + ret, out, err = g.run(server, "cat /proc/mounts | grep %s" + " | awk '{ print $2}'" % brick_root) + if ret != 0: + g.log.error("bricks not available on %s" % server) + else: + servers_bricks_dict[server] = out.strip().split("\n") + + for key, value in servers_bricks_dict.items(): + value.sort() + + return servers_bricks_dict + + +def get_servers_used_bricks_dict(mnode, servers): + """This module returns servers_used_bricks dictionary. + This information is fetched from gluster volume info command. + Args: + servers (list): List of servers for which we need the + list of unused bricks on it. + mnode (str): The node on which gluster volume info command has + to be executed. + Returns: + OrderedDict: key - server + value - list of used bricks + or empty list(if all bricks are free) + Example: + get_servers_used_bricks_dict(g.config['servers'][0]['host'], + g.config['servers']) + """ + if not isinstance(servers, list): + servers = [servers] + + servers_used_bricks_dict = OrderedDict() + for server in servers: + servers_used_bricks_dict[server] = [] + + ret, out, err = g.run(mnode, "gluster volume info | egrep " + "\"^Brick[0-9]+\" | grep -v \"ss_brick\"") + if ret != 0: + g.log.error("error in getting bricklist using gluster v info") + else: + list1 = list2 = [] + list1 = out.strip().split('\n') + for item in list1: + x = re.search(':(.*)/(.*)', item) + list2 = x.group(1).strip().split(':') + if servers_used_bricks_dict.has_key(list2[0]): + value = servers_used_bricks_dict[list2[0]] + value.append(list2[1]) + else: + servers_used_bricks_dict[list2[0]] = [list2[1]] + + for key, value in servers_used_bricks_dict.items(): + value.sort() + + return servers_used_bricks_dict + + +def get_servers_unused_bricks_dict(mnode, servers, servers_info): + """This module returns servers_unused_bricks dictionary. + Gets a list of unused bricks for each server by using functions, + get_servers_bricks_dict() and get_servers_used_bricks_dict() + Args: + mnode (str): The node on which gluster volume info command has + to be executed. + servers (list): List of servers for which we need the + list of unused bricks available on it. + servers_info (dict): dict of server info of each servers + Returns: + OrderedDict: key - server + value - list of unused bricks + Example: + get_servers_unused_bricks_dict(g.config['servers'][0]['host'], + g.config['servers'], + g.config['servers_info']) + """ + if not isinstance(servers, list): + servers = [servers] + dict1 = get_servers_bricks_dict(servers, servers_info) + dict2 = get_servers_used_bricks_dict(mnode, servers) + servers_unused_bricks_dict = OrderedDict() + for key, value in dict1.items(): + if dict2.has_key(key): + unused_bricks = list(set(value) - set(dict2[key])) + servers_unused_bricks_dict[key] = unused_bricks + else: + servers_unused_bricks_dict[key] = value + + for key, value in servers_unused_bricks_dict.items(): + value.sort() + + return servers_unused_bricks_dict + + +def form_bricks_list(mnode, volname, number_of_bricks, servers, servers_info): + """Forms bricks list for create-volume/add-brick given the num_of_bricks + servers and servers_info. + + Args: + mnode (str): The node on which the command has to be run. + volname (str): Volume name for which we require brick-list. + number_of_bricks (int): The number of bricks for which brick list + has to be created. + servers (list): The list of servers from which the bricks + needs to be selected for creating the brick list. + servers_info (dict): dict of server info of each servers. + + Returns: + list - List of bricks to use with volume-create/add-brick + None - if number_of_bricks is greater than unused bricks. + + Example: + form_bricks_path(g.config['servers'](0), "testvol", 6, + g.config['servers'], g.config['servers_info']) + """ + if not isinstance(servers, list): + servers = [servers] + dict_index = 0 + bricks_list = [] + + servers_unused_bricks_dict = get_servers_unused_bricks_dict(mnode, servers, + servers_info) + num_of_unused_bricks = 0 + for each_server_unused_bricks_list in servers_unused_bricks_dict.values(): + num_of_unused_bricks = (num_of_unused_bricks + + len(each_server_unused_bricks_list)) + + if num_of_unused_bricks < number_of_bricks: + g.log.error("Not enough bricks available for creating the bricks") + return None + + brick_index = 0 + vol_info_dict = get_volume_info(mnode, volname) + if vol_info_dict: + brick_index = int(vol_info_dict[volname]['brickCount']) + + for num in range(brick_index, brick_index + number_of_bricks): + # current_server is the server from which brick path will be created + current_server = servers_unused_bricks_dict.keys()[dict_index] + current_server_unused_bricks_list = (servers_unused_bricks_dict.values() + [dict_index]) + brick_path = '' + if current_server_unused_bricks_list: + brick_path = ("%s:%s/%s_brick%s" % + (current_server, + current_server_unused_bricks_list[0], volname, num)) + bricks_list.append(brick_path) + + # Remove the added brick from the current_server_unused_bricks_list + servers_unused_bricks_dict.values()[dict_index].pop(0) + + if dict_index < len(servers_unused_bricks_dict) - 1: + dict_index = dict_index + 1 + else: + dict_index = 0 + + return bricks_list + +def is_rhel6(servers): + """Function to get whether the server is RHEL-6 + + Args: + servers(list): List of server hosts to know the RHEL Version + + Returns: + bool:Returns True, if its RHEL-6 else returns false + """ + + results = g.run_parallel(servers, "cat /etc/redhat-release") + rc = True + for server, ret_values in results.iteritems(): + retcode, out, err = ret_values + if retcode != 0: + g.log.error("Unable to get the RHEL version on server %s" %(server,)) + rc = False + if retcode == 0 and not "release 6" in out: + g.log.error("Server %s is not RHEL-6" %(server,)) + rc = False + return rc + + +def is_rhel7(servers): + """Function to get whether the server is RHEL-7 + + Args: + servers(list): List of server hosts to know the RHEL Version + + Returns: + bool:Returns True, if its RHEL-7 else returns false + """ + + results = g.run_parallel(servers, "cat /etc/redhat-release") + rc = True + for server, ret_values in results.iteritems(): + retcode, out, err = ret_values + if retcode != 0: + g.log.error("Unable to get the RHEL version on server %s" %(server,)) + rc = False + if retcode == 0 and not "release 7" in out: + g.log.error("Server %s is not RHEL-7" %(server,)) + rc = False + return rc + + +def get_disk_usage(mnode, path, user="root"): + """ + This module gets disk usage of the given path + + Args: + path (str): path for which disk usage to be calculated + conn (obj): connection object of the remote node + + Kwargs: + user (str): username + + Returns: + dict: disk usage in dict format on success + None Type, on failure + + Example: + get_disk_usage("abc.com", "/mnt/glusterfs") + """ + + inst = random.randint(10,100) + conn = g.rpyc_get_connection(mnode, user=user, instance=inst) + if conn is None: + g.log.error("Failed to get rpyc connection") + return None + cmd = 'stat -f '+ path + p = conn.modules.subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + ret = p.returncode + if ret != 0: + g.log.error("Failed to execute stat command") + return None + + g.rpyc_close_connection(host=mnode, user=user, instance=inst) + res = ''.join(out) + match = re.match(r'.*Block size:\s(\d+).*Blocks:\sTotal:\s(\d+)\s+?' + r'Free:\s(\d+)\s+?Available:\s(\d+).*Inodes:\s' + r'Total:\s(\d+)\s+?Free:\s(\d+)', res, re.S) + if match is None: + g.log.error("Regex mismatch in get_disk_usage()") + return None + + usage_info = dict() + keys = ['b_size', 'b_total', 'b_free', 'b_avail', 'i_total', 'i_free'] + val = list(match.groups()) + info = dict(zip(keys, val)) + usage_info['total'] = ((int(info['b_total']) * int(info['b_size'])) + / ONE_GB_BYTES) + usage_info['free'] = ((int(info['b_free']) * int(info['b_size'])) + / ONE_GB_BYTES) + usage_info['used_percent'] = (100 - (100.0 * usage_info['free'] + / usage_info['total'])) + usage_info['total_inode'] = int(info['i_total']) + usage_info['free_inode'] = int(info['i_free']) + usage_info['used_percent_inode'] = (100 - (100.0 * usage_info['free_inode'] + / usage_info['total_inode'])) + usage_info['used'] = usage_info['total'] - usage_info['free'] + usage_info['used_inode'] = (usage_info['total_inode'] - + usage_info['free_inode']) + return usage_info + + +def get_disk_used_percent(mnode, dirname): + """ + Module to get disk used percent + + Args: + mnode (str): node on which cmd has to be executed + dirname (str): absolute path of directory + + Returns: + str: used percent for given directory + None Type, on failure + + Example: + get_disk_used_percent("abc.com", "/mnt/glusterfs") + + """ + + output = get_disk_usage(mnode, dirname) + if output is None: + g.log.error("Failed to get disk used percent for %s" + % dirname) + return None + return output['used_percent'] + + +def check_if_dir_is_filled(mnode, dirname, percent_to_fill, + timeout=3600): + """ + Module to check if the directory is filled with given percentage. + + Args: + mnode (str): node to check if directory is filled + dirname (str): absolute path of directory + percent_to_fill (int): percentage to fill the volume + + Kwargs: + timeout (int): overall timeout value for wait till the dir fills + with given percentage + + Returns: + bool: True, if volume is filled with given percent, False otherwise + + Example: + check_if_dir_is_filled("abc.com", "/mnt/glusterfs", 10) + """ + flag = 0 + count = 0 + while (count < timeout): + output = get_disk_usage(mnode, dirname) + used = output['used_percent'] + + if int(percent_to_fill) > int(used): + remaining_to_fill = int(percent_to_fill) - int(used) + + g.log.info("Directory %s used percent: %s" + %(dirname, used)) + if int(percent_to_fill) <= int(used): + flag = 1 + g.rpyc_close_connection(host=mnode) + break + time.sleep(5) + count = count + 5 + else: + g.log.info("Diectory %s is filled with given percent already" + % dirname) + g.rpyc_close_connection(host=mnode) + flag = 1 + break + + if flag: + g.log.info("Directory is filled with given percentage") + return True + else: + g.log.info("Timeout reached before filling directory with given percentage") + return True + return False -- cgit