From dd2382c613db6f3dd72a25c74d5f765006aea31f Mon Sep 17 00:00:00 2001 From: kshithijiyer Date: Wed, 1 Apr 2020 10:44:54 +0530 Subject: [Libfix] Remove rpyc_get_connection() dependency from code Problem: `g.rpyc_get_connection()` has a limitaion where it can't convert python2 calls to python3 calls. Due to this a large number of testcases fail when executed from a python2 machine on a python3 only setup or visa versa with the below stack trace: ``` E ========= Remote Traceback (1) ========= E Traceback (most recent call last): E File "/root/tmp.tL8Eqx7d8l/rpyc/core/protocol.py", line 323, in _dispatch_request E res = self._HANDLERS[handler](self, *args) E File "/root/tmp.tL8Eqx7d8l/rpyc/core/protocol.py", line 591, in _handle_inspect E if hasattr(self._local_objects[id_pack], '____conn__'): E File "/root/tmp.tL8Eqx7d8l/rpyc/lib/colls.py", line 110, in __getitem__ E return self._dict[key][0] E KeyError: (b'rpyc.core.service.SlaveService', 94282642994712, 140067150858560) ``` Solution: The solution here is to modify the code to not use `g.rpyc_get_connection()`. The following changes are done to accomplish it: 1)Remove code which uses g.rpyc_get_connection() and use generic logic in functions: a. do_bricks_exist_in_shd_volfile() b. get_disk_usage() c. mount_volume() d. list_files() f. append_string_to_file() 2)Create files which can be uploaded and executed on clients/servers to avoid rpc calls in functions: a. calculate_hash() b. validate_files_in_dir() 3)Modify setup.py to push the below files to `/usr/share/glustolibs/scripts/`: a.compute_hash.py b.walk_dir.py Change-Id: I00a81a88382bf3f8b366753eebdb2999260788ca Signed-off-by: kshithijiyer --- .../glustolibs/gluster/dht_test_utils.py | 27 +++++-- .../glustolibs/gluster/glusterfile.py | 49 ++++++++----- glustolibs-gluster/glustolibs/gluster/heal_libs.py | 44 ++++++------ glustolibs-gluster/glustolibs/gluster/lib_utils.py | 84 +++++++--------------- glustolibs-gluster/glustolibs/gluster/mount_ops.py | 23 ++---- glustolibs-gluster/scripts/compute_hash.py | 26 +++++++ glustolibs-gluster/scripts/walk_dir.py | 26 +++++++ glustolibs-gluster/setup.py | 6 +- 8 files changed, 161 insertions(+), 124 deletions(-) create mode 100644 glustolibs-gluster/scripts/compute_hash.py create mode 100644 glustolibs-gluster/scripts/walk_dir.py diff --git a/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py b/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py index 8cef3015b..55dcce5c7 100644 --- a/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py +++ b/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py @@ -22,7 +22,7 @@ import os from glusto.core import Glusto as g from glustolibs.gluster.glusterfile import (GlusterFile, calculate_hash, - get_pathinfo) + get_pathinfo, file_exists) from glustolibs.gluster.glusterdir import GlusterDir from glustolibs.gluster.layout import Layout import glustolibs.gluster.constants as k @@ -30,6 +30,7 @@ import glustolibs.gluster.exceptions as gex from glustolibs.gluster.brickdir import BrickDir from glustolibs.gluster.volume_libs import get_subvols, get_volume_type from glustolibs.gluster.gluster_init import get_gluster_version +from glustolibs.misc.misc_libs import upload_scripts def run_layout_tests(mnode, fqpath, layout, test_type): @@ -120,11 +121,25 @@ def validate_files_in_dir(mnode, rootdir, """ layout_cache = {} - conn = g.rpyc_get_connection(mnode) - if conn is None: - g.log.info("Not able to establish connection to node %s" % mnode) + script_path = ("/usr/share/glustolibs/scripts/walk_dir.py") + if not file_exists(mnode, script_path): + if upload_scripts(mnode, script_path, + "/usr/share/glustolibs/scripts/"): + g.log.info("Successfully uploaded script " + "walk_dir.py!") + else: + g.log.error("Faild to upload walk_dir.py!") + return False + else: + g.log.info("compute_hash.py already present!") + + cmd = ("/usr/bin/env python {0} {1}".format(script_path, rootdir)) + ret, out, _ = g.run(mnode, cmd) + if ret: + g.log.error('Unable to run the script on node {0}' + .format(mnode)) return False - for walkies in conn.modules.os.walk(rootdir): + for walkies in eval(out): g.log.info("TESTING DIRECTORY %s..." % walkies[0]) # check directories @@ -161,8 +176,6 @@ def validate_files_in_dir(mnode, rootdir, if test_type & k.TEST_FILE_EXISTS_ON_HASHED_BRICKS: run_hashed_bricks_test(gfile) - - g.rpyc_close_connection(mnode) return True diff --git a/glustolibs-gluster/glustolibs/gluster/glusterfile.py b/glustolibs-gluster/glustolibs/gluster/glusterfile.py index 413a4f9a7..4d712a5f3 100755 --- a/glustolibs-gluster/glustolibs/gluster/glusterfile.py +++ b/glustolibs-gluster/glustolibs/gluster/glusterfile.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2018 Red Hat, Inc. +# Copyright (C) 2018-2020 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,8 +27,8 @@ import os import re from glusto.core import Glusto as g - from glustolibs.gluster.layout import Layout +from glustolibs.misc.misc_libs import upload_scripts def calculate_hash(host, filename): @@ -39,26 +39,43 @@ def calculate_hash(host, filename): Returns: An integer representation of the hash + + TODO: For testcases specifically testing hashing routine + consider using a baseline external Davies-Meyer hash_value.c + Creating comparison hash from same library we are testing + may not be best practice here. (Holloway) """ - # TODO: For testcases specifically testing hashing routine - # consider using a baseline external Davies-Meyer hash_value.c - # Creating comparison hash from same library we are testing - # may not be best practice here. (Holloway) try: # Check if libglusterfs.so.0 is available locally glusterfs = ctypes.cdll.LoadLibrary("libglusterfs.so.0") g.log.debug("Library libglusterfs.so.0 loaded locally") + computed_hash = ( + ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename, + len(filename)))) + hash_value = int(computed_hash.value) except OSError: - conn = g.rpyc_get_connection(host) - glusterfs = \ - conn.modules.ctypes.cdll.LoadLibrary("libglusterfs.so.0") - g.log.debug("Library libglusterfs.so.0 loaded via rpyc") - - computed_hash = \ - ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename, len(filename))) - # conn.close() - - return int(computed_hash.value) + script_path = ("/usr/share/glustolibs/scripts/" + "compute_hash.py") + if not file_exists(host, script_path): + if upload_scripts(host, script_path, + '/usr/share/glustolibs/scripts/'): + g.log.info("Successfully uploaded script " + "compute_hash.py!") + else: + g.log.error('Unable to upload the script to node {0}' + .format(host)) + return 0 + else: + g.log.info("compute_hash.py already present!") + cmd = ("/usr/bin/env python {0} {1}".format(script_path, + filename)) + ret, out, _ = g.run(host, cmd) + if ret: + g.log.error('Unable to run the script on node {0}' + .format(host)) + return 0 + hash_value = int(out.split('\n')[0]) + return hash_value def get_mountpoint(host, fqpath): diff --git a/glustolibs-gluster/glustolibs/gluster/heal_libs.py b/glustolibs-gluster/glustolibs/gluster/heal_libs.py index 2b0e9ed33..504173ae7 100755 --- a/glustolibs-gluster/glustolibs/gluster/heal_libs.py +++ b/glustolibs-gluster/glustolibs/gluster/heal_libs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2016 Red Hat, Inc. +# Copyright (C) 2016-2020 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -395,30 +395,26 @@ def do_bricks_exist_in_shd_volfile(mnode, volname, brick_list): host = brick = None parse = False - # Establish connection to mnode - conn = g.rpyc_get_connection(mnode) - if conn is None: - g.log.info("Not able to establish connection to node %s" % mnode) - return False - try: - fd = conn.builtins.open(GLUSTERSHD) - for each_line in fd: - each_line = each_line.strip() - if volume_clients in each_line: - parse = True - elif "end-volume" in each_line: - if parse: - brick_list_server_vol.append("%s:%s" % (host, brick)) - parse = False - elif parse: - if "option remote-subvolume" in each_line: - brick = each_line.split(" ")[2] - if "option remote-host" in each_line: - host = each_line.split(" ")[2] - - except IOError as e: - g.log.info("I/O error ({0}): {1}".format(e.errno, e.strerror)) + cmd = "cat {0}".format(GLUSTERSHD) + ret, out, _ = g.run(mnode, cmd) + if ret: + g.log.error("Unable to cat the GLUSTERSHD file.") return False + fd = out.split('\n') + + for each_line in fd: + each_line = each_line.strip() + if volume_clients in each_line: + parse = True + elif "end-volume" in each_line: + if parse: + brick_list_server_vol.append("%s:%s" % (host, brick)) + parse = False + elif parse: + if "option remote-subvolume" in each_line: + brick = each_line.split(" ")[2] + if "option remote-host" in each_line: + host = each_line.split(" ")[2] g.log.info("Brick List from volume info : %s" % brick_list) g.log.info("Brick List from glustershd server volume " diff --git a/glustolibs-gluster/glustolibs/gluster/lib_utils.py b/glustolibs-gluster/glustolibs/gluster/lib_utils.py index 2c0dda3dd..7299874d0 100755 --- a/glustolibs-gluster/glustolibs/gluster/lib_utils.py +++ b/glustolibs-gluster/glustolibs/gluster/lib_utils.py @@ -26,8 +26,6 @@ import re import time from collections import OrderedDict import tempfile -import subprocess -import random ONE_GB_BYTES = 1073741824.0 @@ -53,23 +51,16 @@ def append_string_to_file(mnode, filename, str_to_add_in_file, Returns: True, on success, False otherwise """ - try: - conn = g.rpyc_get_connection(mnode, user=user) - if conn is None: - g.log.error("Unable to get connection to 'root' of node %s" - " in append_string_to_file()" % mnode) - return False - - with conn.builtin.open(filename, 'a') as _filehandle: - _filehandle.write(str_to_add_in_file) - - return True - except IOError: - g.log.error("Exception occurred while adding string to " - "file %s in append_string_to_file()", filename) + cmd = "echo '{0}' >> {1}".format(str_to_add_in_file, + filename) + ret, out, err = g.run(mnode, cmd, user) + if ret or out or err: + g.log.error("Unable to append string '{0}' to file " + "'{1}' on node {2} using user {3}" + .format(str_to_add_in_file, filename, + mnode, user)) return False - finally: - g.rpyc_close_connection(host=mnode, user=user) + return True def search_pattern_in_file(mnode, search_pattern, filename, start_str_to_parse, @@ -268,31 +259,19 @@ def list_files(mnode, dir_path, parse_str="", user="root"): NoneType: None if command execution fails, parse errors. list: files with absolute name """ - - try: - conn = g.rpyc_get_connection(mnode, user=user) - if conn is None: - g.log.error("Unable to get connection to 'root' of node %s" - % mnode) - return None - - filepaths = [] - for root, directories, files in conn.modules.os.walk(dir_path): - for filename in files: - if parse_str != "": - if parse_str in filename: - filepath = conn.modules.os.path.join(root, filename) - filepaths.append(filepath) - else: - filepath = conn.modules.os.path.join(root, filename) - filepaths.append(filepath) - return filepaths - except StopIteration: - g.log.error("Exception occurred in list_files()") + if parse_str == "": + cmd = "find {0} -type f".format(dir_path) + else: + cmd = "find {0} -type f | grep {1}".format(dir_path, + parse_str) + ret, out, err = g.run(mnode, cmd, user) + if ret or err: + g.log.error("Unable to get the list of files on path " + "{0} on node {1} using user {2} due to error {3}" + .format(dir_path, mnode, user, err)) return None - - finally: - g.rpyc_close_connection(host=mnode, user=user) + file_list = out.split('\n') + return file_list[0:len(file_list)-1] def get_servers_bricks_dict(servers, servers_info): @@ -544,22 +523,13 @@ def get_disk_usage(mnode, path, user="root"): Example: get_disk_usage("abc.com", "/mnt/glusterfs") """ - - inst = random.randint(10, 100) - conn = g.rpyc_get_connection(mnode, user=user, instance=inst) - if conn is None: - g.log.error("Failed to get rpyc connection") - return None - cmd = 'stat -f ' + path - p = conn.modules.subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = p.communicate() - ret = p.returncode - if ret != 0: - g.log.error("Failed to execute stat command") + cmd = 'stat -f {0}'.format(path) + ret, out, err = g.run(mnode, cmd, user) + if ret: + g.log.error("Unable to get stat of path {0} on node {1} " + "using user {2} due to error {3}".format(path, mnode, + user, err)) return None - - g.rpyc_close_connection(host=mnode, user=user, instance=inst) res = ''.join(out) match = re.match(r'.*Block size:\s(\d+).*Blocks:\sTotal:\s(\d+)\s+?' r'Free:\s(\d+)\s+?Available:\s(\d+).*Inodes:\s' diff --git a/glustolibs-gluster/glustolibs/gluster/mount_ops.py b/glustolibs-gluster/glustolibs/gluster/mount_ops.py index c41ef8528..02dc0a253 100755 --- a/glustolibs-gluster/glustolibs/gluster/mount_ops.py +++ b/glustolibs-gluster/glustolibs/gluster/mount_ops.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015-2016 Red Hat, Inc. +# Copyright (C) 2015-2020 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -356,23 +356,10 @@ def mount_volume(volname, mtype, mpoint, mserver, mclient, options='', # Check if client is running rhel. If so add specific options cifs_options = "" - try: - conn = g.rpyc_get_connection(mclient, user=user) - if conn is None: - g.log.error("Unable to get connection to %s on node %s" - " in mount_volume()", user, mclient) - return (1, '', '') - - os, version, name = conn.modules.platform.linux_distribution() - if "Santiago" in name: - cifs_options = "sec=ntlmssp" - except Exception as e: - g.log.error("Exception occurred while getting the platform " - "of node %s: %s", mclient, str(e)) - return (1, '', '') - finally: - g.rpyc_close_connection(host=mclient, user=user) - + cmd = "cat /etc/redhat-release | grep Santiago" + ret, _, _ = g.run(mclient, cmd, user=user) + if not ret: + cifs_options = "sec=ntlmssp" mcmd = ("mount -t cifs -o username=%s,password=%s,%s " "\\\\\\\\%s\\\\gluster-%s %s" % (smbuser, smbpasswd, cifs_options, mserver, diff --git a/glustolibs-gluster/scripts/compute_hash.py b/glustolibs-gluster/scripts/compute_hash.py new file mode 100644 index 000000000..b5ae2f83b --- /dev/null +++ b/glustolibs-gluster/scripts/compute_hash.py @@ -0,0 +1,26 @@ +# Copyright (C) 2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import print_function +import ctypes +import sys + +filename = sys.argv[1] +glusterfs = ctypes.cdll.LoadLibrary("libglusterfs.so.0") +computed_hash = ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename, + len(filename))) + +print(computed_hash.value) diff --git a/glustolibs-gluster/scripts/walk_dir.py b/glustolibs-gluster/scripts/walk_dir.py new file mode 100644 index 000000000..02d115b0b --- /dev/null +++ b/glustolibs-gluster/scripts/walk_dir.py @@ -0,0 +1,26 @@ +# Copyright (C) 2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import print_function +import os +import sys + +rootdir = sys.argv[1] + +list_of_levels = [] +for level in os.walk(rootdir): + list_of_levels.append(level) +print(list_of_levels) diff --git a/glustolibs-gluster/setup.py b/glustolibs-gluster/setup.py index b3f9cb09c..05e59fde6 100644 --- a/glustolibs-gluster/setup.py +++ b/glustolibs-gluster/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016-2019 Red Hat, Inc. +# Copyright (c) 2016-2020 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -49,6 +49,8 @@ setup( ) try: - dir_util.copy_tree('./gdeploy_configs', '/usr/share/glustolibs/gdeploy_configs') + for srcdir, destdir in (('./gdeploy_configs', '/usr/share/glustolibs/gdeploy_configs'), + ('./scripts', '/usr/share/glustolibs/scripts/')): + dir_util.copy_tree(srcdir, destdir) except: pass -- cgit