summaryrefslogtreecommitdiffstats
path: root/glustolibs-gluster
diff options
context:
space:
mode:
authorkshithijiyer <kshithij.ki@gmail.com>2020-04-01 10:44:54 +0530
committerkshithijiyer <kshithij.ki@gmail.com>2020-04-01 12:17:18 +0530
commitdd2382c613db6f3dd72a25c74d5f765006aea31f (patch)
treefcbdca0fc93539b7eee159bfa3c381167cd0eb1c /glustolibs-gluster
parentd4349ba649c967ed6c16c1efdd9caa56433142f1 (diff)
[Libfix] Remove rpyc_get_connection() dependency from code
Problem: `g.rpyc_get_connection()` has a limitaion where it can't convert python2 calls to python3 calls. Due to this a large number of testcases fail when executed from a python2 machine on a python3 only setup or visa versa with the below stack trace: ``` E ========= Remote Traceback (1) ========= E Traceback (most recent call last): E File "/root/tmp.tL8Eqx7d8l/rpyc/core/protocol.py", line 323, in _dispatch_request E res = self._HANDLERS[handler](self, *args) E File "/root/tmp.tL8Eqx7d8l/rpyc/core/protocol.py", line 591, in _handle_inspect E if hasattr(self._local_objects[id_pack], '____conn__'): E File "/root/tmp.tL8Eqx7d8l/rpyc/lib/colls.py", line 110, in __getitem__ E return self._dict[key][0] E KeyError: (b'rpyc.core.service.SlaveService', 94282642994712, 140067150858560) ``` Solution: The solution here is to modify the code to not use `g.rpyc_get_connection()`. The following changes are done to accomplish it: 1)Remove code which uses g.rpyc_get_connection() and use generic logic in functions: a. do_bricks_exist_in_shd_volfile() b. get_disk_usage() c. mount_volume() d. list_files() f. append_string_to_file() 2)Create files which can be uploaded and executed on clients/servers to avoid rpc calls in functions: a. calculate_hash() b. validate_files_in_dir() 3)Modify setup.py to push the below files to `/usr/share/glustolibs/scripts/`: a.compute_hash.py b.walk_dir.py Change-Id: I00a81a88382bf3f8b366753eebdb2999260788ca Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>
Diffstat (limited to 'glustolibs-gluster')
-rw-r--r--glustolibs-gluster/glustolibs/gluster/dht_test_utils.py27
-rwxr-xr-xglustolibs-gluster/glustolibs/gluster/glusterfile.py49
-rwxr-xr-xglustolibs-gluster/glustolibs/gluster/heal_libs.py44
-rwxr-xr-xglustolibs-gluster/glustolibs/gluster/lib_utils.py84
-rwxr-xr-xglustolibs-gluster/glustolibs/gluster/mount_ops.py23
-rw-r--r--glustolibs-gluster/scripts/compute_hash.py26
-rw-r--r--glustolibs-gluster/scripts/walk_dir.py26
-rw-r--r--glustolibs-gluster/setup.py6
8 files changed, 161 insertions, 124 deletions
diff --git a/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py b/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py
index 8cef3015b..55dcce5c7 100644
--- a/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py
+++ b/glustolibs-gluster/glustolibs/gluster/dht_test_utils.py
@@ -22,7 +22,7 @@ import os
from glusto.core import Glusto as g
from glustolibs.gluster.glusterfile import (GlusterFile, calculate_hash,
- get_pathinfo)
+ get_pathinfo, file_exists)
from glustolibs.gluster.glusterdir import GlusterDir
from glustolibs.gluster.layout import Layout
import glustolibs.gluster.constants as k
@@ -30,6 +30,7 @@ import glustolibs.gluster.exceptions as gex
from glustolibs.gluster.brickdir import BrickDir
from glustolibs.gluster.volume_libs import get_subvols, get_volume_type
from glustolibs.gluster.gluster_init import get_gluster_version
+from glustolibs.misc.misc_libs import upload_scripts
def run_layout_tests(mnode, fqpath, layout, test_type):
@@ -120,11 +121,25 @@ def validate_files_in_dir(mnode, rootdir,
"""
layout_cache = {}
- conn = g.rpyc_get_connection(mnode)
- if conn is None:
- g.log.info("Not able to establish connection to node %s" % mnode)
+ script_path = ("/usr/share/glustolibs/scripts/walk_dir.py")
+ if not file_exists(mnode, script_path):
+ if upload_scripts(mnode, script_path,
+ "/usr/share/glustolibs/scripts/"):
+ g.log.info("Successfully uploaded script "
+ "walk_dir.py!")
+ else:
+ g.log.error("Faild to upload walk_dir.py!")
+ return False
+ else:
+ g.log.info("compute_hash.py already present!")
+
+ cmd = ("/usr/bin/env python {0} {1}".format(script_path, rootdir))
+ ret, out, _ = g.run(mnode, cmd)
+ if ret:
+ g.log.error('Unable to run the script on node {0}'
+ .format(mnode))
return False
- for walkies in conn.modules.os.walk(rootdir):
+ for walkies in eval(out):
g.log.info("TESTING DIRECTORY %s..." % walkies[0])
# check directories
@@ -161,8 +176,6 @@ def validate_files_in_dir(mnode, rootdir,
if test_type & k.TEST_FILE_EXISTS_ON_HASHED_BRICKS:
run_hashed_bricks_test(gfile)
-
- g.rpyc_close_connection(mnode)
return True
diff --git a/glustolibs-gluster/glustolibs/gluster/glusterfile.py b/glustolibs-gluster/glustolibs/gluster/glusterfile.py
index 413a4f9a7..4d712a5f3 100755
--- a/glustolibs-gluster/glustolibs/gluster/glusterfile.py
+++ b/glustolibs-gluster/glustolibs/gluster/glusterfile.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright (C) 2018 Red Hat, Inc. <http://www.redhat.com>
+# Copyright (C) 2018-2020 Red Hat, Inc. <http://www.redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -27,8 +27,8 @@ import os
import re
from glusto.core import Glusto as g
-
from glustolibs.gluster.layout import Layout
+from glustolibs.misc.misc_libs import upload_scripts
def calculate_hash(host, filename):
@@ -39,26 +39,43 @@ def calculate_hash(host, filename):
Returns:
An integer representation of the hash
+
+ TODO: For testcases specifically testing hashing routine
+ consider using a baseline external Davies-Meyer hash_value.c
+ Creating comparison hash from same library we are testing
+ may not be best practice here. (Holloway)
"""
- # TODO: For testcases specifically testing hashing routine
- # consider using a baseline external Davies-Meyer hash_value.c
- # Creating comparison hash from same library we are testing
- # may not be best practice here. (Holloway)
try:
# Check if libglusterfs.so.0 is available locally
glusterfs = ctypes.cdll.LoadLibrary("libglusterfs.so.0")
g.log.debug("Library libglusterfs.so.0 loaded locally")
+ computed_hash = (
+ ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename,
+ len(filename))))
+ hash_value = int(computed_hash.value)
except OSError:
- conn = g.rpyc_get_connection(host)
- glusterfs = \
- conn.modules.ctypes.cdll.LoadLibrary("libglusterfs.so.0")
- g.log.debug("Library libglusterfs.so.0 loaded via rpyc")
-
- computed_hash = \
- ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename, len(filename)))
- # conn.close()
-
- return int(computed_hash.value)
+ script_path = ("/usr/share/glustolibs/scripts/"
+ "compute_hash.py")
+ if not file_exists(host, script_path):
+ if upload_scripts(host, script_path,
+ '/usr/share/glustolibs/scripts/'):
+ g.log.info("Successfully uploaded script "
+ "compute_hash.py!")
+ else:
+ g.log.error('Unable to upload the script to node {0}'
+ .format(host))
+ return 0
+ else:
+ g.log.info("compute_hash.py already present!")
+ cmd = ("/usr/bin/env python {0} {1}".format(script_path,
+ filename))
+ ret, out, _ = g.run(host, cmd)
+ if ret:
+ g.log.error('Unable to run the script on node {0}'
+ .format(host))
+ return 0
+ hash_value = int(out.split('\n')[0])
+ return hash_value
def get_mountpoint(host, fqpath):
diff --git a/glustolibs-gluster/glustolibs/gluster/heal_libs.py b/glustolibs-gluster/glustolibs/gluster/heal_libs.py
index 2b0e9ed33..504173ae7 100755
--- a/glustolibs-gluster/glustolibs/gluster/heal_libs.py
+++ b/glustolibs-gluster/glustolibs/gluster/heal_libs.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright (C) 2016 Red Hat, Inc. <http://www.redhat.com>
+# Copyright (C) 2016-2020 Red Hat, Inc. <http://www.redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -395,30 +395,26 @@ def do_bricks_exist_in_shd_volfile(mnode, volname, brick_list):
host = brick = None
parse = False
- # Establish connection to mnode
- conn = g.rpyc_get_connection(mnode)
- if conn is None:
- g.log.info("Not able to establish connection to node %s" % mnode)
- return False
- try:
- fd = conn.builtins.open(GLUSTERSHD)
- for each_line in fd:
- each_line = each_line.strip()
- if volume_clients in each_line:
- parse = True
- elif "end-volume" in each_line:
- if parse:
- brick_list_server_vol.append("%s:%s" % (host, brick))
- parse = False
- elif parse:
- if "option remote-subvolume" in each_line:
- brick = each_line.split(" ")[2]
- if "option remote-host" in each_line:
- host = each_line.split(" ")[2]
-
- except IOError as e:
- g.log.info("I/O error ({0}): {1}".format(e.errno, e.strerror))
+ cmd = "cat {0}".format(GLUSTERSHD)
+ ret, out, _ = g.run(mnode, cmd)
+ if ret:
+ g.log.error("Unable to cat the GLUSTERSHD file.")
return False
+ fd = out.split('\n')
+
+ for each_line in fd:
+ each_line = each_line.strip()
+ if volume_clients in each_line:
+ parse = True
+ elif "end-volume" in each_line:
+ if parse:
+ brick_list_server_vol.append("%s:%s" % (host, brick))
+ parse = False
+ elif parse:
+ if "option remote-subvolume" in each_line:
+ brick = each_line.split(" ")[2]
+ if "option remote-host" in each_line:
+ host = each_line.split(" ")[2]
g.log.info("Brick List from volume info : %s" % brick_list)
g.log.info("Brick List from glustershd server volume "
diff --git a/glustolibs-gluster/glustolibs/gluster/lib_utils.py b/glustolibs-gluster/glustolibs/gluster/lib_utils.py
index 2c0dda3dd..7299874d0 100755
--- a/glustolibs-gluster/glustolibs/gluster/lib_utils.py
+++ b/glustolibs-gluster/glustolibs/gluster/lib_utils.py
@@ -26,8 +26,6 @@ import re
import time
from collections import OrderedDict
import tempfile
-import subprocess
-import random
ONE_GB_BYTES = 1073741824.0
@@ -53,23 +51,16 @@ def append_string_to_file(mnode, filename, str_to_add_in_file,
Returns:
True, on success, False otherwise
"""
- try:
- conn = g.rpyc_get_connection(mnode, user=user)
- if conn is None:
- g.log.error("Unable to get connection to 'root' of node %s"
- " in append_string_to_file()" % mnode)
- return False
-
- with conn.builtin.open(filename, 'a') as _filehandle:
- _filehandle.write(str_to_add_in_file)
-
- return True
- except IOError:
- g.log.error("Exception occurred while adding string to "
- "file %s in append_string_to_file()", filename)
+ cmd = "echo '{0}' >> {1}".format(str_to_add_in_file,
+ filename)
+ ret, out, err = g.run(mnode, cmd, user)
+ if ret or out or err:
+ g.log.error("Unable to append string '{0}' to file "
+ "'{1}' on node {2} using user {3}"
+ .format(str_to_add_in_file, filename,
+ mnode, user))
return False
- finally:
- g.rpyc_close_connection(host=mnode, user=user)
+ return True
def search_pattern_in_file(mnode, search_pattern, filename, start_str_to_parse,
@@ -268,31 +259,19 @@ def list_files(mnode, dir_path, parse_str="", user="root"):
NoneType: None if command execution fails, parse errors.
list: files with absolute name
"""
-
- try:
- conn = g.rpyc_get_connection(mnode, user=user)
- if conn is None:
- g.log.error("Unable to get connection to 'root' of node %s"
- % mnode)
- return None
-
- filepaths = []
- for root, directories, files in conn.modules.os.walk(dir_path):
- for filename in files:
- if parse_str != "":
- if parse_str in filename:
- filepath = conn.modules.os.path.join(root, filename)
- filepaths.append(filepath)
- else:
- filepath = conn.modules.os.path.join(root, filename)
- filepaths.append(filepath)
- return filepaths
- except StopIteration:
- g.log.error("Exception occurred in list_files()")
+ if parse_str == "":
+ cmd = "find {0} -type f".format(dir_path)
+ else:
+ cmd = "find {0} -type f | grep {1}".format(dir_path,
+ parse_str)
+ ret, out, err = g.run(mnode, cmd, user)
+ if ret or err:
+ g.log.error("Unable to get the list of files on path "
+ "{0} on node {1} using user {2} due to error {3}"
+ .format(dir_path, mnode, user, err))
return None
-
- finally:
- g.rpyc_close_connection(host=mnode, user=user)
+ file_list = out.split('\n')
+ return file_list[0:len(file_list)-1]
def get_servers_bricks_dict(servers, servers_info):
@@ -544,22 +523,13 @@ def get_disk_usage(mnode, path, user="root"):
Example:
get_disk_usage("abc.com", "/mnt/glusterfs")
"""
-
- inst = random.randint(10, 100)
- conn = g.rpyc_get_connection(mnode, user=user, instance=inst)
- if conn is None:
- g.log.error("Failed to get rpyc connection")
- return None
- cmd = 'stat -f ' + path
- p = conn.modules.subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, err = p.communicate()
- ret = p.returncode
- if ret != 0:
- g.log.error("Failed to execute stat command")
+ cmd = 'stat -f {0}'.format(path)
+ ret, out, err = g.run(mnode, cmd, user)
+ if ret:
+ g.log.error("Unable to get stat of path {0} on node {1} "
+ "using user {2} due to error {3}".format(path, mnode,
+ user, err))
return None
-
- g.rpyc_close_connection(host=mnode, user=user, instance=inst)
res = ''.join(out)
match = re.match(r'.*Block size:\s(\d+).*Blocks:\sTotal:\s(\d+)\s+?'
r'Free:\s(\d+)\s+?Available:\s(\d+).*Inodes:\s'
diff --git a/glustolibs-gluster/glustolibs/gluster/mount_ops.py b/glustolibs-gluster/glustolibs/gluster/mount_ops.py
index c41ef8528..02dc0a253 100755
--- a/glustolibs-gluster/glustolibs/gluster/mount_ops.py
+++ b/glustolibs-gluster/glustolibs/gluster/mount_ops.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright (C) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+# Copyright (C) 2015-2020 Red Hat, Inc. <http://www.redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -356,23 +356,10 @@ def mount_volume(volname, mtype, mpoint, mserver, mclient, options='',
# Check if client is running rhel. If so add specific options
cifs_options = ""
- try:
- conn = g.rpyc_get_connection(mclient, user=user)
- if conn is None:
- g.log.error("Unable to get connection to %s on node %s"
- " in mount_volume()", user, mclient)
- return (1, '', '')
-
- os, version, name = conn.modules.platform.linux_distribution()
- if "Santiago" in name:
- cifs_options = "sec=ntlmssp"
- except Exception as e:
- g.log.error("Exception occurred while getting the platform "
- "of node %s: %s", mclient, str(e))
- return (1, '', '')
- finally:
- g.rpyc_close_connection(host=mclient, user=user)
-
+ cmd = "cat /etc/redhat-release | grep Santiago"
+ ret, _, _ = g.run(mclient, cmd, user=user)
+ if not ret:
+ cifs_options = "sec=ntlmssp"
mcmd = ("mount -t cifs -o username=%s,password=%s,%s "
"\\\\\\\\%s\\\\gluster-%s %s" % (smbuser, smbpasswd,
cifs_options, mserver,
diff --git a/glustolibs-gluster/scripts/compute_hash.py b/glustolibs-gluster/scripts/compute_hash.py
new file mode 100644
index 000000000..b5ae2f83b
--- /dev/null
+++ b/glustolibs-gluster/scripts/compute_hash.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from __future__ import print_function
+import ctypes
+import sys
+
+filename = sys.argv[1]
+glusterfs = ctypes.cdll.LoadLibrary("libglusterfs.so.0")
+computed_hash = ctypes.c_uint32(glusterfs.gf_dm_hashfn(filename,
+ len(filename)))
+
+print(computed_hash.value)
diff --git a/glustolibs-gluster/scripts/walk_dir.py b/glustolibs-gluster/scripts/walk_dir.py
new file mode 100644
index 000000000..02d115b0b
--- /dev/null
+++ b/glustolibs-gluster/scripts/walk_dir.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from __future__ import print_function
+import os
+import sys
+
+rootdir = sys.argv[1]
+
+list_of_levels = []
+for level in os.walk(rootdir):
+ list_of_levels.append(level)
+print(list_of_levels)
diff --git a/glustolibs-gluster/setup.py b/glustolibs-gluster/setup.py
index b3f9cb09c..05e59fde6 100644
--- a/glustolibs-gluster/setup.py
+++ b/glustolibs-gluster/setup.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright (c) 2016-2019 Red Hat, Inc.
+# Copyright (c) 2016-2020 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -49,6 +49,8 @@ setup(
)
try:
- dir_util.copy_tree('./gdeploy_configs', '/usr/share/glustolibs/gdeploy_configs')
+ for srcdir, destdir in (('./gdeploy_configs', '/usr/share/glustolibs/gdeploy_configs'),
+ ('./scripts', '/usr/share/glustolibs/scripts/')):
+ dir_util.copy_tree(srcdir, destdir)
except:
pass