summaryrefslogtreecommitdiffstats
path: root/glustolibs-gluster/glustolibs/gluster/lib_utils.py
diff options
context:
space:
mode:
authorkshithijiyer <kshithij.ki@gmail.com>2020-04-01 10:44:54 +0530
committerkshithijiyer <kshithij.ki@gmail.com>2020-04-01 12:17:18 +0530
commitdd2382c613db6f3dd72a25c74d5f765006aea31f (patch)
treefcbdca0fc93539b7eee159bfa3c381167cd0eb1c /glustolibs-gluster/glustolibs/gluster/lib_utils.py
parentd4349ba649c967ed6c16c1efdd9caa56433142f1 (diff)
[Libfix] Remove rpyc_get_connection() dependency from code
Problem: `g.rpyc_get_connection()` has a limitaion where it can't convert python2 calls to python3 calls. Due to this a large number of testcases fail when executed from a python2 machine on a python3 only setup or visa versa with the below stack trace: ``` E ========= Remote Traceback (1) ========= E Traceback (most recent call last): E File "/root/tmp.tL8Eqx7d8l/rpyc/core/protocol.py", line 323, in _dispatch_request E res = self._HANDLERS[handler](self, *args) E File "/root/tmp.tL8Eqx7d8l/rpyc/core/protocol.py", line 591, in _handle_inspect E if hasattr(self._local_objects[id_pack], '____conn__'): E File "/root/tmp.tL8Eqx7d8l/rpyc/lib/colls.py", line 110, in __getitem__ E return self._dict[key][0] E KeyError: (b'rpyc.core.service.SlaveService', 94282642994712, 140067150858560) ``` Solution: The solution here is to modify the code to not use `g.rpyc_get_connection()`. The following changes are done to accomplish it: 1)Remove code which uses g.rpyc_get_connection() and use generic logic in functions: a. do_bricks_exist_in_shd_volfile() b. get_disk_usage() c. mount_volume() d. list_files() f. append_string_to_file() 2)Create files which can be uploaded and executed on clients/servers to avoid rpc calls in functions: a. calculate_hash() b. validate_files_in_dir() 3)Modify setup.py to push the below files to `/usr/share/glustolibs/scripts/`: a.compute_hash.py b.walk_dir.py Change-Id: I00a81a88382bf3f8b366753eebdb2999260788ca Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>
Diffstat (limited to 'glustolibs-gluster/glustolibs/gluster/lib_utils.py')
-rwxr-xr-xglustolibs-gluster/glustolibs/gluster/lib_utils.py84
1 files changed, 27 insertions, 57 deletions
diff --git a/glustolibs-gluster/glustolibs/gluster/lib_utils.py b/glustolibs-gluster/glustolibs/gluster/lib_utils.py
index 2c0dda3..7299874 100755
--- a/glustolibs-gluster/glustolibs/gluster/lib_utils.py
+++ b/glustolibs-gluster/glustolibs/gluster/lib_utils.py
@@ -26,8 +26,6 @@ import re
import time
from collections import OrderedDict
import tempfile
-import subprocess
-import random
ONE_GB_BYTES = 1073741824.0
@@ -53,23 +51,16 @@ def append_string_to_file(mnode, filename, str_to_add_in_file,
Returns:
True, on success, False otherwise
"""
- try:
- conn = g.rpyc_get_connection(mnode, user=user)
- if conn is None:
- g.log.error("Unable to get connection to 'root' of node %s"
- " in append_string_to_file()" % mnode)
- return False
-
- with conn.builtin.open(filename, 'a') as _filehandle:
- _filehandle.write(str_to_add_in_file)
-
- return True
- except IOError:
- g.log.error("Exception occurred while adding string to "
- "file %s in append_string_to_file()", filename)
+ cmd = "echo '{0}' >> {1}".format(str_to_add_in_file,
+ filename)
+ ret, out, err = g.run(mnode, cmd, user)
+ if ret or out or err:
+ g.log.error("Unable to append string '{0}' to file "
+ "'{1}' on node {2} using user {3}"
+ .format(str_to_add_in_file, filename,
+ mnode, user))
return False
- finally:
- g.rpyc_close_connection(host=mnode, user=user)
+ return True
def search_pattern_in_file(mnode, search_pattern, filename, start_str_to_parse,
@@ -268,31 +259,19 @@ def list_files(mnode, dir_path, parse_str="", user="root"):
NoneType: None if command execution fails, parse errors.
list: files with absolute name
"""
-
- try:
- conn = g.rpyc_get_connection(mnode, user=user)
- if conn is None:
- g.log.error("Unable to get connection to 'root' of node %s"
- % mnode)
- return None
-
- filepaths = []
- for root, directories, files in conn.modules.os.walk(dir_path):
- for filename in files:
- if parse_str != "":
- if parse_str in filename:
- filepath = conn.modules.os.path.join(root, filename)
- filepaths.append(filepath)
- else:
- filepath = conn.modules.os.path.join(root, filename)
- filepaths.append(filepath)
- return filepaths
- except StopIteration:
- g.log.error("Exception occurred in list_files()")
+ if parse_str == "":
+ cmd = "find {0} -type f".format(dir_path)
+ else:
+ cmd = "find {0} -type f | grep {1}".format(dir_path,
+ parse_str)
+ ret, out, err = g.run(mnode, cmd, user)
+ if ret or err:
+ g.log.error("Unable to get the list of files on path "
+ "{0} on node {1} using user {2} due to error {3}"
+ .format(dir_path, mnode, user, err))
return None
-
- finally:
- g.rpyc_close_connection(host=mnode, user=user)
+ file_list = out.split('\n')
+ return file_list[0:len(file_list)-1]
def get_servers_bricks_dict(servers, servers_info):
@@ -544,22 +523,13 @@ def get_disk_usage(mnode, path, user="root"):
Example:
get_disk_usage("abc.com", "/mnt/glusterfs")
"""
-
- inst = random.randint(10, 100)
- conn = g.rpyc_get_connection(mnode, user=user, instance=inst)
- if conn is None:
- g.log.error("Failed to get rpyc connection")
- return None
- cmd = 'stat -f ' + path
- p = conn.modules.subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, err = p.communicate()
- ret = p.returncode
- if ret != 0:
- g.log.error("Failed to execute stat command")
+ cmd = 'stat -f {0}'.format(path)
+ ret, out, err = g.run(mnode, cmd, user)
+ if ret:
+ g.log.error("Unable to get stat of path {0} on node {1} "
+ "using user {2} due to error {3}".format(path, mnode,
+ user, err))
return None
-
- g.rpyc_close_connection(host=mnode, user=user, instance=inst)
res = ''.join(out)
match = re.match(r'.*Block size:\s(\d+).*Blocks:\sTotal:\s(\d+)\s+?'
r'Free:\s(\d+)\s+?Available:\s(\d+).*Inodes:\s'