summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKotresh HR <khiremat@redhat.com>2018-10-03 00:45:09 -0400
committerShyamsundar Ranganathan <srangana@redhat.com>2018-10-10 14:09:10 +0000
commit84a3d43079cfe8cf6a374defa86a345124d5901c (patch)
tree1feb533cae4b73ceb595e95e395743c662acfbc8
parent324384ca48944662fecdffe72faa5d2c5872eb5f (diff)
georep: python2 to python3 compat - syscalls
1. ctypes/syscalls A) arguments is expected to be encoded B) Raw conversion of return value from bytearray into string 2. struct pack/unpack - Raw converstion of string to bytearray 3. basestring -> str Updates: #411 Change-Id: I80f939adcdec0ed0022c87c0b76d057ad5559e5a Signed-off-by: Kotresh HR <khiremat@redhat.com> (cherry picked from commit fb6e8d0d0ca21b16d331fa69da9b9dadf6c5c35d)
-rw-r--r--geo-replication/syncdaemon/Makefile.am2
-rw-r--r--geo-replication/syncdaemon/libcxattr.py17
-rw-r--r--geo-replication/syncdaemon/libgfchangelog.py18
-rw-r--r--geo-replication/syncdaemon/monitor.py4
-rw-r--r--geo-replication/syncdaemon/py2py3.py49
-rw-r--r--geo-replication/syncdaemon/resource.py21
-rw-r--r--geo-replication/syncdaemon/syncdutils.py12
7 files changed, 88 insertions, 35 deletions
diff --git a/geo-replication/syncdaemon/Makefile.am b/geo-replication/syncdaemon/Makefile.am
index 19f0bfce1b7..62c5ce7fe30 100644
--- a/geo-replication/syncdaemon/Makefile.am
+++ b/geo-replication/syncdaemon/Makefile.am
@@ -3,6 +3,6 @@ syncdaemondir = $(GLUSTERFS_LIBEXECDIR)/python/syncdaemon
syncdaemon_PYTHON = rconf.py gsyncd.py __init__.py master.py README.md repce.py \
resource.py syncdutils.py monitor.py libcxattr.py gsyncdconfig.py \
libgfchangelog.py changelogagent.py gsyncdstatus.py conf.py logutils.py \
- subcmds.py argsupgrade.py
+ subcmds.py argsupgrade.py py2py3.py
CLEANFILES =
diff --git a/geo-replication/syncdaemon/libcxattr.py b/geo-replication/syncdaemon/libcxattr.py
index 2d186b95c89..7f3f6ce453a 100644
--- a/geo-replication/syncdaemon/libcxattr.py
+++ b/geo-replication/syncdaemon/libcxattr.py
@@ -10,6 +10,8 @@
import os
from ctypes import CDLL, create_string_buffer, get_errno
+import py2py3
+from py2py3 import bytearray_to_str
class Xattr(object):
@@ -38,20 +40,23 @@ class Xattr(object):
@classmethod
def _query_xattr(cls, path, siz, syscall, *a):
if siz:
- buf = create_string_buffer('\0' * siz)
+ buf = create_string_buffer(b'\0' * siz)
else:
buf = None
ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
if ret == -1:
cls.raise_oserr()
if siz:
- return buf.raw[:ret]
+ # py2 and py3 compatibility. Convert bytes array
+ # to string
+ result = bytearray_to_str(buf.raw)
+ return result[:ret]
else:
return ret
@classmethod
def lgetxattr(cls, path, attr, siz=0):
- return cls._query_xattr(path, siz, 'lgetxattr', attr)
+ return cls._query_xattr(path.encode(), siz, 'lgetxattr', attr.encode())
@classmethod
def lgetxattr_buf(cls, path, attr):
@@ -65,7 +70,7 @@ class Xattr(object):
@classmethod
def llistxattr(cls, path, siz=0):
- ret = cls._query_xattr(path, siz, 'llistxattr')
+ ret = cls._query_xattr(path.encode(), siz, 'llistxattr')
if isinstance(ret, str):
ret = ret.strip('\0')
ret = ret.split('\0') if ret else []
@@ -73,13 +78,13 @@ class Xattr(object):
@classmethod
def lsetxattr(cls, path, attr, val):
- ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
+ ret = cls.libc.lsetxattr(path.encode(), attr.encode(), val, len(val), 0)
if ret == -1:
cls.raise_oserr()
@classmethod
def lremovexattr(cls, path, attr):
- ret = cls.libc.lremovexattr(path, attr)
+ ret = cls.libc.lremovexattr(path.encode(), attr.encode())
if ret == -1:
cls.raise_oserr()
diff --git a/geo-replication/syncdaemon/libgfchangelog.py b/geo-replication/syncdaemon/libgfchangelog.py
index da12438d069..cc40fd5475d 100644
--- a/geo-replication/syncdaemon/libgfchangelog.py
+++ b/geo-replication/syncdaemon/libgfchangelog.py
@@ -39,8 +39,8 @@ class Changes(object):
@classmethod
def cl_register(cls, brick, path, log_file, log_level, retries=0):
- ret = cls._get_api('gf_changelog_register')(brick, path,
- log_file,
+ ret = cls._get_api('gf_changelog_register')(brick.encode(), path.encode(),
+ log_file.encode(),
log_level, retries)
if ret == -1:
cls.raise_changelog_err()
@@ -63,14 +63,14 @@ class Changes(object):
def clsort(f):
return f.split('.')[-1]
changes = []
- buf = create_string_buffer('\0', 4096)
+ buf = create_string_buffer(b'\0' * 4096)
call = cls._get_api('gf_changelog_next_change')
while True:
ret = call(buf, 4096)
if ret in (0, -1):
break
- changes.append(buf.raw[:ret - 1])
+ changes.append(buf.raw[:ret - 1].decode())
if ret == -1:
cls.raise_changelog_err()
# cleanup tracker
@@ -79,7 +79,7 @@ class Changes(object):
@classmethod
def cl_done(cls, clfile):
- ret = cls._get_api('gf_changelog_done')(clfile)
+ ret = cls._get_api('gf_changelog_done')(clfile.encode())
if ret == -1:
cls.raise_changelog_err()
@@ -94,7 +94,7 @@ class Changes(object):
@classmethod
def cl_history_changelog(cls, changelog_path, start, end, num_parallel):
actual_end = c_ulong()
- ret = cls._get_api('gf_history_changelog')(changelog_path, start, end,
+ ret = cls._get_api('gf_history_changelog')(changelog_path.encode(), start, end,
num_parallel,
byref(actual_end))
if ret == -1:
@@ -118,14 +118,14 @@ class Changes(object):
return f.split('.')[-1]
changes = []
- buf = create_string_buffer('\0', 4096)
+ buf = create_string_buffer(b'\0' * 4096)
call = cls._get_api('gf_history_changelog_next_change')
while True:
ret = call(buf, 4096)
if ret in (0, -1):
break
- changes.append(buf.raw[:ret - 1])
+ changes.append(buf.raw[:ret - 1].decode())
if ret == -1:
cls.raise_changelog_err()
@@ -133,6 +133,6 @@ class Changes(object):
@classmethod
def cl_history_done(cls, clfile):
- ret = cls._get_api('gf_history_changelog_done')(clfile)
+ ret = cls._get_api('gf_history_changelog_done')(clfile.encode())
if ret == -1:
cls.raise_changelog_err()
diff --git a/geo-replication/syncdaemon/monitor.py b/geo-replication/syncdaemon/monitor.py
index ca2839059a3..c45ef24e59f 100644
--- a/geo-replication/syncdaemon/monitor.py
+++ b/geo-replication/syncdaemon/monitor.py
@@ -26,7 +26,9 @@ from syncdutils import set_term_handler, GsyncdError
from syncdutils import Thread, finalize, Volinfo, VolinfoFromGconf
from syncdutils import gf_event, EVENT_GEOREP_FAULTY, get_up_nodes
from gsyncdstatus import GeorepStatus, set_monitor_status
-from syncdutils import unshare_propagation_supported, pipe
+from syncdutils import unshare_propagation_supported
+import py2py3
+from py2py3 import pipe
ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError
diff --git a/geo-replication/syncdaemon/py2py3.py b/geo-replication/syncdaemon/py2py3.py
new file mode 100644
index 00000000000..4c0e1152aa8
--- /dev/null
+++ b/geo-replication/syncdaemon/py2py3.py
@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+# All python2/python3 compatibility routines
+
+import sys
+import os
+
+
+if sys.version_info >= (3,):
+ def pipe():
+ (r, w) = os.pipe()
+ os.set_inheritable(r, True)
+ os.set_inheritable(w, True)
+ return (r, w)
+
+ # Raw conversion of bytearray to string. Used in the cases where
+ # buffer is created by create_string_buffer which is a 8-bit char
+ # array and passed to syscalls to fetch results. Using encode/decode
+ # doesn't work as it converts to string altering the size.
+ def bytearray_to_str(byte_arr):
+ return ''.join([chr(b) for b in byte_arr])
+
+ # Raw conversion of string to bytes. This is required to convert
+ # back the string into bytearray(c char array) to use in struc
+ # pack/unpacking. Again encode/decode can't be used as it
+ # converts it alters size.
+ def str_to_bytearray(string):
+ return bytes([ord(c) for c in string])
+
+else:
+ def pipe():
+ (r, w) = os.pipe()
+ return (r, w)
+
+ # Raw conversion of bytearray to string
+ def bytearray_to_str(byte_arr):
+ return ''.join([b for b in byte_arr])
+
+ # Raw conversion of string to bytearray
+ def str_to_bytearray(string):
+ return b"".join([c for c in string])
diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index a0edeb7d56b..200b5566e08 100644
--- a/geo-replication/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -40,7 +40,9 @@ from syncdutils import GX_GFID_CANONICAL_LEN
from gsyncdstatus import GeorepStatus
from syncdutils import lf, Popen, sup
from syncdutils import Xattr, matching_disk_gfid, get_gfid_from_mnt
-from syncdutils import unshare_propagation_supported, get_slv_dir_path, pipe
+from syncdutils import unshare_propagation_supported, get_slv_dir_path
+import py2py3
+from py2py3 import pipe, str_to_bytearray
ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
@@ -145,6 +147,7 @@ class Server(object):
if buf == ENOENT:
return buf
else:
+ buf = str_to_bytearray(buf)
m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(
['%02x' % x for x in struct.unpack(cls.GFID_FMTSTR, buf)]))
return '-'.join(m.groups())
@@ -235,6 +238,7 @@ class Server(object):
val = Xattr.lgetxattr(path,
'.'.join([cls.GX_NSPACE, uuid, 'xtime']),
8)
+ val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@@ -257,6 +261,7 @@ class Server(object):
val = Xattr.lgetxattr(path,
'.'.join([cls.GX_NSPACE, uuid, 'stime']),
8)
+ val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@@ -279,6 +284,7 @@ class Server(object):
val = Xattr.lgetxattr(path,
'.'.join([cls.GX_NSPACE, uuid, 'stime']),
8)
+ val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@@ -302,6 +308,7 @@ class Server(object):
'.'.join([cls.GX_NSPACE, uuid,
'entry_stime']),
8)
+ val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@@ -375,7 +382,7 @@ class Server(object):
def entry_pack_reg(gf, bn, mo, uid, gid):
blen = len(bn)
return struct.pack(cls._fmt_mknod(blen),
- uid, gid, gf, mo, bn,
+ uid, gid, gf.encode(), mo, bn.encode(),
stat.S_IMODE(mo), 0, umask())
def entry_pack_reg_stat(gf, bn, st):
@@ -383,14 +390,14 @@ class Server(object):
mo = st['mode']
return struct.pack(cls._fmt_mknod(blen),
st['uid'], st['gid'],
- gf, mo, bn,
+ gf.encode(), mo, bn.encode(),
stat.S_IMODE(mo), 0, umask())
# mkdir
def entry_pack_mkdir(gf, bn, mo, uid, gid):
blen = len(bn)
return struct.pack(cls._fmt_mkdir(blen),
- uid, gid, gf, mo, bn,
+ uid, gid, gf.encode(), mo, bn.encode(),
stat.S_IMODE(mo), umask())
# symlink
@@ -399,7 +406,8 @@ class Server(object):
llen = len(lnk)
return struct.pack(cls._fmt_symlink(blen, llen),
st['uid'], st['gid'],
- gf, st['mode'], bn, lnk)
+ gf.encode(), st['mode'], bn.encode(),
+ lnk.encode())
def entry_purge(op, entry, gfid, e):
# This is an extremely racy code and needs to be fixed ASAP.
@@ -450,7 +458,7 @@ class Server(object):
else:
en = e['entry']
disk_gfid = get_gfid_from_mnt(en)
- if isinstance(disk_gfid, basestring) and \
+ if isinstance(disk_gfid, str) and \
e['gfid'] != disk_gfid:
slv_entry_info['gfid_mismatch'] = True
st = lstat(en)
@@ -1022,6 +1030,7 @@ class GLUSTERServer(Server):
"""generic volume mark fetching/parsing backed"""
fmt_string = cls.NTV_FMTSTR + extra_fields
buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
+ buf = str_to_bytearray(buf)
vm = struct.unpack(fmt_string, buf)
m = re.match(
'(.{8})(.{4})(.{4})(.{4})(.{12})',
diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
index cbb169c6c96..fad1a3e4f76 100644
--- a/geo-replication/syncdaemon/syncdutils.py
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -1021,15 +1021,3 @@ def get_up_nodes(hosts, port):
up_nodes.append(h)
return up_nodes
-
-
-def pipe():
- # Pipe routine for python2 and python3 compatiability
- try:
- (r, w) = os.pipe()
- os.set_inheritable(r, True)
- os.set_inheritable(w, True)
- except AttributeError:
- (r, w) = os.pipe()
-
- return (r, w)