diff options
Diffstat (limited to 'tests/utils')
| -rw-r--r-- | tests/utils/changelog/changelog.h | 6 | ||||
| -rw-r--r-- | tests/utils/changelog/test-changelog-api.c | 98 | ||||
| -rw-r--r-- | tests/utils/changelog/test-history-api.c | 111 | ||||
| -rw-r--r-- | tests/utils/changelogparser.py | 5 | ||||
| -rwxr-xr-x | tests/utils/create-files.py | 9 | ||||
| -rw-r--r-- | tests/utils/get-mdata-xattr.c | 152 | ||||
| -rwxr-xr-x | tests/utils/gfid-access.py | 62 | ||||
| -rw-r--r-- | tests/utils/libcxattr.py | 22 | ||||
| -rw-r--r-- | tests/utils/py2py3.py | 186 |
9 files changed, 623 insertions, 28 deletions
diff --git a/tests/utils/changelog/changelog.h b/tests/utils/changelog/changelog.h index 969a1f370c2..1502b689eb4 100644 --- a/tests/utils/changelog/changelog.h +++ b/tests/utils/changelog/changelog.h @@ -116,4 +116,10 @@ int gf_history_changelog(char *changelog_dir, unsigned long start, unsigned long end, int n_parallel, unsigned long *actual_end); +int +gf_history_changelog_scan(); +ssize_t +gf_history_changelog_next_change(char *bufptr, size_t maxlen); +int +gf_history_changelog_done(char *file); #endif diff --git a/tests/utils/changelog/test-changelog-api.c b/tests/utils/changelog/test-changelog-api.c new file mode 100644 index 00000000000..f4eb066b630 --- /dev/null +++ b/tests/utils/changelog/test-changelog-api.c @@ -0,0 +1,98 @@ +/* + Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +/** + * get set of new changes every 5 seconds (just print the file names) + * + * Compile it using: + * gcc -o getchanges `pkg-config --cflags libgfchangelog` get-changes.c \ + * `pkg-config --libs libgfchangelog` + */ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/un.h> +#include <limits.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <errno.h> + +#include "changelog.h" + +int +main(int argc, char **argv) +{ + int i = 0; + int ret = 0; + ssize_t nr_changes = 0; + ssize_t changes = 0; + char fbuf[PATH_MAX] = { + 0, + }; + + ret = gf_changelog_init(NULL); + if (ret) { + printf("-1"); + fflush(stdout); + return -1; + } + + /* get changes for brick "/d/backends/patchy0" */ + ret = gf_changelog_register("/d/backends/patchy0", "/tmp/scratch_v1", + "/var/log/glusterfs/changes.log", 9, 5); + if (ret) { + printf("-2"); + fflush(stdout); + return -1; + } + + while (1) { + i = 0; + nr_changes = gf_changelog_scan(); + if (nr_changes < 0) { + printf("-4"); + fflush(stdout); + return -1; + } + + if (nr_changes == 0) + goto next; + + while ((changes = gf_changelog_next_change(fbuf, PATH_MAX)) > 0) { + /* process changelog */ + /* ... */ + /* ... */ + /* ... */ + /* done processing */ + + ret = gf_changelog_done(fbuf); + if (ret) { + printf("-5"); + fflush(stdout); + return -1; + } + } + + if (changes == -1) { + printf("-6"); + fflush(stdout); + return -1; + } + + next: + sleep(2); + } + +out: + printf("0"); + fflush(stdout); + return ret; +} diff --git a/tests/utils/changelog/test-history-api.c b/tests/utils/changelog/test-history-api.c new file mode 100644 index 00000000000..d78e387df10 --- /dev/null +++ b/tests/utils/changelog/test-history-api.c @@ -0,0 +1,111 @@ +/* + Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +/** + * get set of new changes every 10 seconds (just print the file names) + * + * Compile it using: + * gcc -o gethistory `pkg-config --cflags libgfchangelog` get-history.c \ + * `pkg-config --libs libgfchangelog` + */ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/un.h> +#include <limits.h> +#include <sys/socket.h> +#include <sys/types.h> + +#include "changelog.h" + +int +main(int argc, char **argv) +{ + int ret = 0; + int i = 0; + unsigned long end_ts = 0; + ssize_t nr_changes = 0; + ssize_t changes = 0; + int start = 0; + int end = 0; + char fbuf[PATH_MAX] = { + 0, + }; + + ret = gf_changelog_init(NULL); + if (ret) { + printf("-1"); + fflush(stdout); + return -1; + } + + ret = gf_changelog_register("/d/backends/patchy0", "/tmp/scratch_v1", + "/var/log/glusterfs/changes.log", 9, 5); + if (ret) { + printf("-2"); + fflush(stdout); + return -1; + } + + start = atoi(argv[1]); + end = atoi(argv[2]); + + ret = gf_history_changelog("/d/backends/patchy0/.glusterfs/changelogs", + start, end, 3, &end_ts); + if (ret < 0) { + printf("-3"); + fflush(stdout); + return -1; + } else if (ret == 1) { + printf("1"); + fflush(stdout); + return 0; + } + + while (1) { + nr_changes = gf_history_changelog_scan(); + if (nr_changes < 0) { + printf("-4"); + fflush(stdout); + return -1; + } + + if (nr_changes == 0) { + goto out; + } + + while ((changes = gf_history_changelog_next_change(fbuf, PATH_MAX)) > + 0) { + /* process changelog */ + /* ... */ + /* ... */ + /* ... */ + /* done processing */ + + ret = gf_history_changelog_done(fbuf); + if (ret) { + printf("-5"); + fflush(stdout); + return -1; + } + } + if (changes == -1) { + printf("-6"); + fflush(stdout); + return -1; + } + } + +out: + printf("0"); + fflush(stdout); + return 0; +} diff --git a/tests/utils/changelogparser.py b/tests/utils/changelogparser.py index e8e252d195f..3b8f81d1bad 100644 --- a/tests/utils/changelogparser.py +++ b/tests/utils/changelogparser.py @@ -125,7 +125,10 @@ class Record(object): return repr(self.__dict__) def __str__(self): - return unicode(self).encode('utf-8') + if sys.version_info >= (3,): + return self.__unicode__() + else: + return unicode(self).encode('utf-8') def get_num_tokens(data, tokens, version=Version.V11): diff --git a/tests/utils/create-files.py b/tests/utils/create-files.py index b2a19610d63..04736e9c73b 100755 --- a/tests/utils/create-files.py +++ b/tests/utils/create-files.py @@ -19,6 +19,11 @@ import argparse datsiz = 0 timr = 0 +def get_ascii_upper_alpha_digits(): + if sys.version_info > (3,0): + return string.ascii_uppercase+string.digits + else: + return string.uppercase+string.digits def setLogger(filename): global logger @@ -111,7 +116,7 @@ def create_tar_file(fil, size, mins, maxs, rand): def get_filename(flen): size = flen - char = string.uppercase+string.digits + char = get_ascii_upper_alpha_digits() st = ''.join(random.choice(char) for i in range(size)) ti = str((hex(int(str(time.time()).split('.')[0])))[2:]) return ti+"%%"+st @@ -175,7 +180,7 @@ def tar_files(files, file_count, inter, size, mins, maxs, def setxattr_files(files, randname, dir_path): - char = string.uppercase+string.digits + char = get_ascii_upper_alpha_digits() if not randname: for k in range(files): v = ''.join(random.choice(char) for i in range(10)) diff --git a/tests/utils/get-mdata-xattr.c b/tests/utils/get-mdata-xattr.c new file mode 100644 index 00000000000..e9f54717263 --- /dev/null +++ b/tests/utils/get-mdata-xattr.c @@ -0,0 +1,152 @@ +/* + Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <stdlib.h> +#include <endian.h> +#include <stdio.h> +#include <time.h> +#include <string.h> +#include <inttypes.h> +#include <sys/types.h> +#include <sys/xattr.h> +#include <errno.h> + +typedef struct gf_timespec_disk { + uint64_t tv_sec; + uint64_t tv_nsec; +} gf_timespec_disk_t; + +/* posix_mdata_t on disk structure */ +typedef struct __attribute__((__packed__)) posix_mdata_disk { + /* version of structure, bumped up if any new member is added */ + uint8_t version; + /* flags indicates valid fields in the structure */ + uint64_t flags; + gf_timespec_disk_t ctime; + gf_timespec_disk_t mtime; + gf_timespec_disk_t atime; +} posix_mdata_disk_t; + +/* In memory representation posix metadata xattr */ +typedef struct { + /* version of structure, bumped up if any new member is added */ + uint8_t version; + /* flags indicates valid fields in the structure */ + uint64_t flags; + struct timespec ctime; + struct timespec mtime; + struct timespec atime; +} posix_mdata_t; + +#define GF_XATTR_MDATA_KEY "trusted.glusterfs.mdata" + +/* posix_mdata_from_disk converts posix_mdata_disk_t into host byte order + */ +static inline void +posix_mdata_from_disk(posix_mdata_t *out, posix_mdata_disk_t *in) +{ + out->version = in->version; + out->flags = be64toh(in->flags); + + out->ctime.tv_sec = be64toh(in->ctime.tv_sec); + out->ctime.tv_nsec = be64toh(in->ctime.tv_nsec); + + out->mtime.tv_sec = be64toh(in->mtime.tv_sec); + out->mtime.tv_nsec = be64toh(in->mtime.tv_nsec); + + out->atime.tv_sec = be64toh(in->atime.tv_sec); + out->atime.tv_nsec = be64toh(in->atime.tv_nsec); +} + +/* posix_fetch_mdata_xattr fetches the posix_mdata_t from disk */ +static int +posix_fetch_mdata_xattr(const char *real_path, posix_mdata_t *metadata) +{ + size_t size = -1; + char *value = NULL; + char gfid_str[64] = {0}; + + char *key = GF_XATTR_MDATA_KEY; + + if (!metadata || !real_path) { + goto err; + } + + /* Get size */ + size = lgetxattr(real_path, key, NULL, 0); + if (size == -1) { + goto err; + } + + value = calloc(size + 1, sizeof(char)); + if (!value) { + goto err; + } + + /* Get xattr value */ + size = lgetxattr(real_path, key, value, size); + if (size == -1) { + goto err; + } + posix_mdata_from_disk(metadata, (posix_mdata_disk_t *)value); + +out: + if (value) + free(value); + return 0; +err: + if (value) + free(value); + return -1; +} + +int +main(int argc, char *argv[]) +{ + posix_mdata_t metadata; + uint64_t result; + + if (argc != 3) { + /* + Usage: get_mdata_xattr -c|-m|-a <file-name> + where -c --> ctime + -m --> mtime + -a --> atime + */ + printf("-1"); + goto err; + } + + if (posix_fetch_mdata_xattr(argv[2], &metadata)) { + printf("-1"); + goto err; + } + + switch (argv[1][1]) { + case 'c': + result = metadata.ctime.tv_sec; + break; + case 'm': + result = metadata.mtime.tv_sec; + break; + case 'a': + result = metadata.atime.tv_sec; + break; + default: + printf("-1"); + goto err; + } + printf("%" PRIu64, result); + fflush(stdout); + return 0; +err: + fflush(stdout); + return -1; +} diff --git a/tests/utils/gfid-access.py b/tests/utils/gfid-access.py index 556d2b4c65b..c35c1223df6 100755 --- a/tests/utils/gfid-access.py +++ b/tests/utils/gfid-access.py @@ -33,23 +33,51 @@ def _fmt_mkdir(l): def _fmt_symlink(l1, l2): return "!II%dsI%ds%ds" % (37, l1+1, l2+1) -def entry_pack_reg(gf, bn, mo, uid, gid): - blen = len(bn) - return struct.pack(_fmt_mknod(blen), - uid, gid, gf, mo, bn, - stat.S_IMODE(mo), 0, umask()) - -def entry_pack_dir(gf, bn, mo, uid, gid): - blen = len(bn) - return struct.pack(_fmt_mkdir(blen), - uid, gid, gf, mo, bn, - stat.S_IMODE(mo), umask()) - -def entry_pack_symlink(gf, bn, lnk, mo, uid, gid): - blen = len(bn) - llen = len(lnk) - return struct.pack(_fmt_symlink(blen, llen), - uid, gid, gf, mo, bn, lnk) + +if sys.version_info > (3,): + def entry_pack_reg(gf, bn, mo, uid, gid): + bn_encoded = bn.encode() + blen = len(bn_encoded) + return struct.pack(_fmt_mknod(blen), + uid, gid, gf.encode(), mo, bn_encoded, + stat.S_IMODE(mo), 0, umask()) + + # mkdir + def entry_pack_dir(gf, bn, mo, uid, gid): + bn_encoded = bn.encode() + blen = len(bn_encoded) + return struct.pack(_fmt_mkdir(blen), + uid, gid, gf.encode(), mo, bn_encoded, + stat.S_IMODE(mo), umask()) + # symlink + def entry_pack_symlink(gf, bn, lnk, st): + bn_encoded = bn.encode() + blen = len(bn_encoded) + lnk_encoded = lnk.encode() + llen = len(lnk_encoded) + return struct.pack(_fmt_symlink(blen, llen), + st['uid'], st['gid'], + gf.encode(), st['mode'], bn_encoded, + lnk_encoded) + +else: + def entry_pack_reg(gf, bn, mo, uid, gid): + blen = len(bn) + return struct.pack(_fmt_mknod(blen), + uid, gid, gf, mo, bn, + stat.S_IMODE(mo), 0, umask()) + + def entry_pack_dir(gf, bn, mo, uid, gid): + blen = len(bn) + return struct.pack(_fmt_mkdir(blen), + uid, gid, gf, mo, bn, + stat.S_IMODE(mo), umask()) + + def entry_pack_symlink(gf, bn, lnk, mo, uid, gid): + blen = len(bn) + llen = len(lnk) + return struct.pack(_fmt_symlink(blen, llen), + uid, gid, gf, mo, bn, lnk) if __name__ == '__main__': if len(sys.argv) < 9: diff --git a/tests/utils/libcxattr.py b/tests/utils/libcxattr.py index fd0b08378fc..3f3ed1fffbb 100644 --- a/tests/utils/libcxattr.py +++ b/tests/utils/libcxattr.py @@ -10,7 +10,9 @@ import os import sys -from ctypes import CDLL, c_int, create_string_buffer +from ctypes import CDLL, c_int +from py2py3 import bytearray_to_str, gr_create_string_buffer +from py2py3 import gr_query_xattr, gr_lsetxattr, gr_lremovexattr class Xattr(object): @@ -47,20 +49,23 @@ class Xattr(object): @classmethod def _query_xattr(cls, path, siz, syscall, *a): if siz: - buf = create_string_buffer('\0' * siz) + buf = gr_create_string_buffer(siz) else: buf = None ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz))) if ret == -1: cls.raise_oserr() if siz: - return buf.raw[:ret] + # py2 and py3 compatibility. Convert bytes array + # to string + result = bytearray_to_str(buf.raw) + return result[:ret] else: return ret @classmethod def lgetxattr(cls, path, attr, siz=0): - return cls._query_xattr(path, siz, 'lgetxattr', attr) + return gr_query_xattr(cls, path, siz, 'lgetxattr', attr) @classmethod def lgetxattr_buf(cls, path, attr): @@ -74,20 +79,21 @@ class Xattr(object): @classmethod def llistxattr(cls, path, siz=0): - ret = cls._query_xattr(path, siz, 'llistxattr') + ret = gr_query_xattr(cls, path, siz, 'llistxattr') if isinstance(ret, str): - ret = ret.split('\0') + ret = ret.strip('\0') + ret = ret.split('\0') if ret else [] return ret @classmethod def lsetxattr(cls, path, attr, val): - ret = cls.libc.lsetxattr(path, attr, val, len(val), 0) + ret = gr_lsetxattr(cls, path, attr, val) if ret == -1: cls.raise_oserr() @classmethod def lremovexattr(cls, path, attr): - ret = cls.libc.lremovexattr(path, attr) + ret = gr_lremovexattr(cls, path, attr) if ret == -1: cls.raise_oserr() diff --git a/tests/utils/py2py3.py b/tests/utils/py2py3.py new file mode 100644 index 00000000000..63aca10fd26 --- /dev/null +++ b/tests/utils/py2py3.py @@ -0,0 +1,186 @@ +# +# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. + +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. +# + +# All python2/python3 compatibility routines + +import sys +import os +import stat +import struct +from ctypes import create_string_buffer + +def umask(): + return os.umask(0) + +if sys.version_info >= (3,): + def pipe(): + (r, w) = os.pipe() + os.set_inheritable(r, True) + os.set_inheritable(w, True) + return (r, w) + + # Raw conversion of bytearray to string. Used in the cases where + # buffer is created by create_string_buffer which is a 8-bit char + # array and passed to syscalls to fetch results. Using encode/decode + # doesn't work as it converts to string altering the size. + def bytearray_to_str(byte_arr): + return ''.join([chr(b) for b in byte_arr]) + + # Raw conversion of string to bytes. This is required to convert + # back the string into bytearray(c char array) to use in struc + # pack/unpacking. Again encode/decode can't be used as it + # converts it alters size. + def str_to_bytearray(string): + return bytes([ord(c) for c in string]) + + def gr_create_string_buffer(size): + return create_string_buffer(b'\0', size) + + def gr_query_xattr(cls, path, size, syscall, attr=None): + if attr: + return cls._query_xattr(path.encode(), size, syscall, + attr.encode()) + else: + return cls._query_xattr(path.encode(), size, syscall) + + def gr_lsetxattr(cls, path, attr, val): + return cls.libc.lsetxattr(path.encode(), attr.encode(), val, + len(val), 0) + + def gr_lremovexattr(cls, path, attr): + return cls.libc.lremovexattr(path.encode(), attr.encode()) + + def gr_cl_register(cls, brick, path, log_file, log_level, retries): + return cls._get_api('gf_changelog_register')(brick.encode(), + path.encode(), + log_file.encode(), + log_level, retries) + + def gr_cl_done(cls, clfile): + return cls._get_api('gf_changelog_done')(clfile.encode()) + + def gr_cl_history_changelog(cls, changelog_path, start, end, num_parallel, + actual_end): + return cls._get_api('gf_history_changelog')(changelog_path.encode(), + start, end, num_parallel, + actual_end) + + def gr_cl_history_done(cls, clfile): + return cls._get_api('gf_history_changelog_done')(clfile.encode()) + + # regular file + + def entry_pack_reg(cls, gf, bn, mo, uid, gid): + bn_encoded = bn.encode() + blen = len(bn_encoded) + return struct.pack(cls._fmt_mknod(blen), + uid, gid, gf.encode(), mo, bn_encoded, + stat.S_IMODE(mo), 0, umask()) + + def entry_pack_reg_stat(cls, gf, bn, st): + bn_encoded = bn.encode() + blen = len(bn_encoded) + mo = st['mode'] + return struct.pack(cls._fmt_mknod(blen), + st['uid'], st['gid'], + gf.encode(), mo, bn_encoded, + stat.S_IMODE(mo), 0, umask()) + # mkdir + + def entry_pack_mkdir(cls, gf, bn, mo, uid, gid): + bn_encoded = bn.encode() + blen = len(bn_encoded) + return struct.pack(cls._fmt_mkdir(blen), + uid, gid, gf.encode(), mo, bn_encoded, + stat.S_IMODE(mo), umask()) + # symlink + + def entry_pack_symlink(cls, gf, bn, lnk, st): + bn_encoded = bn.encode() + blen = len(bn_encoded) + lnk_encoded = lnk.encode() + llen = len(lnk_encoded) + return struct.pack(cls._fmt_symlink(blen, llen), + st['uid'], st['gid'], + gf.encode(), st['mode'], bn_encoded, + lnk_encoded) +else: + def pipe(): + (r, w) = os.pipe() + return (r, w) + + # Raw conversion of bytearray to string + def bytearray_to_str(byte_arr): + return byte_arr + + # Raw conversion of string to bytearray + def str_to_bytearray(string): + return string + + def gr_create_string_buffer(size): + return create_string_buffer('\0', size) + + def gr_query_xattr(cls, path, size, syscall, attr=None): + if attr: + return cls._query_xattr(path, size, syscall, attr) + else: + return cls._query_xattr(path, size, syscall) + + def gr_lsetxattr(cls, path, attr, val): + return cls.libc.lsetxattr(path, attr, val, len(val), 0) + + def gr_lremovexattr(cls, path, attr): + return cls.libc.lremovexattr(path, attr) + + def gr_cl_register(cls, brick, path, log_file, log_level, retries): + return cls._get_api('gf_changelog_register')(brick, path, log_file, + log_level, retries) + + def gr_cl_done(cls, clfile): + return cls._get_api('gf_changelog_done')(clfile) + + def gr_cl_history_changelog(cls, changelog_path, start, end, num_parallel, + actual_end): + return cls._get_api('gf_history_changelog')(changelog_path, start, end, + num_parallel, actual_end) + + def gr_cl_history_done(cls, clfile): + return cls._get_api('gf_history_changelog_done')(clfile) + + # regular file + + def entry_pack_reg(cls, gf, bn, mo, uid, gid): + blen = len(bn) + return struct.pack(cls._fmt_mknod(blen), + uid, gid, gf, mo, bn, + stat.S_IMODE(mo), 0, umask()) + + def entry_pack_reg_stat(cls, gf, bn, st): + blen = len(bn) + mo = st['mode'] + return struct.pack(cls._fmt_mknod(blen), + st['uid'], st['gid'], + gf, mo, bn, + stat.S_IMODE(mo), 0, umask()) + # mkdir + + def entry_pack_mkdir(cls, gf, bn, mo, uid, gid): + blen = len(bn) + return struct.pack(cls._fmt_mkdir(blen), + uid, gid, gf, mo, bn, + stat.S_IMODE(mo), umask()) + # symlink + + def entry_pack_symlink(cls, gf, bn, lnk, st): + blen = len(bn) + llen = len(lnk) + return struct.pack(cls._fmt_symlink(blen, llen), + st['uid'], st['gid'], + gf, st['mode'], bn, lnk) |