summaryrefslogtreecommitdiffstats
path: root/tests/utils
diff options
context:
space:
mode:
Diffstat (limited to 'tests/utils')
-rw-r--r--tests/utils/arequal-checksum.c633
-rw-r--r--tests/utils/changelog/changelog.h125
-rw-r--r--tests/utils/changelog/get-history.c71
-rw-r--r--tests/utils/changelog/test-changelog-api.c98
-rw-r--r--tests/utils/changelog/test-history-api.c111
-rw-r--r--tests/utils/changelogparser.py236
-rwxr-xr-xtests/utils/create-files.py642
-rw-r--r--tests/utils/get-mdata-xattr.c152
-rwxr-xr-xtests/utils/getfattr.py133
-rwxr-xr-xtests/utils/gfid-access.py124
-rw-r--r--tests/utils/libcxattr.py108
-rwxr-xr-xtests/utils/pidof.py45
-rw-r--r--tests/utils/py2py3.py186
-rwxr-xr-xtests/utils/setfattr.py77
-rwxr-xr-xtests/utils/testn.sh16
15 files changed, 2629 insertions, 128 deletions
diff --git a/tests/utils/arequal-checksum.c b/tests/utils/arequal-checksum.c
new file mode 100644
index 00000000000..b51a054162b
--- /dev/null
+++ b/tests/utils/arequal-checksum.c
@@ -0,0 +1,633 @@
+/*
+ Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#define _XOPEN_SOURCE 500
+
+#include <ftw.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <dirent.h>
+#include <stdlib.h>
+#include <libgen.h>
+#include <stdint.h>
+#include <dirent.h>
+#include <argp.h>
+
+/*
+ * FTW_ACTIONRETVAL is a GNU libc extension. It is used here to skip
+ * hierarchies. On other systems we will still walk the tree, ignoring
+ * entries.
+ */
+#ifndef FTW_ACTIONRETVAL
+#define FTW_ACTIONRETVAL 0
+#endif
+
+int debug = 0;
+
+typedef struct {
+ char test_directory[4096];
+ char **ignored_directory;
+ unsigned int directories_ignored;
+} arequal_config_t;
+
+static arequal_config_t arequal_config;
+
+static error_t
+arequal_parse_opts(int key, char *arg, struct argp_state *_state);
+
+static struct argp_option arequal_options[] = {
+ {"ignore", 'i', "IGNORED", 0, "entry in the given path to be ignored"},
+ {"path", 'p', "PATH", 0, "path where arequal has to be run"},
+ {0, 0, 0, 0, 0}};
+
+#define DBG(fmt...) \
+ do { \
+ if (debug) { \
+ fprintf(stderr, "D "); \
+ fprintf(stderr, fmt); \
+ } \
+ } while (0)
+
+void
+add_to_list(char *arg);
+void
+get_absolute_path(char directory[], char *arg);
+
+static int
+roof(int a, int b)
+{
+ return ((((a) + (b)-1) / ((b) ? (b) : 1)) * (b));
+}
+
+void
+add_to_list(char *arg)
+{
+ char *string = NULL;
+ int index = 0;
+
+ index = arequal_config.directories_ignored - 1;
+ string = strdup(arg);
+
+ if (!arequal_config.ignored_directory) {
+ arequal_config.ignored_directory = calloc(1, sizeof(char *));
+ } else
+ arequal_config.ignored_directory = realloc(
+ arequal_config.ignored_directory, sizeof(char *) * (index + 1));
+
+ arequal_config.ignored_directory[index] = string;
+}
+
+static error_t
+arequal_parse_opts(int key, char *arg, struct argp_state *_state)
+{
+ switch (key) {
+ case 'i': {
+ arequal_config.directories_ignored++;
+ add_to_list(arg);
+ } break;
+ case 'p': {
+ if (arg[0] == '/')
+ strcpy(arequal_config.test_directory, arg);
+ else
+ get_absolute_path(arequal_config.test_directory, arg);
+
+ if (arequal_config
+ .test_directory[strlen(arequal_config.test_directory) -
+ 1] == '/')
+ arequal_config
+ .test_directory[strlen(arequal_config.test_directory) - 1] =
+ '\0';
+ } break;
+
+ case ARGP_KEY_NO_ARGS:
+ break;
+ case ARGP_KEY_ARG:
+ break;
+ case ARGP_KEY_END:
+ if (_state->argc == 1) {
+ argp_usage(_state);
+ }
+ }
+
+ return 0;
+}
+
+void
+get_absolute_path(char directory[], char *arg)
+{
+ char cwd[4096] = {
+ 0,
+ };
+
+ if (getcwd(cwd, sizeof(cwd)) == NULL)
+ printf("some error in getting cwd\n");
+
+ if (strcmp(arg, ".") != 0) {
+ if (cwd[strlen(cwd)] != '/')
+ cwd[strlen(cwd)] = '/';
+ strcat(cwd, arg);
+ }
+ strcpy(directory, cwd);
+}
+
+static struct argp argp = {
+ arequal_options, arequal_parse_opts, "",
+ "arequal - Tool which calculates the checksum of all the entries"
+ "present in a given directory"};
+
+/* All this runs in single thread, hence using 'global' variables */
+
+unsigned long long avg_uid_file = 0;
+unsigned long long avg_uid_dir = 0;
+unsigned long long avg_uid_symlink = 0;
+unsigned long long avg_uid_other = 0;
+
+unsigned long long avg_gid_file = 0;
+unsigned long long avg_gid_dir = 0;
+unsigned long long avg_gid_symlink = 0;
+unsigned long long avg_gid_other = 0;
+
+unsigned long long avg_mode_file = 0;
+unsigned long long avg_mode_dir = 0;
+unsigned long long avg_mode_symlink = 0;
+unsigned long long avg_mode_other = 0;
+
+unsigned long long global_ctime_checksum = 0;
+
+unsigned long long count_dir = 0;
+unsigned long long count_file = 0;
+unsigned long long count_symlink = 0;
+unsigned long long count_other = 0;
+
+unsigned long long checksum_file1 = 0;
+unsigned long long checksum_file2 = 0;
+unsigned long long checksum_dir = 0;
+unsigned long long checksum_symlink = 0;
+unsigned long long checksum_other = 0;
+
+unsigned long long
+checksum_path(const char *path)
+{
+ unsigned long long csum = 0;
+ unsigned long long *nums = 0;
+ int len = 0;
+ int cnt = 0;
+
+ len = roof(strlen(path), sizeof(csum));
+ cnt = len / sizeof(csum);
+
+ nums = __builtin_alloca(len);
+ memset(nums, 0, len);
+ strcpy((char *)nums, path);
+
+ while (cnt) {
+ csum ^= *nums;
+ nums++;
+ cnt--;
+ }
+
+ return csum;
+}
+
+int
+checksum_md5(const char *path, const struct stat *sb)
+{
+ uint64_t this_data_checksum = 0;
+ FILE *filep = NULL;
+ char *cmd = NULL;
+ char strvalue[17] = {
+ 0,
+ };
+ int ret = -1;
+ int len = 0;
+ const char *pos = NULL;
+ char *cpos = NULL;
+
+ /* Have to escape single-quotes in filename.
+ * First, calculate the size of the buffer I'll need.
+ */
+ for (pos = path; *pos; pos++) {
+ if (*pos == '\'')
+ len += 4;
+ else
+ len += 1;
+ }
+
+ cmd = malloc(sizeof(char) * (len + 20));
+ cmd[0] = '\0';
+
+ /* Now, build the command with single quotes escaped. */
+
+ cpos = cmd;
+#if defined(linux)
+ strcpy(cpos, "md5sum '");
+ cpos += 8;
+#elif defined(__NetBSD__)
+ strcpy(cpos, "md5 -n '");
+ cpos += 8;
+#elif defined(__FreeBSD__) || defined(__APPLE__)
+ strcpy(cpos, "md5 -q '");
+ cpos += 8;
+#else
+#error "Please add system-specific md5 command"
+#endif
+
+ /* Add the file path, with every single quotes replaced with this sequence:
+ * '\''
+ */
+
+ for (pos = path; *pos; pos++) {
+ if (*pos == '\'') {
+ strcpy(cpos, "'\\''");
+ cpos += 4;
+ } else {
+ *cpos = *pos;
+ cpos++;
+ }
+ }
+
+ /* Add on the trailing single-quote and null-terminate. */
+ strcpy(cpos, "'");
+
+ filep = popen(cmd, "r");
+ if (!filep) {
+ perror(path);
+ goto out;
+ }
+
+ if (fread(strvalue, sizeof(char), 16, filep) != 16) {
+ fprintf(stderr, "%s: short read\n", path);
+ goto out;
+ }
+
+ this_data_checksum = strtoull(strvalue, NULL, 16);
+ if (-1 == this_data_checksum) {
+ fprintf(stderr, "%s: %s\n", strvalue, strerror(errno));
+ goto out;
+ }
+ checksum_file1 ^= this_data_checksum;
+
+ if (fread(strvalue, sizeof(char), 16, filep) != 16) {
+ fprintf(stderr, "%s: short read\n", path);
+ goto out;
+ }
+
+ this_data_checksum = strtoull(strvalue, NULL, 16);
+ if (-1 == this_data_checksum) {
+ fprintf(stderr, "%s: %s\n", strvalue, strerror(errno));
+ goto out;
+ }
+ checksum_file2 ^= this_data_checksum;
+
+ ret = 0;
+out:
+ if (filep)
+ pclose(filep);
+
+ if (cmd)
+ free(cmd);
+
+ return ret;
+}
+
+int
+checksum_filenames(const char *path, const struct stat *sb)
+{
+ DIR *dirp = NULL;
+ struct dirent *entry = NULL;
+ unsigned long long csum = 0;
+ int i = 0;
+ int found = 0;
+
+ dirp = opendir(path);
+ if (!dirp) {
+ perror(path);
+ goto out;
+ }
+
+ errno = 0;
+ while ((entry = readdir(dirp))) {
+ /* do not calculate the checksum of the entries which user has
+ told to ignore and proceed to other siblings.*/
+ if (arequal_config.ignored_directory) {
+ for (i = 0; i < arequal_config.directories_ignored; i++) {
+ if ((strcmp(entry->d_name,
+ arequal_config.ignored_directory[i]) == 0)) {
+ found = 1;
+ DBG("ignoring the entry %s\n", entry->d_name);
+ break;
+ }
+ }
+ if (found == 1) {
+ found = 0;
+ continue;
+ }
+ }
+ csum = checksum_path(entry->d_name);
+ checksum_dir ^= csum;
+ }
+
+ if (errno) {
+ perror(path);
+ goto out;
+ }
+
+out:
+ if (dirp)
+ closedir(dirp);
+
+ return 0;
+}
+
+int
+process_file(const char *path, const struct stat *sb)
+{
+ int ret = 0;
+
+ count_file++;
+
+ avg_uid_file ^= sb->st_uid;
+ avg_gid_file ^= sb->st_gid;
+ avg_mode_file ^= sb->st_mode;
+
+ ret = checksum_md5(path, sb);
+
+ return ret;
+}
+
+int
+process_dir(const char *path, const struct stat *sb)
+{
+ unsigned long long csum = 0;
+
+ count_dir++;
+
+ avg_uid_dir ^= sb->st_uid;
+ avg_gid_dir ^= sb->st_gid;
+ avg_mode_dir ^= sb->st_mode;
+
+ csum = checksum_filenames(path, sb);
+
+ checksum_dir ^= csum;
+
+ return 0;
+}
+
+int
+process_symlink(const char *path, const struct stat *sb)
+{
+ int ret = 0;
+ char buf[4096] = {
+ 0,
+ };
+ unsigned long long csum = 0;
+
+ count_symlink++;
+
+ avg_uid_symlink ^= sb->st_uid;
+ avg_gid_symlink ^= sb->st_gid;
+ avg_mode_symlink ^= sb->st_mode;
+
+ ret = readlink(path, buf, 4096);
+ if (ret < 0) {
+ perror(path);
+ goto out;
+ }
+
+ DBG("readlink (%s) => %s\n", path, buf);
+
+ csum = checksum_path(buf);
+
+ DBG("checksum_path (%s) => %llx\n", buf, csum);
+
+ checksum_symlink ^= csum;
+
+ ret = 0;
+out:
+ return ret;
+}
+
+int
+process_other(const char *path, const struct stat *sb)
+{
+ count_other++;
+
+ avg_uid_other ^= sb->st_uid;
+ avg_gid_other ^= sb->st_gid;
+ avg_mode_other ^= sb->st_mode;
+
+ checksum_other ^= sb->st_rdev;
+
+ return 0;
+}
+
+static int
+ignore_entry(const char *bname, const char *dname)
+{
+ int i;
+
+ for (i = 0; i < arequal_config.directories_ignored; i++) {
+ if (strcmp(bname, arequal_config.ignored_directory[i]) == 0 &&
+ strncmp(arequal_config.test_directory, dname,
+ strlen(arequal_config.test_directory)) == 0)
+ return 1;
+ }
+
+ return 0;
+}
+
+int
+process_entry(const char *path, const struct stat *sb, int typeflag,
+ struct FTW *ftwbuf)
+{
+ int ret = 0;
+ char *name = NULL;
+ char *bname = NULL;
+ char *dname = NULL;
+ int i = 0;
+
+ /* The if condition below helps in ignoring some directories in
+ the given path. If the name of the entry is one of the directory
+ names that the user told to ignore, then that directory will not
+ be processed and will return FTW_SKIP_SUBTREE to nftw which will
+ not crawl this directory and move on to other siblings.
+ Note that for nftw to recognize FTW_SKIP_TREE, FTW_ACTIONRETVAL
+ should be passed as an argument to nftw.
+
+ This mainly helps in calculating the checksum of network filesystems
+ (client-server), where the server might have some hidden directories
+ for managing the filesystem. So to calculate the sanity of filesystem
+ one has to get the checksum of the client and then the export directory
+ of server by telling arequal to ignore some of the directories which
+ are not part of the namespace.
+ */
+
+ if (arequal_config.ignored_directory) {
+#ifndef FTW_SKIP_SUBTREE
+ char *cp;
+
+ name = strdup(path);
+ dname = dirname(name);
+
+ for (cp = strtok(name, "/"); cp; cp = strtok(NULL, "/")) {
+ if (ignore_entry(cp, dname)) {
+ DBG("ignoring %s\n", path);
+ if (name)
+ free(name);
+ return 0;
+ }
+ }
+#else /* FTW_SKIP_SUBTREE */
+ name = strdup(path);
+
+ name[strlen(name)] = '\0';
+
+ bname = strrchr(name, '/');
+ if (bname)
+ bname++;
+
+ dname = dirname(name);
+ if (ignore_entry(bname, dname)) {
+ DBG("ignoring %s\n", bname);
+ ret = FTW_SKIP_SUBTREE;
+ if (name)
+ free(name);
+ return ret;
+ }
+#endif /* FTW_SKIP_SUBTREE */
+ }
+
+ DBG("processing entry %s\n", path);
+
+ switch ((S_IFMT & sb->st_mode)) {
+ case S_IFDIR:
+ ret = process_dir(path, sb);
+ break;
+ case S_IFREG:
+ ret = process_file(path, sb);
+ break;
+ case S_IFLNK:
+ ret = process_symlink(path, sb);
+ break;
+ default:
+ ret = process_other(path, sb);
+ break;
+ }
+
+ if (name)
+ free(name);
+ return ret;
+}
+
+int
+display_counts(FILE *fp)
+{
+ fprintf(fp, "\n");
+ fprintf(fp, "Entry counts\n");
+ fprintf(fp, "Regular files : %lld\n", count_file);
+ fprintf(fp, "Directories : %lld\n", count_dir);
+ fprintf(fp, "Symbolic links : %lld\n", count_symlink);
+ fprintf(fp, "Other : %lld\n", count_other);
+ fprintf(fp, "Total : %lld\n",
+ (count_file + count_dir + count_symlink + count_other));
+
+ return 0;
+}
+
+int
+display_checksums(FILE *fp)
+{
+ fprintf(fp, "\n");
+ fprintf(fp, "Checksums\n");
+ fprintf(fp, "Regular files : %llx%llx\n", checksum_file1, checksum_file2);
+ fprintf(fp, "Directories : %llx\n", checksum_dir);
+ fprintf(fp, "Symbolic links : %llx\n", checksum_symlink);
+ fprintf(fp, "Other : %llx\n", checksum_other);
+ fprintf(fp, "Total : %llx\n",
+ (checksum_file1 ^ checksum_file2 ^ checksum_dir ^ checksum_symlink ^
+ checksum_other));
+
+ return 0;
+}
+
+int
+display_metadata(FILE *fp)
+{
+ fprintf(fp, "\n");
+ fprintf(fp, "Metadata checksums\n");
+ fprintf(fp, "Regular files : %llx\n",
+ (avg_uid_file + 13) * (avg_gid_file + 11) * (avg_mode_file + 7));
+ fprintf(fp, "Directories : %llx\n",
+ (avg_uid_dir + 13) * (avg_gid_dir + 11) * (avg_mode_dir + 7));
+ fprintf(fp, "Symbolic links : %llx\n",
+ (avg_uid_symlink + 13) * (avg_gid_symlink + 11) *
+ (avg_mode_symlink + 7));
+ fprintf(fp, "Other : %llx\n",
+ (avg_uid_other + 13) * (avg_gid_other + 11) * (avg_mode_other + 7));
+
+ return 0;
+}
+
+int
+display_stats(FILE *fp)
+{
+ display_counts(fp);
+
+ display_metadata(fp);
+
+ display_checksums(fp);
+
+ return 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+ int ret = 0;
+ int i = 0;
+
+ ret = argp_parse(&argp, argc, argv, 0, 0, NULL);
+ if (ret != 0) {
+ fprintf(stderr, "parsing arguments failed\n");
+ return -2;
+ }
+
+ /* Use FTW_ACTIONRETVAL to take decision on what to do depending upon */
+ /* the return value of the callback function */
+ /* (process_entry in this case) */
+ ret = nftw(arequal_config.test_directory, process_entry, 30,
+ FTW_ACTIONRETVAL | FTW_PHYS | FTW_MOUNT);
+ if (ret != 0) {
+ fprintf(stderr, "ftw (%s) returned %d (%s), terminating\n", argv[1],
+ ret, strerror(errno));
+ return 1;
+ }
+
+ display_stats(stdout);
+
+ if (arequal_config.ignored_directory) {
+ for (i = 0; i < arequal_config.directories_ignored; i++) {
+ if (arequal_config.ignored_directory[i])
+ free(arequal_config.ignored_directory[i]);
+ }
+ free(arequal_config.ignored_directory);
+ }
+
+ return 0;
+}
diff --git a/tests/utils/changelog/changelog.h b/tests/utils/changelog/changelog.h
new file mode 100644
index 00000000000..1502b689eb4
--- /dev/null
+++ b/tests/utils/changelog/changelog.h
@@ -0,0 +1,125 @@
+/*
+ Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_H
+#define _GF_CHANGELOG_H
+
+struct gf_brick_spec;
+
+/**
+ * Max bit shiter for event selection
+ */
+#define CHANGELOG_EV_SELECTION_RANGE 5
+
+#define CHANGELOG_OP_TYPE_JOURNAL (1 << 0)
+#define CHANGELOG_OP_TYPE_OPEN (1 << 1)
+#define CHANGELOG_OP_TYPE_CREATE (1 << 2)
+#define CHANGELOG_OP_TYPE_RELEASE (1 << 3)
+#define CHANGELOG_OP_TYPE_BR_RELEASE \
+ (1 << 4) /* logical release (last close()), \
+ sent by bitrot stub */
+#define CHANGELOG_OP_TYPE_MAX (1 << CHANGELOG_EV_SELECTION_RANGE)
+
+struct ev_open {
+ unsigned char gfid[16];
+ int32_t flags;
+};
+
+struct ev_creat {
+ unsigned char gfid[16];
+ int32_t flags;
+};
+
+struct ev_release {
+ unsigned char gfid[16];
+};
+
+struct ev_release_br {
+ unsigned long version;
+ unsigned char gfid[16];
+ int32_t sign_info;
+};
+
+struct ev_changelog {
+ char path[PATH_MAX];
+};
+
+typedef struct changelog_event {
+ unsigned int ev_type;
+
+ union {
+ struct ev_open open;
+ struct ev_creat create;
+ struct ev_release release;
+ struct ev_changelog journal;
+ struct ev_release_br releasebr;
+ } u;
+} changelog_event_t;
+
+#define CHANGELOG_EV_SIZE (sizeof(changelog_event_t))
+
+/**
+ * event callback, connected & disconnection defs
+ */
+typedef void(CALLBACK)(void *, char *, void *, changelog_event_t *);
+typedef void *(INIT)(void *, struct gf_brick_spec *);
+typedef void(FINI)(void *, char *, void *);
+typedef void(CONNECT)(void *, char *, void *);
+typedef void(DISCONNECT)(void *, char *, void *);
+
+struct gf_brick_spec {
+ char *brick_path;
+ unsigned int filter;
+
+ INIT *init;
+ FINI *fini;
+ CALLBACK *callback;
+ CONNECT *connected;
+ DISCONNECT *disconnected;
+
+ void *ptr;
+};
+
+/* API set */
+
+int
+gf_changelog_register(char *brick_path, char *scratch_dir, char *log_file,
+ int log_levl, int max_reconnects);
+ssize_t
+gf_changelog_scan();
+
+int
+gf_changelog_start_fresh();
+
+ssize_t
+gf_changelog_next_change(char *bufptr, size_t maxlen);
+
+int
+gf_changelog_done(char *file);
+
+/* newer flexible API */
+int
+gf_changelog_init(void *xl);
+
+int
+gf_changelog_register_generic(struct gf_brick_spec *bricks, int count,
+ int ordered, char *logfile, int lvl, void *xl);
+
+int
+gf_history_changelog(char *changelog_dir, unsigned long start,
+ unsigned long end, int n_parallel,
+ unsigned long *actual_end);
+int
+gf_history_changelog_scan();
+ssize_t
+gf_history_changelog_next_change(char *bufptr, size_t maxlen);
+int
+gf_history_changelog_done(char *file);
+#endif
diff --git a/tests/utils/changelog/get-history.c b/tests/utils/changelog/get-history.c
new file mode 100644
index 00000000000..9963ab76958
--- /dev/null
+++ b/tests/utils/changelog/get-history.c
@@ -0,0 +1,71 @@
+/*
+ Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 10 seconds (just print the file names)
+ *
+ * Compile it using:
+ * gcc -o gethistory `pkg-config --cflags libgfchangelog` get-history.c \
+ * `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+
+#include "changelog.h"
+
+int
+main(int argc, char **argv)
+{
+ int ret = 0;
+ unsigned long end_ts = 0;
+ int start = 0;
+ int end = 0;
+
+ ret = gf_changelog_init(NULL);
+ if (ret) {
+ printf("-1");
+ fflush(stdout);
+ return -1;
+ }
+
+ ret = gf_changelog_register("/d/backends/patchy0", "/tmp/scratch_v1",
+ "/var/log/glusterfs/changes.log", 9, 5);
+ if (ret) {
+ printf("-2");
+ fflush(stdout);
+ return -1;
+ }
+
+ start = atoi(argv[1]);
+ end = atoi(argv[2]);
+
+ ret = gf_history_changelog("/d/backends/patchy0/.glusterfs/changelogs",
+ start, end, 3, &end_ts);
+ if (ret < 0) {
+ printf("-3");
+ fflush(stdout);
+ return -1;
+ } else if (ret == 1) {
+ printf("1");
+ fflush(stdout);
+ return 0;
+ }
+
+out:
+ printf("0");
+ fflush(stdout);
+ return 0;
+}
diff --git a/tests/utils/changelog/test-changelog-api.c b/tests/utils/changelog/test-changelog-api.c
new file mode 100644
index 00000000000..f4eb066b630
--- /dev/null
+++ b/tests/utils/changelog/test-changelog-api.c
@@ -0,0 +1,98 @@
+/*
+ Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 5 seconds (just print the file names)
+ *
+ * Compile it using:
+ * gcc -o getchanges `pkg-config --cflags libgfchangelog` get-changes.c \
+ * `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <errno.h>
+
+#include "changelog.h"
+
+int
+main(int argc, char **argv)
+{
+ int i = 0;
+ int ret = 0;
+ ssize_t nr_changes = 0;
+ ssize_t changes = 0;
+ char fbuf[PATH_MAX] = {
+ 0,
+ };
+
+ ret = gf_changelog_init(NULL);
+ if (ret) {
+ printf("-1");
+ fflush(stdout);
+ return -1;
+ }
+
+ /* get changes for brick "/d/backends/patchy0" */
+ ret = gf_changelog_register("/d/backends/patchy0", "/tmp/scratch_v1",
+ "/var/log/glusterfs/changes.log", 9, 5);
+ if (ret) {
+ printf("-2");
+ fflush(stdout);
+ return -1;
+ }
+
+ while (1) {
+ i = 0;
+ nr_changes = gf_changelog_scan();
+ if (nr_changes < 0) {
+ printf("-4");
+ fflush(stdout);
+ return -1;
+ }
+
+ if (nr_changes == 0)
+ goto next;
+
+ while ((changes = gf_changelog_next_change(fbuf, PATH_MAX)) > 0) {
+ /* process changelog */
+ /* ... */
+ /* ... */
+ /* ... */
+ /* done processing */
+
+ ret = gf_changelog_done(fbuf);
+ if (ret) {
+ printf("-5");
+ fflush(stdout);
+ return -1;
+ }
+ }
+
+ if (changes == -1) {
+ printf("-6");
+ fflush(stdout);
+ return -1;
+ }
+
+ next:
+ sleep(2);
+ }
+
+out:
+ printf("0");
+ fflush(stdout);
+ return ret;
+}
diff --git a/tests/utils/changelog/test-history-api.c b/tests/utils/changelog/test-history-api.c
new file mode 100644
index 00000000000..d78e387df10
--- /dev/null
+++ b/tests/utils/changelog/test-history-api.c
@@ -0,0 +1,111 @@
+/*
+ Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 10 seconds (just print the file names)
+ *
+ * Compile it using:
+ * gcc -o gethistory `pkg-config --cflags libgfchangelog` get-history.c \
+ * `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+
+#include "changelog.h"
+
+int
+main(int argc, char **argv)
+{
+ int ret = 0;
+ int i = 0;
+ unsigned long end_ts = 0;
+ ssize_t nr_changes = 0;
+ ssize_t changes = 0;
+ int start = 0;
+ int end = 0;
+ char fbuf[PATH_MAX] = {
+ 0,
+ };
+
+ ret = gf_changelog_init(NULL);
+ if (ret) {
+ printf("-1");
+ fflush(stdout);
+ return -1;
+ }
+
+ ret = gf_changelog_register("/d/backends/patchy0", "/tmp/scratch_v1",
+ "/var/log/glusterfs/changes.log", 9, 5);
+ if (ret) {
+ printf("-2");
+ fflush(stdout);
+ return -1;
+ }
+
+ start = atoi(argv[1]);
+ end = atoi(argv[2]);
+
+ ret = gf_history_changelog("/d/backends/patchy0/.glusterfs/changelogs",
+ start, end, 3, &end_ts);
+ if (ret < 0) {
+ printf("-3");
+ fflush(stdout);
+ return -1;
+ } else if (ret == 1) {
+ printf("1");
+ fflush(stdout);
+ return 0;
+ }
+
+ while (1) {
+ nr_changes = gf_history_changelog_scan();
+ if (nr_changes < 0) {
+ printf("-4");
+ fflush(stdout);
+ return -1;
+ }
+
+ if (nr_changes == 0) {
+ goto out;
+ }
+
+ while ((changes = gf_history_changelog_next_change(fbuf, PATH_MAX)) >
+ 0) {
+ /* process changelog */
+ /* ... */
+ /* ... */
+ /* ... */
+ /* done processing */
+
+ ret = gf_history_changelog_done(fbuf);
+ if (ret) {
+ printf("-5");
+ fflush(stdout);
+ return -1;
+ }
+ }
+ if (changes == -1) {
+ printf("-6");
+ fflush(stdout);
+ return -1;
+ }
+ }
+
+out:
+ printf("0");
+ fflush(stdout);
+ return 0;
+}
diff --git a/tests/utils/changelogparser.py b/tests/utils/changelogparser.py
new file mode 100644
index 00000000000..3b8f81d1bad
--- /dev/null
+++ b/tests/utils/changelogparser.py
@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+"""
+Why?
+
+Converts this
+
+GlusterFS Changelog | version: v1.1 | encoding : 2
+E0b99ef11-4b79-4cd0-9730-b5a0e8c4a8c0^@4^@16877^@0^@0^@00000000-0000-0000-0000-
+000000000001/dir1^@Ec5250af6-720e-4bfe-b938-827614304f39^@23^@33188^@0^@0^@0b99
+ef11-4b79-4cd0-9730-b5a0e8c4a8c0/hello.txt^@Dc5250af6-720e-4bfe-b938-827614304f
+39^@Dc5250af6-720e-4bfe-b938-827614304f39^@
+
+
+to human readable :)
+
+E 0b99ef11-4b79-4cd0-9730-b5a0e8c4a8c0 MKDIR 16877 0 000000000-0000-0000-0000
+ -000000000001/dir1
+E c5250af6-720e-4bfe-b938-827614304f39 CREATE 33188 0 0 0b99ef11-4b79-4cd0-9730
+ -b5a0e8c4a8c0/hello.txt
+D c5250af6-720e-4bfe-b938-827614304f39
+D c5250af6-720e-4bfe-b938-827614304f39
+
+
+"""
+import sys
+import codecs
+
+ENTRY = 'E'
+META = 'M'
+DATA = 'D'
+SEP = "\x00"
+
+GF_FOP = [
+ "NULL", "STAT", "READLINK", "MKNOD", "MKDIR", "UNLINK",
+ "RMDIR", "SYMLINK", "RENAME", "LINK", "TRUNCATE", "OPEN",
+ "READ", "WRITE", "STATFS", "FLUSH", "FSYNC", "SETXATTR",
+ "GETXATTR", "REMOVEXATTR", "OPENDIR", "FSYNCDIR", "ACCESS",
+ "CREATE", "FTRUNCATE", "FSTAT", "LK", "LOOKUP", "READDIR",
+ "INODELK", "FINODELK", "ENTRYLK", "FENTRYLK", "XATTROP",
+ "FXATTROP", "FSETXATTR", "FGETXATTR", "RCHECKSUM", "SETATTR",
+ "FSETATTR", "READDIRP", "GETSPEC", "FORGET", "RELEASE",
+ "RELEASEDIR", "FREMOVEXATTR", "FALLOCATE", "DISCARD", "ZEROFILL"]
+
+
+class NumTokens_V11(object):
+ E = 7
+ M = 3
+ D = 2
+ NULL = 3
+ MKNOD = 7
+ MKDIR = 7
+ UNLINK = 4
+ RMDIR = 4
+ SYMLINK = 4
+ RENAME = 5
+ LINK = 4
+ SETXATTR = 3
+ REMOVEXATTR = 3
+ CREATE = 7
+ SETATTR = 3
+ FTRUNCATE = 3
+ FXATTROP = 3
+
+
+class NumTokens_V12(NumTokens_V11):
+ UNLINK = 5
+ RMDIR = 5
+
+
+class Version:
+ V11 = "v1.1"
+ V12 = "v1.2"
+
+
+class Record(object):
+ def __init__(self, **kwargs):
+ self.ts = kwargs.get("ts", None)
+ self.fop_type = kwargs.get("fop_type", None)
+ self.gfid = kwargs.get("gfid", None)
+ self.path = kwargs.get("path", None)
+ self.fop = kwargs.get("fop", None)
+ self.path1 = kwargs.get("path1", None)
+ self.path2 = kwargs.get("path2", None)
+ self.mode = kwargs.get("mode", None)
+ self.uid = kwargs.get("uid", None)
+ self.gid = kwargs.get("gid", None)
+
+ def create_mknod_mkdir(self, **kwargs):
+ self.path = kwargs.get("path", None)
+ self.fop = kwargs.get("fop", None)
+ self.mode = kwargs.get("mode", None)
+ self.uid = kwargs.get("uid", None)
+ self.gid = kwargs.get("gid", None)
+
+ def metadata(self, **kwargs):
+ self.fop = kwargs.get("fop", None)
+
+ def rename(self, **kwargs):
+ self.fop = kwargs.get("fop", None)
+ self.path1 = kwargs.get("path1", None)
+ self.path2 = kwargs.get("path2", None)
+
+ def link_symlink_unlink_rmdir(self, **kwargs):
+ self.path = kwargs.get("path", None)
+ self.fop = kwargs.get("fop", None)
+
+ def __unicode__(self):
+ if self.fop_type == "D":
+ return u"{ts} {fop_type} {gfid}".format(**self.__dict__)
+ elif self.fop_type == "M":
+ return u"{ts} {fop_type} {gfid} {fop}".format(**self.__dict__)
+ elif self.fop_type == "E":
+ if self.fop in ["CREATE", "MKNOD", "MKDIR"]:
+ return (u"{ts} {fop_type} {gfid} {fop} "
+ u"{path} {mode} {uid} {gid}".format(**self.__dict__))
+ elif self.fop == "RENAME":
+ return (u"{ts} {fop_type} {gfid} {fop} "
+ u"{path1} {path2}".format(**self.__dict__))
+ elif self.fop in ["LINK", "SYMLINK", "UNLINK", "RMDIR"]:
+ return (u"{ts} {fop_type} {gfid} {fop} "
+ u"{path}".format(**self.__dict__))
+ else:
+ return repr(self.__dict__)
+ else:
+ return repr(self.__dict__)
+
+ def __str__(self):
+ if sys.version_info >= (3,):
+ return self.__unicode__()
+ else:
+ return unicode(self).encode('utf-8')
+
+
+def get_num_tokens(data, tokens, version=Version.V11):
+ if version == Version.V11:
+ cls_numtokens = NumTokens_V11
+ elif version == Version.V12:
+ cls_numtokens = NumTokens_V12
+ else:
+ sys.stderr.write("Unknown Changelog Version\n")
+ sys.exit(1)
+
+ if data[tokens[0]] in [ENTRY, META]:
+ if len(tokens) >= 3:
+ return getattr(cls_numtokens, GF_FOP[int(data[tokens[2]])])
+ else:
+ return None
+ else:
+ return getattr(cls_numtokens, data[tokens[0]])
+
+
+def process_record(data, tokens, changelog_ts, callback):
+ if data[tokens[0]] in [ENTRY, META]:
+ try:
+ tokens[2] = GF_FOP[int(data[tokens[2]])]
+ except ValueError:
+ tokens[2] = "NULL"
+
+ if not changelog_ts:
+ ts1 = int(changelog_ts)
+ else:
+ ts1=""
+ record = Record(ts=ts1, fop_type=data[tokens[0]],
+ gfid=data[tokens[1]])
+ if data[tokens[0]] == META:
+ record.metadata(fop=tokens[2])
+ elif data[tokens[0]] == ENTRY:
+ if tokens[2] in ["CREATE", "MKNOD", "MKDIR"]:
+ record.create_mknod_mkdir(fop=tokens[2],
+ path=data[tokens[6]],
+ mode=int(data[tokens[3]]),
+ uid=int(data[tokens[4]]),
+ gid=int(data[tokens[5]]))
+ elif tokens[2] == "RENAME":
+ record.rename(fop=tokens[2],
+ path1=data[tokens[3]],
+ path2=data[tokens[4]])
+ if tokens[2] in ["LINK", "SYMLINK", "UNLINK", "RMDIR"]:
+ record.link_symlink_unlink_rmdir(fop=tokens[2],
+ path=data[tokens[3]])
+ callback(record)
+
+
+def default_callback(record):
+ sys.stdout.write(u"{0}\n".format(record))
+
+
+def parse(filename, callback=default_callback):
+ data = None
+ tokens = []
+ changelog_ts = filename.rsplit(".")[-1]
+ with codecs.open(filename, mode="rb", encoding="utf-8") as f:
+ # GlusterFS Changelog | version: v1.1 | encoding : 2
+ header = f.readline()
+ version = header.split()[4]
+
+ data = f.readline()
+
+ slice_start = 0
+ in_record = False
+
+ prev_char = ""
+ next_char = ""
+ for i, c in enumerate(data):
+ next_char = ""
+ if len(data) >= (i + 2):
+ next_char = data[i+1]
+
+ if not in_record and c in [ENTRY, META, DATA]:
+ tokens.append(slice(slice_start, i+1))
+ slice_start = i+1
+ in_record = True
+ continue
+
+ if c == SEP and ((prev_char != SEP and next_char == SEP) or
+ (prev_char == SEP and next_char != SEP) or
+ (prev_char != SEP and next_char != SEP)):
+ tokens.append(slice(slice_start, i))
+ slice_start = i+1
+
+ num_tokens = get_num_tokens(data, tokens, version)
+
+ if num_tokens == len(tokens):
+ process_record(data, tokens, changelog_ts, callback)
+ in_record = False
+ tokens = []
+
+ prev_char = c
+
+ # process last record
+ if slice_start < (len(data) - 1):
+ tokens.append(slice(slice_start, len(data)))
+ process_record(data, tokens, changelog_ts, callback)
+ tokens = []
+
+parse(sys.argv[1])
diff --git a/tests/utils/create-files.py b/tests/utils/create-files.py
index 0d937eff978..04736e9c73b 100755
--- a/tests/utils/create-files.py
+++ b/tests/utils/create-files.py
@@ -1,111 +1,320 @@
-#!/usr/bin/python
# This script was developed by Vijaykumar Koppad (vkoppad@redhat.com)
# The latest version of this script can found at
# http://github.com/vijaykumar-koppad/crefi
from __future__ import with_statement
-import sys
import os
import re
-import random
-from optparse import OptionParser
+import sys
import time
-import string
import errno
+import xattr
+import string
+import random
+import logging
+import tarfile
+import argparse
+
+datsiz = 0
+timr = 0
+
+def get_ascii_upper_alpha_digits():
+ if sys.version_info > (3,0):
+ return string.ascii_uppercase+string.digits
+ else:
+ return string.uppercase+string.digits
+
+def setLogger(filename):
+ global logger
+ logger = logging.getLogger(filename)
+ logger.setLevel(logging.DEBUG)
+ return
+
+
+def setupLogger(filename):
+ logger = logging.getLogger(filename)
+ logger.setLevel(logging.DEBUG)
+ formatter = logging.Formatter('%(asctime)s - %(message)s')
+ ch = logging.StreamHandler()
+ ch.setLevel(logging.INFO)
+ ch.setFormatter(formatter)
+ logger.addHandler(ch)
+ return logger
+
def os_rd(src, size):
- fd = os.open(src,os.O_RDONLY)
+ global datsiz
+ fd = os.open(src, os.O_RDONLY)
data = os.read(fd, size)
os.close(fd)
+ datsiz = datsiz + size
return data
+
def os_wr(dest, data):
- fd = os.open(dest,os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0644)
+ global timr
+ st = time.time()
+ fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
os.write(fd, data)
os.close(fd)
+ ed = time.time()
+ timr = timr+(ed-st)
return
-def create_sparse_file(fil):
- if option.size:
- option.random = False
- size = option.size
+
+def create_sparse_file(fil, size, mins, maxs, rand):
+ if rand:
+ size = random.randint(mins, maxs)
else:
- size = random.randint(option.min, option.max)
+ size = size
data = os_rd("/dev/zero", size)
os_wr(fil, data)
return
-def create_binary_file(fil):
- if option.size:
- option.random = False
- size = option.size
+
+def create_binary_file(fil, size, mins, maxs, rand):
+ if rand:
+ size = random.randint(mins, maxs)
else:
- size = random.randint(option.min, option.max)
+ size = size
data = os_rd("/dev/urandom", size)
os_wr(fil, data)
return
-def create_txt_file(fil):
- if option.size:
- option.random = False
- size = option.size
- else:
- size = random.randint(option.min, option.max)
+
+def create_txt_file(fil, size, mins, maxs, rand):
+ if rand:
+ size = random.randint(mins, maxs)
if size < 500*1024:
data = os_rd("/etc/services", size)
os_wr(fil, data)
else:
- data = os_rd("/etc/services", 500*1024)
+ data = os_rd("/etc/services", 512*1024)
file_size = 0
- fd = os.open(fil,os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0644)
+ fd = os.open(fil, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
while file_size < size:
os.write(fd, data)
file_size += 500*1024
os.close(fd)
return
-def get_filename():
- size = option.flen
- char = string.uppercase+string.digits
+
+def create_tar_file(fil, size, mins, maxs, rand):
+ if rand:
+ size = random.randint(mins, maxs)
+ else:
+ size = size
+ data = os_rd("/dev/urandom", size)
+ os_wr(fil, data)
+ tar = tarfile.open(fil+".tar.gz", "w:gz")
+ tar.add(fil)
+ tar.close()
+ os.unlink(fil)
+ return
+
+
+def get_filename(flen):
+ size = flen
+ char = get_ascii_upper_alpha_digits()
st = ''.join(random.choice(char) for i in range(size))
ti = str((hex(int(str(time.time()).split('.')[0])))[2:])
- return ti+"~~"+st
+ return ti+"%%"+st
+
+
+def text_files(files, file_count, inter, size, mins, maxs, rand,
+ flen, randname, dir_path):
+ global datsiz, timr
+ for k in range(files):
+ if not file_count % inter:
+ logger.info("Total files created -- "+str(file_count))
+ if not randname:
+ fil = dir_path+"/"+"file"+str(k)
+ else:
+ fil = dir_path+"/"+get_filename(flen)
+ create_txt_file(fil, size, mins, maxs, rand)
+ file_count += 1
+ return file_count
+
-def text_files(files, file_count):
+def sparse_files(files, file_count, inter, size, mins, maxs,
+ rand, flen, randname, dir_path):
for k in range(files):
- if not file_count%option.inter:
- print file_count
- fil = get_filename()
- create_txt_file(fil)
+ if not file_count % inter:
+ logger.info("Total files created -- "+str(file_count))
+ if not randname:
+ fil = dir_path+"/"+"file"+str(k)
+ else:
+ fil = dir_path+"/"+get_filename(flen)
+ create_sparse_file(fil, size, mins, maxs, rand)
file_count += 1
return file_count
-def sparse_files(files, file_count):
+
+def binary_files(files, file_count, inter, size, mins, maxs,
+ rand, flen, randname, dir_path):
for k in range(files):
- if not file_count%option.inter:
- print file_count
- fil = get_filename()
- create_sparse_file(fil)
+ if not file_count % inter:
+ logger.info("Total files created -- "+str(file_count))
+ if not randname:
+ fil = dir_path+"/"+"file"+str(k)
+ else:
+ fil = dir_path+"/"+get_filename(flen)
+ create_binary_file(fil, size, mins, maxs, rand)
file_count += 1
return file_count
-def binary_files(files, file_count):
+
+def tar_files(files, file_count, inter, size, mins, maxs,
+ rand, flen, randname, dir_path):
for k in range(files):
- if not file_count%option.inter:
- print file_count
- fil = get_filename()
- create_binary_file(fil)
+ if not file_count % inter:
+ logger.info("Total files created -- "+str(file_count))
+ if not randname:
+ fil = dir_path+"/"+"file"+str(k)
+ else:
+ fil = dir_path+"/"+get_filename(flen)
+ create_tar_file(fil, size, mins, maxs, rand)
file_count += 1
return file_count
+
+def setxattr_files(files, randname, dir_path):
+ char = get_ascii_upper_alpha_digits()
+ if not randname:
+ for k in range(files):
+ v = ''.join(random.choice(char) for i in range(10))
+ n = "user."+v
+ xattr.setxattr(dir_path+"/"+"file"+str(k), n, v)
+ else:
+ dirs = os.listdir(dir_path+"/")
+ for fil in dirs:
+ v = ''.join(random.choice(char) for i in range(10))
+ n = "user."+v
+ xattr.setxattr(dir_path+"/"+fil, n, v)
+ return
+
+
+def rename_files(files, flen, randname, dir_path):
+ if not randname:
+ for k in range(files):
+ os.rename(dir_path + "/" + "file" + str(k),
+ dir_path + "/" + "file" + str(files+k))
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ if not os.path.isdir(fil):
+ newfil = get_filename(flen)
+ os.rename(dir_path + "/" + fil,
+ dir_path + "/" + newfil)
+ return
+
+
+def truncate_files(files, mins, maxs, randname, dir_path):
+ if not randname:
+ for k in range(files):
+ byts = random.randint(mins, maxs)
+ fd = os.open(dir_path + "/" + "file" + str(k), os.O_WRONLY)
+ os.ftruncate(fd, byts)
+ os.close(fd)
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ if not os.path.isdir(dir_path+"/"+fil):
+ byts = random.randint(mins, maxs)
+ fd = os.open(dir_path+"/"+fil, os.O_WRONLY)
+ os.ftruncate(fd, byts)
+ os.close(fd)
+ return
+
+
+def chmod_files(files, flen, randname, dir_path):
+ if not randname:
+ for k in range(files):
+ mod = random.randint(0, 511)
+ os.chmod(dir_path+"/"+"file"+str(k), mod)
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ mod = random.randint(0, 511)
+ os.chmod(dir_path+"/"+fil, mod)
+ return
+
+def random_og(path):
+ u = random.randint(1025, 65536)
+ g = -1
+ os.chown(path, u, g)
+
+def chown_files(files, flen, randname, dir_path):
+ if not randname:
+ for k in range(files):
+ random_og(dir_path+"/"+"file"+str(k))
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ random_og(dir_path+"/"+fil)
+ return
+
+
+def chgrp_files(files, flen, randname, dir_path):
+ if not randname:
+ for k in range(files):
+ random_og(dir_path+"/"+"file"+str(k))
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ random_og(dir_path+"/"+fil)
+ return
+
+
+def symlink_files(files, flen, randname, dir_path):
+ try:
+ os.makedirs(dir_path+"/"+"symlink_to_files")
+ except OSError as ex:
+ if ex.errno is not errno.EEXIST:
+ raise
+ if not randname:
+ for k in range(files):
+ src_file = "file"+str(k)
+ os.symlink(dir_path+"/"+src_file,
+ dir_path+"/"+"symlink_to_files/file"+str(k)+"_sym")
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ newfil = get_filename(flen)
+ os.symlink(dir_path+"/"+fil,
+ dir_path+"/"+"symlink_to_files/"+newfil)
+ return
+
+
+def hardlink_files(files, flen, randname, dir_path):
+ try:
+ os.makedirs(dir_path+"/"+"hardlink_to_files")
+ except OSError as ex:
+ if ex.errno is not errno.EEXIST:
+ raise
+ if not randname:
+ for k in range(files):
+ src_file = "file"+str(k)
+ os.link(dir_path+"/"+src_file,
+ dir_path+"/"+"hardlink_to_files/file"+str(k)+"_hard")
+ else:
+ dirs = os.listdir(dir_path)
+ for fil in dirs:
+ if not os.path.isdir(dir_path+"/"+fil):
+ newfil = get_filename(flen)
+ os.link(dir_path+"/"+fil,
+ dir_path+"/"+"hardlink_to_files/"+newfil)
+ return
+
+
def human2bytes(size):
size_short = {
- 1024 : ['K','KB','KiB','k','kB','kiB'],
- 1024*1024 : ['M','MB','MiB'],
- 1024*1024*1024 : ['G','GB','GiB']
-}
- num = re.search('(\d+)',size).group()
+ 1024: ['K', 'KB', 'KiB', 'k', 'kB', 'kiB'],
+ 1024*1024: ['M', 'MB', 'MiB'],
+ 1024*1024*1024: ['G', 'GB', 'GiB']
+ }
+ num = re.search('(\d+)', size).group()
ext = size[len(num):]
num = int(num)
if ext == '':
@@ -115,93 +324,270 @@ def human2bytes(size):
size = num*value
return size
-def multipledir(mnt_pnt,brdth,depth,files):
+
+def bytes2human(byts):
+ abbr = {
+ 1 << 30: "GB",
+ 1 << 20: "MB",
+ 1 << 10: "KB",
+ 1: "bytes"
+ }
+ if byts == 1:
+ return '1 bytes'
+ for factor, suffix in abbr.items():
+ if byts >= factor:
+ break
+ return "%.3f %s" % (byts / factor, suffix)
+
+
+def multipledir(mnt_pnt, brdth, depth, files, fop, file_type="text",
+ inter="1000", size="100K", mins="10K", maxs="500K",
+ rand=False, l=10, randname=False):
files_count = 1
+ size = human2bytes(size)
+ maxs = human2bytes(maxs)
+ mins = human2bytes(mins)
for i in range(brdth):
- breadth = mnt_pnt+"/"+str(i)
- try:
- os.makedirs(breadth)
- except OSError as ex:
- if not ex.errno is errno.EEXIST:
- raise
- os.chdir(breadth)
- dir_depth = breadth
- print breadth
+ dir_path = mnt_pnt
for j in range(depth):
- dir_depth = dir_depth+"/"+str(j)
+ dir_path = dir_path+"/"+"level"+str(j)+str(i)
try:
- os.makedirs(dir_depth)
+ os.makedirs(dir_path)
except OSError as ex:
- if not ex.errno is errno.EEXIST:
+ if ex.errno is not errno.EEXIST:
raise
- os.chdir(dir_depth)
- if option.file_type == "text":
- files_count = text_files(files, files_count)
- elif option.file_type == "sparse":
- files_count = sparse_files(files, files_count)
- elif option.file_type == "binary":
- files_count = binary_files(files, files_count)
- else:
- print "Not a valid file type"
- sys.exit(1)
-
-def singledir(mnt_pnt, files):
+
+ if fop == "create":
+ logger.info("Entering the directory level"+str(j)+str(i))
+ if file_type == "text":
+ files_count = text_files(files, files_count, inter, size,
+ mins, maxs, rand, l, randname,
+ dir_path)
+ elif file_type == "sparse":
+ files_count = sparse_files(files, files_count, inter, size,
+ mins, maxs, rand, l, randname,
+ dir_path)
+ elif file_type == "binary":
+ files_count = binary_files(files, files_count, inter, size,
+ mins, maxs, rand, l, randname,
+ dir_path)
+ elif file_type == "tar":
+ files_count = tar_files(files, files_count, inter, size,
+ mins, maxs, rand, l, randname,
+ dir_path)
+ else:
+ logger.error("Not a valid file type")
+ sys.exit(1)
+
+ elif fop == "rename":
+ logger.info("Started renaming files for the files 0 to " +
+ str(files)+" in the directory level"+str(j) +
+ str(i)+" ...")
+ rename_files(files, l, randname, dir_path)
+ logger.info("Finished renaming files for the files 0 to " +
+ str(files)+" in the directory level"+str(j)+str(i))
+
+ elif fop == "chmod":
+ logger.info("Started changing permission of files for the " +
+ "files 0 to "+str(files)+" in the directory level"
+ + str(j)+str(i)+" ...")
+ chmod_files(files, l, randname, dir_path)
+ logger.info("Finished changing permission of files for " +
+ "the files 0 to "+str(files) +
+ " in the directory level"+str(j)+str(i))
+
+ elif fop == "chown":
+ logger.info("Started changing ownership of files for the " +
+ "files 0 to " + str(files) +
+ " in the directory level"+str(j)+str(i)+" ...")
+ chown_files(files, l, randname, dir_path)
+ logger.info("Finished changing ownership of files for " +
+ "the files 0 to "+str(files) +
+ " in the directory level"+str(j)+str(i))
+
+ elif fop == "chgrp":
+ logger.info("Started changing group ownership of files for " +
+ "the files 0 to " + str(files) +
+ " in the directory level"+str(j)+str(i)+" ...")
+ chgrp_files(files, l, randname, dir_path)
+ logger.info("Finished changing group ownership of files for " +
+ "the files 0 to "+str(files) +
+ " in the directory level"+str(j)+str(i))
+
+ elif fop == "symlink":
+ logger.info("Started creating symlink to the files 0 to " +
+ str(files)+" in the directory level" +
+ str(j)+str(i)+"...")
+ symlink_files(files, l, randname, dir_path)
+ logger.info("Finished creating symlink to the files 0 to " +
+ str(files) + " in the directory level" +
+ str(j)+str(i))
+
+ elif fop == "hardlink":
+ logger.info("Started creating hardlink to the files 0 to " +
+ str(files)+" in the directory level" +
+ str(j)+str(i)+"...")
+ hardlink_files(files, l, randname, dir_path)
+ logger.info("Finished creating hardlink to the files 0 to " +
+ str(files) + " in the directory level" +
+ str(j)+str(i))
+
+ elif fop == "truncate":
+ logger.info("Started truncating the files 0 to " +
+ str(files)+" in the directory level" +
+ str(j)+str(i)+"...")
+ truncate_files(files, mins, maxs, randname, dir_path)
+ logger.info("Finished truncating the files 0 to " +
+ str(files)+" in the directory level" +
+ str(j)+str(i))
+
+ elif fop == "setxattr":
+ logger.info("Started setxattr to the files 0 to " +
+ str(files)+" in the directory level" +
+ str(j)+str(i)+"...")
+ setxattr_files(files, randname, dir_path)
+ logger.info("Finished setxattr to the files 0 to " +
+ str(files)+" in the directory level" +
+ str(j)+str(i))
+
+ if fop == "create":
+ thrpt = datsiz / timr
+ logger.info("finished creating files with throughput ---- " +
+ bytes2human(thrpt)+"ps")
+
+
+def singledir(mnt_pnt, files, fop, file_type="text", inter="1000", size="100K",
+ mins="10K", maxs="500K", rand=False, l=10, randname=False):
+
files_count = 1
- os.chdir(mnt_pnt)
- if option.file_type == "text":
- files_count = text_files(files, files_count)
- elif option.file_type == "sparse":
- files_count = sparse_files(files, files_count)
- elif option.file_type == "binary":
- files_count = binary_files(files, files_count)
- else:
- print "Not a valid file type"
- sys.exit(1)
+ size = human2bytes(size)
+ maxs = human2bytes(maxs)
+ mins = human2bytes(mins)
+ if fop == "create":
+ if file_type == "text":
+ files_count = text_files(files, files_count, inter, size, mins,
+ maxs, rand, l, randname, mnt_pnt)
+ elif file_type == "sparse":
+ files_count = sparse_files(files, files_count, inter, size, mins,
+ maxs, rand, l, randname, mnt_pnt)
+ elif file_type == "binary":
+ files_count = binary_files(files, files_count, inter, size, mins,
+ maxs, rand, l, randname, mnt_pnt)
+ elif file_type == "tar":
+ files_count = tar_files(files, files_count, inter, size, mins,
+ maxs, rand, l, randname, mnt_pnt)
+ else:
+ logger.info("Not a valid file type")
+ sys.exit(1)
+ thrpt = datsiz / timr
+ logger.info("finished creating files with avg throughput ---- " +
+ bytes2human(thrpt)+"ps")
+
+ elif fop == "rename":
+ logger.info("Started renaming files for the files 0 to " +
+ str(files) + "...")
+ rename_files(files, l, randname, mnt_pnt)
+ logger.info("Finished renaming files for the files 0 to "+str(files))
+
+ elif fop == "chmod":
+ logger.info("Started changing permission for the files 0 to " +
+ str(files)+" ...")
+ chmod_files(files, l, randname, mnt_pnt)
+ logger.info("Finished changing permission files for the files 0 to " +
+ str(files))
+
+ elif fop == "chown":
+ logger.info("Started changing ownership for the files 0 to " +
+ str(files)+"...")
+ chown_files(files, l, randname, mnt_pnt)
+ logger.info("Finished changing ownership for the files 0 to " +
+ str(files))
+
+ elif fop == "chgrp":
+ logger.info("Started changing group ownership for the files 0 to " +
+ str(files)+"...")
+ chgrp_files(files, l, randname, mnt_pnt)
+ logger.info("Finished changing group ownership for the files 0 to " +
+ str(files))
+
+ elif fop == "symlink":
+ logger.info("Started creating symlink to the files 0 to " +
+ str(files)+"...")
+ symlink_files(files, l, randname, mnt_pnt)
+ logger.info("Finished creating symlink to the files 0 to " +
+ str(files))
+
+ elif fop == "hardlink":
+ logger.info("Started creating hardlink to the files 0 to " +
+ str(files)+"...")
+ hardlink_files(files, l, randname, mnt_pnt)
+ logger.info("Finished creating hardlink to the files 0 to " +
+ str(files))
+
+ elif fop == "truncate":
+ logger.info("Started truncating the files 0 to " + str(files)+"...")
+ truncate_files(files, mins, maxs, randname, mnt_pnt)
+ logger.info("Finished truncating the files 0 to " + str(files))
+
+ elif fop == "setxattr":
+ logger.info("Started setxattr to the files 0 to " + str(files)+"...")
+ setxattr_files(files, randname, mnt_pnt)
+ logger.info("Finished setxattr to the files 0 to " + str(files))
+
if __name__ == '__main__':
usage = "usage: %prog [option] <MNT_PT>"
- parser = OptionParser(usage=usage)
- parser.add_option("-n", dest="files",type="int" ,default=100,
- help="number of files in each level [default: %default]")
- parser.add_option("--size", action = "store",type="string",
- help="size of the files to be used")
- parser.add_option("--random", action="store_true", default=True,
- help="random size of the file between --min and --max "
- "[default: %default]")
- parser.add_option("--max", action = "store",type="string", default="500K",
- help="maximum size of the files, if random is True "
- "[default: %default]")
- parser.add_option("--min", action = "store",type="string", default="10K",
- help="minimum size of the files, if random is True "
- "[default: %default]" )
- parser.add_option("--single", action="store_true", dest="dir",default=True,
- help="create files in single directory [default: %default]" )
- parser.add_option("--multi", action="store_false", dest="dir",
- help="create files in multiple directories")
- parser.add_option("-b", dest="brdth",type="int",default=5,
- help="number of directories in one level(works with --multi)[default: %default]")
- parser.add_option("-d", dest="depth",type="int",default=5,
- help="number of levels of directories(works with --multi)[default: %default]")
- parser.add_option("-l", dest="flen",type="int" ,default=10,
- help="number of bytes for filename "
- "[default: %default]")
- parser.add_option("-t","--type", action="store", type="string" , dest="file_type",default="text",
- help="type of the file to be created (text, sparse, binary) [default: %default]" )
- parser.add_option("-I", dest="inter", type="int", default=100,
- help="print number files created of interval [defailt: %dafault]")
- (option,args) = parser.parse_args()
- if not args:
- print "usage: <script> [option] <MNT_PT>"
- print ""
- sys.exit(1)
- args[0] = os.path.abspath(args[0])
- if option.size:
- option.size = human2bytes(option.size)
- else:
- option.max = human2bytes(option.max)
- option.min = human2bytes(option.min)
- if option.dir:
- singledir(args[0], option.files)
+ parser = argparse.ArgumentParser(formatter_class=argparse.
+ ArgumentDefaultsHelpFormatter)
+ parser.add_argument("-n", dest="files", type=int, default=100,
+ help="number of files in each level ")
+ parser.add_argument("--size", action="store", default="100k",
+ help="size of the files to be used ")
+ parser.add_argument("--random", action="store_true", default=False,
+ help="random size of the file between --min and --max")
+ parser.add_argument("--max", action="store", default="500K",
+ help="maximum size of the files, if random is True")
+ parser.add_argument("--min", action="store", default="10K",
+ help="minimum size of the files, if random is True")
+ parser.add_argument("--single", action="store_true", dest="dir",
+ default=True, help="create files in single directory")
+ parser.add_argument("--multi", action="store_false", dest="dir",
+ help="create files in multiple directories")
+ parser.add_argument("-b", dest="brdth", type=int, default=5,
+ help="number of directories in one level(works " +
+ "with --multi) ")
+ parser.add_argument("-d", dest="depth", type=int, default=5,
+ help="number of levels of directories (works " +
+ "with --multi) ")
+ parser.add_argument("-l", dest="flen", type=int, default=10,
+ help="number of bytes for filename ( Used only when " +
+ "randname is enabled) ")
+ parser.add_argument("-t", action="store", dest="file_type",
+ default="text", choices=["text", "sparse", "binary",
+ "tar"],
+ help="type of the file to be created ()")
+ parser.add_argument("-I", dest="inter", type=int, default=100,
+ help="print number files created of interval")
+ parser.add_argument("--fop", action="store", dest="fop", default="create",
+ choices=["create", "rename", "chmod", "chown", "chgrp",
+ "symlink", "hardlink", "truncate",
+ "setxattr"],
+ help="fop to be performed on the files")
+ parser.add_argument("-R", dest="randname", action="store_false",
+ default=True, help="To disable random file name " +
+ "(default: Enabled)")
+ parser.add_argument("mntpnt", help="Mount point")
+
+ args = parser.parse_args()
+ logger = setupLogger("testlost")
+ args.mntpnt = os.path.abspath(args.mntpnt)
+
+ if args.dir:
+ singledir(args.mntpnt, args.files, args.fop, args.file_type,
+ args.inter, args.size, args.min, args.max,
+ args.random, args.flen, args.randname)
else:
- multipledir(args[0], option.brdth, option.depth, option.files)
- print "creation of files completed.\n"
+ multipledir(args.mntpnt, args.brdth, args.depth, args.files,
+ args.fop, args.file_type, args.inter, args.size,
+ args.min, args.max, args.random, args.flen,
+ args.randname)
diff --git a/tests/utils/get-mdata-xattr.c b/tests/utils/get-mdata-xattr.c
new file mode 100644
index 00000000000..e9f54717263
--- /dev/null
+++ b/tests/utils/get-mdata-xattr.c
@@ -0,0 +1,152 @@
+/*
+ Copyright (c) 2019 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#include <stdlib.h>
+#include <endian.h>
+#include <stdio.h>
+#include <time.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/xattr.h>
+#include <errno.h>
+
+typedef struct gf_timespec_disk {
+ uint64_t tv_sec;
+ uint64_t tv_nsec;
+} gf_timespec_disk_t;
+
+/* posix_mdata_t on disk structure */
+typedef struct __attribute__((__packed__)) posix_mdata_disk {
+ /* version of structure, bumped up if any new member is added */
+ uint8_t version;
+ /* flags indicates valid fields in the structure */
+ uint64_t flags;
+ gf_timespec_disk_t ctime;
+ gf_timespec_disk_t mtime;
+ gf_timespec_disk_t atime;
+} posix_mdata_disk_t;
+
+/* In memory representation posix metadata xattr */
+typedef struct {
+ /* version of structure, bumped up if any new member is added */
+ uint8_t version;
+ /* flags indicates valid fields in the structure */
+ uint64_t flags;
+ struct timespec ctime;
+ struct timespec mtime;
+ struct timespec atime;
+} posix_mdata_t;
+
+#define GF_XATTR_MDATA_KEY "trusted.glusterfs.mdata"
+
+/* posix_mdata_from_disk converts posix_mdata_disk_t into host byte order
+ */
+static inline void
+posix_mdata_from_disk(posix_mdata_t *out, posix_mdata_disk_t *in)
+{
+ out->version = in->version;
+ out->flags = be64toh(in->flags);
+
+ out->ctime.tv_sec = be64toh(in->ctime.tv_sec);
+ out->ctime.tv_nsec = be64toh(in->ctime.tv_nsec);
+
+ out->mtime.tv_sec = be64toh(in->mtime.tv_sec);
+ out->mtime.tv_nsec = be64toh(in->mtime.tv_nsec);
+
+ out->atime.tv_sec = be64toh(in->atime.tv_sec);
+ out->atime.tv_nsec = be64toh(in->atime.tv_nsec);
+}
+
+/* posix_fetch_mdata_xattr fetches the posix_mdata_t from disk */
+static int
+posix_fetch_mdata_xattr(const char *real_path, posix_mdata_t *metadata)
+{
+ size_t size = -1;
+ char *value = NULL;
+ char gfid_str[64] = {0};
+
+ char *key = GF_XATTR_MDATA_KEY;
+
+ if (!metadata || !real_path) {
+ goto err;
+ }
+
+ /* Get size */
+ size = lgetxattr(real_path, key, NULL, 0);
+ if (size == -1) {
+ goto err;
+ }
+
+ value = calloc(size + 1, sizeof(char));
+ if (!value) {
+ goto err;
+ }
+
+ /* Get xattr value */
+ size = lgetxattr(real_path, key, value, size);
+ if (size == -1) {
+ goto err;
+ }
+ posix_mdata_from_disk(metadata, (posix_mdata_disk_t *)value);
+
+out:
+ if (value)
+ free(value);
+ return 0;
+err:
+ if (value)
+ free(value);
+ return -1;
+}
+
+int
+main(int argc, char *argv[])
+{
+ posix_mdata_t metadata;
+ uint64_t result;
+
+ if (argc != 3) {
+ /*
+ Usage: get_mdata_xattr -c|-m|-a <file-name>
+ where -c --> ctime
+ -m --> mtime
+ -a --> atime
+ */
+ printf("-1");
+ goto err;
+ }
+
+ if (posix_fetch_mdata_xattr(argv[2], &metadata)) {
+ printf("-1");
+ goto err;
+ }
+
+ switch (argv[1][1]) {
+ case 'c':
+ result = metadata.ctime.tv_sec;
+ break;
+ case 'm':
+ result = metadata.mtime.tv_sec;
+ break;
+ case 'a':
+ result = metadata.atime.tv_sec;
+ break;
+ default:
+ printf("-1");
+ goto err;
+ }
+ printf("%" PRIu64, result);
+ fflush(stdout);
+ return 0;
+err:
+ fflush(stdout);
+ return -1;
+}
diff --git a/tests/utils/getfattr.py b/tests/utils/getfattr.py
new file mode 100755
index 00000000000..3eb40e1c887
--- /dev/null
+++ b/tests/utils/getfattr.py
@@ -0,0 +1,133 @@
+
+from __future__ import print_function
+import os
+import sys
+from optparse import OptionParser
+
+import xattr
+
+def handle_textencoding(attr):
+ ### required for Python's handling of NULL strings.
+ attr_null_replace = (attr.encode('hex').decode('hex')).replace('\x00',
+ '\\000')
+ return attr_null_replace
+
+def getfattr(path, option):
+ attr = xattr.getxattr(path, option.name)
+ encoded_attr = attr
+
+ if option.encoding == "text":
+ ## special case handle it.
+ encoded_attr = handle_textencoding(attr)
+ else:
+ encoded_attr = attr.encode(option.encoding)
+
+ if option.onlyvalues:
+ print (encoded_attr)
+ return
+
+ print_getfattr (path, option, encoded_attr)
+ return
+
+def print_getfattr (path, option, encoded_attr=None):
+ if encoded_attr:
+ if option.encoding == "hex":
+ print(("%s=0x%s" % (option.name, encoded_attr)))
+ elif option.encoding == "base64":
+ print(("%s=0s%s" % (option.name, encoded_attr)))
+ else:
+ print(("%s=\"%s\"" % (option.name, encoded_attr)))
+ else:
+ print(option.name)
+
+ return
+
+def print_header (path, absnames):
+ if absnames:
+ print(("# file: %s" % path))
+ else:
+ print ("getfattr: Removing leading '/' from absolute path names")
+ print(("# file: %s" % path[1:]))
+
+if __name__ == '__main__':
+ usage = "usage: %prog [-n name|-d] [-e en] [-m pattern] path...."
+ parser = OptionParser(usage=usage)
+ parser.add_option("-n", action="store", dest="name", type="string",
+ help="Dump the value of the named extended attribute"
+ " extended attribute.")
+ parser.add_option("-d", action="store_true", dest="dump",
+ help="Dump the values of all extended attributes"
+ " associated with pathname.")
+ parser.add_option("-e", action="store", dest="encoding", type="string",
+ default="base64",
+ help="Encode values after retrieving"
+ " them. Valid values of [en] are `text`, `hex`,"
+ " and `base64`. Values encoded as text strings are"
+ " enclosed in double quotes (\"), while strings"
+ " encoded as hexadecimal and base64 are prefixed with"
+ " 0x and 0s, respectively.")
+ parser.add_option("-m", action="store", dest="pattern", type="string",
+ help="Only include attributes with names matching the"
+ " regular expression pattern. The default value for"
+ " pattern is \"^user\\.\", which includes all the"
+ " attributes in the user namespace. Specify \"-\" for"
+ " including all attributes. Refer to attr(5) for a more"
+ " detailed discussion of namespaces.")
+ parser.add_option("--absolute-names", action="store_true", dest="absnames",
+ help="Do not strip leading slash characters ('/')."
+ " The default behaviour is to strip leading slash characters.")
+ parser.add_option("--only-values", action="store_true", dest="onlyvalues",
+ help="Dump out the raw extended attribute value(s)"
+ " without encoding them.")
+
+ (option, args) = parser.parse_args()
+ if not args:
+ print ("Usage: getfattr [-hRLP] [-n name|-d] [-e en] [-m pattern]"
+ " path...")
+ print ("Try `getfattr --help' for more information.")
+ sys.exit(1)
+
+ if option.dump and option.name:
+ print ("-d and -n are mutually exclusive...")
+ sys.exit(1)
+
+ if option.pattern and option.name:
+ print ("-m and -n are mutually exclusive...")
+ sys.exit(1)
+
+ if option.encoding:
+ if (not (option.encoding.strip() == "hex" or
+ option.encoding.strip() == "base64" or
+ option.encoding.strip() == "text")):
+ print(("unrecognized encoding parameter... %s, please use"
+ " `text`, `base64` or `hex`" % option.encoding))
+ sys.exit(1)
+
+ args[0] = os.path.abspath(args[0])
+
+ if option.name:
+ print_header(args[0], option.absnames)
+ try:
+ getfattr(args[0], option)
+ except KeyError as err:
+ print(("Invalid key %s" % err))
+ sys.exit(1)
+ except IOError as err:
+ print (err)
+ sys.exit(1)
+
+ if option.pattern:
+ print_header(args[0], option.absnames)
+ try:
+ xattrs = xattr.listxattr(args[0])
+ for attr in xattrs:
+ if option.dump:
+ option.name = attr.encode('utf-8')
+ getfattr(args[0], option)
+ else:
+ option.name = attr.encode('utf-8')
+ print_getfattr(args[0], option, None)
+
+ except IOError as err:
+ print (err)
+ sys.exit(1)
diff --git a/tests/utils/gfid-access.py b/tests/utils/gfid-access.py
new file mode 100755
index 00000000000..c35c1223df6
--- /dev/null
+++ b/tests/utils/gfid-access.py
@@ -0,0 +1,124 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+from __future__ import print_function
+import os
+import sys
+import stat
+import time
+import struct
+import random
+import libcxattr
+
+from errno import EEXIST
+
+Xattr = libcxattr.Xattr()
+
+def umask():
+ return os.umask(0)
+
+def _fmt_mknod(l):
+ return "!II%dsI%dsIII" % (37, l+1)
+
+def _fmt_mkdir(l):
+ return "!II%dsI%dsII" % (37, l+1)
+
+def _fmt_symlink(l1, l2):
+ return "!II%dsI%ds%ds" % (37, l1+1, l2+1)
+
+
+if sys.version_info > (3,):
+ def entry_pack_reg(gf, bn, mo, uid, gid):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ return struct.pack(_fmt_mknod(blen),
+ uid, gid, gf.encode(), mo, bn_encoded,
+ stat.S_IMODE(mo), 0, umask())
+
+ # mkdir
+ def entry_pack_dir(gf, bn, mo, uid, gid):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ return struct.pack(_fmt_mkdir(blen),
+ uid, gid, gf.encode(), mo, bn_encoded,
+ stat.S_IMODE(mo), umask())
+ # symlink
+ def entry_pack_symlink(gf, bn, lnk, st):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ lnk_encoded = lnk.encode()
+ llen = len(lnk_encoded)
+ return struct.pack(_fmt_symlink(blen, llen),
+ st['uid'], st['gid'],
+ gf.encode(), st['mode'], bn_encoded,
+ lnk_encoded)
+
+else:
+ def entry_pack_reg(gf, bn, mo, uid, gid):
+ blen = len(bn)
+ return struct.pack(_fmt_mknod(blen),
+ uid, gid, gf, mo, bn,
+ stat.S_IMODE(mo), 0, umask())
+
+ def entry_pack_dir(gf, bn, mo, uid, gid):
+ blen = len(bn)
+ return struct.pack(_fmt_mkdir(blen),
+ uid, gid, gf, mo, bn,
+ stat.S_IMODE(mo), umask())
+
+ def entry_pack_symlink(gf, bn, lnk, mo, uid, gid):
+ blen = len(bn)
+ llen = len(lnk)
+ return struct.pack(_fmt_symlink(blen, llen),
+ uid, gid, gf, mo, bn, lnk)
+
+if __name__ == '__main__':
+ if len(sys.argv) < 9:
+ print(("USAGE: %s <mount> <pargfid|ROOT> <filename> <GFID> <file type>"
+ " <uid> <gid> <file permission(octal str)>" % (sys.argv[0])))
+ sys.exit(-1) # nothing to do
+ mtpt = sys.argv[1]
+ pargfid = sys.argv[2]
+ fname = sys.argv[3]
+ randomgfid = sys.argv[4]
+ ftype = sys.argv[5]
+ uid = int(sys.argv[6])
+ gid = int(sys.argv[7])
+ perm = int(sys.argv[8], 8)
+
+ os.chdir(mtpt)
+ if pargfid == 'ROOT':
+ pargfid = '.gfid/00000000-0000-0000-0000-000000000001'
+ else:
+ pargfid = '.gfid/' + pargfid
+
+ blob = None
+
+ # entry op: use non-zero uid/gid (to catch gfid-access xlator bugs)
+ if ftype == 'file':
+ mode = stat.S_IFREG | perm
+ blob = entry_pack_reg(randomgfid, fname, mode, uid, gid)
+ elif ftype =='dir':
+ mode = stat.S_IFDIR | perm
+ blob = entry_pack_dir(randomgfid, fname, mode, uid, gid)
+ else: # not yet...
+ sys.exit(-1)
+
+ if blob == None:
+ sys.exit(-1)
+ try:
+ Xattr.lsetxattr(pargfid, 'glusterfs.gfid.newfile', blob)
+ except OSError:
+ ex = sys.exc_info()[1]
+ if not ex.errno in [EEXIST]:
+ raise
+ sys.exit(-1)
+ print("File creation OK")
+ sys.exit(0)
diff --git a/tests/utils/libcxattr.py b/tests/utils/libcxattr.py
new file mode 100644
index 00000000000..3f3ed1fffbb
--- /dev/null
+++ b/tests/utils/libcxattr.py
@@ -0,0 +1,108 @@
+#
+# Copyright (c) 2011-2014 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+import os
+import sys
+from ctypes import CDLL, c_int
+from py2py3 import bytearray_to_str, gr_create_string_buffer
+from py2py3 import gr_query_xattr, gr_lsetxattr, gr_lremovexattr
+
+
+class Xattr(object):
+
+ """singleton that wraps the extended attributes system
+ interface for python using ctypes
+
+ Just implement it to the degree we need it, in particular
+ - we need just the l*xattr variants, ie. we never want symlinks to be
+ followed
+ - don't need size discovery for getxattr, as we always know the exact
+ sizes we expect
+ """
+
+ if sys.hexversion >= 0x02060000:
+ from ctypes import DEFAULT_MODE
+ libc = CDLL("libc.so.6", DEFAULT_MODE, None, True)
+ else:
+ libc = CDLL("libc.so.6")
+
+ @classmethod
+ def geterrno(cls):
+ if sys.hexversion >= 0x02060000:
+ from ctypes import get_errno
+ return get_errno()
+ # breaks on NetBSD
+ return c_int.in_dll(cls.libc, 'errno').value
+
+ @classmethod
+ def raise_oserr(cls):
+ errn = cls.geterrno()
+ raise OSError(errn, os.strerror(errn))
+
+ @classmethod
+ def _query_xattr(cls, path, siz, syscall, *a):
+ if siz:
+ buf = gr_create_string_buffer(siz)
+ else:
+ buf = None
+ ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
+ if ret == -1:
+ cls.raise_oserr()
+ if siz:
+ # py2 and py3 compatibility. Convert bytes array
+ # to string
+ result = bytearray_to_str(buf.raw)
+ return result[:ret]
+ else:
+ return ret
+
+ @classmethod
+ def lgetxattr(cls, path, attr, siz=0):
+ return gr_query_xattr(cls, path, siz, 'lgetxattr', attr)
+
+ @classmethod
+ def lgetxattr_buf(cls, path, attr):
+ """lgetxattr variant with size discovery"""
+ size = cls.lgetxattr(path, attr)
+ if size == -1:
+ cls.raise_oserr()
+ if size == 0:
+ return ''
+ return cls.lgetxattr(path, attr, size)
+
+ @classmethod
+ def llistxattr(cls, path, siz=0):
+ ret = gr_query_xattr(cls, path, siz, 'llistxattr')
+ if isinstance(ret, str):
+ ret = ret.strip('\0')
+ ret = ret.split('\0') if ret else []
+ return ret
+
+ @classmethod
+ def lsetxattr(cls, path, attr, val):
+ ret = gr_lsetxattr(cls, path, attr, val)
+ if ret == -1:
+ cls.raise_oserr()
+
+ @classmethod
+ def lremovexattr(cls, path, attr):
+ ret = gr_lremovexattr(cls, path, attr)
+ if ret == -1:
+ cls.raise_oserr()
+
+ @classmethod
+ def llistxattr_buf(cls, path):
+ """listxattr variant with size discovery"""
+ size = cls.llistxattr(path)
+ if size == -1:
+ cls.raise_oserr()
+ if size == 0:
+ return []
+ return cls.llistxattr(path, size)
diff --git a/tests/utils/pidof.py b/tests/utils/pidof.py
new file mode 100755
index 00000000000..4b7071c0a48
--- /dev/null
+++ b/tests/utils/pidof.py
@@ -0,0 +1,45 @@
+
+from __future__ import print_function
+import sys
+
+try:
+ import psutil
+except ImportError:
+ print("Please install psutil --> pip install psutil")
+ sys.exit(1)
+
+def pmap_find(p, name):
+ for m in p.memory_maps(grouped=True):
+ if m.path.endswith("%s.so" % name):
+ return True
+ continue
+ return False
+
+def pidof(processname):
+ for p in psutil.process_iter():
+ if p.pid == 0:
+ continue
+ if "gluster" in processname:
+ if processname == "glusterd" and pmap_find(p, "glusterd"):
+ print((p.pid))
+ if processname == "glusterfs" and pmap_find(p, "client"):
+ print((p.pid))
+ if processname == "glusterfsd" and pmap_find(p, "posix-acl"):
+ print((p.pid))
+ continue
+ if processname.strip() == p.name():
+ print((p.pid))
+
+def main(argv):
+ if len(argv) < 2:
+ sys.stderr.write("Usage: %s <processname>\n" % (argv[0],))
+ return 1
+ try:
+ pidof(argv[1])
+ except Exception as err:
+ print(err)
+ sys.stderr.write("Please be root - %s\n" % err);
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/tests/utils/py2py3.py b/tests/utils/py2py3.py
new file mode 100644
index 00000000000..63aca10fd26
--- /dev/null
+++ b/tests/utils/py2py3.py
@@ -0,0 +1,186 @@
+#
+# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+# This file is part of GlusterFS.
+
+# This file is licensed to you under your choice of the GNU Lesser
+# General Public License, version 3 or any later version (LGPLv3 or
+# later), or the GNU General Public License, version 2 (GPLv2), in all
+# cases as published by the Free Software Foundation.
+#
+
+# All python2/python3 compatibility routines
+
+import sys
+import os
+import stat
+import struct
+from ctypes import create_string_buffer
+
+def umask():
+ return os.umask(0)
+
+if sys.version_info >= (3,):
+ def pipe():
+ (r, w) = os.pipe()
+ os.set_inheritable(r, True)
+ os.set_inheritable(w, True)
+ return (r, w)
+
+ # Raw conversion of bytearray to string. Used in the cases where
+ # buffer is created by create_string_buffer which is a 8-bit char
+ # array and passed to syscalls to fetch results. Using encode/decode
+ # doesn't work as it converts to string altering the size.
+ def bytearray_to_str(byte_arr):
+ return ''.join([chr(b) for b in byte_arr])
+
+ # Raw conversion of string to bytes. This is required to convert
+ # back the string into bytearray(c char array) to use in struc
+ # pack/unpacking. Again encode/decode can't be used as it
+ # converts it alters size.
+ def str_to_bytearray(string):
+ return bytes([ord(c) for c in string])
+
+ def gr_create_string_buffer(size):
+ return create_string_buffer(b'\0', size)
+
+ def gr_query_xattr(cls, path, size, syscall, attr=None):
+ if attr:
+ return cls._query_xattr(path.encode(), size, syscall,
+ attr.encode())
+ else:
+ return cls._query_xattr(path.encode(), size, syscall)
+
+ def gr_lsetxattr(cls, path, attr, val):
+ return cls.libc.lsetxattr(path.encode(), attr.encode(), val,
+ len(val), 0)
+
+ def gr_lremovexattr(cls, path, attr):
+ return cls.libc.lremovexattr(path.encode(), attr.encode())
+
+ def gr_cl_register(cls, brick, path, log_file, log_level, retries):
+ return cls._get_api('gf_changelog_register')(brick.encode(),
+ path.encode(),
+ log_file.encode(),
+ log_level, retries)
+
+ def gr_cl_done(cls, clfile):
+ return cls._get_api('gf_changelog_done')(clfile.encode())
+
+ def gr_cl_history_changelog(cls, changelog_path, start, end, num_parallel,
+ actual_end):
+ return cls._get_api('gf_history_changelog')(changelog_path.encode(),
+ start, end, num_parallel,
+ actual_end)
+
+ def gr_cl_history_done(cls, clfile):
+ return cls._get_api('gf_history_changelog_done')(clfile.encode())
+
+ # regular file
+
+ def entry_pack_reg(cls, gf, bn, mo, uid, gid):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ return struct.pack(cls._fmt_mknod(blen),
+ uid, gid, gf.encode(), mo, bn_encoded,
+ stat.S_IMODE(mo), 0, umask())
+
+ def entry_pack_reg_stat(cls, gf, bn, st):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ mo = st['mode']
+ return struct.pack(cls._fmt_mknod(blen),
+ st['uid'], st['gid'],
+ gf.encode(), mo, bn_encoded,
+ stat.S_IMODE(mo), 0, umask())
+ # mkdir
+
+ def entry_pack_mkdir(cls, gf, bn, mo, uid, gid):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ return struct.pack(cls._fmt_mkdir(blen),
+ uid, gid, gf.encode(), mo, bn_encoded,
+ stat.S_IMODE(mo), umask())
+ # symlink
+
+ def entry_pack_symlink(cls, gf, bn, lnk, st):
+ bn_encoded = bn.encode()
+ blen = len(bn_encoded)
+ lnk_encoded = lnk.encode()
+ llen = len(lnk_encoded)
+ return struct.pack(cls._fmt_symlink(blen, llen),
+ st['uid'], st['gid'],
+ gf.encode(), st['mode'], bn_encoded,
+ lnk_encoded)
+else:
+ def pipe():
+ (r, w) = os.pipe()
+ return (r, w)
+
+ # Raw conversion of bytearray to string
+ def bytearray_to_str(byte_arr):
+ return byte_arr
+
+ # Raw conversion of string to bytearray
+ def str_to_bytearray(string):
+ return string
+
+ def gr_create_string_buffer(size):
+ return create_string_buffer('\0', size)
+
+ def gr_query_xattr(cls, path, size, syscall, attr=None):
+ if attr:
+ return cls._query_xattr(path, size, syscall, attr)
+ else:
+ return cls._query_xattr(path, size, syscall)
+
+ def gr_lsetxattr(cls, path, attr, val):
+ return cls.libc.lsetxattr(path, attr, val, len(val), 0)
+
+ def gr_lremovexattr(cls, path, attr):
+ return cls.libc.lremovexattr(path, attr)
+
+ def gr_cl_register(cls, brick, path, log_file, log_level, retries):
+ return cls._get_api('gf_changelog_register')(brick, path, log_file,
+ log_level, retries)
+
+ def gr_cl_done(cls, clfile):
+ return cls._get_api('gf_changelog_done')(clfile)
+
+ def gr_cl_history_changelog(cls, changelog_path, start, end, num_parallel,
+ actual_end):
+ return cls._get_api('gf_history_changelog')(changelog_path, start, end,
+ num_parallel, actual_end)
+
+ def gr_cl_history_done(cls, clfile):
+ return cls._get_api('gf_history_changelog_done')(clfile)
+
+ # regular file
+
+ def entry_pack_reg(cls, gf, bn, mo, uid, gid):
+ blen = len(bn)
+ return struct.pack(cls._fmt_mknod(blen),
+ uid, gid, gf, mo, bn,
+ stat.S_IMODE(mo), 0, umask())
+
+ def entry_pack_reg_stat(cls, gf, bn, st):
+ blen = len(bn)
+ mo = st['mode']
+ return struct.pack(cls._fmt_mknod(blen),
+ st['uid'], st['gid'],
+ gf, mo, bn,
+ stat.S_IMODE(mo), 0, umask())
+ # mkdir
+
+ def entry_pack_mkdir(cls, gf, bn, mo, uid, gid):
+ blen = len(bn)
+ return struct.pack(cls._fmt_mkdir(blen),
+ uid, gid, gf, mo, bn,
+ stat.S_IMODE(mo), umask())
+ # symlink
+
+ def entry_pack_symlink(cls, gf, bn, lnk, st):
+ blen = len(bn)
+ llen = len(lnk)
+ return struct.pack(cls._fmt_symlink(blen, llen),
+ st['uid'], st['gid'],
+ gf, st['mode'], bn, lnk)
diff --git a/tests/utils/setfattr.py b/tests/utils/setfattr.py
new file mode 100755
index 00000000000..8b7b6abacc0
--- /dev/null
+++ b/tests/utils/setfattr.py
@@ -0,0 +1,77 @@
+
+import os
+import sys
+from optparse import OptionParser
+
+import xattr
+
+def convert(string):
+ tmp_string = string
+ if (string[0] == '0' and
+ (string[1] == 's' or
+ string[1] == 'S')):
+ tmp_string = string.strip('%s%s' %
+ (string[0],
+ string[1]))
+ return tmp_string.decode('base64')
+
+ if (string[0] == '0' and
+ (string[1] == 'x' or
+ string[1] == 'X')):
+ tmp_string = string.split('%s%s' %
+ (string[0],
+ string[1]))
+ return tmp_string[1].decode('hex')
+
+ return tmp_string
+
+if __name__ == '__main__':
+ usage = "usage: %prog [-n name] [-v value] [-x name]"
+ parser = OptionParser(usage=usage)
+ parser.add_option("-n", action="store", dest="name", type="string",
+ help="Specifies the name of the extended attribute to set.")
+ parser.add_option("-v", action="store", dest="value", type="string",
+ help="Specifies the new value of the extended attribute."
+ " There are three methods available for encoding the value."
+ " If the given string is enclosed in double quotes, the"
+ " inner string is treated as text. In that case,"
+ " backslashes and double quotes have special meanings"
+ " and need to be escaped by a preceding backslash. Any"
+ " control characters can be encoded as a backslash"
+ " followed by three digits as its ASCII code in octal."
+ " If the given string begins with 0x or 0X, it expresses"
+ " a hexadecimal number. If the given string begins with"
+ " 0s or 0S, base64 encoding is expected.")
+ parser.add_option("-x", action="store", dest="xname", type="string",
+ help="Remove the named extended attribute entirely.")
+
+ (option, args) = parser.parse_args()
+ if not args:
+ print ("Usage: setfattr {-n name} [-v value] file...")
+ print (" setfattr {-x name} file...")
+ print ("Try `setfattr --help' for more information.")
+ sys.exit(1)
+
+ if option.name and option.xname:
+ print ("-n and -x are mutually exclusive...")
+ sys.exit(1)
+
+ if option.name:
+ if option.value is None:
+ print ("-n option requires -v value...")
+
+ args[0] = os.path.abspath(args[0])
+
+ if option.name and option.value:
+ try:
+ xattr.setxattr(args[0], option.name, convert(option.value))
+ except Exception as err:
+ print (err)
+ sys.exit(1)
+
+ if option.xname:
+ try:
+ xattr.removexattr(args[0], option.xname)
+ except Exception as err:
+ print (err)
+ sys.exit(1)
diff --git a/tests/utils/testn.sh b/tests/utils/testn.sh
new file mode 100755
index 00000000000..079351d8529
--- /dev/null
+++ b/tests/utils/testn.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# Use this script to identify the command and line-number of test-cases.
+#
+
+if [ -z "${1}" -a -z "${2}" ]
+then
+ echo "Usage: ${0} path/to/test/case.t testnumber"
+ exit 1
+elif [ -z "${2}" ]
+then
+ echo "ERROR: The second parameter to ${0} should be a number."
+ exit 2
+fi
+
+awk '{print FNR " " $0}' ${1} | egrep '^[[:digit:]]+[[:space:]]*(EXPECT|TEST|EXPECT_WITHIN|EXPECT_KEYWORD)' | sed -n ${2}p
generated by cgit (git 2.34.1) at 2025-12-28 17:45:48 +0000